1 /* 2 * Resizable virtual memory filesystem for Linux. 3 * 4 * Copyright (C) 2000 Linus Torvalds. 5 * 2000 Transmeta Corp. 6 * 2000-2001 Christoph Rohland 7 * 2000-2001 SAP AG 8 * 2002 Red Hat Inc. 9 * Copyright (C) 2002-2011 Hugh Dickins. 10 * Copyright (C) 2011 Google Inc. 11 * Copyright (C) 2002-2005 VERITAS Software Corporation. 12 * Copyright (C) 2004 Andi Kleen, SuSE Labs 13 * 14 * Extended attribute support for tmpfs: 15 * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net> 16 * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com> 17 * 18 * tiny-shmem: 19 * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com> 20 * 21 * This file is released under the GPL. 22 */ 23 24 #include <linux/fs.h> 25 #include <linux/init.h> 26 #include <linux/vfs.h> 27 #include <linux/mount.h> 28 #include <linux/ramfs.h> 29 #include <linux/pagemap.h> 30 #include <linux/file.h> 31 #include <linux/mm.h> 32 #include <linux/random.h> 33 #include <linux/sched/signal.h> 34 #include <linux/export.h> 35 #include <linux/swap.h> 36 #include <linux/uio.h> 37 #include <linux/khugepaged.h> 38 #include <linux/hugetlb.h> 39 #include <linux/frontswap.h> 40 #include <linux/fs_parser.h> 41 #include <linux/swapfile.h> 42 43 static struct vfsmount *shm_mnt; 44 45 #ifdef CONFIG_SHMEM 46 /* 47 * This virtual memory filesystem is heavily based on the ramfs. It 48 * extends ramfs by the ability to use swap and honor resource limits 49 * which makes it a completely usable filesystem. 50 */ 51 52 #include <linux/xattr.h> 53 #include <linux/exportfs.h> 54 #include <linux/posix_acl.h> 55 #include <linux/posix_acl_xattr.h> 56 #include <linux/mman.h> 57 #include <linux/string.h> 58 #include <linux/slab.h> 59 #include <linux/backing-dev.h> 60 #include <linux/shmem_fs.h> 61 #include <linux/writeback.h> 62 #include <linux/pagevec.h> 63 #include <linux/percpu_counter.h> 64 #include <linux/falloc.h> 65 #include <linux/splice.h> 66 #include <linux/security.h> 67 #include <linux/swapops.h> 68 #include <linux/mempolicy.h> 69 #include <linux/namei.h> 70 #include <linux/ctype.h> 71 #include <linux/migrate.h> 72 #include <linux/highmem.h> 73 #include <linux/seq_file.h> 74 #include <linux/magic.h> 75 #include <linux/syscalls.h> 76 #include <linux/fcntl.h> 77 #include <uapi/linux/memfd.h> 78 #include <linux/userfaultfd_k.h> 79 #include <linux/rmap.h> 80 #include <linux/uuid.h> 81 82 #include <linux/uaccess.h> 83 84 #include "internal.h" 85 86 #define BLOCKS_PER_PAGE (PAGE_SIZE/512) 87 #define VM_ACCT(size) (PAGE_ALIGN(size) >> PAGE_SHIFT) 88 89 /* Pretend that each entry is of this size in directory's i_size */ 90 #define BOGO_DIRENT_SIZE 20 91 92 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */ 93 #define SHORT_SYMLINK_LEN 128 94 95 /* 96 * shmem_fallocate communicates with shmem_fault or shmem_writepage via 97 * inode->i_private (with i_rwsem making sure that it has only one user at 98 * a time): we would prefer not to enlarge the shmem inode just for that. 99 */ 100 struct shmem_falloc { 101 wait_queue_head_t *waitq; /* faults into hole wait for punch to end */ 102 pgoff_t start; /* start of range currently being fallocated */ 103 pgoff_t next; /* the next page offset to be fallocated */ 104 pgoff_t nr_falloced; /* how many new pages have been fallocated */ 105 pgoff_t nr_unswapped; /* how often writepage refused to swap out */ 106 }; 107 108 struct shmem_options { 109 unsigned long long blocks; 110 unsigned long long inodes; 111 struct mempolicy *mpol; 112 kuid_t uid; 113 kgid_t gid; 114 umode_t mode; 115 bool full_inums; 116 int huge; 117 int seen; 118 #define SHMEM_SEEN_BLOCKS 1 119 #define SHMEM_SEEN_INODES 2 120 #define SHMEM_SEEN_HUGE 4 121 #define SHMEM_SEEN_INUMS 8 122 }; 123 124 #ifdef CONFIG_TMPFS 125 static unsigned long shmem_default_max_blocks(void) 126 { 127 return totalram_pages() / 2; 128 } 129 130 static unsigned long shmem_default_max_inodes(void) 131 { 132 unsigned long nr_pages = totalram_pages(); 133 134 return min(nr_pages - totalhigh_pages(), nr_pages / 2); 135 } 136 #endif 137 138 static int shmem_swapin_page(struct inode *inode, pgoff_t index, 139 struct page **pagep, enum sgp_type sgp, 140 gfp_t gfp, struct vm_area_struct *vma, 141 vm_fault_t *fault_type); 142 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 143 struct page **pagep, enum sgp_type sgp, 144 gfp_t gfp, struct vm_area_struct *vma, 145 struct vm_fault *vmf, vm_fault_t *fault_type); 146 147 int shmem_getpage(struct inode *inode, pgoff_t index, 148 struct page **pagep, enum sgp_type sgp) 149 { 150 return shmem_getpage_gfp(inode, index, pagep, sgp, 151 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL); 152 } 153 154 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) 155 { 156 return sb->s_fs_info; 157 } 158 159 /* 160 * shmem_file_setup pre-accounts the whole fixed size of a VM object, 161 * for shared memory and for shared anonymous (/dev/zero) mappings 162 * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1), 163 * consistent with the pre-accounting of private mappings ... 164 */ 165 static inline int shmem_acct_size(unsigned long flags, loff_t size) 166 { 167 return (flags & VM_NORESERVE) ? 168 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); 169 } 170 171 static inline void shmem_unacct_size(unsigned long flags, loff_t size) 172 { 173 if (!(flags & VM_NORESERVE)) 174 vm_unacct_memory(VM_ACCT(size)); 175 } 176 177 static inline int shmem_reacct_size(unsigned long flags, 178 loff_t oldsize, loff_t newsize) 179 { 180 if (!(flags & VM_NORESERVE)) { 181 if (VM_ACCT(newsize) > VM_ACCT(oldsize)) 182 return security_vm_enough_memory_mm(current->mm, 183 VM_ACCT(newsize) - VM_ACCT(oldsize)); 184 else if (VM_ACCT(newsize) < VM_ACCT(oldsize)) 185 vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize)); 186 } 187 return 0; 188 } 189 190 /* 191 * ... whereas tmpfs objects are accounted incrementally as 192 * pages are allocated, in order to allow large sparse files. 193 * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM, 194 * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM. 195 */ 196 static inline int shmem_acct_block(unsigned long flags, long pages) 197 { 198 if (!(flags & VM_NORESERVE)) 199 return 0; 200 201 return security_vm_enough_memory_mm(current->mm, 202 pages * VM_ACCT(PAGE_SIZE)); 203 } 204 205 static inline void shmem_unacct_blocks(unsigned long flags, long pages) 206 { 207 if (flags & VM_NORESERVE) 208 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); 209 } 210 211 static inline bool shmem_inode_acct_block(struct inode *inode, long pages) 212 { 213 struct shmem_inode_info *info = SHMEM_I(inode); 214 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 215 216 if (shmem_acct_block(info->flags, pages)) 217 return false; 218 219 if (sbinfo->max_blocks) { 220 if (percpu_counter_compare(&sbinfo->used_blocks, 221 sbinfo->max_blocks - pages) > 0) 222 goto unacct; 223 percpu_counter_add(&sbinfo->used_blocks, pages); 224 } 225 226 return true; 227 228 unacct: 229 shmem_unacct_blocks(info->flags, pages); 230 return false; 231 } 232 233 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages) 234 { 235 struct shmem_inode_info *info = SHMEM_I(inode); 236 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 237 238 if (sbinfo->max_blocks) 239 percpu_counter_sub(&sbinfo->used_blocks, pages); 240 shmem_unacct_blocks(info->flags, pages); 241 } 242 243 static const struct super_operations shmem_ops; 244 const struct address_space_operations shmem_aops; 245 static const struct file_operations shmem_file_operations; 246 static const struct inode_operations shmem_inode_operations; 247 static const struct inode_operations shmem_dir_inode_operations; 248 static const struct inode_operations shmem_special_inode_operations; 249 static const struct vm_operations_struct shmem_vm_ops; 250 static struct file_system_type shmem_fs_type; 251 252 bool vma_is_shmem(struct vm_area_struct *vma) 253 { 254 return vma->vm_ops == &shmem_vm_ops; 255 } 256 257 static LIST_HEAD(shmem_swaplist); 258 static DEFINE_MUTEX(shmem_swaplist_mutex); 259 260 /* 261 * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and 262 * produces a novel ino for the newly allocated inode. 263 * 264 * It may also be called when making a hard link to permit the space needed by 265 * each dentry. However, in that case, no new inode number is needed since that 266 * internally draws from another pool of inode numbers (currently global 267 * get_next_ino()). This case is indicated by passing NULL as inop. 268 */ 269 #define SHMEM_INO_BATCH 1024 270 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop) 271 { 272 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 273 ino_t ino; 274 275 if (!(sb->s_flags & SB_KERNMOUNT)) { 276 raw_spin_lock(&sbinfo->stat_lock); 277 if (sbinfo->max_inodes) { 278 if (!sbinfo->free_inodes) { 279 raw_spin_unlock(&sbinfo->stat_lock); 280 return -ENOSPC; 281 } 282 sbinfo->free_inodes--; 283 } 284 if (inop) { 285 ino = sbinfo->next_ino++; 286 if (unlikely(is_zero_ino(ino))) 287 ino = sbinfo->next_ino++; 288 if (unlikely(!sbinfo->full_inums && 289 ino > UINT_MAX)) { 290 /* 291 * Emulate get_next_ino uint wraparound for 292 * compatibility 293 */ 294 if (IS_ENABLED(CONFIG_64BIT)) 295 pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n", 296 __func__, MINOR(sb->s_dev)); 297 sbinfo->next_ino = 1; 298 ino = sbinfo->next_ino++; 299 } 300 *inop = ino; 301 } 302 raw_spin_unlock(&sbinfo->stat_lock); 303 } else if (inop) { 304 /* 305 * __shmem_file_setup, one of our callers, is lock-free: it 306 * doesn't hold stat_lock in shmem_reserve_inode since 307 * max_inodes is always 0, and is called from potentially 308 * unknown contexts. As such, use a per-cpu batched allocator 309 * which doesn't require the per-sb stat_lock unless we are at 310 * the batch boundary. 311 * 312 * We don't need to worry about inode{32,64} since SB_KERNMOUNT 313 * shmem mounts are not exposed to userspace, so we don't need 314 * to worry about things like glibc compatibility. 315 */ 316 ino_t *next_ino; 317 318 next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu()); 319 ino = *next_ino; 320 if (unlikely(ino % SHMEM_INO_BATCH == 0)) { 321 raw_spin_lock(&sbinfo->stat_lock); 322 ino = sbinfo->next_ino; 323 sbinfo->next_ino += SHMEM_INO_BATCH; 324 raw_spin_unlock(&sbinfo->stat_lock); 325 if (unlikely(is_zero_ino(ino))) 326 ino++; 327 } 328 *inop = ino; 329 *next_ino = ++ino; 330 put_cpu(); 331 } 332 333 return 0; 334 } 335 336 static void shmem_free_inode(struct super_block *sb) 337 { 338 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 339 if (sbinfo->max_inodes) { 340 raw_spin_lock(&sbinfo->stat_lock); 341 sbinfo->free_inodes++; 342 raw_spin_unlock(&sbinfo->stat_lock); 343 } 344 } 345 346 /** 347 * shmem_recalc_inode - recalculate the block usage of an inode 348 * @inode: inode to recalc 349 * 350 * We have to calculate the free blocks since the mm can drop 351 * undirtied hole pages behind our back. 352 * 353 * But normally info->alloced == inode->i_mapping->nrpages + info->swapped 354 * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped) 355 * 356 * It has to be called with the spinlock held. 357 */ 358 static void shmem_recalc_inode(struct inode *inode) 359 { 360 struct shmem_inode_info *info = SHMEM_I(inode); 361 long freed; 362 363 freed = info->alloced - info->swapped - inode->i_mapping->nrpages; 364 if (freed > 0) { 365 info->alloced -= freed; 366 inode->i_blocks -= freed * BLOCKS_PER_PAGE; 367 shmem_inode_unacct_blocks(inode, freed); 368 } 369 } 370 371 bool shmem_charge(struct inode *inode, long pages) 372 { 373 struct shmem_inode_info *info = SHMEM_I(inode); 374 unsigned long flags; 375 376 if (!shmem_inode_acct_block(inode, pages)) 377 return false; 378 379 /* nrpages adjustment first, then shmem_recalc_inode() when balanced */ 380 inode->i_mapping->nrpages += pages; 381 382 spin_lock_irqsave(&info->lock, flags); 383 info->alloced += pages; 384 inode->i_blocks += pages * BLOCKS_PER_PAGE; 385 shmem_recalc_inode(inode); 386 spin_unlock_irqrestore(&info->lock, flags); 387 388 return true; 389 } 390 391 void shmem_uncharge(struct inode *inode, long pages) 392 { 393 struct shmem_inode_info *info = SHMEM_I(inode); 394 unsigned long flags; 395 396 /* nrpages adjustment done by __delete_from_page_cache() or caller */ 397 398 spin_lock_irqsave(&info->lock, flags); 399 info->alloced -= pages; 400 inode->i_blocks -= pages * BLOCKS_PER_PAGE; 401 shmem_recalc_inode(inode); 402 spin_unlock_irqrestore(&info->lock, flags); 403 404 shmem_inode_unacct_blocks(inode, pages); 405 } 406 407 /* 408 * Replace item expected in xarray by a new item, while holding xa_lock. 409 */ 410 static int shmem_replace_entry(struct address_space *mapping, 411 pgoff_t index, void *expected, void *replacement) 412 { 413 XA_STATE(xas, &mapping->i_pages, index); 414 void *item; 415 416 VM_BUG_ON(!expected); 417 VM_BUG_ON(!replacement); 418 item = xas_load(&xas); 419 if (item != expected) 420 return -ENOENT; 421 xas_store(&xas, replacement); 422 return 0; 423 } 424 425 /* 426 * Sometimes, before we decide whether to proceed or to fail, we must check 427 * that an entry was not already brought back from swap by a racing thread. 428 * 429 * Checking page is not enough: by the time a SwapCache page is locked, it 430 * might be reused, and again be SwapCache, using the same swap as before. 431 */ 432 static bool shmem_confirm_swap(struct address_space *mapping, 433 pgoff_t index, swp_entry_t swap) 434 { 435 return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap); 436 } 437 438 /* 439 * Definitions for "huge tmpfs": tmpfs mounted with the huge= option 440 * 441 * SHMEM_HUGE_NEVER: 442 * disables huge pages for the mount; 443 * SHMEM_HUGE_ALWAYS: 444 * enables huge pages for the mount; 445 * SHMEM_HUGE_WITHIN_SIZE: 446 * only allocate huge pages if the page will be fully within i_size, 447 * also respect fadvise()/madvise() hints; 448 * SHMEM_HUGE_ADVISE: 449 * only allocate huge pages if requested with fadvise()/madvise(); 450 */ 451 452 #define SHMEM_HUGE_NEVER 0 453 #define SHMEM_HUGE_ALWAYS 1 454 #define SHMEM_HUGE_WITHIN_SIZE 2 455 #define SHMEM_HUGE_ADVISE 3 456 457 /* 458 * Special values. 459 * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled: 460 * 461 * SHMEM_HUGE_DENY: 462 * disables huge on shm_mnt and all mounts, for emergency use; 463 * SHMEM_HUGE_FORCE: 464 * enables huge on shm_mnt and all mounts, w/o needing option, for testing; 465 * 466 */ 467 #define SHMEM_HUGE_DENY (-1) 468 #define SHMEM_HUGE_FORCE (-2) 469 470 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 471 /* ifdef here to avoid bloating shmem.o when not necessary */ 472 473 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER; 474 475 bool shmem_is_huge(struct vm_area_struct *vma, 476 struct inode *inode, pgoff_t index) 477 { 478 loff_t i_size; 479 480 if (shmem_huge == SHMEM_HUGE_DENY) 481 return false; 482 if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) || 483 test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))) 484 return false; 485 if (shmem_huge == SHMEM_HUGE_FORCE) 486 return true; 487 488 switch (SHMEM_SB(inode->i_sb)->huge) { 489 case SHMEM_HUGE_ALWAYS: 490 return true; 491 case SHMEM_HUGE_WITHIN_SIZE: 492 index = round_up(index + 1, HPAGE_PMD_NR); 493 i_size = round_up(i_size_read(inode), PAGE_SIZE); 494 if (i_size >> PAGE_SHIFT >= index) 495 return true; 496 fallthrough; 497 case SHMEM_HUGE_ADVISE: 498 if (vma && (vma->vm_flags & VM_HUGEPAGE)) 499 return true; 500 fallthrough; 501 default: 502 return false; 503 } 504 } 505 506 #if defined(CONFIG_SYSFS) 507 static int shmem_parse_huge(const char *str) 508 { 509 if (!strcmp(str, "never")) 510 return SHMEM_HUGE_NEVER; 511 if (!strcmp(str, "always")) 512 return SHMEM_HUGE_ALWAYS; 513 if (!strcmp(str, "within_size")) 514 return SHMEM_HUGE_WITHIN_SIZE; 515 if (!strcmp(str, "advise")) 516 return SHMEM_HUGE_ADVISE; 517 if (!strcmp(str, "deny")) 518 return SHMEM_HUGE_DENY; 519 if (!strcmp(str, "force")) 520 return SHMEM_HUGE_FORCE; 521 return -EINVAL; 522 } 523 #endif 524 525 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS) 526 static const char *shmem_format_huge(int huge) 527 { 528 switch (huge) { 529 case SHMEM_HUGE_NEVER: 530 return "never"; 531 case SHMEM_HUGE_ALWAYS: 532 return "always"; 533 case SHMEM_HUGE_WITHIN_SIZE: 534 return "within_size"; 535 case SHMEM_HUGE_ADVISE: 536 return "advise"; 537 case SHMEM_HUGE_DENY: 538 return "deny"; 539 case SHMEM_HUGE_FORCE: 540 return "force"; 541 default: 542 VM_BUG_ON(1); 543 return "bad_val"; 544 } 545 } 546 #endif 547 548 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 549 struct shrink_control *sc, unsigned long nr_to_split) 550 { 551 LIST_HEAD(list), *pos, *next; 552 LIST_HEAD(to_remove); 553 struct inode *inode; 554 struct shmem_inode_info *info; 555 struct page *page; 556 unsigned long batch = sc ? sc->nr_to_scan : 128; 557 int removed = 0, split = 0; 558 559 if (list_empty(&sbinfo->shrinklist)) 560 return SHRINK_STOP; 561 562 spin_lock(&sbinfo->shrinklist_lock); 563 list_for_each_safe(pos, next, &sbinfo->shrinklist) { 564 info = list_entry(pos, struct shmem_inode_info, shrinklist); 565 566 /* pin the inode */ 567 inode = igrab(&info->vfs_inode); 568 569 /* inode is about to be evicted */ 570 if (!inode) { 571 list_del_init(&info->shrinklist); 572 removed++; 573 goto next; 574 } 575 576 /* Check if there's anything to gain */ 577 if (round_up(inode->i_size, PAGE_SIZE) == 578 round_up(inode->i_size, HPAGE_PMD_SIZE)) { 579 list_move(&info->shrinklist, &to_remove); 580 removed++; 581 goto next; 582 } 583 584 list_move(&info->shrinklist, &list); 585 next: 586 if (!--batch) 587 break; 588 } 589 spin_unlock(&sbinfo->shrinklist_lock); 590 591 list_for_each_safe(pos, next, &to_remove) { 592 info = list_entry(pos, struct shmem_inode_info, shrinklist); 593 inode = &info->vfs_inode; 594 list_del_init(&info->shrinklist); 595 iput(inode); 596 } 597 598 list_for_each_safe(pos, next, &list) { 599 int ret; 600 601 info = list_entry(pos, struct shmem_inode_info, shrinklist); 602 inode = &info->vfs_inode; 603 604 if (nr_to_split && split >= nr_to_split) 605 goto leave; 606 607 page = find_get_page(inode->i_mapping, 608 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT); 609 if (!page) 610 goto drop; 611 612 /* No huge page at the end of the file: nothing to split */ 613 if (!PageTransHuge(page)) { 614 put_page(page); 615 goto drop; 616 } 617 618 /* 619 * Leave the inode on the list if we failed to lock 620 * the page at this time. 621 * 622 * Waiting for the lock may lead to deadlock in the 623 * reclaim path. 624 */ 625 if (!trylock_page(page)) { 626 put_page(page); 627 goto leave; 628 } 629 630 ret = split_huge_page(page); 631 unlock_page(page); 632 put_page(page); 633 634 /* If split failed leave the inode on the list */ 635 if (ret) 636 goto leave; 637 638 split++; 639 drop: 640 list_del_init(&info->shrinklist); 641 removed++; 642 leave: 643 iput(inode); 644 } 645 646 spin_lock(&sbinfo->shrinklist_lock); 647 list_splice_tail(&list, &sbinfo->shrinklist); 648 sbinfo->shrinklist_len -= removed; 649 spin_unlock(&sbinfo->shrinklist_lock); 650 651 return split; 652 } 653 654 static long shmem_unused_huge_scan(struct super_block *sb, 655 struct shrink_control *sc) 656 { 657 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 658 659 if (!READ_ONCE(sbinfo->shrinklist_len)) 660 return SHRINK_STOP; 661 662 return shmem_unused_huge_shrink(sbinfo, sc, 0); 663 } 664 665 static long shmem_unused_huge_count(struct super_block *sb, 666 struct shrink_control *sc) 667 { 668 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 669 return READ_ONCE(sbinfo->shrinklist_len); 670 } 671 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */ 672 673 #define shmem_huge SHMEM_HUGE_DENY 674 675 bool shmem_is_huge(struct vm_area_struct *vma, 676 struct inode *inode, pgoff_t index) 677 { 678 return false; 679 } 680 681 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo, 682 struct shrink_control *sc, unsigned long nr_to_split) 683 { 684 return 0; 685 } 686 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 687 688 /* 689 * Like add_to_page_cache_locked, but error if expected item has gone. 690 */ 691 static int shmem_add_to_page_cache(struct page *page, 692 struct address_space *mapping, 693 pgoff_t index, void *expected, gfp_t gfp, 694 struct mm_struct *charge_mm) 695 { 696 XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page)); 697 unsigned long i = 0; 698 unsigned long nr = compound_nr(page); 699 int error; 700 701 VM_BUG_ON_PAGE(PageTail(page), page); 702 VM_BUG_ON_PAGE(index != round_down(index, nr), page); 703 VM_BUG_ON_PAGE(!PageLocked(page), page); 704 VM_BUG_ON_PAGE(!PageSwapBacked(page), page); 705 VM_BUG_ON(expected && PageTransHuge(page)); 706 707 page_ref_add(page, nr); 708 page->mapping = mapping; 709 page->index = index; 710 711 if (!PageSwapCache(page)) { 712 error = mem_cgroup_charge(page, charge_mm, gfp); 713 if (error) { 714 if (PageTransHuge(page)) { 715 count_vm_event(THP_FILE_FALLBACK); 716 count_vm_event(THP_FILE_FALLBACK_CHARGE); 717 } 718 goto error; 719 } 720 } 721 cgroup_throttle_swaprate(page, gfp); 722 723 do { 724 void *entry; 725 xas_lock_irq(&xas); 726 entry = xas_find_conflict(&xas); 727 if (entry != expected) 728 xas_set_err(&xas, -EEXIST); 729 xas_create_range(&xas); 730 if (xas_error(&xas)) 731 goto unlock; 732 next: 733 xas_store(&xas, page); 734 if (++i < nr) { 735 xas_next(&xas); 736 goto next; 737 } 738 if (PageTransHuge(page)) { 739 count_vm_event(THP_FILE_ALLOC); 740 __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr); 741 } 742 mapping->nrpages += nr; 743 __mod_lruvec_page_state(page, NR_FILE_PAGES, nr); 744 __mod_lruvec_page_state(page, NR_SHMEM, nr); 745 unlock: 746 xas_unlock_irq(&xas); 747 } while (xas_nomem(&xas, gfp)); 748 749 if (xas_error(&xas)) { 750 error = xas_error(&xas); 751 goto error; 752 } 753 754 return 0; 755 error: 756 page->mapping = NULL; 757 page_ref_sub(page, nr); 758 return error; 759 } 760 761 /* 762 * Like delete_from_page_cache, but substitutes swap for page. 763 */ 764 static void shmem_delete_from_page_cache(struct page *page, void *radswap) 765 { 766 struct address_space *mapping = page->mapping; 767 int error; 768 769 VM_BUG_ON_PAGE(PageCompound(page), page); 770 771 xa_lock_irq(&mapping->i_pages); 772 error = shmem_replace_entry(mapping, page->index, page, radswap); 773 page->mapping = NULL; 774 mapping->nrpages--; 775 __dec_lruvec_page_state(page, NR_FILE_PAGES); 776 __dec_lruvec_page_state(page, NR_SHMEM); 777 xa_unlock_irq(&mapping->i_pages); 778 put_page(page); 779 BUG_ON(error); 780 } 781 782 /* 783 * Remove swap entry from page cache, free the swap and its page cache. 784 */ 785 static int shmem_free_swap(struct address_space *mapping, 786 pgoff_t index, void *radswap) 787 { 788 void *old; 789 790 old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0); 791 if (old != radswap) 792 return -ENOENT; 793 free_swap_and_cache(radix_to_swp_entry(radswap)); 794 return 0; 795 } 796 797 /* 798 * Determine (in bytes) how many of the shmem object's pages mapped by the 799 * given offsets are swapped out. 800 * 801 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 802 * as long as the inode doesn't go away and racy results are not a problem. 803 */ 804 unsigned long shmem_partial_swap_usage(struct address_space *mapping, 805 pgoff_t start, pgoff_t end) 806 { 807 XA_STATE(xas, &mapping->i_pages, start); 808 struct page *page; 809 unsigned long swapped = 0; 810 811 rcu_read_lock(); 812 xas_for_each(&xas, page, end - 1) { 813 if (xas_retry(&xas, page)) 814 continue; 815 if (xa_is_value(page)) 816 swapped++; 817 818 if (need_resched()) { 819 xas_pause(&xas); 820 cond_resched_rcu(); 821 } 822 } 823 824 rcu_read_unlock(); 825 826 return swapped << PAGE_SHIFT; 827 } 828 829 /* 830 * Determine (in bytes) how many of the shmem object's pages mapped by the 831 * given vma is swapped out. 832 * 833 * This is safe to call without i_rwsem or the i_pages lock thanks to RCU, 834 * as long as the inode doesn't go away and racy results are not a problem. 835 */ 836 unsigned long shmem_swap_usage(struct vm_area_struct *vma) 837 { 838 struct inode *inode = file_inode(vma->vm_file); 839 struct shmem_inode_info *info = SHMEM_I(inode); 840 struct address_space *mapping = inode->i_mapping; 841 unsigned long swapped; 842 843 /* Be careful as we don't hold info->lock */ 844 swapped = READ_ONCE(info->swapped); 845 846 /* 847 * The easier cases are when the shmem object has nothing in swap, or 848 * the vma maps it whole. Then we can simply use the stats that we 849 * already track. 850 */ 851 if (!swapped) 852 return 0; 853 854 if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size) 855 return swapped << PAGE_SHIFT; 856 857 /* Here comes the more involved part */ 858 return shmem_partial_swap_usage(mapping, 859 linear_page_index(vma, vma->vm_start), 860 linear_page_index(vma, vma->vm_end)); 861 } 862 863 /* 864 * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists. 865 */ 866 void shmem_unlock_mapping(struct address_space *mapping) 867 { 868 struct pagevec pvec; 869 pgoff_t index = 0; 870 871 pagevec_init(&pvec); 872 /* 873 * Minor point, but we might as well stop if someone else SHM_LOCKs it. 874 */ 875 while (!mapping_unevictable(mapping)) { 876 if (!pagevec_lookup(&pvec, mapping, &index)) 877 break; 878 check_move_unevictable_pages(&pvec); 879 pagevec_release(&pvec); 880 cond_resched(); 881 } 882 } 883 884 /* 885 * Check whether a hole-punch or truncation needs to split a huge page, 886 * returning true if no split was required, or the split has been successful. 887 * 888 * Eviction (or truncation to 0 size) should never need to split a huge page; 889 * but in rare cases might do so, if shmem_undo_range() failed to trylock on 890 * head, and then succeeded to trylock on tail. 891 * 892 * A split can only succeed when there are no additional references on the 893 * huge page: so the split below relies upon find_get_entries() having stopped 894 * when it found a subpage of the huge page, without getting further references. 895 */ 896 static bool shmem_punch_compound(struct page *page, pgoff_t start, pgoff_t end) 897 { 898 if (!PageTransCompound(page)) 899 return true; 900 901 /* Just proceed to delete a huge page wholly within the range punched */ 902 if (PageHead(page) && 903 page->index >= start && page->index + HPAGE_PMD_NR <= end) 904 return true; 905 906 /* Try to split huge page, so we can truly punch the hole or truncate */ 907 return split_huge_page(page) >= 0; 908 } 909 910 /* 911 * Remove range of pages and swap entries from page cache, and free them. 912 * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate. 913 */ 914 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend, 915 bool unfalloc) 916 { 917 struct address_space *mapping = inode->i_mapping; 918 struct shmem_inode_info *info = SHMEM_I(inode); 919 pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT; 920 pgoff_t end = (lend + 1) >> PAGE_SHIFT; 921 unsigned int partial_start = lstart & (PAGE_SIZE - 1); 922 unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1); 923 struct pagevec pvec; 924 pgoff_t indices[PAGEVEC_SIZE]; 925 long nr_swaps_freed = 0; 926 pgoff_t index; 927 int i; 928 929 if (lend == -1) 930 end = -1; /* unsigned, so actually very big */ 931 932 if (info->fallocend > start && info->fallocend <= end && !unfalloc) 933 info->fallocend = start; 934 935 pagevec_init(&pvec); 936 index = start; 937 while (index < end && find_lock_entries(mapping, index, end - 1, 938 &pvec, indices)) { 939 for (i = 0; i < pagevec_count(&pvec); i++) { 940 struct page *page = pvec.pages[i]; 941 942 index = indices[i]; 943 944 if (xa_is_value(page)) { 945 if (unfalloc) 946 continue; 947 nr_swaps_freed += !shmem_free_swap(mapping, 948 index, page); 949 continue; 950 } 951 index += thp_nr_pages(page) - 1; 952 953 if (!unfalloc || !PageUptodate(page)) 954 truncate_inode_page(mapping, page); 955 unlock_page(page); 956 } 957 pagevec_remove_exceptionals(&pvec); 958 pagevec_release(&pvec); 959 cond_resched(); 960 index++; 961 } 962 963 if (partial_start) { 964 struct page *page = NULL; 965 shmem_getpage(inode, start - 1, &page, SGP_READ); 966 if (page) { 967 unsigned int top = PAGE_SIZE; 968 if (start > end) { 969 top = partial_end; 970 partial_end = 0; 971 } 972 zero_user_segment(page, partial_start, top); 973 set_page_dirty(page); 974 unlock_page(page); 975 put_page(page); 976 } 977 } 978 if (partial_end) { 979 struct page *page = NULL; 980 shmem_getpage(inode, end, &page, SGP_READ); 981 if (page) { 982 zero_user_segment(page, 0, partial_end); 983 set_page_dirty(page); 984 unlock_page(page); 985 put_page(page); 986 } 987 } 988 if (start >= end) 989 return; 990 991 index = start; 992 while (index < end) { 993 cond_resched(); 994 995 if (!find_get_entries(mapping, index, end - 1, &pvec, 996 indices)) { 997 /* If all gone or hole-punch or unfalloc, we're done */ 998 if (index == start || end != -1) 999 break; 1000 /* But if truncating, restart to make sure all gone */ 1001 index = start; 1002 continue; 1003 } 1004 for (i = 0; i < pagevec_count(&pvec); i++) { 1005 struct page *page = pvec.pages[i]; 1006 1007 index = indices[i]; 1008 if (xa_is_value(page)) { 1009 if (unfalloc) 1010 continue; 1011 if (shmem_free_swap(mapping, index, page)) { 1012 /* Swap was replaced by page: retry */ 1013 index--; 1014 break; 1015 } 1016 nr_swaps_freed++; 1017 continue; 1018 } 1019 1020 lock_page(page); 1021 1022 if (!unfalloc || !PageUptodate(page)) { 1023 if (page_mapping(page) != mapping) { 1024 /* Page was replaced by swap: retry */ 1025 unlock_page(page); 1026 index--; 1027 break; 1028 } 1029 VM_BUG_ON_PAGE(PageWriteback(page), page); 1030 if (shmem_punch_compound(page, start, end)) 1031 truncate_inode_page(mapping, page); 1032 else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) { 1033 /* Wipe the page and don't get stuck */ 1034 clear_highpage(page); 1035 flush_dcache_page(page); 1036 set_page_dirty(page); 1037 if (index < 1038 round_up(start, HPAGE_PMD_NR)) 1039 start = index + 1; 1040 } 1041 } 1042 unlock_page(page); 1043 } 1044 pagevec_remove_exceptionals(&pvec); 1045 pagevec_release(&pvec); 1046 index++; 1047 } 1048 1049 spin_lock_irq(&info->lock); 1050 info->swapped -= nr_swaps_freed; 1051 shmem_recalc_inode(inode); 1052 spin_unlock_irq(&info->lock); 1053 } 1054 1055 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 1056 { 1057 shmem_undo_range(inode, lstart, lend, false); 1058 inode->i_ctime = inode->i_mtime = current_time(inode); 1059 } 1060 EXPORT_SYMBOL_GPL(shmem_truncate_range); 1061 1062 static int shmem_getattr(struct user_namespace *mnt_userns, 1063 const struct path *path, struct kstat *stat, 1064 u32 request_mask, unsigned int query_flags) 1065 { 1066 struct inode *inode = path->dentry->d_inode; 1067 struct shmem_inode_info *info = SHMEM_I(inode); 1068 1069 if (info->alloced - info->swapped != inode->i_mapping->nrpages) { 1070 spin_lock_irq(&info->lock); 1071 shmem_recalc_inode(inode); 1072 spin_unlock_irq(&info->lock); 1073 } 1074 generic_fillattr(&init_user_ns, inode, stat); 1075 1076 if (shmem_is_huge(NULL, inode, 0)) 1077 stat->blksize = HPAGE_PMD_SIZE; 1078 1079 return 0; 1080 } 1081 1082 static int shmem_setattr(struct user_namespace *mnt_userns, 1083 struct dentry *dentry, struct iattr *attr) 1084 { 1085 struct inode *inode = d_inode(dentry); 1086 struct shmem_inode_info *info = SHMEM_I(inode); 1087 int error; 1088 1089 error = setattr_prepare(&init_user_ns, dentry, attr); 1090 if (error) 1091 return error; 1092 1093 if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) { 1094 loff_t oldsize = inode->i_size; 1095 loff_t newsize = attr->ia_size; 1096 1097 /* protected by i_rwsem */ 1098 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) || 1099 (newsize > oldsize && (info->seals & F_SEAL_GROW))) 1100 return -EPERM; 1101 1102 if (newsize != oldsize) { 1103 error = shmem_reacct_size(SHMEM_I(inode)->flags, 1104 oldsize, newsize); 1105 if (error) 1106 return error; 1107 i_size_write(inode, newsize); 1108 inode->i_ctime = inode->i_mtime = current_time(inode); 1109 } 1110 if (newsize <= oldsize) { 1111 loff_t holebegin = round_up(newsize, PAGE_SIZE); 1112 if (oldsize > holebegin) 1113 unmap_mapping_range(inode->i_mapping, 1114 holebegin, 0, 1); 1115 if (info->alloced) 1116 shmem_truncate_range(inode, 1117 newsize, (loff_t)-1); 1118 /* unmap again to remove racily COWed private pages */ 1119 if (oldsize > holebegin) 1120 unmap_mapping_range(inode->i_mapping, 1121 holebegin, 0, 1); 1122 } 1123 } 1124 1125 setattr_copy(&init_user_ns, inode, attr); 1126 if (attr->ia_valid & ATTR_MODE) 1127 error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode); 1128 return error; 1129 } 1130 1131 static void shmem_evict_inode(struct inode *inode) 1132 { 1133 struct shmem_inode_info *info = SHMEM_I(inode); 1134 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 1135 1136 if (shmem_mapping(inode->i_mapping)) { 1137 shmem_unacct_size(info->flags, inode->i_size); 1138 inode->i_size = 0; 1139 shmem_truncate_range(inode, 0, (loff_t)-1); 1140 if (!list_empty(&info->shrinklist)) { 1141 spin_lock(&sbinfo->shrinklist_lock); 1142 if (!list_empty(&info->shrinklist)) { 1143 list_del_init(&info->shrinklist); 1144 sbinfo->shrinklist_len--; 1145 } 1146 spin_unlock(&sbinfo->shrinklist_lock); 1147 } 1148 while (!list_empty(&info->swaplist)) { 1149 /* Wait while shmem_unuse() is scanning this inode... */ 1150 wait_var_event(&info->stop_eviction, 1151 !atomic_read(&info->stop_eviction)); 1152 mutex_lock(&shmem_swaplist_mutex); 1153 /* ...but beware of the race if we peeked too early */ 1154 if (!atomic_read(&info->stop_eviction)) 1155 list_del_init(&info->swaplist); 1156 mutex_unlock(&shmem_swaplist_mutex); 1157 } 1158 } 1159 1160 simple_xattrs_free(&info->xattrs); 1161 WARN_ON(inode->i_blocks); 1162 shmem_free_inode(inode->i_sb); 1163 clear_inode(inode); 1164 } 1165 1166 static int shmem_find_swap_entries(struct address_space *mapping, 1167 pgoff_t start, unsigned int nr_entries, 1168 struct page **entries, pgoff_t *indices, 1169 unsigned int type, bool frontswap) 1170 { 1171 XA_STATE(xas, &mapping->i_pages, start); 1172 struct page *page; 1173 swp_entry_t entry; 1174 unsigned int ret = 0; 1175 1176 if (!nr_entries) 1177 return 0; 1178 1179 rcu_read_lock(); 1180 xas_for_each(&xas, page, ULONG_MAX) { 1181 if (xas_retry(&xas, page)) 1182 continue; 1183 1184 if (!xa_is_value(page)) 1185 continue; 1186 1187 entry = radix_to_swp_entry(page); 1188 if (swp_type(entry) != type) 1189 continue; 1190 if (frontswap && 1191 !frontswap_test(swap_info[type], swp_offset(entry))) 1192 continue; 1193 1194 indices[ret] = xas.xa_index; 1195 entries[ret] = page; 1196 1197 if (need_resched()) { 1198 xas_pause(&xas); 1199 cond_resched_rcu(); 1200 } 1201 if (++ret == nr_entries) 1202 break; 1203 } 1204 rcu_read_unlock(); 1205 1206 return ret; 1207 } 1208 1209 /* 1210 * Move the swapped pages for an inode to page cache. Returns the count 1211 * of pages swapped in, or the error in case of failure. 1212 */ 1213 static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec, 1214 pgoff_t *indices) 1215 { 1216 int i = 0; 1217 int ret = 0; 1218 int error = 0; 1219 struct address_space *mapping = inode->i_mapping; 1220 1221 for (i = 0; i < pvec.nr; i++) { 1222 struct page *page = pvec.pages[i]; 1223 1224 if (!xa_is_value(page)) 1225 continue; 1226 error = shmem_swapin_page(inode, indices[i], 1227 &page, SGP_CACHE, 1228 mapping_gfp_mask(mapping), 1229 NULL, NULL); 1230 if (error == 0) { 1231 unlock_page(page); 1232 put_page(page); 1233 ret++; 1234 } 1235 if (error == -ENOMEM) 1236 break; 1237 error = 0; 1238 } 1239 return error ? error : ret; 1240 } 1241 1242 /* 1243 * If swap found in inode, free it and move page from swapcache to filecache. 1244 */ 1245 static int shmem_unuse_inode(struct inode *inode, unsigned int type, 1246 bool frontswap, unsigned long *fs_pages_to_unuse) 1247 { 1248 struct address_space *mapping = inode->i_mapping; 1249 pgoff_t start = 0; 1250 struct pagevec pvec; 1251 pgoff_t indices[PAGEVEC_SIZE]; 1252 bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0); 1253 int ret = 0; 1254 1255 pagevec_init(&pvec); 1256 do { 1257 unsigned int nr_entries = PAGEVEC_SIZE; 1258 1259 if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE) 1260 nr_entries = *fs_pages_to_unuse; 1261 1262 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries, 1263 pvec.pages, indices, 1264 type, frontswap); 1265 if (pvec.nr == 0) { 1266 ret = 0; 1267 break; 1268 } 1269 1270 ret = shmem_unuse_swap_entries(inode, pvec, indices); 1271 if (ret < 0) 1272 break; 1273 1274 if (frontswap_partial) { 1275 *fs_pages_to_unuse -= ret; 1276 if (*fs_pages_to_unuse == 0) { 1277 ret = FRONTSWAP_PAGES_UNUSED; 1278 break; 1279 } 1280 } 1281 1282 start = indices[pvec.nr - 1]; 1283 } while (true); 1284 1285 return ret; 1286 } 1287 1288 /* 1289 * Read all the shared memory data that resides in the swap 1290 * device 'type' back into memory, so the swap device can be 1291 * unused. 1292 */ 1293 int shmem_unuse(unsigned int type, bool frontswap, 1294 unsigned long *fs_pages_to_unuse) 1295 { 1296 struct shmem_inode_info *info, *next; 1297 int error = 0; 1298 1299 if (list_empty(&shmem_swaplist)) 1300 return 0; 1301 1302 mutex_lock(&shmem_swaplist_mutex); 1303 list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) { 1304 if (!info->swapped) { 1305 list_del_init(&info->swaplist); 1306 continue; 1307 } 1308 /* 1309 * Drop the swaplist mutex while searching the inode for swap; 1310 * but before doing so, make sure shmem_evict_inode() will not 1311 * remove placeholder inode from swaplist, nor let it be freed 1312 * (igrab() would protect from unlink, but not from unmount). 1313 */ 1314 atomic_inc(&info->stop_eviction); 1315 mutex_unlock(&shmem_swaplist_mutex); 1316 1317 error = shmem_unuse_inode(&info->vfs_inode, type, frontswap, 1318 fs_pages_to_unuse); 1319 cond_resched(); 1320 1321 mutex_lock(&shmem_swaplist_mutex); 1322 next = list_next_entry(info, swaplist); 1323 if (!info->swapped) 1324 list_del_init(&info->swaplist); 1325 if (atomic_dec_and_test(&info->stop_eviction)) 1326 wake_up_var(&info->stop_eviction); 1327 if (error) 1328 break; 1329 } 1330 mutex_unlock(&shmem_swaplist_mutex); 1331 1332 return error; 1333 } 1334 1335 /* 1336 * Move the page from the page cache to the swap cache. 1337 */ 1338 static int shmem_writepage(struct page *page, struct writeback_control *wbc) 1339 { 1340 struct shmem_inode_info *info; 1341 struct address_space *mapping; 1342 struct inode *inode; 1343 swp_entry_t swap; 1344 pgoff_t index; 1345 1346 /* 1347 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or 1348 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages, 1349 * and its shmem_writeback() needs them to be split when swapping. 1350 */ 1351 if (PageTransCompound(page)) { 1352 /* Ensure the subpages are still dirty */ 1353 SetPageDirty(page); 1354 if (split_huge_page(page) < 0) 1355 goto redirty; 1356 ClearPageDirty(page); 1357 } 1358 1359 BUG_ON(!PageLocked(page)); 1360 mapping = page->mapping; 1361 index = page->index; 1362 inode = mapping->host; 1363 info = SHMEM_I(inode); 1364 if (info->flags & VM_LOCKED) 1365 goto redirty; 1366 if (!total_swap_pages) 1367 goto redirty; 1368 1369 /* 1370 * Our capabilities prevent regular writeback or sync from ever calling 1371 * shmem_writepage; but a stacking filesystem might use ->writepage of 1372 * its underlying filesystem, in which case tmpfs should write out to 1373 * swap only in response to memory pressure, and not for the writeback 1374 * threads or sync. 1375 */ 1376 if (!wbc->for_reclaim) { 1377 WARN_ON_ONCE(1); /* Still happens? Tell us about it! */ 1378 goto redirty; 1379 } 1380 1381 /* 1382 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC 1383 * value into swapfile.c, the only way we can correctly account for a 1384 * fallocated page arriving here is now to initialize it and write it. 1385 * 1386 * That's okay for a page already fallocated earlier, but if we have 1387 * not yet completed the fallocation, then (a) we want to keep track 1388 * of this page in case we have to undo it, and (b) it may not be a 1389 * good idea to continue anyway, once we're pushing into swap. So 1390 * reactivate the page, and let shmem_fallocate() quit when too many. 1391 */ 1392 if (!PageUptodate(page)) { 1393 if (inode->i_private) { 1394 struct shmem_falloc *shmem_falloc; 1395 spin_lock(&inode->i_lock); 1396 shmem_falloc = inode->i_private; 1397 if (shmem_falloc && 1398 !shmem_falloc->waitq && 1399 index >= shmem_falloc->start && 1400 index < shmem_falloc->next) 1401 shmem_falloc->nr_unswapped++; 1402 else 1403 shmem_falloc = NULL; 1404 spin_unlock(&inode->i_lock); 1405 if (shmem_falloc) 1406 goto redirty; 1407 } 1408 clear_highpage(page); 1409 flush_dcache_page(page); 1410 SetPageUptodate(page); 1411 } 1412 1413 swap = get_swap_page(page); 1414 if (!swap.val) 1415 goto redirty; 1416 1417 /* 1418 * Add inode to shmem_unuse()'s list of swapped-out inodes, 1419 * if it's not already there. Do it now before the page is 1420 * moved to swap cache, when its pagelock no longer protects 1421 * the inode from eviction. But don't unlock the mutex until 1422 * we've incremented swapped, because shmem_unuse_inode() will 1423 * prune a !swapped inode from the swaplist under this mutex. 1424 */ 1425 mutex_lock(&shmem_swaplist_mutex); 1426 if (list_empty(&info->swaplist)) 1427 list_add(&info->swaplist, &shmem_swaplist); 1428 1429 if (add_to_swap_cache(page, swap, 1430 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN, 1431 NULL) == 0) { 1432 spin_lock_irq(&info->lock); 1433 shmem_recalc_inode(inode); 1434 info->swapped++; 1435 spin_unlock_irq(&info->lock); 1436 1437 swap_shmem_alloc(swap); 1438 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap)); 1439 1440 mutex_unlock(&shmem_swaplist_mutex); 1441 BUG_ON(page_mapped(page)); 1442 swap_writepage(page, wbc); 1443 return 0; 1444 } 1445 1446 mutex_unlock(&shmem_swaplist_mutex); 1447 put_swap_page(page, swap); 1448 redirty: 1449 set_page_dirty(page); 1450 if (wbc->for_reclaim) 1451 return AOP_WRITEPAGE_ACTIVATE; /* Return with page locked */ 1452 unlock_page(page); 1453 return 0; 1454 } 1455 1456 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS) 1457 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1458 { 1459 char buffer[64]; 1460 1461 if (!mpol || mpol->mode == MPOL_DEFAULT) 1462 return; /* show nothing */ 1463 1464 mpol_to_str(buffer, sizeof(buffer), mpol); 1465 1466 seq_printf(seq, ",mpol=%s", buffer); 1467 } 1468 1469 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1470 { 1471 struct mempolicy *mpol = NULL; 1472 if (sbinfo->mpol) { 1473 raw_spin_lock(&sbinfo->stat_lock); /* prevent replace/use races */ 1474 mpol = sbinfo->mpol; 1475 mpol_get(mpol); 1476 raw_spin_unlock(&sbinfo->stat_lock); 1477 } 1478 return mpol; 1479 } 1480 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */ 1481 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol) 1482 { 1483 } 1484 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo) 1485 { 1486 return NULL; 1487 } 1488 #endif /* CONFIG_NUMA && CONFIG_TMPFS */ 1489 #ifndef CONFIG_NUMA 1490 #define vm_policy vm_private_data 1491 #endif 1492 1493 static void shmem_pseudo_vma_init(struct vm_area_struct *vma, 1494 struct shmem_inode_info *info, pgoff_t index) 1495 { 1496 /* Create a pseudo vma that just contains the policy */ 1497 vma_init(vma, NULL); 1498 /* Bias interleave by inode number to distribute better across nodes */ 1499 vma->vm_pgoff = index + info->vfs_inode.i_ino; 1500 vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index); 1501 } 1502 1503 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma) 1504 { 1505 /* Drop reference taken by mpol_shared_policy_lookup() */ 1506 mpol_cond_put(vma->vm_policy); 1507 } 1508 1509 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp, 1510 struct shmem_inode_info *info, pgoff_t index) 1511 { 1512 struct vm_area_struct pvma; 1513 struct page *page; 1514 struct vm_fault vmf = { 1515 .vma = &pvma, 1516 }; 1517 1518 shmem_pseudo_vma_init(&pvma, info, index); 1519 page = swap_cluster_readahead(swap, gfp, &vmf); 1520 shmem_pseudo_vma_destroy(&pvma); 1521 1522 return page; 1523 } 1524 1525 /* 1526 * Make sure huge_gfp is always more limited than limit_gfp. 1527 * Some of the flags set permissions, while others set limitations. 1528 */ 1529 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp) 1530 { 1531 gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; 1532 gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; 1533 gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; 1534 gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK); 1535 1536 /* Allow allocations only from the originally specified zones. */ 1537 result |= zoneflags; 1538 1539 /* 1540 * Minimize the result gfp by taking the union with the deny flags, 1541 * and the intersection of the allow flags. 1542 */ 1543 result |= (limit_gfp & denyflags); 1544 result |= (huge_gfp & limit_gfp) & allowflags; 1545 1546 return result; 1547 } 1548 1549 static struct page *shmem_alloc_hugepage(gfp_t gfp, 1550 struct shmem_inode_info *info, pgoff_t index) 1551 { 1552 struct vm_area_struct pvma; 1553 struct address_space *mapping = info->vfs_inode.i_mapping; 1554 pgoff_t hindex; 1555 struct page *page; 1556 1557 hindex = round_down(index, HPAGE_PMD_NR); 1558 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1, 1559 XA_PRESENT)) 1560 return NULL; 1561 1562 shmem_pseudo_vma_init(&pvma, info, hindex); 1563 page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), 1564 true); 1565 shmem_pseudo_vma_destroy(&pvma); 1566 if (page) 1567 prep_transhuge_page(page); 1568 else 1569 count_vm_event(THP_FILE_FALLBACK); 1570 return page; 1571 } 1572 1573 static struct page *shmem_alloc_page(gfp_t gfp, 1574 struct shmem_inode_info *info, pgoff_t index) 1575 { 1576 struct vm_area_struct pvma; 1577 struct page *page; 1578 1579 shmem_pseudo_vma_init(&pvma, info, index); 1580 page = alloc_page_vma(gfp, &pvma, 0); 1581 shmem_pseudo_vma_destroy(&pvma); 1582 1583 return page; 1584 } 1585 1586 static struct page *shmem_alloc_and_acct_page(gfp_t gfp, 1587 struct inode *inode, 1588 pgoff_t index, bool huge) 1589 { 1590 struct shmem_inode_info *info = SHMEM_I(inode); 1591 struct page *page; 1592 int nr; 1593 int err = -ENOSPC; 1594 1595 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 1596 huge = false; 1597 nr = huge ? HPAGE_PMD_NR : 1; 1598 1599 if (!shmem_inode_acct_block(inode, nr)) 1600 goto failed; 1601 1602 if (huge) 1603 page = shmem_alloc_hugepage(gfp, info, index); 1604 else 1605 page = shmem_alloc_page(gfp, info, index); 1606 if (page) { 1607 __SetPageLocked(page); 1608 __SetPageSwapBacked(page); 1609 return page; 1610 } 1611 1612 err = -ENOMEM; 1613 shmem_inode_unacct_blocks(inode, nr); 1614 failed: 1615 return ERR_PTR(err); 1616 } 1617 1618 /* 1619 * When a page is moved from swapcache to shmem filecache (either by the 1620 * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of 1621 * shmem_unuse_inode()), it may have been read in earlier from swap, in 1622 * ignorance of the mapping it belongs to. If that mapping has special 1623 * constraints (like the gma500 GEM driver, which requires RAM below 4GB), 1624 * we may need to copy to a suitable page before moving to filecache. 1625 * 1626 * In a future release, this may well be extended to respect cpuset and 1627 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page(); 1628 * but for now it is a simple matter of zone. 1629 */ 1630 static bool shmem_should_replace_page(struct page *page, gfp_t gfp) 1631 { 1632 return page_zonenum(page) > gfp_zone(gfp); 1633 } 1634 1635 static int shmem_replace_page(struct page **pagep, gfp_t gfp, 1636 struct shmem_inode_info *info, pgoff_t index) 1637 { 1638 struct page *oldpage, *newpage; 1639 struct address_space *swap_mapping; 1640 swp_entry_t entry; 1641 pgoff_t swap_index; 1642 int error; 1643 1644 oldpage = *pagep; 1645 entry.val = page_private(oldpage); 1646 swap_index = swp_offset(entry); 1647 swap_mapping = page_mapping(oldpage); 1648 1649 /* 1650 * We have arrived here because our zones are constrained, so don't 1651 * limit chance of success by further cpuset and node constraints. 1652 */ 1653 gfp &= ~GFP_CONSTRAINT_MASK; 1654 newpage = shmem_alloc_page(gfp, info, index); 1655 if (!newpage) 1656 return -ENOMEM; 1657 1658 get_page(newpage); 1659 copy_highpage(newpage, oldpage); 1660 flush_dcache_page(newpage); 1661 1662 __SetPageLocked(newpage); 1663 __SetPageSwapBacked(newpage); 1664 SetPageUptodate(newpage); 1665 set_page_private(newpage, entry.val); 1666 SetPageSwapCache(newpage); 1667 1668 /* 1669 * Our caller will very soon move newpage out of swapcache, but it's 1670 * a nice clean interface for us to replace oldpage by newpage there. 1671 */ 1672 xa_lock_irq(&swap_mapping->i_pages); 1673 error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage); 1674 if (!error) { 1675 mem_cgroup_migrate(oldpage, newpage); 1676 __inc_lruvec_page_state(newpage, NR_FILE_PAGES); 1677 __dec_lruvec_page_state(oldpage, NR_FILE_PAGES); 1678 } 1679 xa_unlock_irq(&swap_mapping->i_pages); 1680 1681 if (unlikely(error)) { 1682 /* 1683 * Is this possible? I think not, now that our callers check 1684 * both PageSwapCache and page_private after getting page lock; 1685 * but be defensive. Reverse old to newpage for clear and free. 1686 */ 1687 oldpage = newpage; 1688 } else { 1689 lru_cache_add(newpage); 1690 *pagep = newpage; 1691 } 1692 1693 ClearPageSwapCache(oldpage); 1694 set_page_private(oldpage, 0); 1695 1696 unlock_page(oldpage); 1697 put_page(oldpage); 1698 put_page(oldpage); 1699 return error; 1700 } 1701 1702 /* 1703 * Swap in the page pointed to by *pagep. 1704 * Caller has to make sure that *pagep contains a valid swapped page. 1705 * Returns 0 and the page in pagep if success. On failure, returns the 1706 * error code and NULL in *pagep. 1707 */ 1708 static int shmem_swapin_page(struct inode *inode, pgoff_t index, 1709 struct page **pagep, enum sgp_type sgp, 1710 gfp_t gfp, struct vm_area_struct *vma, 1711 vm_fault_t *fault_type) 1712 { 1713 struct address_space *mapping = inode->i_mapping; 1714 struct shmem_inode_info *info = SHMEM_I(inode); 1715 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL; 1716 struct page *page; 1717 swp_entry_t swap; 1718 int error; 1719 1720 VM_BUG_ON(!*pagep || !xa_is_value(*pagep)); 1721 swap = radix_to_swp_entry(*pagep); 1722 *pagep = NULL; 1723 1724 /* Look it up and read it in.. */ 1725 page = lookup_swap_cache(swap, NULL, 0); 1726 if (!page) { 1727 /* Or update major stats only when swapin succeeds?? */ 1728 if (fault_type) { 1729 *fault_type |= VM_FAULT_MAJOR; 1730 count_vm_event(PGMAJFAULT); 1731 count_memcg_event_mm(charge_mm, PGMAJFAULT); 1732 } 1733 /* Here we actually start the io */ 1734 page = shmem_swapin(swap, gfp, info, index); 1735 if (!page) { 1736 error = -ENOMEM; 1737 goto failed; 1738 } 1739 } 1740 1741 /* We have to do this with page locked to prevent races */ 1742 lock_page(page); 1743 if (!PageSwapCache(page) || page_private(page) != swap.val || 1744 !shmem_confirm_swap(mapping, index, swap)) { 1745 error = -EEXIST; 1746 goto unlock; 1747 } 1748 if (!PageUptodate(page)) { 1749 error = -EIO; 1750 goto failed; 1751 } 1752 wait_on_page_writeback(page); 1753 1754 /* 1755 * Some architectures may have to restore extra metadata to the 1756 * physical page after reading from swap. 1757 */ 1758 arch_swap_restore(swap, page); 1759 1760 if (shmem_should_replace_page(page, gfp)) { 1761 error = shmem_replace_page(&page, gfp, info, index); 1762 if (error) 1763 goto failed; 1764 } 1765 1766 error = shmem_add_to_page_cache(page, mapping, index, 1767 swp_to_radix_entry(swap), gfp, 1768 charge_mm); 1769 if (error) 1770 goto failed; 1771 1772 spin_lock_irq(&info->lock); 1773 info->swapped--; 1774 shmem_recalc_inode(inode); 1775 spin_unlock_irq(&info->lock); 1776 1777 if (sgp == SGP_WRITE) 1778 mark_page_accessed(page); 1779 1780 delete_from_swap_cache(page); 1781 set_page_dirty(page); 1782 swap_free(swap); 1783 1784 *pagep = page; 1785 return 0; 1786 failed: 1787 if (!shmem_confirm_swap(mapping, index, swap)) 1788 error = -EEXIST; 1789 unlock: 1790 if (page) { 1791 unlock_page(page); 1792 put_page(page); 1793 } 1794 1795 return error; 1796 } 1797 1798 /* 1799 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate 1800 * 1801 * If we allocate a new one we do not mark it dirty. That's up to the 1802 * vm. If we swap it in we mark it dirty since we also free the swap 1803 * entry since a page cannot live in both the swap and page cache. 1804 * 1805 * vma, vmf, and fault_type are only supplied by shmem_fault: 1806 * otherwise they are NULL. 1807 */ 1808 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index, 1809 struct page **pagep, enum sgp_type sgp, gfp_t gfp, 1810 struct vm_area_struct *vma, struct vm_fault *vmf, 1811 vm_fault_t *fault_type) 1812 { 1813 struct address_space *mapping = inode->i_mapping; 1814 struct shmem_inode_info *info = SHMEM_I(inode); 1815 struct shmem_sb_info *sbinfo; 1816 struct mm_struct *charge_mm; 1817 struct page *page; 1818 pgoff_t hindex = index; 1819 gfp_t huge_gfp; 1820 int error; 1821 int once = 0; 1822 int alloced = 0; 1823 1824 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT)) 1825 return -EFBIG; 1826 repeat: 1827 if (sgp <= SGP_CACHE && 1828 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1829 return -EINVAL; 1830 } 1831 1832 sbinfo = SHMEM_SB(inode->i_sb); 1833 charge_mm = vma ? vma->vm_mm : NULL; 1834 1835 page = pagecache_get_page(mapping, index, 1836 FGP_ENTRY | FGP_HEAD | FGP_LOCK, 0); 1837 1838 if (page && vma && userfaultfd_minor(vma)) { 1839 if (!xa_is_value(page)) { 1840 unlock_page(page); 1841 put_page(page); 1842 } 1843 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR); 1844 return 0; 1845 } 1846 1847 if (xa_is_value(page)) { 1848 error = shmem_swapin_page(inode, index, &page, 1849 sgp, gfp, vma, fault_type); 1850 if (error == -EEXIST) 1851 goto repeat; 1852 1853 *pagep = page; 1854 return error; 1855 } 1856 1857 if (page) { 1858 hindex = page->index; 1859 if (sgp == SGP_WRITE) 1860 mark_page_accessed(page); 1861 if (PageUptodate(page)) 1862 goto out; 1863 /* fallocated page */ 1864 if (sgp != SGP_READ) 1865 goto clear; 1866 unlock_page(page); 1867 put_page(page); 1868 } 1869 1870 /* 1871 * SGP_READ: succeed on hole, with NULL page, letting caller zero. 1872 * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail. 1873 */ 1874 *pagep = NULL; 1875 if (sgp == SGP_READ) 1876 return 0; 1877 if (sgp == SGP_NOALLOC) 1878 return -ENOENT; 1879 1880 /* 1881 * Fast cache lookup and swap lookup did not find it: allocate. 1882 */ 1883 1884 if (vma && userfaultfd_missing(vma)) { 1885 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING); 1886 return 0; 1887 } 1888 1889 /* Never use a huge page for shmem_symlink() */ 1890 if (S_ISLNK(inode->i_mode)) 1891 goto alloc_nohuge; 1892 if (!shmem_is_huge(vma, inode, index)) 1893 goto alloc_nohuge; 1894 1895 huge_gfp = vma_thp_gfp_mask(vma); 1896 huge_gfp = limit_gfp_mask(huge_gfp, gfp); 1897 page = shmem_alloc_and_acct_page(huge_gfp, inode, index, true); 1898 if (IS_ERR(page)) { 1899 alloc_nohuge: 1900 page = shmem_alloc_and_acct_page(gfp, inode, 1901 index, false); 1902 } 1903 if (IS_ERR(page)) { 1904 int retry = 5; 1905 1906 error = PTR_ERR(page); 1907 page = NULL; 1908 if (error != -ENOSPC) 1909 goto unlock; 1910 /* 1911 * Try to reclaim some space by splitting a huge page 1912 * beyond i_size on the filesystem. 1913 */ 1914 while (retry--) { 1915 int ret; 1916 1917 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1); 1918 if (ret == SHRINK_STOP) 1919 break; 1920 if (ret) 1921 goto alloc_nohuge; 1922 } 1923 goto unlock; 1924 } 1925 1926 if (PageTransHuge(page)) 1927 hindex = round_down(index, HPAGE_PMD_NR); 1928 else 1929 hindex = index; 1930 1931 if (sgp == SGP_WRITE) 1932 __SetPageReferenced(page); 1933 1934 error = shmem_add_to_page_cache(page, mapping, hindex, 1935 NULL, gfp & GFP_RECLAIM_MASK, 1936 charge_mm); 1937 if (error) 1938 goto unacct; 1939 lru_cache_add(page); 1940 1941 spin_lock_irq(&info->lock); 1942 info->alloced += compound_nr(page); 1943 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page); 1944 shmem_recalc_inode(inode); 1945 spin_unlock_irq(&info->lock); 1946 alloced = true; 1947 1948 if (PageTransHuge(page) && 1949 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) < 1950 hindex + HPAGE_PMD_NR - 1) { 1951 /* 1952 * Part of the huge page is beyond i_size: subject 1953 * to shrink under memory pressure. 1954 */ 1955 spin_lock(&sbinfo->shrinklist_lock); 1956 /* 1957 * _careful to defend against unlocked access to 1958 * ->shrink_list in shmem_unused_huge_shrink() 1959 */ 1960 if (list_empty_careful(&info->shrinklist)) { 1961 list_add_tail(&info->shrinklist, 1962 &sbinfo->shrinklist); 1963 sbinfo->shrinklist_len++; 1964 } 1965 spin_unlock(&sbinfo->shrinklist_lock); 1966 } 1967 1968 /* 1969 * Let SGP_FALLOC use the SGP_WRITE optimization on a new page. 1970 */ 1971 if (sgp == SGP_FALLOC) 1972 sgp = SGP_WRITE; 1973 clear: 1974 /* 1975 * Let SGP_WRITE caller clear ends if write does not fill page; 1976 * but SGP_FALLOC on a page fallocated earlier must initialize 1977 * it now, lest undo on failure cancel our earlier guarantee. 1978 */ 1979 if (sgp != SGP_WRITE && !PageUptodate(page)) { 1980 int i; 1981 1982 for (i = 0; i < compound_nr(page); i++) { 1983 clear_highpage(page + i); 1984 flush_dcache_page(page + i); 1985 } 1986 SetPageUptodate(page); 1987 } 1988 1989 /* Perhaps the file has been truncated since we checked */ 1990 if (sgp <= SGP_CACHE && 1991 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) { 1992 if (alloced) { 1993 ClearPageDirty(page); 1994 delete_from_page_cache(page); 1995 spin_lock_irq(&info->lock); 1996 shmem_recalc_inode(inode); 1997 spin_unlock_irq(&info->lock); 1998 } 1999 error = -EINVAL; 2000 goto unlock; 2001 } 2002 out: 2003 *pagep = page + index - hindex; 2004 return 0; 2005 2006 /* 2007 * Error recovery. 2008 */ 2009 unacct: 2010 shmem_inode_unacct_blocks(inode, compound_nr(page)); 2011 2012 if (PageTransHuge(page)) { 2013 unlock_page(page); 2014 put_page(page); 2015 goto alloc_nohuge; 2016 } 2017 unlock: 2018 if (page) { 2019 unlock_page(page); 2020 put_page(page); 2021 } 2022 if (error == -ENOSPC && !once++) { 2023 spin_lock_irq(&info->lock); 2024 shmem_recalc_inode(inode); 2025 spin_unlock_irq(&info->lock); 2026 goto repeat; 2027 } 2028 if (error == -EEXIST) 2029 goto repeat; 2030 return error; 2031 } 2032 2033 /* 2034 * This is like autoremove_wake_function, but it removes the wait queue 2035 * entry unconditionally - even if something else had already woken the 2036 * target. 2037 */ 2038 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key) 2039 { 2040 int ret = default_wake_function(wait, mode, sync, key); 2041 list_del_init(&wait->entry); 2042 return ret; 2043 } 2044 2045 static vm_fault_t shmem_fault(struct vm_fault *vmf) 2046 { 2047 struct vm_area_struct *vma = vmf->vma; 2048 struct inode *inode = file_inode(vma->vm_file); 2049 gfp_t gfp = mapping_gfp_mask(inode->i_mapping); 2050 int err; 2051 vm_fault_t ret = VM_FAULT_LOCKED; 2052 2053 /* 2054 * Trinity finds that probing a hole which tmpfs is punching can 2055 * prevent the hole-punch from ever completing: which in turn 2056 * locks writers out with its hold on i_rwsem. So refrain from 2057 * faulting pages into the hole while it's being punched. Although 2058 * shmem_undo_range() does remove the additions, it may be unable to 2059 * keep up, as each new page needs its own unmap_mapping_range() call, 2060 * and the i_mmap tree grows ever slower to scan if new vmas are added. 2061 * 2062 * It does not matter if we sometimes reach this check just before the 2063 * hole-punch begins, so that one fault then races with the punch: 2064 * we just need to make racing faults a rare case. 2065 * 2066 * The implementation below would be much simpler if we just used a 2067 * standard mutex or completion: but we cannot take i_rwsem in fault, 2068 * and bloating every shmem inode for this unlikely case would be sad. 2069 */ 2070 if (unlikely(inode->i_private)) { 2071 struct shmem_falloc *shmem_falloc; 2072 2073 spin_lock(&inode->i_lock); 2074 shmem_falloc = inode->i_private; 2075 if (shmem_falloc && 2076 shmem_falloc->waitq && 2077 vmf->pgoff >= shmem_falloc->start && 2078 vmf->pgoff < shmem_falloc->next) { 2079 struct file *fpin; 2080 wait_queue_head_t *shmem_falloc_waitq; 2081 DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function); 2082 2083 ret = VM_FAULT_NOPAGE; 2084 fpin = maybe_unlock_mmap_for_io(vmf, NULL); 2085 if (fpin) 2086 ret = VM_FAULT_RETRY; 2087 2088 shmem_falloc_waitq = shmem_falloc->waitq; 2089 prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait, 2090 TASK_UNINTERRUPTIBLE); 2091 spin_unlock(&inode->i_lock); 2092 schedule(); 2093 2094 /* 2095 * shmem_falloc_waitq points into the shmem_fallocate() 2096 * stack of the hole-punching task: shmem_falloc_waitq 2097 * is usually invalid by the time we reach here, but 2098 * finish_wait() does not dereference it in that case; 2099 * though i_lock needed lest racing with wake_up_all(). 2100 */ 2101 spin_lock(&inode->i_lock); 2102 finish_wait(shmem_falloc_waitq, &shmem_fault_wait); 2103 spin_unlock(&inode->i_lock); 2104 2105 if (fpin) 2106 fput(fpin); 2107 return ret; 2108 } 2109 spin_unlock(&inode->i_lock); 2110 } 2111 2112 err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, SGP_CACHE, 2113 gfp, vma, vmf, &ret); 2114 if (err) 2115 return vmf_error(err); 2116 return ret; 2117 } 2118 2119 unsigned long shmem_get_unmapped_area(struct file *file, 2120 unsigned long uaddr, unsigned long len, 2121 unsigned long pgoff, unsigned long flags) 2122 { 2123 unsigned long (*get_area)(struct file *, 2124 unsigned long, unsigned long, unsigned long, unsigned long); 2125 unsigned long addr; 2126 unsigned long offset; 2127 unsigned long inflated_len; 2128 unsigned long inflated_addr; 2129 unsigned long inflated_offset; 2130 2131 if (len > TASK_SIZE) 2132 return -ENOMEM; 2133 2134 get_area = current->mm->get_unmapped_area; 2135 addr = get_area(file, uaddr, len, pgoff, flags); 2136 2137 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) 2138 return addr; 2139 if (IS_ERR_VALUE(addr)) 2140 return addr; 2141 if (addr & ~PAGE_MASK) 2142 return addr; 2143 if (addr > TASK_SIZE - len) 2144 return addr; 2145 2146 if (shmem_huge == SHMEM_HUGE_DENY) 2147 return addr; 2148 if (len < HPAGE_PMD_SIZE) 2149 return addr; 2150 if (flags & MAP_FIXED) 2151 return addr; 2152 /* 2153 * Our priority is to support MAP_SHARED mapped hugely; 2154 * and support MAP_PRIVATE mapped hugely too, until it is COWed. 2155 * But if caller specified an address hint and we allocated area there 2156 * successfully, respect that as before. 2157 */ 2158 if (uaddr == addr) 2159 return addr; 2160 2161 if (shmem_huge != SHMEM_HUGE_FORCE) { 2162 struct super_block *sb; 2163 2164 if (file) { 2165 VM_BUG_ON(file->f_op != &shmem_file_operations); 2166 sb = file_inode(file)->i_sb; 2167 } else { 2168 /* 2169 * Called directly from mm/mmap.c, or drivers/char/mem.c 2170 * for "/dev/zero", to create a shared anonymous object. 2171 */ 2172 if (IS_ERR(shm_mnt)) 2173 return addr; 2174 sb = shm_mnt->mnt_sb; 2175 } 2176 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER) 2177 return addr; 2178 } 2179 2180 offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1); 2181 if (offset && offset + len < 2 * HPAGE_PMD_SIZE) 2182 return addr; 2183 if ((addr & (HPAGE_PMD_SIZE-1)) == offset) 2184 return addr; 2185 2186 inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE; 2187 if (inflated_len > TASK_SIZE) 2188 return addr; 2189 if (inflated_len < len) 2190 return addr; 2191 2192 inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags); 2193 if (IS_ERR_VALUE(inflated_addr)) 2194 return addr; 2195 if (inflated_addr & ~PAGE_MASK) 2196 return addr; 2197 2198 inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1); 2199 inflated_addr += offset - inflated_offset; 2200 if (inflated_offset > offset) 2201 inflated_addr += HPAGE_PMD_SIZE; 2202 2203 if (inflated_addr > TASK_SIZE - len) 2204 return addr; 2205 return inflated_addr; 2206 } 2207 2208 #ifdef CONFIG_NUMA 2209 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol) 2210 { 2211 struct inode *inode = file_inode(vma->vm_file); 2212 return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol); 2213 } 2214 2215 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma, 2216 unsigned long addr) 2217 { 2218 struct inode *inode = file_inode(vma->vm_file); 2219 pgoff_t index; 2220 2221 index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff; 2222 return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index); 2223 } 2224 #endif 2225 2226 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 2227 { 2228 struct inode *inode = file_inode(file); 2229 struct shmem_inode_info *info = SHMEM_I(inode); 2230 int retval = -ENOMEM; 2231 2232 /* 2233 * What serializes the accesses to info->flags? 2234 * ipc_lock_object() when called from shmctl_do_lock(), 2235 * no serialization needed when called from shm_destroy(). 2236 */ 2237 if (lock && !(info->flags & VM_LOCKED)) { 2238 if (!user_shm_lock(inode->i_size, ucounts)) 2239 goto out_nomem; 2240 info->flags |= VM_LOCKED; 2241 mapping_set_unevictable(file->f_mapping); 2242 } 2243 if (!lock && (info->flags & VM_LOCKED) && ucounts) { 2244 user_shm_unlock(inode->i_size, ucounts); 2245 info->flags &= ~VM_LOCKED; 2246 mapping_clear_unevictable(file->f_mapping); 2247 } 2248 retval = 0; 2249 2250 out_nomem: 2251 return retval; 2252 } 2253 2254 static int shmem_mmap(struct file *file, struct vm_area_struct *vma) 2255 { 2256 struct shmem_inode_info *info = SHMEM_I(file_inode(file)); 2257 int ret; 2258 2259 ret = seal_check_future_write(info->seals, vma); 2260 if (ret) 2261 return ret; 2262 2263 /* arm64 - allow memory tagging on RAM-based files */ 2264 vma->vm_flags |= VM_MTE_ALLOWED; 2265 2266 file_accessed(file); 2267 vma->vm_ops = &shmem_vm_ops; 2268 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 2269 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 2270 (vma->vm_end & HPAGE_PMD_MASK)) { 2271 khugepaged_enter(vma, vma->vm_flags); 2272 } 2273 return 0; 2274 } 2275 2276 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir, 2277 umode_t mode, dev_t dev, unsigned long flags) 2278 { 2279 struct inode *inode; 2280 struct shmem_inode_info *info; 2281 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 2282 ino_t ino; 2283 2284 if (shmem_reserve_inode(sb, &ino)) 2285 return NULL; 2286 2287 inode = new_inode(sb); 2288 if (inode) { 2289 inode->i_ino = ino; 2290 inode_init_owner(&init_user_ns, inode, dir, mode); 2291 inode->i_blocks = 0; 2292 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode); 2293 inode->i_generation = prandom_u32(); 2294 info = SHMEM_I(inode); 2295 memset(info, 0, (char *)inode - (char *)info); 2296 spin_lock_init(&info->lock); 2297 atomic_set(&info->stop_eviction, 0); 2298 info->seals = F_SEAL_SEAL; 2299 info->flags = flags & VM_NORESERVE; 2300 INIT_LIST_HEAD(&info->shrinklist); 2301 INIT_LIST_HEAD(&info->swaplist); 2302 simple_xattrs_init(&info->xattrs); 2303 cache_no_acl(inode); 2304 2305 switch (mode & S_IFMT) { 2306 default: 2307 inode->i_op = &shmem_special_inode_operations; 2308 init_special_inode(inode, mode, dev); 2309 break; 2310 case S_IFREG: 2311 inode->i_mapping->a_ops = &shmem_aops; 2312 inode->i_op = &shmem_inode_operations; 2313 inode->i_fop = &shmem_file_operations; 2314 mpol_shared_policy_init(&info->policy, 2315 shmem_get_sbmpol(sbinfo)); 2316 break; 2317 case S_IFDIR: 2318 inc_nlink(inode); 2319 /* Some things misbehave if size == 0 on a directory */ 2320 inode->i_size = 2 * BOGO_DIRENT_SIZE; 2321 inode->i_op = &shmem_dir_inode_operations; 2322 inode->i_fop = &simple_dir_operations; 2323 break; 2324 case S_IFLNK: 2325 /* 2326 * Must not load anything in the rbtree, 2327 * mpol_free_shared_policy will not be called. 2328 */ 2329 mpol_shared_policy_init(&info->policy, NULL); 2330 break; 2331 } 2332 2333 lockdep_annotate_inode_mutex_key(inode); 2334 } else 2335 shmem_free_inode(sb); 2336 return inode; 2337 } 2338 2339 #ifdef CONFIG_USERFAULTFD 2340 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm, 2341 pmd_t *dst_pmd, 2342 struct vm_area_struct *dst_vma, 2343 unsigned long dst_addr, 2344 unsigned long src_addr, 2345 bool zeropage, 2346 struct page **pagep) 2347 { 2348 struct inode *inode = file_inode(dst_vma->vm_file); 2349 struct shmem_inode_info *info = SHMEM_I(inode); 2350 struct address_space *mapping = inode->i_mapping; 2351 gfp_t gfp = mapping_gfp_mask(mapping); 2352 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr); 2353 void *page_kaddr; 2354 struct page *page; 2355 int ret; 2356 pgoff_t max_off; 2357 2358 if (!shmem_inode_acct_block(inode, 1)) { 2359 /* 2360 * We may have got a page, returned -ENOENT triggering a retry, 2361 * and now we find ourselves with -ENOMEM. Release the page, to 2362 * avoid a BUG_ON in our caller. 2363 */ 2364 if (unlikely(*pagep)) { 2365 put_page(*pagep); 2366 *pagep = NULL; 2367 } 2368 return -ENOMEM; 2369 } 2370 2371 if (!*pagep) { 2372 ret = -ENOMEM; 2373 page = shmem_alloc_page(gfp, info, pgoff); 2374 if (!page) 2375 goto out_unacct_blocks; 2376 2377 if (!zeropage) { /* COPY */ 2378 page_kaddr = kmap_atomic(page); 2379 ret = copy_from_user(page_kaddr, 2380 (const void __user *)src_addr, 2381 PAGE_SIZE); 2382 kunmap_atomic(page_kaddr); 2383 2384 /* fallback to copy_from_user outside mmap_lock */ 2385 if (unlikely(ret)) { 2386 *pagep = page; 2387 ret = -ENOENT; 2388 /* don't free the page */ 2389 goto out_unacct_blocks; 2390 } 2391 } else { /* ZEROPAGE */ 2392 clear_highpage(page); 2393 } 2394 } else { 2395 page = *pagep; 2396 *pagep = NULL; 2397 } 2398 2399 VM_BUG_ON(PageLocked(page)); 2400 VM_BUG_ON(PageSwapBacked(page)); 2401 __SetPageLocked(page); 2402 __SetPageSwapBacked(page); 2403 __SetPageUptodate(page); 2404 2405 ret = -EFAULT; 2406 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE); 2407 if (unlikely(pgoff >= max_off)) 2408 goto out_release; 2409 2410 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL, 2411 gfp & GFP_RECLAIM_MASK, dst_mm); 2412 if (ret) 2413 goto out_release; 2414 2415 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr, 2416 page, true, false); 2417 if (ret) 2418 goto out_delete_from_cache; 2419 2420 spin_lock_irq(&info->lock); 2421 info->alloced++; 2422 inode->i_blocks += BLOCKS_PER_PAGE; 2423 shmem_recalc_inode(inode); 2424 spin_unlock_irq(&info->lock); 2425 2426 SetPageDirty(page); 2427 unlock_page(page); 2428 return 0; 2429 out_delete_from_cache: 2430 delete_from_page_cache(page); 2431 out_release: 2432 unlock_page(page); 2433 put_page(page); 2434 out_unacct_blocks: 2435 shmem_inode_unacct_blocks(inode, 1); 2436 return ret; 2437 } 2438 #endif /* CONFIG_USERFAULTFD */ 2439 2440 #ifdef CONFIG_TMPFS 2441 static const struct inode_operations shmem_symlink_inode_operations; 2442 static const struct inode_operations shmem_short_symlink_operations; 2443 2444 #ifdef CONFIG_TMPFS_XATTR 2445 static int shmem_initxattrs(struct inode *, const struct xattr *, void *); 2446 #else 2447 #define shmem_initxattrs NULL 2448 #endif 2449 2450 static int 2451 shmem_write_begin(struct file *file, struct address_space *mapping, 2452 loff_t pos, unsigned len, unsigned flags, 2453 struct page **pagep, void **fsdata) 2454 { 2455 struct inode *inode = mapping->host; 2456 struct shmem_inode_info *info = SHMEM_I(inode); 2457 pgoff_t index = pos >> PAGE_SHIFT; 2458 2459 /* i_rwsem is held by caller */ 2460 if (unlikely(info->seals & (F_SEAL_GROW | 2461 F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) { 2462 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) 2463 return -EPERM; 2464 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size) 2465 return -EPERM; 2466 } 2467 2468 return shmem_getpage(inode, index, pagep, SGP_WRITE); 2469 } 2470 2471 static int 2472 shmem_write_end(struct file *file, struct address_space *mapping, 2473 loff_t pos, unsigned len, unsigned copied, 2474 struct page *page, void *fsdata) 2475 { 2476 struct inode *inode = mapping->host; 2477 2478 if (pos + copied > inode->i_size) 2479 i_size_write(inode, pos + copied); 2480 2481 if (!PageUptodate(page)) { 2482 struct page *head = compound_head(page); 2483 if (PageTransCompound(page)) { 2484 int i; 2485 2486 for (i = 0; i < HPAGE_PMD_NR; i++) { 2487 if (head + i == page) 2488 continue; 2489 clear_highpage(head + i); 2490 flush_dcache_page(head + i); 2491 } 2492 } 2493 if (copied < PAGE_SIZE) { 2494 unsigned from = pos & (PAGE_SIZE - 1); 2495 zero_user_segments(page, 0, from, 2496 from + copied, PAGE_SIZE); 2497 } 2498 SetPageUptodate(head); 2499 } 2500 set_page_dirty(page); 2501 unlock_page(page); 2502 put_page(page); 2503 2504 return copied; 2505 } 2506 2507 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 2508 { 2509 struct file *file = iocb->ki_filp; 2510 struct inode *inode = file_inode(file); 2511 struct address_space *mapping = inode->i_mapping; 2512 pgoff_t index; 2513 unsigned long offset; 2514 enum sgp_type sgp = SGP_READ; 2515 int error = 0; 2516 ssize_t retval = 0; 2517 loff_t *ppos = &iocb->ki_pos; 2518 2519 /* 2520 * Might this read be for a stacking filesystem? Then when reading 2521 * holes of a sparse file, we actually need to allocate those pages, 2522 * and even mark them dirty, so it cannot exceed the max_blocks limit. 2523 */ 2524 if (!iter_is_iovec(to)) 2525 sgp = SGP_CACHE; 2526 2527 index = *ppos >> PAGE_SHIFT; 2528 offset = *ppos & ~PAGE_MASK; 2529 2530 for (;;) { 2531 struct page *page = NULL; 2532 pgoff_t end_index; 2533 unsigned long nr, ret; 2534 loff_t i_size = i_size_read(inode); 2535 2536 end_index = i_size >> PAGE_SHIFT; 2537 if (index > end_index) 2538 break; 2539 if (index == end_index) { 2540 nr = i_size & ~PAGE_MASK; 2541 if (nr <= offset) 2542 break; 2543 } 2544 2545 error = shmem_getpage(inode, index, &page, sgp); 2546 if (error) { 2547 if (error == -EINVAL) 2548 error = 0; 2549 break; 2550 } 2551 if (page) { 2552 if (sgp == SGP_CACHE) 2553 set_page_dirty(page); 2554 unlock_page(page); 2555 } 2556 2557 /* 2558 * We must evaluate after, since reads (unlike writes) 2559 * are called without i_rwsem protection against truncate 2560 */ 2561 nr = PAGE_SIZE; 2562 i_size = i_size_read(inode); 2563 end_index = i_size >> PAGE_SHIFT; 2564 if (index == end_index) { 2565 nr = i_size & ~PAGE_MASK; 2566 if (nr <= offset) { 2567 if (page) 2568 put_page(page); 2569 break; 2570 } 2571 } 2572 nr -= offset; 2573 2574 if (page) { 2575 /* 2576 * If users can be writing to this page using arbitrary 2577 * virtual addresses, take care about potential aliasing 2578 * before reading the page on the kernel side. 2579 */ 2580 if (mapping_writably_mapped(mapping)) 2581 flush_dcache_page(page); 2582 /* 2583 * Mark the page accessed if we read the beginning. 2584 */ 2585 if (!offset) 2586 mark_page_accessed(page); 2587 } else { 2588 page = ZERO_PAGE(0); 2589 get_page(page); 2590 } 2591 2592 /* 2593 * Ok, we have the page, and it's up-to-date, so 2594 * now we can copy it to user space... 2595 */ 2596 ret = copy_page_to_iter(page, offset, nr, to); 2597 retval += ret; 2598 offset += ret; 2599 index += offset >> PAGE_SHIFT; 2600 offset &= ~PAGE_MASK; 2601 2602 put_page(page); 2603 if (!iov_iter_count(to)) 2604 break; 2605 if (ret < nr) { 2606 error = -EFAULT; 2607 break; 2608 } 2609 cond_resched(); 2610 } 2611 2612 *ppos = ((loff_t) index << PAGE_SHIFT) + offset; 2613 file_accessed(file); 2614 return retval ? retval : error; 2615 } 2616 2617 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence) 2618 { 2619 struct address_space *mapping = file->f_mapping; 2620 struct inode *inode = mapping->host; 2621 2622 if (whence != SEEK_DATA && whence != SEEK_HOLE) 2623 return generic_file_llseek_size(file, offset, whence, 2624 MAX_LFS_FILESIZE, i_size_read(inode)); 2625 if (offset < 0) 2626 return -ENXIO; 2627 2628 inode_lock(inode); 2629 /* We're holding i_rwsem so we can access i_size directly */ 2630 offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence); 2631 if (offset >= 0) 2632 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE); 2633 inode_unlock(inode); 2634 return offset; 2635 } 2636 2637 static long shmem_fallocate(struct file *file, int mode, loff_t offset, 2638 loff_t len) 2639 { 2640 struct inode *inode = file_inode(file); 2641 struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); 2642 struct shmem_inode_info *info = SHMEM_I(inode); 2643 struct shmem_falloc shmem_falloc; 2644 pgoff_t start, index, end, undo_fallocend; 2645 int error; 2646 2647 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE)) 2648 return -EOPNOTSUPP; 2649 2650 inode_lock(inode); 2651 2652 if (mode & FALLOC_FL_PUNCH_HOLE) { 2653 struct address_space *mapping = file->f_mapping; 2654 loff_t unmap_start = round_up(offset, PAGE_SIZE); 2655 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1; 2656 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq); 2657 2658 /* protected by i_rwsem */ 2659 if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) { 2660 error = -EPERM; 2661 goto out; 2662 } 2663 2664 shmem_falloc.waitq = &shmem_falloc_waitq; 2665 shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT; 2666 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT; 2667 spin_lock(&inode->i_lock); 2668 inode->i_private = &shmem_falloc; 2669 spin_unlock(&inode->i_lock); 2670 2671 if ((u64)unmap_end > (u64)unmap_start) 2672 unmap_mapping_range(mapping, unmap_start, 2673 1 + unmap_end - unmap_start, 0); 2674 shmem_truncate_range(inode, offset, offset + len - 1); 2675 /* No need to unmap again: hole-punching leaves COWed pages */ 2676 2677 spin_lock(&inode->i_lock); 2678 inode->i_private = NULL; 2679 wake_up_all(&shmem_falloc_waitq); 2680 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head)); 2681 spin_unlock(&inode->i_lock); 2682 error = 0; 2683 goto out; 2684 } 2685 2686 /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */ 2687 error = inode_newsize_ok(inode, offset + len); 2688 if (error) 2689 goto out; 2690 2691 if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) { 2692 error = -EPERM; 2693 goto out; 2694 } 2695 2696 start = offset >> PAGE_SHIFT; 2697 end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT; 2698 /* Try to avoid a swapstorm if len is impossible to satisfy */ 2699 if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) { 2700 error = -ENOSPC; 2701 goto out; 2702 } 2703 2704 shmem_falloc.waitq = NULL; 2705 shmem_falloc.start = start; 2706 shmem_falloc.next = start; 2707 shmem_falloc.nr_falloced = 0; 2708 shmem_falloc.nr_unswapped = 0; 2709 spin_lock(&inode->i_lock); 2710 inode->i_private = &shmem_falloc; 2711 spin_unlock(&inode->i_lock); 2712 2713 /* 2714 * info->fallocend is only relevant when huge pages might be 2715 * involved: to prevent split_huge_page() freeing fallocated 2716 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size. 2717 */ 2718 undo_fallocend = info->fallocend; 2719 if (info->fallocend < end) 2720 info->fallocend = end; 2721 2722 for (index = start; index < end; ) { 2723 struct page *page; 2724 2725 /* 2726 * Good, the fallocate(2) manpage permits EINTR: we may have 2727 * been interrupted because we are using up too much memory. 2728 */ 2729 if (signal_pending(current)) 2730 error = -EINTR; 2731 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced) 2732 error = -ENOMEM; 2733 else 2734 error = shmem_getpage(inode, index, &page, SGP_FALLOC); 2735 if (error) { 2736 info->fallocend = undo_fallocend; 2737 /* Remove the !PageUptodate pages we added */ 2738 if (index > start) { 2739 shmem_undo_range(inode, 2740 (loff_t)start << PAGE_SHIFT, 2741 ((loff_t)index << PAGE_SHIFT) - 1, true); 2742 } 2743 goto undone; 2744 } 2745 2746 index++; 2747 /* 2748 * Here is a more important optimization than it appears: 2749 * a second SGP_FALLOC on the same huge page will clear it, 2750 * making it PageUptodate and un-undoable if we fail later. 2751 */ 2752 if (PageTransCompound(page)) { 2753 index = round_up(index, HPAGE_PMD_NR); 2754 /* Beware 32-bit wraparound */ 2755 if (!index) 2756 index--; 2757 } 2758 2759 /* 2760 * Inform shmem_writepage() how far we have reached. 2761 * No need for lock or barrier: we have the page lock. 2762 */ 2763 if (!PageUptodate(page)) 2764 shmem_falloc.nr_falloced += index - shmem_falloc.next; 2765 shmem_falloc.next = index; 2766 2767 /* 2768 * If !PageUptodate, leave it that way so that freeable pages 2769 * can be recognized if we need to rollback on error later. 2770 * But set_page_dirty so that memory pressure will swap rather 2771 * than free the pages we are allocating (and SGP_CACHE pages 2772 * might still be clean: we now need to mark those dirty too). 2773 */ 2774 set_page_dirty(page); 2775 unlock_page(page); 2776 put_page(page); 2777 cond_resched(); 2778 } 2779 2780 if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) 2781 i_size_write(inode, offset + len); 2782 inode->i_ctime = current_time(inode); 2783 undone: 2784 spin_lock(&inode->i_lock); 2785 inode->i_private = NULL; 2786 spin_unlock(&inode->i_lock); 2787 out: 2788 inode_unlock(inode); 2789 return error; 2790 } 2791 2792 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf) 2793 { 2794 struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb); 2795 2796 buf->f_type = TMPFS_MAGIC; 2797 buf->f_bsize = PAGE_SIZE; 2798 buf->f_namelen = NAME_MAX; 2799 if (sbinfo->max_blocks) { 2800 buf->f_blocks = sbinfo->max_blocks; 2801 buf->f_bavail = 2802 buf->f_bfree = sbinfo->max_blocks - 2803 percpu_counter_sum(&sbinfo->used_blocks); 2804 } 2805 if (sbinfo->max_inodes) { 2806 buf->f_files = sbinfo->max_inodes; 2807 buf->f_ffree = sbinfo->free_inodes; 2808 } 2809 /* else leave those fields 0 like simple_statfs */ 2810 2811 buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b); 2812 2813 return 0; 2814 } 2815 2816 /* 2817 * File creation. Allocate an inode, and we're done.. 2818 */ 2819 static int 2820 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir, 2821 struct dentry *dentry, umode_t mode, dev_t dev) 2822 { 2823 struct inode *inode; 2824 int error = -ENOSPC; 2825 2826 inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE); 2827 if (inode) { 2828 error = simple_acl_create(dir, inode); 2829 if (error) 2830 goto out_iput; 2831 error = security_inode_init_security(inode, dir, 2832 &dentry->d_name, 2833 shmem_initxattrs, NULL); 2834 if (error && error != -EOPNOTSUPP) 2835 goto out_iput; 2836 2837 error = 0; 2838 dir->i_size += BOGO_DIRENT_SIZE; 2839 dir->i_ctime = dir->i_mtime = current_time(dir); 2840 d_instantiate(dentry, inode); 2841 dget(dentry); /* Extra count - pin the dentry in core */ 2842 } 2843 return error; 2844 out_iput: 2845 iput(inode); 2846 return error; 2847 } 2848 2849 static int 2850 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir, 2851 struct dentry *dentry, umode_t mode) 2852 { 2853 struct inode *inode; 2854 int error = -ENOSPC; 2855 2856 inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE); 2857 if (inode) { 2858 error = security_inode_init_security(inode, dir, 2859 NULL, 2860 shmem_initxattrs, NULL); 2861 if (error && error != -EOPNOTSUPP) 2862 goto out_iput; 2863 error = simple_acl_create(dir, inode); 2864 if (error) 2865 goto out_iput; 2866 d_tmpfile(dentry, inode); 2867 } 2868 return error; 2869 out_iput: 2870 iput(inode); 2871 return error; 2872 } 2873 2874 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir, 2875 struct dentry *dentry, umode_t mode) 2876 { 2877 int error; 2878 2879 if ((error = shmem_mknod(&init_user_ns, dir, dentry, 2880 mode | S_IFDIR, 0))) 2881 return error; 2882 inc_nlink(dir); 2883 return 0; 2884 } 2885 2886 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir, 2887 struct dentry *dentry, umode_t mode, bool excl) 2888 { 2889 return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0); 2890 } 2891 2892 /* 2893 * Link a file.. 2894 */ 2895 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry) 2896 { 2897 struct inode *inode = d_inode(old_dentry); 2898 int ret = 0; 2899 2900 /* 2901 * No ordinary (disk based) filesystem counts links as inodes; 2902 * but each new link needs a new dentry, pinning lowmem, and 2903 * tmpfs dentries cannot be pruned until they are unlinked. 2904 * But if an O_TMPFILE file is linked into the tmpfs, the 2905 * first link must skip that, to get the accounting right. 2906 */ 2907 if (inode->i_nlink) { 2908 ret = shmem_reserve_inode(inode->i_sb, NULL); 2909 if (ret) 2910 goto out; 2911 } 2912 2913 dir->i_size += BOGO_DIRENT_SIZE; 2914 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2915 inc_nlink(inode); 2916 ihold(inode); /* New dentry reference */ 2917 dget(dentry); /* Extra pinning count for the created dentry */ 2918 d_instantiate(dentry, inode); 2919 out: 2920 return ret; 2921 } 2922 2923 static int shmem_unlink(struct inode *dir, struct dentry *dentry) 2924 { 2925 struct inode *inode = d_inode(dentry); 2926 2927 if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) 2928 shmem_free_inode(inode->i_sb); 2929 2930 dir->i_size -= BOGO_DIRENT_SIZE; 2931 inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode); 2932 drop_nlink(inode); 2933 dput(dentry); /* Undo the count from "create" - this does all the work */ 2934 return 0; 2935 } 2936 2937 static int shmem_rmdir(struct inode *dir, struct dentry *dentry) 2938 { 2939 if (!simple_empty(dentry)) 2940 return -ENOTEMPTY; 2941 2942 drop_nlink(d_inode(dentry)); 2943 drop_nlink(dir); 2944 return shmem_unlink(dir, dentry); 2945 } 2946 2947 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry) 2948 { 2949 bool old_is_dir = d_is_dir(old_dentry); 2950 bool new_is_dir = d_is_dir(new_dentry); 2951 2952 if (old_dir != new_dir && old_is_dir != new_is_dir) { 2953 if (old_is_dir) { 2954 drop_nlink(old_dir); 2955 inc_nlink(new_dir); 2956 } else { 2957 drop_nlink(new_dir); 2958 inc_nlink(old_dir); 2959 } 2960 } 2961 old_dir->i_ctime = old_dir->i_mtime = 2962 new_dir->i_ctime = new_dir->i_mtime = 2963 d_inode(old_dentry)->i_ctime = 2964 d_inode(new_dentry)->i_ctime = current_time(old_dir); 2965 2966 return 0; 2967 } 2968 2969 static int shmem_whiteout(struct user_namespace *mnt_userns, 2970 struct inode *old_dir, struct dentry *old_dentry) 2971 { 2972 struct dentry *whiteout; 2973 int error; 2974 2975 whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name); 2976 if (!whiteout) 2977 return -ENOMEM; 2978 2979 error = shmem_mknod(&init_user_ns, old_dir, whiteout, 2980 S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV); 2981 dput(whiteout); 2982 if (error) 2983 return error; 2984 2985 /* 2986 * Cheat and hash the whiteout while the old dentry is still in 2987 * place, instead of playing games with FS_RENAME_DOES_D_MOVE. 2988 * 2989 * d_lookup() will consistently find one of them at this point, 2990 * not sure which one, but that isn't even important. 2991 */ 2992 d_rehash(whiteout); 2993 return 0; 2994 } 2995 2996 /* 2997 * The VFS layer already does all the dentry stuff for rename, 2998 * we just have to decrement the usage count for the target if 2999 * it exists so that the VFS layer correctly free's it when it 3000 * gets overwritten. 3001 */ 3002 static int shmem_rename2(struct user_namespace *mnt_userns, 3003 struct inode *old_dir, struct dentry *old_dentry, 3004 struct inode *new_dir, struct dentry *new_dentry, 3005 unsigned int flags) 3006 { 3007 struct inode *inode = d_inode(old_dentry); 3008 int they_are_dirs = S_ISDIR(inode->i_mode); 3009 3010 if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT)) 3011 return -EINVAL; 3012 3013 if (flags & RENAME_EXCHANGE) 3014 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry); 3015 3016 if (!simple_empty(new_dentry)) 3017 return -ENOTEMPTY; 3018 3019 if (flags & RENAME_WHITEOUT) { 3020 int error; 3021 3022 error = shmem_whiteout(&init_user_ns, old_dir, old_dentry); 3023 if (error) 3024 return error; 3025 } 3026 3027 if (d_really_is_positive(new_dentry)) { 3028 (void) shmem_unlink(new_dir, new_dentry); 3029 if (they_are_dirs) { 3030 drop_nlink(d_inode(new_dentry)); 3031 drop_nlink(old_dir); 3032 } 3033 } else if (they_are_dirs) { 3034 drop_nlink(old_dir); 3035 inc_nlink(new_dir); 3036 } 3037 3038 old_dir->i_size -= BOGO_DIRENT_SIZE; 3039 new_dir->i_size += BOGO_DIRENT_SIZE; 3040 old_dir->i_ctime = old_dir->i_mtime = 3041 new_dir->i_ctime = new_dir->i_mtime = 3042 inode->i_ctime = current_time(old_dir); 3043 return 0; 3044 } 3045 3046 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir, 3047 struct dentry *dentry, const char *symname) 3048 { 3049 int error; 3050 int len; 3051 struct inode *inode; 3052 struct page *page; 3053 3054 len = strlen(symname) + 1; 3055 if (len > PAGE_SIZE) 3056 return -ENAMETOOLONG; 3057 3058 inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0, 3059 VM_NORESERVE); 3060 if (!inode) 3061 return -ENOSPC; 3062 3063 error = security_inode_init_security(inode, dir, &dentry->d_name, 3064 shmem_initxattrs, NULL); 3065 if (error && error != -EOPNOTSUPP) { 3066 iput(inode); 3067 return error; 3068 } 3069 3070 inode->i_size = len-1; 3071 if (len <= SHORT_SYMLINK_LEN) { 3072 inode->i_link = kmemdup(symname, len, GFP_KERNEL); 3073 if (!inode->i_link) { 3074 iput(inode); 3075 return -ENOMEM; 3076 } 3077 inode->i_op = &shmem_short_symlink_operations; 3078 } else { 3079 inode_nohighmem(inode); 3080 error = shmem_getpage(inode, 0, &page, SGP_WRITE); 3081 if (error) { 3082 iput(inode); 3083 return error; 3084 } 3085 inode->i_mapping->a_ops = &shmem_aops; 3086 inode->i_op = &shmem_symlink_inode_operations; 3087 memcpy(page_address(page), symname, len); 3088 SetPageUptodate(page); 3089 set_page_dirty(page); 3090 unlock_page(page); 3091 put_page(page); 3092 } 3093 dir->i_size += BOGO_DIRENT_SIZE; 3094 dir->i_ctime = dir->i_mtime = current_time(dir); 3095 d_instantiate(dentry, inode); 3096 dget(dentry); 3097 return 0; 3098 } 3099 3100 static void shmem_put_link(void *arg) 3101 { 3102 mark_page_accessed(arg); 3103 put_page(arg); 3104 } 3105 3106 static const char *shmem_get_link(struct dentry *dentry, 3107 struct inode *inode, 3108 struct delayed_call *done) 3109 { 3110 struct page *page = NULL; 3111 int error; 3112 if (!dentry) { 3113 page = find_get_page(inode->i_mapping, 0); 3114 if (!page) 3115 return ERR_PTR(-ECHILD); 3116 if (!PageUptodate(page)) { 3117 put_page(page); 3118 return ERR_PTR(-ECHILD); 3119 } 3120 } else { 3121 error = shmem_getpage(inode, 0, &page, SGP_READ); 3122 if (error) 3123 return ERR_PTR(error); 3124 unlock_page(page); 3125 } 3126 set_delayed_call(done, shmem_put_link, page); 3127 return page_address(page); 3128 } 3129 3130 #ifdef CONFIG_TMPFS_XATTR 3131 /* 3132 * Superblocks without xattr inode operations may get some security.* xattr 3133 * support from the LSM "for free". As soon as we have any other xattrs 3134 * like ACLs, we also need to implement the security.* handlers at 3135 * filesystem level, though. 3136 */ 3137 3138 /* 3139 * Callback for security_inode_init_security() for acquiring xattrs. 3140 */ 3141 static int shmem_initxattrs(struct inode *inode, 3142 const struct xattr *xattr_array, 3143 void *fs_info) 3144 { 3145 struct shmem_inode_info *info = SHMEM_I(inode); 3146 const struct xattr *xattr; 3147 struct simple_xattr *new_xattr; 3148 size_t len; 3149 3150 for (xattr = xattr_array; xattr->name != NULL; xattr++) { 3151 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len); 3152 if (!new_xattr) 3153 return -ENOMEM; 3154 3155 len = strlen(xattr->name) + 1; 3156 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len, 3157 GFP_KERNEL); 3158 if (!new_xattr->name) { 3159 kvfree(new_xattr); 3160 return -ENOMEM; 3161 } 3162 3163 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX, 3164 XATTR_SECURITY_PREFIX_LEN); 3165 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN, 3166 xattr->name, len); 3167 3168 simple_xattr_list_add(&info->xattrs, new_xattr); 3169 } 3170 3171 return 0; 3172 } 3173 3174 static int shmem_xattr_handler_get(const struct xattr_handler *handler, 3175 struct dentry *unused, struct inode *inode, 3176 const char *name, void *buffer, size_t size) 3177 { 3178 struct shmem_inode_info *info = SHMEM_I(inode); 3179 3180 name = xattr_full_name(handler, name); 3181 return simple_xattr_get(&info->xattrs, name, buffer, size); 3182 } 3183 3184 static int shmem_xattr_handler_set(const struct xattr_handler *handler, 3185 struct user_namespace *mnt_userns, 3186 struct dentry *unused, struct inode *inode, 3187 const char *name, const void *value, 3188 size_t size, int flags) 3189 { 3190 struct shmem_inode_info *info = SHMEM_I(inode); 3191 3192 name = xattr_full_name(handler, name); 3193 return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL); 3194 } 3195 3196 static const struct xattr_handler shmem_security_xattr_handler = { 3197 .prefix = XATTR_SECURITY_PREFIX, 3198 .get = shmem_xattr_handler_get, 3199 .set = shmem_xattr_handler_set, 3200 }; 3201 3202 static const struct xattr_handler shmem_trusted_xattr_handler = { 3203 .prefix = XATTR_TRUSTED_PREFIX, 3204 .get = shmem_xattr_handler_get, 3205 .set = shmem_xattr_handler_set, 3206 }; 3207 3208 static const struct xattr_handler *shmem_xattr_handlers[] = { 3209 #ifdef CONFIG_TMPFS_POSIX_ACL 3210 &posix_acl_access_xattr_handler, 3211 &posix_acl_default_xattr_handler, 3212 #endif 3213 &shmem_security_xattr_handler, 3214 &shmem_trusted_xattr_handler, 3215 NULL 3216 }; 3217 3218 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size) 3219 { 3220 struct shmem_inode_info *info = SHMEM_I(d_inode(dentry)); 3221 return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size); 3222 } 3223 #endif /* CONFIG_TMPFS_XATTR */ 3224 3225 static const struct inode_operations shmem_short_symlink_operations = { 3226 .get_link = simple_get_link, 3227 #ifdef CONFIG_TMPFS_XATTR 3228 .listxattr = shmem_listxattr, 3229 #endif 3230 }; 3231 3232 static const struct inode_operations shmem_symlink_inode_operations = { 3233 .get_link = shmem_get_link, 3234 #ifdef CONFIG_TMPFS_XATTR 3235 .listxattr = shmem_listxattr, 3236 #endif 3237 }; 3238 3239 static struct dentry *shmem_get_parent(struct dentry *child) 3240 { 3241 return ERR_PTR(-ESTALE); 3242 } 3243 3244 static int shmem_match(struct inode *ino, void *vfh) 3245 { 3246 __u32 *fh = vfh; 3247 __u64 inum = fh[2]; 3248 inum = (inum << 32) | fh[1]; 3249 return ino->i_ino == inum && fh[0] == ino->i_generation; 3250 } 3251 3252 /* Find any alias of inode, but prefer a hashed alias */ 3253 static struct dentry *shmem_find_alias(struct inode *inode) 3254 { 3255 struct dentry *alias = d_find_alias(inode); 3256 3257 return alias ?: d_find_any_alias(inode); 3258 } 3259 3260 3261 static struct dentry *shmem_fh_to_dentry(struct super_block *sb, 3262 struct fid *fid, int fh_len, int fh_type) 3263 { 3264 struct inode *inode; 3265 struct dentry *dentry = NULL; 3266 u64 inum; 3267 3268 if (fh_len < 3) 3269 return NULL; 3270 3271 inum = fid->raw[2]; 3272 inum = (inum << 32) | fid->raw[1]; 3273 3274 inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]), 3275 shmem_match, fid->raw); 3276 if (inode) { 3277 dentry = shmem_find_alias(inode); 3278 iput(inode); 3279 } 3280 3281 return dentry; 3282 } 3283 3284 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len, 3285 struct inode *parent) 3286 { 3287 if (*len < 3) { 3288 *len = 3; 3289 return FILEID_INVALID; 3290 } 3291 3292 if (inode_unhashed(inode)) { 3293 /* Unfortunately insert_inode_hash is not idempotent, 3294 * so as we hash inodes here rather than at creation 3295 * time, we need a lock to ensure we only try 3296 * to do it once 3297 */ 3298 static DEFINE_SPINLOCK(lock); 3299 spin_lock(&lock); 3300 if (inode_unhashed(inode)) 3301 __insert_inode_hash(inode, 3302 inode->i_ino + inode->i_generation); 3303 spin_unlock(&lock); 3304 } 3305 3306 fh[0] = inode->i_generation; 3307 fh[1] = inode->i_ino; 3308 fh[2] = ((__u64)inode->i_ino) >> 32; 3309 3310 *len = 3; 3311 return 1; 3312 } 3313 3314 static const struct export_operations shmem_export_ops = { 3315 .get_parent = shmem_get_parent, 3316 .encode_fh = shmem_encode_fh, 3317 .fh_to_dentry = shmem_fh_to_dentry, 3318 }; 3319 3320 enum shmem_param { 3321 Opt_gid, 3322 Opt_huge, 3323 Opt_mode, 3324 Opt_mpol, 3325 Opt_nr_blocks, 3326 Opt_nr_inodes, 3327 Opt_size, 3328 Opt_uid, 3329 Opt_inode32, 3330 Opt_inode64, 3331 }; 3332 3333 static const struct constant_table shmem_param_enums_huge[] = { 3334 {"never", SHMEM_HUGE_NEVER }, 3335 {"always", SHMEM_HUGE_ALWAYS }, 3336 {"within_size", SHMEM_HUGE_WITHIN_SIZE }, 3337 {"advise", SHMEM_HUGE_ADVISE }, 3338 {} 3339 }; 3340 3341 const struct fs_parameter_spec shmem_fs_parameters[] = { 3342 fsparam_u32 ("gid", Opt_gid), 3343 fsparam_enum ("huge", Opt_huge, shmem_param_enums_huge), 3344 fsparam_u32oct("mode", Opt_mode), 3345 fsparam_string("mpol", Opt_mpol), 3346 fsparam_string("nr_blocks", Opt_nr_blocks), 3347 fsparam_string("nr_inodes", Opt_nr_inodes), 3348 fsparam_string("size", Opt_size), 3349 fsparam_u32 ("uid", Opt_uid), 3350 fsparam_flag ("inode32", Opt_inode32), 3351 fsparam_flag ("inode64", Opt_inode64), 3352 {} 3353 }; 3354 3355 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param) 3356 { 3357 struct shmem_options *ctx = fc->fs_private; 3358 struct fs_parse_result result; 3359 unsigned long long size; 3360 char *rest; 3361 int opt; 3362 3363 opt = fs_parse(fc, shmem_fs_parameters, param, &result); 3364 if (opt < 0) 3365 return opt; 3366 3367 switch (opt) { 3368 case Opt_size: 3369 size = memparse(param->string, &rest); 3370 if (*rest == '%') { 3371 size <<= PAGE_SHIFT; 3372 size *= totalram_pages(); 3373 do_div(size, 100); 3374 rest++; 3375 } 3376 if (*rest) 3377 goto bad_value; 3378 ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE); 3379 ctx->seen |= SHMEM_SEEN_BLOCKS; 3380 break; 3381 case Opt_nr_blocks: 3382 ctx->blocks = memparse(param->string, &rest); 3383 if (*rest) 3384 goto bad_value; 3385 ctx->seen |= SHMEM_SEEN_BLOCKS; 3386 break; 3387 case Opt_nr_inodes: 3388 ctx->inodes = memparse(param->string, &rest); 3389 if (*rest) 3390 goto bad_value; 3391 ctx->seen |= SHMEM_SEEN_INODES; 3392 break; 3393 case Opt_mode: 3394 ctx->mode = result.uint_32 & 07777; 3395 break; 3396 case Opt_uid: 3397 ctx->uid = make_kuid(current_user_ns(), result.uint_32); 3398 if (!uid_valid(ctx->uid)) 3399 goto bad_value; 3400 break; 3401 case Opt_gid: 3402 ctx->gid = make_kgid(current_user_ns(), result.uint_32); 3403 if (!gid_valid(ctx->gid)) 3404 goto bad_value; 3405 break; 3406 case Opt_huge: 3407 ctx->huge = result.uint_32; 3408 if (ctx->huge != SHMEM_HUGE_NEVER && 3409 !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 3410 has_transparent_hugepage())) 3411 goto unsupported_parameter; 3412 ctx->seen |= SHMEM_SEEN_HUGE; 3413 break; 3414 case Opt_mpol: 3415 if (IS_ENABLED(CONFIG_NUMA)) { 3416 mpol_put(ctx->mpol); 3417 ctx->mpol = NULL; 3418 if (mpol_parse_str(param->string, &ctx->mpol)) 3419 goto bad_value; 3420 break; 3421 } 3422 goto unsupported_parameter; 3423 case Opt_inode32: 3424 ctx->full_inums = false; 3425 ctx->seen |= SHMEM_SEEN_INUMS; 3426 break; 3427 case Opt_inode64: 3428 if (sizeof(ino_t) < 8) { 3429 return invalfc(fc, 3430 "Cannot use inode64 with <64bit inums in kernel\n"); 3431 } 3432 ctx->full_inums = true; 3433 ctx->seen |= SHMEM_SEEN_INUMS; 3434 break; 3435 } 3436 return 0; 3437 3438 unsupported_parameter: 3439 return invalfc(fc, "Unsupported parameter '%s'", param->key); 3440 bad_value: 3441 return invalfc(fc, "Bad value for '%s'", param->key); 3442 } 3443 3444 static int shmem_parse_options(struct fs_context *fc, void *data) 3445 { 3446 char *options = data; 3447 3448 if (options) { 3449 int err = security_sb_eat_lsm_opts(options, &fc->security); 3450 if (err) 3451 return err; 3452 } 3453 3454 while (options != NULL) { 3455 char *this_char = options; 3456 for (;;) { 3457 /* 3458 * NUL-terminate this option: unfortunately, 3459 * mount options form a comma-separated list, 3460 * but mpol's nodelist may also contain commas. 3461 */ 3462 options = strchr(options, ','); 3463 if (options == NULL) 3464 break; 3465 options++; 3466 if (!isdigit(*options)) { 3467 options[-1] = '\0'; 3468 break; 3469 } 3470 } 3471 if (*this_char) { 3472 char *value = strchr(this_char, '='); 3473 size_t len = 0; 3474 int err; 3475 3476 if (value) { 3477 *value++ = '\0'; 3478 len = strlen(value); 3479 } 3480 err = vfs_parse_fs_string(fc, this_char, value, len); 3481 if (err < 0) 3482 return err; 3483 } 3484 } 3485 return 0; 3486 } 3487 3488 /* 3489 * Reconfigure a shmem filesystem. 3490 * 3491 * Note that we disallow change from limited->unlimited blocks/inodes while any 3492 * are in use; but we must separately disallow unlimited->limited, because in 3493 * that case we have no record of how much is already in use. 3494 */ 3495 static int shmem_reconfigure(struct fs_context *fc) 3496 { 3497 struct shmem_options *ctx = fc->fs_private; 3498 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb); 3499 unsigned long inodes; 3500 struct mempolicy *mpol = NULL; 3501 const char *err; 3502 3503 raw_spin_lock(&sbinfo->stat_lock); 3504 inodes = sbinfo->max_inodes - sbinfo->free_inodes; 3505 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) { 3506 if (!sbinfo->max_blocks) { 3507 err = "Cannot retroactively limit size"; 3508 goto out; 3509 } 3510 if (percpu_counter_compare(&sbinfo->used_blocks, 3511 ctx->blocks) > 0) { 3512 err = "Too small a size for current use"; 3513 goto out; 3514 } 3515 } 3516 if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) { 3517 if (!sbinfo->max_inodes) { 3518 err = "Cannot retroactively limit inodes"; 3519 goto out; 3520 } 3521 if (ctx->inodes < inodes) { 3522 err = "Too few inodes for current use"; 3523 goto out; 3524 } 3525 } 3526 3527 if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums && 3528 sbinfo->next_ino > UINT_MAX) { 3529 err = "Current inum too high to switch to 32-bit inums"; 3530 goto out; 3531 } 3532 3533 if (ctx->seen & SHMEM_SEEN_HUGE) 3534 sbinfo->huge = ctx->huge; 3535 if (ctx->seen & SHMEM_SEEN_INUMS) 3536 sbinfo->full_inums = ctx->full_inums; 3537 if (ctx->seen & SHMEM_SEEN_BLOCKS) 3538 sbinfo->max_blocks = ctx->blocks; 3539 if (ctx->seen & SHMEM_SEEN_INODES) { 3540 sbinfo->max_inodes = ctx->inodes; 3541 sbinfo->free_inodes = ctx->inodes - inodes; 3542 } 3543 3544 /* 3545 * Preserve previous mempolicy unless mpol remount option was specified. 3546 */ 3547 if (ctx->mpol) { 3548 mpol = sbinfo->mpol; 3549 sbinfo->mpol = ctx->mpol; /* transfers initial ref */ 3550 ctx->mpol = NULL; 3551 } 3552 raw_spin_unlock(&sbinfo->stat_lock); 3553 mpol_put(mpol); 3554 return 0; 3555 out: 3556 raw_spin_unlock(&sbinfo->stat_lock); 3557 return invalfc(fc, "%s", err); 3558 } 3559 3560 static int shmem_show_options(struct seq_file *seq, struct dentry *root) 3561 { 3562 struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb); 3563 3564 if (sbinfo->max_blocks != shmem_default_max_blocks()) 3565 seq_printf(seq, ",size=%luk", 3566 sbinfo->max_blocks << (PAGE_SHIFT - 10)); 3567 if (sbinfo->max_inodes != shmem_default_max_inodes()) 3568 seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes); 3569 if (sbinfo->mode != (0777 | S_ISVTX)) 3570 seq_printf(seq, ",mode=%03ho", sbinfo->mode); 3571 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID)) 3572 seq_printf(seq, ",uid=%u", 3573 from_kuid_munged(&init_user_ns, sbinfo->uid)); 3574 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID)) 3575 seq_printf(seq, ",gid=%u", 3576 from_kgid_munged(&init_user_ns, sbinfo->gid)); 3577 3578 /* 3579 * Showing inode{64,32} might be useful even if it's the system default, 3580 * since then people don't have to resort to checking both here and 3581 * /proc/config.gz to confirm 64-bit inums were successfully applied 3582 * (which may not even exist if IKCONFIG_PROC isn't enabled). 3583 * 3584 * We hide it when inode64 isn't the default and we are using 32-bit 3585 * inodes, since that probably just means the feature isn't even under 3586 * consideration. 3587 * 3588 * As such: 3589 * 3590 * +-----------------+-----------------+ 3591 * | TMPFS_INODE64=y | TMPFS_INODE64=n | 3592 * +------------------+-----------------+-----------------+ 3593 * | full_inums=true | show | show | 3594 * | full_inums=false | show | hide | 3595 * +------------------+-----------------+-----------------+ 3596 * 3597 */ 3598 if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums) 3599 seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32)); 3600 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3601 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */ 3602 if (sbinfo->huge) 3603 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge)); 3604 #endif 3605 shmem_show_mpol(seq, sbinfo->mpol); 3606 return 0; 3607 } 3608 3609 #endif /* CONFIG_TMPFS */ 3610 3611 static void shmem_put_super(struct super_block *sb) 3612 { 3613 struct shmem_sb_info *sbinfo = SHMEM_SB(sb); 3614 3615 free_percpu(sbinfo->ino_batch); 3616 percpu_counter_destroy(&sbinfo->used_blocks); 3617 mpol_put(sbinfo->mpol); 3618 kfree(sbinfo); 3619 sb->s_fs_info = NULL; 3620 } 3621 3622 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc) 3623 { 3624 struct shmem_options *ctx = fc->fs_private; 3625 struct inode *inode; 3626 struct shmem_sb_info *sbinfo; 3627 3628 /* Round up to L1_CACHE_BYTES to resist false sharing */ 3629 sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info), 3630 L1_CACHE_BYTES), GFP_KERNEL); 3631 if (!sbinfo) 3632 return -ENOMEM; 3633 3634 sb->s_fs_info = sbinfo; 3635 3636 #ifdef CONFIG_TMPFS 3637 /* 3638 * Per default we only allow half of the physical ram per 3639 * tmpfs instance, limiting inodes to one per page of lowmem; 3640 * but the internal instance is left unlimited. 3641 */ 3642 if (!(sb->s_flags & SB_KERNMOUNT)) { 3643 if (!(ctx->seen & SHMEM_SEEN_BLOCKS)) 3644 ctx->blocks = shmem_default_max_blocks(); 3645 if (!(ctx->seen & SHMEM_SEEN_INODES)) 3646 ctx->inodes = shmem_default_max_inodes(); 3647 if (!(ctx->seen & SHMEM_SEEN_INUMS)) 3648 ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64); 3649 } else { 3650 sb->s_flags |= SB_NOUSER; 3651 } 3652 sb->s_export_op = &shmem_export_ops; 3653 sb->s_flags |= SB_NOSEC; 3654 #else 3655 sb->s_flags |= SB_NOUSER; 3656 #endif 3657 sbinfo->max_blocks = ctx->blocks; 3658 sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes; 3659 if (sb->s_flags & SB_KERNMOUNT) { 3660 sbinfo->ino_batch = alloc_percpu(ino_t); 3661 if (!sbinfo->ino_batch) 3662 goto failed; 3663 } 3664 sbinfo->uid = ctx->uid; 3665 sbinfo->gid = ctx->gid; 3666 sbinfo->full_inums = ctx->full_inums; 3667 sbinfo->mode = ctx->mode; 3668 sbinfo->huge = ctx->huge; 3669 sbinfo->mpol = ctx->mpol; 3670 ctx->mpol = NULL; 3671 3672 raw_spin_lock_init(&sbinfo->stat_lock); 3673 if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL)) 3674 goto failed; 3675 spin_lock_init(&sbinfo->shrinklist_lock); 3676 INIT_LIST_HEAD(&sbinfo->shrinklist); 3677 3678 sb->s_maxbytes = MAX_LFS_FILESIZE; 3679 sb->s_blocksize = PAGE_SIZE; 3680 sb->s_blocksize_bits = PAGE_SHIFT; 3681 sb->s_magic = TMPFS_MAGIC; 3682 sb->s_op = &shmem_ops; 3683 sb->s_time_gran = 1; 3684 #ifdef CONFIG_TMPFS_XATTR 3685 sb->s_xattr = shmem_xattr_handlers; 3686 #endif 3687 #ifdef CONFIG_TMPFS_POSIX_ACL 3688 sb->s_flags |= SB_POSIXACL; 3689 #endif 3690 uuid_gen(&sb->s_uuid); 3691 3692 inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE); 3693 if (!inode) 3694 goto failed; 3695 inode->i_uid = sbinfo->uid; 3696 inode->i_gid = sbinfo->gid; 3697 sb->s_root = d_make_root(inode); 3698 if (!sb->s_root) 3699 goto failed; 3700 return 0; 3701 3702 failed: 3703 shmem_put_super(sb); 3704 return -ENOMEM; 3705 } 3706 3707 static int shmem_get_tree(struct fs_context *fc) 3708 { 3709 return get_tree_nodev(fc, shmem_fill_super); 3710 } 3711 3712 static void shmem_free_fc(struct fs_context *fc) 3713 { 3714 struct shmem_options *ctx = fc->fs_private; 3715 3716 if (ctx) { 3717 mpol_put(ctx->mpol); 3718 kfree(ctx); 3719 } 3720 } 3721 3722 static const struct fs_context_operations shmem_fs_context_ops = { 3723 .free = shmem_free_fc, 3724 .get_tree = shmem_get_tree, 3725 #ifdef CONFIG_TMPFS 3726 .parse_monolithic = shmem_parse_options, 3727 .parse_param = shmem_parse_one, 3728 .reconfigure = shmem_reconfigure, 3729 #endif 3730 }; 3731 3732 static struct kmem_cache *shmem_inode_cachep; 3733 3734 static struct inode *shmem_alloc_inode(struct super_block *sb) 3735 { 3736 struct shmem_inode_info *info; 3737 info = kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL); 3738 if (!info) 3739 return NULL; 3740 return &info->vfs_inode; 3741 } 3742 3743 static void shmem_free_in_core_inode(struct inode *inode) 3744 { 3745 if (S_ISLNK(inode->i_mode)) 3746 kfree(inode->i_link); 3747 kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode)); 3748 } 3749 3750 static void shmem_destroy_inode(struct inode *inode) 3751 { 3752 if (S_ISREG(inode->i_mode)) 3753 mpol_free_shared_policy(&SHMEM_I(inode)->policy); 3754 } 3755 3756 static void shmem_init_inode(void *foo) 3757 { 3758 struct shmem_inode_info *info = foo; 3759 inode_init_once(&info->vfs_inode); 3760 } 3761 3762 static void shmem_init_inodecache(void) 3763 { 3764 shmem_inode_cachep = kmem_cache_create("shmem_inode_cache", 3765 sizeof(struct shmem_inode_info), 3766 0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode); 3767 } 3768 3769 static void shmem_destroy_inodecache(void) 3770 { 3771 kmem_cache_destroy(shmem_inode_cachep); 3772 } 3773 3774 const struct address_space_operations shmem_aops = { 3775 .writepage = shmem_writepage, 3776 .set_page_dirty = __set_page_dirty_no_writeback, 3777 #ifdef CONFIG_TMPFS 3778 .write_begin = shmem_write_begin, 3779 .write_end = shmem_write_end, 3780 #endif 3781 #ifdef CONFIG_MIGRATION 3782 .migratepage = migrate_page, 3783 #endif 3784 .error_remove_page = generic_error_remove_page, 3785 }; 3786 EXPORT_SYMBOL(shmem_aops); 3787 3788 static const struct file_operations shmem_file_operations = { 3789 .mmap = shmem_mmap, 3790 .get_unmapped_area = shmem_get_unmapped_area, 3791 #ifdef CONFIG_TMPFS 3792 .llseek = shmem_file_llseek, 3793 .read_iter = shmem_file_read_iter, 3794 .write_iter = generic_file_write_iter, 3795 .fsync = noop_fsync, 3796 .splice_read = generic_file_splice_read, 3797 .splice_write = iter_file_splice_write, 3798 .fallocate = shmem_fallocate, 3799 #endif 3800 }; 3801 3802 static const struct inode_operations shmem_inode_operations = { 3803 .getattr = shmem_getattr, 3804 .setattr = shmem_setattr, 3805 #ifdef CONFIG_TMPFS_XATTR 3806 .listxattr = shmem_listxattr, 3807 .set_acl = simple_set_acl, 3808 #endif 3809 }; 3810 3811 static const struct inode_operations shmem_dir_inode_operations = { 3812 #ifdef CONFIG_TMPFS 3813 .create = shmem_create, 3814 .lookup = simple_lookup, 3815 .link = shmem_link, 3816 .unlink = shmem_unlink, 3817 .symlink = shmem_symlink, 3818 .mkdir = shmem_mkdir, 3819 .rmdir = shmem_rmdir, 3820 .mknod = shmem_mknod, 3821 .rename = shmem_rename2, 3822 .tmpfile = shmem_tmpfile, 3823 #endif 3824 #ifdef CONFIG_TMPFS_XATTR 3825 .listxattr = shmem_listxattr, 3826 #endif 3827 #ifdef CONFIG_TMPFS_POSIX_ACL 3828 .setattr = shmem_setattr, 3829 .set_acl = simple_set_acl, 3830 #endif 3831 }; 3832 3833 static const struct inode_operations shmem_special_inode_operations = { 3834 #ifdef CONFIG_TMPFS_XATTR 3835 .listxattr = shmem_listxattr, 3836 #endif 3837 #ifdef CONFIG_TMPFS_POSIX_ACL 3838 .setattr = shmem_setattr, 3839 .set_acl = simple_set_acl, 3840 #endif 3841 }; 3842 3843 static const struct super_operations shmem_ops = { 3844 .alloc_inode = shmem_alloc_inode, 3845 .free_inode = shmem_free_in_core_inode, 3846 .destroy_inode = shmem_destroy_inode, 3847 #ifdef CONFIG_TMPFS 3848 .statfs = shmem_statfs, 3849 .show_options = shmem_show_options, 3850 #endif 3851 .evict_inode = shmem_evict_inode, 3852 .drop_inode = generic_delete_inode, 3853 .put_super = shmem_put_super, 3854 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3855 .nr_cached_objects = shmem_unused_huge_count, 3856 .free_cached_objects = shmem_unused_huge_scan, 3857 #endif 3858 }; 3859 3860 static const struct vm_operations_struct shmem_vm_ops = { 3861 .fault = shmem_fault, 3862 .map_pages = filemap_map_pages, 3863 #ifdef CONFIG_NUMA 3864 .set_policy = shmem_set_policy, 3865 .get_policy = shmem_get_policy, 3866 #endif 3867 }; 3868 3869 int shmem_init_fs_context(struct fs_context *fc) 3870 { 3871 struct shmem_options *ctx; 3872 3873 ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL); 3874 if (!ctx) 3875 return -ENOMEM; 3876 3877 ctx->mode = 0777 | S_ISVTX; 3878 ctx->uid = current_fsuid(); 3879 ctx->gid = current_fsgid(); 3880 3881 fc->fs_private = ctx; 3882 fc->ops = &shmem_fs_context_ops; 3883 return 0; 3884 } 3885 3886 static struct file_system_type shmem_fs_type = { 3887 .owner = THIS_MODULE, 3888 .name = "tmpfs", 3889 .init_fs_context = shmem_init_fs_context, 3890 #ifdef CONFIG_TMPFS 3891 .parameters = shmem_fs_parameters, 3892 #endif 3893 .kill_sb = kill_litter_super, 3894 .fs_flags = FS_USERNS_MOUNT | FS_THP_SUPPORT, 3895 }; 3896 3897 int __init shmem_init(void) 3898 { 3899 int error; 3900 3901 shmem_init_inodecache(); 3902 3903 error = register_filesystem(&shmem_fs_type); 3904 if (error) { 3905 pr_err("Could not register tmpfs\n"); 3906 goto out2; 3907 } 3908 3909 shm_mnt = kern_mount(&shmem_fs_type); 3910 if (IS_ERR(shm_mnt)) { 3911 error = PTR_ERR(shm_mnt); 3912 pr_err("Could not kern_mount tmpfs\n"); 3913 goto out1; 3914 } 3915 3916 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 3917 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY) 3918 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3919 else 3920 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */ 3921 #endif 3922 return 0; 3923 3924 out1: 3925 unregister_filesystem(&shmem_fs_type); 3926 out2: 3927 shmem_destroy_inodecache(); 3928 shm_mnt = ERR_PTR(error); 3929 return error; 3930 } 3931 3932 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS) 3933 static ssize_t shmem_enabled_show(struct kobject *kobj, 3934 struct kobj_attribute *attr, char *buf) 3935 { 3936 static const int values[] = { 3937 SHMEM_HUGE_ALWAYS, 3938 SHMEM_HUGE_WITHIN_SIZE, 3939 SHMEM_HUGE_ADVISE, 3940 SHMEM_HUGE_NEVER, 3941 SHMEM_HUGE_DENY, 3942 SHMEM_HUGE_FORCE, 3943 }; 3944 int len = 0; 3945 int i; 3946 3947 for (i = 0; i < ARRAY_SIZE(values); i++) { 3948 len += sysfs_emit_at(buf, len, 3949 shmem_huge == values[i] ? "%s[%s]" : "%s%s", 3950 i ? " " : "", 3951 shmem_format_huge(values[i])); 3952 } 3953 3954 len += sysfs_emit_at(buf, len, "\n"); 3955 3956 return len; 3957 } 3958 3959 static ssize_t shmem_enabled_store(struct kobject *kobj, 3960 struct kobj_attribute *attr, const char *buf, size_t count) 3961 { 3962 char tmp[16]; 3963 int huge; 3964 3965 if (count + 1 > sizeof(tmp)) 3966 return -EINVAL; 3967 memcpy(tmp, buf, count); 3968 tmp[count] = '\0'; 3969 if (count && tmp[count - 1] == '\n') 3970 tmp[count - 1] = '\0'; 3971 3972 huge = shmem_parse_huge(tmp); 3973 if (huge == -EINVAL) 3974 return -EINVAL; 3975 if (!has_transparent_hugepage() && 3976 huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY) 3977 return -EINVAL; 3978 3979 shmem_huge = huge; 3980 if (shmem_huge > SHMEM_HUGE_DENY) 3981 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge; 3982 return count; 3983 } 3984 3985 struct kobj_attribute shmem_enabled_attr = 3986 __ATTR(shmem_enabled, 0644, shmem_enabled_show, shmem_enabled_store); 3987 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */ 3988 3989 #else /* !CONFIG_SHMEM */ 3990 3991 /* 3992 * tiny-shmem: simple shmemfs and tmpfs using ramfs code 3993 * 3994 * This is intended for small system where the benefits of the full 3995 * shmem code (swap-backed and resource-limited) are outweighed by 3996 * their complexity. On systems without swap this code should be 3997 * effectively equivalent, but much lighter weight. 3998 */ 3999 4000 static struct file_system_type shmem_fs_type = { 4001 .name = "tmpfs", 4002 .init_fs_context = ramfs_init_fs_context, 4003 .parameters = ramfs_fs_parameters, 4004 .kill_sb = kill_litter_super, 4005 .fs_flags = FS_USERNS_MOUNT, 4006 }; 4007 4008 int __init shmem_init(void) 4009 { 4010 BUG_ON(register_filesystem(&shmem_fs_type) != 0); 4011 4012 shm_mnt = kern_mount(&shmem_fs_type); 4013 BUG_ON(IS_ERR(shm_mnt)); 4014 4015 return 0; 4016 } 4017 4018 int shmem_unuse(unsigned int type, bool frontswap, 4019 unsigned long *fs_pages_to_unuse) 4020 { 4021 return 0; 4022 } 4023 4024 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) 4025 { 4026 return 0; 4027 } 4028 4029 void shmem_unlock_mapping(struct address_space *mapping) 4030 { 4031 } 4032 4033 #ifdef CONFIG_MMU 4034 unsigned long shmem_get_unmapped_area(struct file *file, 4035 unsigned long addr, unsigned long len, 4036 unsigned long pgoff, unsigned long flags) 4037 { 4038 return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); 4039 } 4040 #endif 4041 4042 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend) 4043 { 4044 truncate_inode_pages_range(inode->i_mapping, lstart, lend); 4045 } 4046 EXPORT_SYMBOL_GPL(shmem_truncate_range); 4047 4048 #define shmem_vm_ops generic_file_vm_ops 4049 #define shmem_file_operations ramfs_file_operations 4050 #define shmem_get_inode(sb, dir, mode, dev, flags) ramfs_get_inode(sb, dir, mode, dev) 4051 #define shmem_acct_size(flags, size) 0 4052 #define shmem_unacct_size(flags, size) do {} while (0) 4053 4054 #endif /* CONFIG_SHMEM */ 4055 4056 /* common code */ 4057 4058 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size, 4059 unsigned long flags, unsigned int i_flags) 4060 { 4061 struct inode *inode; 4062 struct file *res; 4063 4064 if (IS_ERR(mnt)) 4065 return ERR_CAST(mnt); 4066 4067 if (size < 0 || size > MAX_LFS_FILESIZE) 4068 return ERR_PTR(-EINVAL); 4069 4070 if (shmem_acct_size(flags, size)) 4071 return ERR_PTR(-ENOMEM); 4072 4073 inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0, 4074 flags); 4075 if (unlikely(!inode)) { 4076 shmem_unacct_size(flags, size); 4077 return ERR_PTR(-ENOSPC); 4078 } 4079 inode->i_flags |= i_flags; 4080 inode->i_size = size; 4081 clear_nlink(inode); /* It is unlinked */ 4082 res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size)); 4083 if (!IS_ERR(res)) 4084 res = alloc_file_pseudo(inode, mnt, name, O_RDWR, 4085 &shmem_file_operations); 4086 if (IS_ERR(res)) 4087 iput(inode); 4088 return res; 4089 } 4090 4091 /** 4092 * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be 4093 * kernel internal. There will be NO LSM permission checks against the 4094 * underlying inode. So users of this interface must do LSM checks at a 4095 * higher layer. The users are the big_key and shm implementations. LSM 4096 * checks are provided at the key or shm level rather than the inode. 4097 * @name: name for dentry (to be seen in /proc/<pid>/maps 4098 * @size: size to be set for the file 4099 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4100 */ 4101 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags) 4102 { 4103 return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE); 4104 } 4105 4106 /** 4107 * shmem_file_setup - get an unlinked file living in tmpfs 4108 * @name: name for dentry (to be seen in /proc/<pid>/maps 4109 * @size: size to be set for the file 4110 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4111 */ 4112 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags) 4113 { 4114 return __shmem_file_setup(shm_mnt, name, size, flags, 0); 4115 } 4116 EXPORT_SYMBOL_GPL(shmem_file_setup); 4117 4118 /** 4119 * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs 4120 * @mnt: the tmpfs mount where the file will be created 4121 * @name: name for dentry (to be seen in /proc/<pid>/maps 4122 * @size: size to be set for the file 4123 * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size 4124 */ 4125 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name, 4126 loff_t size, unsigned long flags) 4127 { 4128 return __shmem_file_setup(mnt, name, size, flags, 0); 4129 } 4130 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt); 4131 4132 /** 4133 * shmem_zero_setup - setup a shared anonymous mapping 4134 * @vma: the vma to be mmapped is prepared by do_mmap 4135 */ 4136 int shmem_zero_setup(struct vm_area_struct *vma) 4137 { 4138 struct file *file; 4139 loff_t size = vma->vm_end - vma->vm_start; 4140 4141 /* 4142 * Cloning a new file under mmap_lock leads to a lock ordering conflict 4143 * between XFS directory reading and selinux: since this file is only 4144 * accessible to the user through its mapping, use S_PRIVATE flag to 4145 * bypass file security, in the same way as shmem_kernel_file_setup(). 4146 */ 4147 file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags); 4148 if (IS_ERR(file)) 4149 return PTR_ERR(file); 4150 4151 if (vma->vm_file) 4152 fput(vma->vm_file); 4153 vma->vm_file = file; 4154 vma->vm_ops = &shmem_vm_ops; 4155 4156 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) && 4157 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) < 4158 (vma->vm_end & HPAGE_PMD_MASK)) { 4159 khugepaged_enter(vma, vma->vm_flags); 4160 } 4161 4162 return 0; 4163 } 4164 4165 /** 4166 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags. 4167 * @mapping: the page's address_space 4168 * @index: the page index 4169 * @gfp: the page allocator flags to use if allocating 4170 * 4171 * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)", 4172 * with any new page allocations done using the specified allocation flags. 4173 * But read_cache_page_gfp() uses the ->readpage() method: which does not 4174 * suit tmpfs, since it may have pages in swapcache, and needs to find those 4175 * for itself; although drivers/gpu/drm i915 and ttm rely upon this support. 4176 * 4177 * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in 4178 * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily. 4179 */ 4180 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping, 4181 pgoff_t index, gfp_t gfp) 4182 { 4183 #ifdef CONFIG_SHMEM 4184 struct inode *inode = mapping->host; 4185 struct page *page; 4186 int error; 4187 4188 BUG_ON(!shmem_mapping(mapping)); 4189 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE, 4190 gfp, NULL, NULL, NULL); 4191 if (error) 4192 page = ERR_PTR(error); 4193 else 4194 unlock_page(page); 4195 return page; 4196 #else 4197 /* 4198 * The tiny !SHMEM case uses ramfs without swap 4199 */ 4200 return read_cache_page_gfp(mapping, index, gfp); 4201 #endif 4202 } 4203 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp); 4204