xref: /openbmc/linux/mm/shmem.c (revision 923e2f0e7c30db5c1ee5d680050ab781e6c114fb)
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *		 2000 Transmeta Corp.
6  *		 2000-2001 Christoph Rohland
7  *		 2000-2001 SAP AG
8  *		 2002 Red Hat Inc.
9  * Copyright (C) 2002-2011 Hugh Dickins.
10  * Copyright (C) 2011 Google Inc.
11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
13  *
14  * Extended attribute support for tmpfs:
15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17  *
18  * tiny-shmem:
19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20  *
21  * This file is released under the GPL.
22  */
23 
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/fileattr.h>
32 #include <linux/mm.h>
33 #include <linux/random.h>
34 #include <linux/sched/signal.h>
35 #include <linux/export.h>
36 #include <linux/swap.h>
37 #include <linux/uio.h>
38 #include <linux/hugetlb.h>
39 #include <linux/fs_parser.h>
40 #include <linux/swapfile.h>
41 #include "swap.h"
42 
43 static struct vfsmount *shm_mnt;
44 
45 #ifdef CONFIG_SHMEM
46 /*
47  * This virtual memory filesystem is heavily based on the ramfs. It
48  * extends ramfs by the ability to use swap and honor resource limits
49  * which makes it a completely usable filesystem.
50  */
51 
52 #include <linux/xattr.h>
53 #include <linux/exportfs.h>
54 #include <linux/posix_acl.h>
55 #include <linux/posix_acl_xattr.h>
56 #include <linux/mman.h>
57 #include <linux/string.h>
58 #include <linux/slab.h>
59 #include <linux/backing-dev.h>
60 #include <linux/shmem_fs.h>
61 #include <linux/writeback.h>
62 #include <linux/pagevec.h>
63 #include <linux/percpu_counter.h>
64 #include <linux/falloc.h>
65 #include <linux/splice.h>
66 #include <linux/security.h>
67 #include <linux/swapops.h>
68 #include <linux/mempolicy.h>
69 #include <linux/namei.h>
70 #include <linux/ctype.h>
71 #include <linux/migrate.h>
72 #include <linux/highmem.h>
73 #include <linux/seq_file.h>
74 #include <linux/magic.h>
75 #include <linux/syscalls.h>
76 #include <linux/fcntl.h>
77 #include <uapi/linux/memfd.h>
78 #include <linux/userfaultfd_k.h>
79 #include <linux/rmap.h>
80 #include <linux/uuid.h>
81 
82 #include <linux/uaccess.h>
83 
84 #include "internal.h"
85 
86 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
87 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
88 
89 /* Pretend that each entry is of this size in directory's i_size */
90 #define BOGO_DIRENT_SIZE 20
91 
92 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
93 #define SHORT_SYMLINK_LEN 128
94 
95 /*
96  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
97  * inode->i_private (with i_rwsem making sure that it has only one user at
98  * a time): we would prefer not to enlarge the shmem inode just for that.
99  */
100 struct shmem_falloc {
101 	wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
102 	pgoff_t start;		/* start of range currently being fallocated */
103 	pgoff_t next;		/* the next page offset to be fallocated */
104 	pgoff_t nr_falloced;	/* how many new pages have been fallocated */
105 	pgoff_t nr_unswapped;	/* how often writepage refused to swap out */
106 };
107 
108 struct shmem_options {
109 	unsigned long long blocks;
110 	unsigned long long inodes;
111 	struct mempolicy *mpol;
112 	kuid_t uid;
113 	kgid_t gid;
114 	umode_t mode;
115 	bool full_inums;
116 	int huge;
117 	int seen;
118 #define SHMEM_SEEN_BLOCKS 1
119 #define SHMEM_SEEN_INODES 2
120 #define SHMEM_SEEN_HUGE 4
121 #define SHMEM_SEEN_INUMS 8
122 };
123 
124 #ifdef CONFIG_TMPFS
125 static unsigned long shmem_default_max_blocks(void)
126 {
127 	return totalram_pages() / 2;
128 }
129 
130 static unsigned long shmem_default_max_inodes(void)
131 {
132 	unsigned long nr_pages = totalram_pages();
133 
134 	return min(nr_pages - totalhigh_pages(), nr_pages / 2);
135 }
136 #endif
137 
138 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
139 			     struct folio **foliop, enum sgp_type sgp,
140 			     gfp_t gfp, struct vm_area_struct *vma,
141 			     vm_fault_t *fault_type);
142 
143 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
144 {
145 	return sb->s_fs_info;
146 }
147 
148 /*
149  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
150  * for shared memory and for shared anonymous (/dev/zero) mappings
151  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
152  * consistent with the pre-accounting of private mappings ...
153  */
154 static inline int shmem_acct_size(unsigned long flags, loff_t size)
155 {
156 	return (flags & VM_NORESERVE) ?
157 		0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
158 }
159 
160 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
161 {
162 	if (!(flags & VM_NORESERVE))
163 		vm_unacct_memory(VM_ACCT(size));
164 }
165 
166 static inline int shmem_reacct_size(unsigned long flags,
167 		loff_t oldsize, loff_t newsize)
168 {
169 	if (!(flags & VM_NORESERVE)) {
170 		if (VM_ACCT(newsize) > VM_ACCT(oldsize))
171 			return security_vm_enough_memory_mm(current->mm,
172 					VM_ACCT(newsize) - VM_ACCT(oldsize));
173 		else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
174 			vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
175 	}
176 	return 0;
177 }
178 
179 /*
180  * ... whereas tmpfs objects are accounted incrementally as
181  * pages are allocated, in order to allow large sparse files.
182  * shmem_get_folio reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
183  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
184  */
185 static inline int shmem_acct_block(unsigned long flags, long pages)
186 {
187 	if (!(flags & VM_NORESERVE))
188 		return 0;
189 
190 	return security_vm_enough_memory_mm(current->mm,
191 			pages * VM_ACCT(PAGE_SIZE));
192 }
193 
194 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
195 {
196 	if (flags & VM_NORESERVE)
197 		vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
198 }
199 
200 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
201 {
202 	struct shmem_inode_info *info = SHMEM_I(inode);
203 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
204 
205 	if (shmem_acct_block(info->flags, pages))
206 		return false;
207 
208 	if (sbinfo->max_blocks) {
209 		if (percpu_counter_compare(&sbinfo->used_blocks,
210 					   sbinfo->max_blocks - pages) > 0)
211 			goto unacct;
212 		percpu_counter_add(&sbinfo->used_blocks, pages);
213 	}
214 
215 	return true;
216 
217 unacct:
218 	shmem_unacct_blocks(info->flags, pages);
219 	return false;
220 }
221 
222 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
223 {
224 	struct shmem_inode_info *info = SHMEM_I(inode);
225 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
226 
227 	if (sbinfo->max_blocks)
228 		percpu_counter_sub(&sbinfo->used_blocks, pages);
229 	shmem_unacct_blocks(info->flags, pages);
230 }
231 
232 static const struct super_operations shmem_ops;
233 const struct address_space_operations shmem_aops;
234 static const struct file_operations shmem_file_operations;
235 static const struct inode_operations shmem_inode_operations;
236 static const struct inode_operations shmem_dir_inode_operations;
237 static const struct inode_operations shmem_special_inode_operations;
238 static const struct vm_operations_struct shmem_vm_ops;
239 static struct file_system_type shmem_fs_type;
240 
241 bool vma_is_shmem(struct vm_area_struct *vma)
242 {
243 	return vma->vm_ops == &shmem_vm_ops;
244 }
245 
246 static LIST_HEAD(shmem_swaplist);
247 static DEFINE_MUTEX(shmem_swaplist_mutex);
248 
249 /*
250  * shmem_reserve_inode() performs bookkeeping to reserve a shmem inode, and
251  * produces a novel ino for the newly allocated inode.
252  *
253  * It may also be called when making a hard link to permit the space needed by
254  * each dentry. However, in that case, no new inode number is needed since that
255  * internally draws from another pool of inode numbers (currently global
256  * get_next_ino()). This case is indicated by passing NULL as inop.
257  */
258 #define SHMEM_INO_BATCH 1024
259 static int shmem_reserve_inode(struct super_block *sb, ino_t *inop)
260 {
261 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
262 	ino_t ino;
263 
264 	if (!(sb->s_flags & SB_KERNMOUNT)) {
265 		raw_spin_lock(&sbinfo->stat_lock);
266 		if (sbinfo->max_inodes) {
267 			if (!sbinfo->free_inodes) {
268 				raw_spin_unlock(&sbinfo->stat_lock);
269 				return -ENOSPC;
270 			}
271 			sbinfo->free_inodes--;
272 		}
273 		if (inop) {
274 			ino = sbinfo->next_ino++;
275 			if (unlikely(is_zero_ino(ino)))
276 				ino = sbinfo->next_ino++;
277 			if (unlikely(!sbinfo->full_inums &&
278 				     ino > UINT_MAX)) {
279 				/*
280 				 * Emulate get_next_ino uint wraparound for
281 				 * compatibility
282 				 */
283 				if (IS_ENABLED(CONFIG_64BIT))
284 					pr_warn("%s: inode number overflow on device %d, consider using inode64 mount option\n",
285 						__func__, MINOR(sb->s_dev));
286 				sbinfo->next_ino = 1;
287 				ino = sbinfo->next_ino++;
288 			}
289 			*inop = ino;
290 		}
291 		raw_spin_unlock(&sbinfo->stat_lock);
292 	} else if (inop) {
293 		/*
294 		 * __shmem_file_setup, one of our callers, is lock-free: it
295 		 * doesn't hold stat_lock in shmem_reserve_inode since
296 		 * max_inodes is always 0, and is called from potentially
297 		 * unknown contexts. As such, use a per-cpu batched allocator
298 		 * which doesn't require the per-sb stat_lock unless we are at
299 		 * the batch boundary.
300 		 *
301 		 * We don't need to worry about inode{32,64} since SB_KERNMOUNT
302 		 * shmem mounts are not exposed to userspace, so we don't need
303 		 * to worry about things like glibc compatibility.
304 		 */
305 		ino_t *next_ino;
306 
307 		next_ino = per_cpu_ptr(sbinfo->ino_batch, get_cpu());
308 		ino = *next_ino;
309 		if (unlikely(ino % SHMEM_INO_BATCH == 0)) {
310 			raw_spin_lock(&sbinfo->stat_lock);
311 			ino = sbinfo->next_ino;
312 			sbinfo->next_ino += SHMEM_INO_BATCH;
313 			raw_spin_unlock(&sbinfo->stat_lock);
314 			if (unlikely(is_zero_ino(ino)))
315 				ino++;
316 		}
317 		*inop = ino;
318 		*next_ino = ++ino;
319 		put_cpu();
320 	}
321 
322 	return 0;
323 }
324 
325 static void shmem_free_inode(struct super_block *sb)
326 {
327 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
328 	if (sbinfo->max_inodes) {
329 		raw_spin_lock(&sbinfo->stat_lock);
330 		sbinfo->free_inodes++;
331 		raw_spin_unlock(&sbinfo->stat_lock);
332 	}
333 }
334 
335 /**
336  * shmem_recalc_inode - recalculate the block usage of an inode
337  * @inode: inode to recalc
338  *
339  * We have to calculate the free blocks since the mm can drop
340  * undirtied hole pages behind our back.
341  *
342  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
343  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
344  *
345  * It has to be called with the spinlock held.
346  */
347 static void shmem_recalc_inode(struct inode *inode)
348 {
349 	struct shmem_inode_info *info = SHMEM_I(inode);
350 	long freed;
351 
352 	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
353 	if (freed > 0) {
354 		info->alloced -= freed;
355 		inode->i_blocks -= freed * BLOCKS_PER_PAGE;
356 		shmem_inode_unacct_blocks(inode, freed);
357 	}
358 }
359 
360 bool shmem_charge(struct inode *inode, long pages)
361 {
362 	struct shmem_inode_info *info = SHMEM_I(inode);
363 	unsigned long flags;
364 
365 	if (!shmem_inode_acct_block(inode, pages))
366 		return false;
367 
368 	/* nrpages adjustment first, then shmem_recalc_inode() when balanced */
369 	inode->i_mapping->nrpages += pages;
370 
371 	spin_lock_irqsave(&info->lock, flags);
372 	info->alloced += pages;
373 	inode->i_blocks += pages * BLOCKS_PER_PAGE;
374 	shmem_recalc_inode(inode);
375 	spin_unlock_irqrestore(&info->lock, flags);
376 
377 	return true;
378 }
379 
380 void shmem_uncharge(struct inode *inode, long pages)
381 {
382 	struct shmem_inode_info *info = SHMEM_I(inode);
383 	unsigned long flags;
384 
385 	/* nrpages adjustment done by __filemap_remove_folio() or caller */
386 
387 	spin_lock_irqsave(&info->lock, flags);
388 	info->alloced -= pages;
389 	inode->i_blocks -= pages * BLOCKS_PER_PAGE;
390 	shmem_recalc_inode(inode);
391 	spin_unlock_irqrestore(&info->lock, flags);
392 
393 	shmem_inode_unacct_blocks(inode, pages);
394 }
395 
396 /*
397  * Replace item expected in xarray by a new item, while holding xa_lock.
398  */
399 static int shmem_replace_entry(struct address_space *mapping,
400 			pgoff_t index, void *expected, void *replacement)
401 {
402 	XA_STATE(xas, &mapping->i_pages, index);
403 	void *item;
404 
405 	VM_BUG_ON(!expected);
406 	VM_BUG_ON(!replacement);
407 	item = xas_load(&xas);
408 	if (item != expected)
409 		return -ENOENT;
410 	xas_store(&xas, replacement);
411 	return 0;
412 }
413 
414 /*
415  * Sometimes, before we decide whether to proceed or to fail, we must check
416  * that an entry was not already brought back from swap by a racing thread.
417  *
418  * Checking page is not enough: by the time a SwapCache page is locked, it
419  * might be reused, and again be SwapCache, using the same swap as before.
420  */
421 static bool shmem_confirm_swap(struct address_space *mapping,
422 			       pgoff_t index, swp_entry_t swap)
423 {
424 	return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
425 }
426 
427 /*
428  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
429  *
430  * SHMEM_HUGE_NEVER:
431  *	disables huge pages for the mount;
432  * SHMEM_HUGE_ALWAYS:
433  *	enables huge pages for the mount;
434  * SHMEM_HUGE_WITHIN_SIZE:
435  *	only allocate huge pages if the page will be fully within i_size,
436  *	also respect fadvise()/madvise() hints;
437  * SHMEM_HUGE_ADVISE:
438  *	only allocate huge pages if requested with fadvise()/madvise();
439  */
440 
441 #define SHMEM_HUGE_NEVER	0
442 #define SHMEM_HUGE_ALWAYS	1
443 #define SHMEM_HUGE_WITHIN_SIZE	2
444 #define SHMEM_HUGE_ADVISE	3
445 
446 /*
447  * Special values.
448  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
449  *
450  * SHMEM_HUGE_DENY:
451  *	disables huge on shm_mnt and all mounts, for emergency use;
452  * SHMEM_HUGE_FORCE:
453  *	enables huge on shm_mnt and all mounts, w/o needing option, for testing;
454  *
455  */
456 #define SHMEM_HUGE_DENY		(-1)
457 #define SHMEM_HUGE_FORCE	(-2)
458 
459 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
460 /* ifdef here to avoid bloating shmem.o when not necessary */
461 
462 static int shmem_huge __read_mostly = SHMEM_HUGE_NEVER;
463 
464 bool shmem_is_huge(struct vm_area_struct *vma,
465 		   struct inode *inode, pgoff_t index)
466 {
467 	loff_t i_size;
468 
469 	if (!S_ISREG(inode->i_mode))
470 		return false;
471 	if (shmem_huge == SHMEM_HUGE_DENY)
472 		return false;
473 	if (vma && ((vma->vm_flags & VM_NOHUGEPAGE) ||
474 	    test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags)))
475 		return false;
476 	if (shmem_huge == SHMEM_HUGE_FORCE)
477 		return true;
478 
479 	switch (SHMEM_SB(inode->i_sb)->huge) {
480 	case SHMEM_HUGE_ALWAYS:
481 		return true;
482 	case SHMEM_HUGE_WITHIN_SIZE:
483 		index = round_up(index + 1, HPAGE_PMD_NR);
484 		i_size = round_up(i_size_read(inode), PAGE_SIZE);
485 		if (i_size >> PAGE_SHIFT >= index)
486 			return true;
487 		fallthrough;
488 	case SHMEM_HUGE_ADVISE:
489 		if (vma && (vma->vm_flags & VM_HUGEPAGE))
490 			return true;
491 		fallthrough;
492 	default:
493 		return false;
494 	}
495 }
496 
497 #if defined(CONFIG_SYSFS)
498 static int shmem_parse_huge(const char *str)
499 {
500 	if (!strcmp(str, "never"))
501 		return SHMEM_HUGE_NEVER;
502 	if (!strcmp(str, "always"))
503 		return SHMEM_HUGE_ALWAYS;
504 	if (!strcmp(str, "within_size"))
505 		return SHMEM_HUGE_WITHIN_SIZE;
506 	if (!strcmp(str, "advise"))
507 		return SHMEM_HUGE_ADVISE;
508 	if (!strcmp(str, "deny"))
509 		return SHMEM_HUGE_DENY;
510 	if (!strcmp(str, "force"))
511 		return SHMEM_HUGE_FORCE;
512 	return -EINVAL;
513 }
514 #endif
515 
516 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
517 static const char *shmem_format_huge(int huge)
518 {
519 	switch (huge) {
520 	case SHMEM_HUGE_NEVER:
521 		return "never";
522 	case SHMEM_HUGE_ALWAYS:
523 		return "always";
524 	case SHMEM_HUGE_WITHIN_SIZE:
525 		return "within_size";
526 	case SHMEM_HUGE_ADVISE:
527 		return "advise";
528 	case SHMEM_HUGE_DENY:
529 		return "deny";
530 	case SHMEM_HUGE_FORCE:
531 		return "force";
532 	default:
533 		VM_BUG_ON(1);
534 		return "bad_val";
535 	}
536 }
537 #endif
538 
539 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
540 		struct shrink_control *sc, unsigned long nr_to_split)
541 {
542 	LIST_HEAD(list), *pos, *next;
543 	LIST_HEAD(to_remove);
544 	struct inode *inode;
545 	struct shmem_inode_info *info;
546 	struct folio *folio;
547 	unsigned long batch = sc ? sc->nr_to_scan : 128;
548 	int split = 0;
549 
550 	if (list_empty(&sbinfo->shrinklist))
551 		return SHRINK_STOP;
552 
553 	spin_lock(&sbinfo->shrinklist_lock);
554 	list_for_each_safe(pos, next, &sbinfo->shrinklist) {
555 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
556 
557 		/* pin the inode */
558 		inode = igrab(&info->vfs_inode);
559 
560 		/* inode is about to be evicted */
561 		if (!inode) {
562 			list_del_init(&info->shrinklist);
563 			goto next;
564 		}
565 
566 		/* Check if there's anything to gain */
567 		if (round_up(inode->i_size, PAGE_SIZE) ==
568 				round_up(inode->i_size, HPAGE_PMD_SIZE)) {
569 			list_move(&info->shrinklist, &to_remove);
570 			goto next;
571 		}
572 
573 		list_move(&info->shrinklist, &list);
574 next:
575 		sbinfo->shrinklist_len--;
576 		if (!--batch)
577 			break;
578 	}
579 	spin_unlock(&sbinfo->shrinklist_lock);
580 
581 	list_for_each_safe(pos, next, &to_remove) {
582 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
583 		inode = &info->vfs_inode;
584 		list_del_init(&info->shrinklist);
585 		iput(inode);
586 	}
587 
588 	list_for_each_safe(pos, next, &list) {
589 		int ret;
590 		pgoff_t index;
591 
592 		info = list_entry(pos, struct shmem_inode_info, shrinklist);
593 		inode = &info->vfs_inode;
594 
595 		if (nr_to_split && split >= nr_to_split)
596 			goto move_back;
597 
598 		index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
599 		folio = filemap_get_folio(inode->i_mapping, index);
600 		if (!folio)
601 			goto drop;
602 
603 		/* No huge page at the end of the file: nothing to split */
604 		if (!folio_test_large(folio)) {
605 			folio_put(folio);
606 			goto drop;
607 		}
608 
609 		/*
610 		 * Move the inode on the list back to shrinklist if we failed
611 		 * to lock the page at this time.
612 		 *
613 		 * Waiting for the lock may lead to deadlock in the
614 		 * reclaim path.
615 		 */
616 		if (!folio_trylock(folio)) {
617 			folio_put(folio);
618 			goto move_back;
619 		}
620 
621 		ret = split_folio(folio);
622 		folio_unlock(folio);
623 		folio_put(folio);
624 
625 		/* If split failed move the inode on the list back to shrinklist */
626 		if (ret)
627 			goto move_back;
628 
629 		split++;
630 drop:
631 		list_del_init(&info->shrinklist);
632 		goto put;
633 move_back:
634 		/*
635 		 * Make sure the inode is either on the global list or deleted
636 		 * from any local list before iput() since it could be deleted
637 		 * in another thread once we put the inode (then the local list
638 		 * is corrupted).
639 		 */
640 		spin_lock(&sbinfo->shrinklist_lock);
641 		list_move(&info->shrinklist, &sbinfo->shrinklist);
642 		sbinfo->shrinklist_len++;
643 		spin_unlock(&sbinfo->shrinklist_lock);
644 put:
645 		iput(inode);
646 	}
647 
648 	return split;
649 }
650 
651 static long shmem_unused_huge_scan(struct super_block *sb,
652 		struct shrink_control *sc)
653 {
654 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
655 
656 	if (!READ_ONCE(sbinfo->shrinklist_len))
657 		return SHRINK_STOP;
658 
659 	return shmem_unused_huge_shrink(sbinfo, sc, 0);
660 }
661 
662 static long shmem_unused_huge_count(struct super_block *sb,
663 		struct shrink_control *sc)
664 {
665 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
666 	return READ_ONCE(sbinfo->shrinklist_len);
667 }
668 #else /* !CONFIG_TRANSPARENT_HUGEPAGE */
669 
670 #define shmem_huge SHMEM_HUGE_DENY
671 
672 bool shmem_is_huge(struct vm_area_struct *vma,
673 		   struct inode *inode, pgoff_t index)
674 {
675 	return false;
676 }
677 
678 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
679 		struct shrink_control *sc, unsigned long nr_to_split)
680 {
681 	return 0;
682 }
683 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
684 
685 /*
686  * Like filemap_add_folio, but error if expected item has gone.
687  */
688 static int shmem_add_to_page_cache(struct folio *folio,
689 				   struct address_space *mapping,
690 				   pgoff_t index, void *expected, gfp_t gfp,
691 				   struct mm_struct *charge_mm)
692 {
693 	XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
694 	long nr = folio_nr_pages(folio);
695 	int error;
696 
697 	VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
698 	VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
699 	VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
700 	VM_BUG_ON(expected && folio_test_large(folio));
701 
702 	folio_ref_add(folio, nr);
703 	folio->mapping = mapping;
704 	folio->index = index;
705 
706 	if (!folio_test_swapcache(folio)) {
707 		error = mem_cgroup_charge(folio, charge_mm, gfp);
708 		if (error) {
709 			if (folio_test_pmd_mappable(folio)) {
710 				count_vm_event(THP_FILE_FALLBACK);
711 				count_vm_event(THP_FILE_FALLBACK_CHARGE);
712 			}
713 			goto error;
714 		}
715 	}
716 	folio_throttle_swaprate(folio, gfp);
717 
718 	do {
719 		xas_lock_irq(&xas);
720 		if (expected != xas_find_conflict(&xas)) {
721 			xas_set_err(&xas, -EEXIST);
722 			goto unlock;
723 		}
724 		if (expected && xas_find_conflict(&xas)) {
725 			xas_set_err(&xas, -EEXIST);
726 			goto unlock;
727 		}
728 		xas_store(&xas, folio);
729 		if (xas_error(&xas))
730 			goto unlock;
731 		if (folio_test_pmd_mappable(folio)) {
732 			count_vm_event(THP_FILE_ALLOC);
733 			__lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
734 		}
735 		mapping->nrpages += nr;
736 		__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
737 		__lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
738 unlock:
739 		xas_unlock_irq(&xas);
740 	} while (xas_nomem(&xas, gfp));
741 
742 	if (xas_error(&xas)) {
743 		error = xas_error(&xas);
744 		goto error;
745 	}
746 
747 	return 0;
748 error:
749 	folio->mapping = NULL;
750 	folio_ref_sub(folio, nr);
751 	return error;
752 }
753 
754 /*
755  * Like delete_from_page_cache, but substitutes swap for @folio.
756  */
757 static void shmem_delete_from_page_cache(struct folio *folio, void *radswap)
758 {
759 	struct address_space *mapping = folio->mapping;
760 	long nr = folio_nr_pages(folio);
761 	int error;
762 
763 	xa_lock_irq(&mapping->i_pages);
764 	error = shmem_replace_entry(mapping, folio->index, folio, radswap);
765 	folio->mapping = NULL;
766 	mapping->nrpages -= nr;
767 	__lruvec_stat_mod_folio(folio, NR_FILE_PAGES, -nr);
768 	__lruvec_stat_mod_folio(folio, NR_SHMEM, -nr);
769 	xa_unlock_irq(&mapping->i_pages);
770 	folio_put(folio);
771 	BUG_ON(error);
772 }
773 
774 /*
775  * Remove swap entry from page cache, free the swap and its page cache.
776  */
777 static int shmem_free_swap(struct address_space *mapping,
778 			   pgoff_t index, void *radswap)
779 {
780 	void *old;
781 
782 	old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
783 	if (old != radswap)
784 		return -ENOENT;
785 	free_swap_and_cache(radix_to_swp_entry(radswap));
786 	return 0;
787 }
788 
789 /*
790  * Determine (in bytes) how many of the shmem object's pages mapped by the
791  * given offsets are swapped out.
792  *
793  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
794  * as long as the inode doesn't go away and racy results are not a problem.
795  */
796 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
797 						pgoff_t start, pgoff_t end)
798 {
799 	XA_STATE(xas, &mapping->i_pages, start);
800 	struct page *page;
801 	unsigned long swapped = 0;
802 
803 	rcu_read_lock();
804 	xas_for_each(&xas, page, end - 1) {
805 		if (xas_retry(&xas, page))
806 			continue;
807 		if (xa_is_value(page))
808 			swapped++;
809 
810 		if (need_resched()) {
811 			xas_pause(&xas);
812 			cond_resched_rcu();
813 		}
814 	}
815 
816 	rcu_read_unlock();
817 
818 	return swapped << PAGE_SHIFT;
819 }
820 
821 /*
822  * Determine (in bytes) how many of the shmem object's pages mapped by the
823  * given vma is swapped out.
824  *
825  * This is safe to call without i_rwsem or the i_pages lock thanks to RCU,
826  * as long as the inode doesn't go away and racy results are not a problem.
827  */
828 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
829 {
830 	struct inode *inode = file_inode(vma->vm_file);
831 	struct shmem_inode_info *info = SHMEM_I(inode);
832 	struct address_space *mapping = inode->i_mapping;
833 	unsigned long swapped;
834 
835 	/* Be careful as we don't hold info->lock */
836 	swapped = READ_ONCE(info->swapped);
837 
838 	/*
839 	 * The easier cases are when the shmem object has nothing in swap, or
840 	 * the vma maps it whole. Then we can simply use the stats that we
841 	 * already track.
842 	 */
843 	if (!swapped)
844 		return 0;
845 
846 	if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
847 		return swapped << PAGE_SHIFT;
848 
849 	/* Here comes the more involved part */
850 	return shmem_partial_swap_usage(mapping, vma->vm_pgoff,
851 					vma->vm_pgoff + vma_pages(vma));
852 }
853 
854 /*
855  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
856  */
857 void shmem_unlock_mapping(struct address_space *mapping)
858 {
859 	struct folio_batch fbatch;
860 	pgoff_t index = 0;
861 
862 	folio_batch_init(&fbatch);
863 	/*
864 	 * Minor point, but we might as well stop if someone else SHM_LOCKs it.
865 	 */
866 	while (!mapping_unevictable(mapping) &&
867 	       filemap_get_folios(mapping, &index, ~0UL, &fbatch)) {
868 		check_move_unevictable_folios(&fbatch);
869 		folio_batch_release(&fbatch);
870 		cond_resched();
871 	}
872 }
873 
874 static struct folio *shmem_get_partial_folio(struct inode *inode, pgoff_t index)
875 {
876 	struct folio *folio;
877 
878 	/*
879 	 * At first avoid shmem_get_folio(,,,SGP_READ): that fails
880 	 * beyond i_size, and reports fallocated pages as holes.
881 	 */
882 	folio = __filemap_get_folio(inode->i_mapping, index,
883 					FGP_ENTRY | FGP_LOCK, 0);
884 	if (!xa_is_value(folio))
885 		return folio;
886 	/*
887 	 * But read a page back from swap if any of it is within i_size
888 	 * (although in some cases this is just a waste of time).
889 	 */
890 	folio = NULL;
891 	shmem_get_folio(inode, index, &folio, SGP_READ);
892 	return folio;
893 }
894 
895 /*
896  * Remove range of pages and swap entries from page cache, and free them.
897  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
898  */
899 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
900 								 bool unfalloc)
901 {
902 	struct address_space *mapping = inode->i_mapping;
903 	struct shmem_inode_info *info = SHMEM_I(inode);
904 	pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
905 	pgoff_t end = (lend + 1) >> PAGE_SHIFT;
906 	struct folio_batch fbatch;
907 	pgoff_t indices[PAGEVEC_SIZE];
908 	struct folio *folio;
909 	bool same_folio;
910 	long nr_swaps_freed = 0;
911 	pgoff_t index;
912 	int i;
913 
914 	if (lend == -1)
915 		end = -1;	/* unsigned, so actually very big */
916 
917 	if (info->fallocend > start && info->fallocend <= end && !unfalloc)
918 		info->fallocend = start;
919 
920 	folio_batch_init(&fbatch);
921 	index = start;
922 	while (index < end && find_lock_entries(mapping, index, end - 1,
923 			&fbatch, indices)) {
924 		for (i = 0; i < folio_batch_count(&fbatch); i++) {
925 			folio = fbatch.folios[i];
926 
927 			index = indices[i];
928 
929 			if (xa_is_value(folio)) {
930 				if (unfalloc)
931 					continue;
932 				nr_swaps_freed += !shmem_free_swap(mapping,
933 								index, folio);
934 				continue;
935 			}
936 			index += folio_nr_pages(folio) - 1;
937 
938 			if (!unfalloc || !folio_test_uptodate(folio))
939 				truncate_inode_folio(mapping, folio);
940 			folio_unlock(folio);
941 		}
942 		folio_batch_remove_exceptionals(&fbatch);
943 		folio_batch_release(&fbatch);
944 		cond_resched();
945 		index++;
946 	}
947 
948 	same_folio = (lstart >> PAGE_SHIFT) == (lend >> PAGE_SHIFT);
949 	folio = shmem_get_partial_folio(inode, lstart >> PAGE_SHIFT);
950 	if (folio) {
951 		same_folio = lend < folio_pos(folio) + folio_size(folio);
952 		folio_mark_dirty(folio);
953 		if (!truncate_inode_partial_folio(folio, lstart, lend)) {
954 			start = folio->index + folio_nr_pages(folio);
955 			if (same_folio)
956 				end = folio->index;
957 		}
958 		folio_unlock(folio);
959 		folio_put(folio);
960 		folio = NULL;
961 	}
962 
963 	if (!same_folio)
964 		folio = shmem_get_partial_folio(inode, lend >> PAGE_SHIFT);
965 	if (folio) {
966 		folio_mark_dirty(folio);
967 		if (!truncate_inode_partial_folio(folio, lstart, lend))
968 			end = folio->index;
969 		folio_unlock(folio);
970 		folio_put(folio);
971 	}
972 
973 	index = start;
974 	while (index < end) {
975 		cond_resched();
976 
977 		if (!find_get_entries(mapping, index, end - 1, &fbatch,
978 				indices)) {
979 			/* If all gone or hole-punch or unfalloc, we're done */
980 			if (index == start || end != -1)
981 				break;
982 			/* But if truncating, restart to make sure all gone */
983 			index = start;
984 			continue;
985 		}
986 		for (i = 0; i < folio_batch_count(&fbatch); i++) {
987 			folio = fbatch.folios[i];
988 
989 			index = indices[i];
990 			if (xa_is_value(folio)) {
991 				if (unfalloc)
992 					continue;
993 				if (shmem_free_swap(mapping, index, folio)) {
994 					/* Swap was replaced by page: retry */
995 					index--;
996 					break;
997 				}
998 				nr_swaps_freed++;
999 				continue;
1000 			}
1001 
1002 			folio_lock(folio);
1003 
1004 			if (!unfalloc || !folio_test_uptodate(folio)) {
1005 				if (folio_mapping(folio) != mapping) {
1006 					/* Page was replaced by swap: retry */
1007 					folio_unlock(folio);
1008 					index--;
1009 					break;
1010 				}
1011 				VM_BUG_ON_FOLIO(folio_test_writeback(folio),
1012 						folio);
1013 				truncate_inode_folio(mapping, folio);
1014 			}
1015 			index = folio->index + folio_nr_pages(folio) - 1;
1016 			folio_unlock(folio);
1017 		}
1018 		folio_batch_remove_exceptionals(&fbatch);
1019 		folio_batch_release(&fbatch);
1020 		index++;
1021 	}
1022 
1023 	spin_lock_irq(&info->lock);
1024 	info->swapped -= nr_swaps_freed;
1025 	shmem_recalc_inode(inode);
1026 	spin_unlock_irq(&info->lock);
1027 }
1028 
1029 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
1030 {
1031 	shmem_undo_range(inode, lstart, lend, false);
1032 	inode->i_ctime = inode->i_mtime = current_time(inode);
1033 }
1034 EXPORT_SYMBOL_GPL(shmem_truncate_range);
1035 
1036 static int shmem_getattr(struct user_namespace *mnt_userns,
1037 			 const struct path *path, struct kstat *stat,
1038 			 u32 request_mask, unsigned int query_flags)
1039 {
1040 	struct inode *inode = path->dentry->d_inode;
1041 	struct shmem_inode_info *info = SHMEM_I(inode);
1042 
1043 	if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
1044 		spin_lock_irq(&info->lock);
1045 		shmem_recalc_inode(inode);
1046 		spin_unlock_irq(&info->lock);
1047 	}
1048 	if (info->fsflags & FS_APPEND_FL)
1049 		stat->attributes |= STATX_ATTR_APPEND;
1050 	if (info->fsflags & FS_IMMUTABLE_FL)
1051 		stat->attributes |= STATX_ATTR_IMMUTABLE;
1052 	if (info->fsflags & FS_NODUMP_FL)
1053 		stat->attributes |= STATX_ATTR_NODUMP;
1054 	stat->attributes_mask |= (STATX_ATTR_APPEND |
1055 			STATX_ATTR_IMMUTABLE |
1056 			STATX_ATTR_NODUMP);
1057 	generic_fillattr(&init_user_ns, inode, stat);
1058 
1059 	if (shmem_is_huge(NULL, inode, 0))
1060 		stat->blksize = HPAGE_PMD_SIZE;
1061 
1062 	if (request_mask & STATX_BTIME) {
1063 		stat->result_mask |= STATX_BTIME;
1064 		stat->btime.tv_sec = info->i_crtime.tv_sec;
1065 		stat->btime.tv_nsec = info->i_crtime.tv_nsec;
1066 	}
1067 
1068 	return 0;
1069 }
1070 
1071 static int shmem_setattr(struct user_namespace *mnt_userns,
1072 			 struct dentry *dentry, struct iattr *attr)
1073 {
1074 	struct inode *inode = d_inode(dentry);
1075 	struct shmem_inode_info *info = SHMEM_I(inode);
1076 	int error;
1077 
1078 	error = setattr_prepare(&init_user_ns, dentry, attr);
1079 	if (error)
1080 		return error;
1081 
1082 	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1083 		loff_t oldsize = inode->i_size;
1084 		loff_t newsize = attr->ia_size;
1085 
1086 		/* protected by i_rwsem */
1087 		if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1088 		    (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1089 			return -EPERM;
1090 
1091 		if (newsize != oldsize) {
1092 			error = shmem_reacct_size(SHMEM_I(inode)->flags,
1093 					oldsize, newsize);
1094 			if (error)
1095 				return error;
1096 			i_size_write(inode, newsize);
1097 			inode->i_ctime = inode->i_mtime = current_time(inode);
1098 		}
1099 		if (newsize <= oldsize) {
1100 			loff_t holebegin = round_up(newsize, PAGE_SIZE);
1101 			if (oldsize > holebegin)
1102 				unmap_mapping_range(inode->i_mapping,
1103 							holebegin, 0, 1);
1104 			if (info->alloced)
1105 				shmem_truncate_range(inode,
1106 							newsize, (loff_t)-1);
1107 			/* unmap again to remove racily COWed private pages */
1108 			if (oldsize > holebegin)
1109 				unmap_mapping_range(inode->i_mapping,
1110 							holebegin, 0, 1);
1111 		}
1112 	}
1113 
1114 	setattr_copy(&init_user_ns, inode, attr);
1115 	if (attr->ia_valid & ATTR_MODE)
1116 		error = posix_acl_chmod(&init_user_ns, inode, inode->i_mode);
1117 	return error;
1118 }
1119 
1120 static void shmem_evict_inode(struct inode *inode)
1121 {
1122 	struct shmem_inode_info *info = SHMEM_I(inode);
1123 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1124 
1125 	if (shmem_mapping(inode->i_mapping)) {
1126 		shmem_unacct_size(info->flags, inode->i_size);
1127 		inode->i_size = 0;
1128 		mapping_set_exiting(inode->i_mapping);
1129 		shmem_truncate_range(inode, 0, (loff_t)-1);
1130 		if (!list_empty(&info->shrinklist)) {
1131 			spin_lock(&sbinfo->shrinklist_lock);
1132 			if (!list_empty(&info->shrinklist)) {
1133 				list_del_init(&info->shrinklist);
1134 				sbinfo->shrinklist_len--;
1135 			}
1136 			spin_unlock(&sbinfo->shrinklist_lock);
1137 		}
1138 		while (!list_empty(&info->swaplist)) {
1139 			/* Wait while shmem_unuse() is scanning this inode... */
1140 			wait_var_event(&info->stop_eviction,
1141 				       !atomic_read(&info->stop_eviction));
1142 			mutex_lock(&shmem_swaplist_mutex);
1143 			/* ...but beware of the race if we peeked too early */
1144 			if (!atomic_read(&info->stop_eviction))
1145 				list_del_init(&info->swaplist);
1146 			mutex_unlock(&shmem_swaplist_mutex);
1147 		}
1148 	}
1149 
1150 	simple_xattrs_free(&info->xattrs);
1151 	WARN_ON(inode->i_blocks);
1152 	shmem_free_inode(inode->i_sb);
1153 	clear_inode(inode);
1154 }
1155 
1156 static int shmem_find_swap_entries(struct address_space *mapping,
1157 				   pgoff_t start, struct folio_batch *fbatch,
1158 				   pgoff_t *indices, unsigned int type)
1159 {
1160 	XA_STATE(xas, &mapping->i_pages, start);
1161 	struct folio *folio;
1162 	swp_entry_t entry;
1163 
1164 	rcu_read_lock();
1165 	xas_for_each(&xas, folio, ULONG_MAX) {
1166 		if (xas_retry(&xas, folio))
1167 			continue;
1168 
1169 		if (!xa_is_value(folio))
1170 			continue;
1171 
1172 		entry = radix_to_swp_entry(folio);
1173 		/*
1174 		 * swapin error entries can be found in the mapping. But they're
1175 		 * deliberately ignored here as we've done everything we can do.
1176 		 */
1177 		if (swp_type(entry) != type)
1178 			continue;
1179 
1180 		indices[folio_batch_count(fbatch)] = xas.xa_index;
1181 		if (!folio_batch_add(fbatch, folio))
1182 			break;
1183 
1184 		if (need_resched()) {
1185 			xas_pause(&xas);
1186 			cond_resched_rcu();
1187 		}
1188 	}
1189 	rcu_read_unlock();
1190 
1191 	return xas.xa_index;
1192 }
1193 
1194 /*
1195  * Move the swapped pages for an inode to page cache. Returns the count
1196  * of pages swapped in, or the error in case of failure.
1197  */
1198 static int shmem_unuse_swap_entries(struct inode *inode,
1199 		struct folio_batch *fbatch, pgoff_t *indices)
1200 {
1201 	int i = 0;
1202 	int ret = 0;
1203 	int error = 0;
1204 	struct address_space *mapping = inode->i_mapping;
1205 
1206 	for (i = 0; i < folio_batch_count(fbatch); i++) {
1207 		struct folio *folio = fbatch->folios[i];
1208 
1209 		if (!xa_is_value(folio))
1210 			continue;
1211 		error = shmem_swapin_folio(inode, indices[i],
1212 					  &folio, SGP_CACHE,
1213 					  mapping_gfp_mask(mapping),
1214 					  NULL, NULL);
1215 		if (error == 0) {
1216 			folio_unlock(folio);
1217 			folio_put(folio);
1218 			ret++;
1219 		}
1220 		if (error == -ENOMEM)
1221 			break;
1222 		error = 0;
1223 	}
1224 	return error ? error : ret;
1225 }
1226 
1227 /*
1228  * If swap found in inode, free it and move page from swapcache to filecache.
1229  */
1230 static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1231 {
1232 	struct address_space *mapping = inode->i_mapping;
1233 	pgoff_t start = 0;
1234 	struct folio_batch fbatch;
1235 	pgoff_t indices[PAGEVEC_SIZE];
1236 	int ret = 0;
1237 
1238 	do {
1239 		folio_batch_init(&fbatch);
1240 		shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1241 		if (folio_batch_count(&fbatch) == 0) {
1242 			ret = 0;
1243 			break;
1244 		}
1245 
1246 		ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1247 		if (ret < 0)
1248 			break;
1249 
1250 		start = indices[folio_batch_count(&fbatch) - 1];
1251 	} while (true);
1252 
1253 	return ret;
1254 }
1255 
1256 /*
1257  * Read all the shared memory data that resides in the swap
1258  * device 'type' back into memory, so the swap device can be
1259  * unused.
1260  */
1261 int shmem_unuse(unsigned int type)
1262 {
1263 	struct shmem_inode_info *info, *next;
1264 	int error = 0;
1265 
1266 	if (list_empty(&shmem_swaplist))
1267 		return 0;
1268 
1269 	mutex_lock(&shmem_swaplist_mutex);
1270 	list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1271 		if (!info->swapped) {
1272 			list_del_init(&info->swaplist);
1273 			continue;
1274 		}
1275 		/*
1276 		 * Drop the swaplist mutex while searching the inode for swap;
1277 		 * but before doing so, make sure shmem_evict_inode() will not
1278 		 * remove placeholder inode from swaplist, nor let it be freed
1279 		 * (igrab() would protect from unlink, but not from unmount).
1280 		 */
1281 		atomic_inc(&info->stop_eviction);
1282 		mutex_unlock(&shmem_swaplist_mutex);
1283 
1284 		error = shmem_unuse_inode(&info->vfs_inode, type);
1285 		cond_resched();
1286 
1287 		mutex_lock(&shmem_swaplist_mutex);
1288 		next = list_next_entry(info, swaplist);
1289 		if (!info->swapped)
1290 			list_del_init(&info->swaplist);
1291 		if (atomic_dec_and_test(&info->stop_eviction))
1292 			wake_up_var(&info->stop_eviction);
1293 		if (error)
1294 			break;
1295 	}
1296 	mutex_unlock(&shmem_swaplist_mutex);
1297 
1298 	return error;
1299 }
1300 
1301 /*
1302  * Move the page from the page cache to the swap cache.
1303  */
1304 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1305 {
1306 	struct folio *folio = page_folio(page);
1307 	struct shmem_inode_info *info;
1308 	struct address_space *mapping;
1309 	struct inode *inode;
1310 	swp_entry_t swap;
1311 	pgoff_t index;
1312 
1313 	/*
1314 	 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or
1315 	 * "force", drivers/gpu/drm/i915/gem/i915_gem_shmem.c gets huge pages,
1316 	 * and its shmem_writeback() needs them to be split when swapping.
1317 	 */
1318 	if (folio_test_large(folio)) {
1319 		/* Ensure the subpages are still dirty */
1320 		folio_test_set_dirty(folio);
1321 		if (split_huge_page(page) < 0)
1322 			goto redirty;
1323 		folio = page_folio(page);
1324 		folio_clear_dirty(folio);
1325 	}
1326 
1327 	BUG_ON(!folio_test_locked(folio));
1328 	mapping = folio->mapping;
1329 	index = folio->index;
1330 	inode = mapping->host;
1331 	info = SHMEM_I(inode);
1332 	if (info->flags & VM_LOCKED)
1333 		goto redirty;
1334 	if (!total_swap_pages)
1335 		goto redirty;
1336 
1337 	/*
1338 	 * Our capabilities prevent regular writeback or sync from ever calling
1339 	 * shmem_writepage; but a stacking filesystem might use ->writepage of
1340 	 * its underlying filesystem, in which case tmpfs should write out to
1341 	 * swap only in response to memory pressure, and not for the writeback
1342 	 * threads or sync.
1343 	 */
1344 	if (!wbc->for_reclaim) {
1345 		WARN_ON_ONCE(1);	/* Still happens? Tell us about it! */
1346 		goto redirty;
1347 	}
1348 
1349 	/*
1350 	 * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1351 	 * value into swapfile.c, the only way we can correctly account for a
1352 	 * fallocated folio arriving here is now to initialize it and write it.
1353 	 *
1354 	 * That's okay for a folio already fallocated earlier, but if we have
1355 	 * not yet completed the fallocation, then (a) we want to keep track
1356 	 * of this folio in case we have to undo it, and (b) it may not be a
1357 	 * good idea to continue anyway, once we're pushing into swap.  So
1358 	 * reactivate the folio, and let shmem_fallocate() quit when too many.
1359 	 */
1360 	if (!folio_test_uptodate(folio)) {
1361 		if (inode->i_private) {
1362 			struct shmem_falloc *shmem_falloc;
1363 			spin_lock(&inode->i_lock);
1364 			shmem_falloc = inode->i_private;
1365 			if (shmem_falloc &&
1366 			    !shmem_falloc->waitq &&
1367 			    index >= shmem_falloc->start &&
1368 			    index < shmem_falloc->next)
1369 				shmem_falloc->nr_unswapped++;
1370 			else
1371 				shmem_falloc = NULL;
1372 			spin_unlock(&inode->i_lock);
1373 			if (shmem_falloc)
1374 				goto redirty;
1375 		}
1376 		folio_zero_range(folio, 0, folio_size(folio));
1377 		flush_dcache_folio(folio);
1378 		folio_mark_uptodate(folio);
1379 	}
1380 
1381 	swap = folio_alloc_swap(folio);
1382 	if (!swap.val)
1383 		goto redirty;
1384 
1385 	/*
1386 	 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1387 	 * if it's not already there.  Do it now before the folio is
1388 	 * moved to swap cache, when its pagelock no longer protects
1389 	 * the inode from eviction.  But don't unlock the mutex until
1390 	 * we've incremented swapped, because shmem_unuse_inode() will
1391 	 * prune a !swapped inode from the swaplist under this mutex.
1392 	 */
1393 	mutex_lock(&shmem_swaplist_mutex);
1394 	if (list_empty(&info->swaplist))
1395 		list_add(&info->swaplist, &shmem_swaplist);
1396 
1397 	if (add_to_swap_cache(folio, swap,
1398 			__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN,
1399 			NULL) == 0) {
1400 		spin_lock_irq(&info->lock);
1401 		shmem_recalc_inode(inode);
1402 		info->swapped++;
1403 		spin_unlock_irq(&info->lock);
1404 
1405 		swap_shmem_alloc(swap);
1406 		shmem_delete_from_page_cache(folio, swp_to_radix_entry(swap));
1407 
1408 		mutex_unlock(&shmem_swaplist_mutex);
1409 		BUG_ON(folio_mapped(folio));
1410 		swap_writepage(&folio->page, wbc);
1411 		return 0;
1412 	}
1413 
1414 	mutex_unlock(&shmem_swaplist_mutex);
1415 	put_swap_folio(folio, swap);
1416 redirty:
1417 	folio_mark_dirty(folio);
1418 	if (wbc->for_reclaim)
1419 		return AOP_WRITEPAGE_ACTIVATE;	/* Return with folio locked */
1420 	folio_unlock(folio);
1421 	return 0;
1422 }
1423 
1424 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1425 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1426 {
1427 	char buffer[64];
1428 
1429 	if (!mpol || mpol->mode == MPOL_DEFAULT)
1430 		return;		/* show nothing */
1431 
1432 	mpol_to_str(buffer, sizeof(buffer), mpol);
1433 
1434 	seq_printf(seq, ",mpol=%s", buffer);
1435 }
1436 
1437 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1438 {
1439 	struct mempolicy *mpol = NULL;
1440 	if (sbinfo->mpol) {
1441 		raw_spin_lock(&sbinfo->stat_lock);	/* prevent replace/use races */
1442 		mpol = sbinfo->mpol;
1443 		mpol_get(mpol);
1444 		raw_spin_unlock(&sbinfo->stat_lock);
1445 	}
1446 	return mpol;
1447 }
1448 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1449 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1450 {
1451 }
1452 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1453 {
1454 	return NULL;
1455 }
1456 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1457 #ifndef CONFIG_NUMA
1458 #define vm_policy vm_private_data
1459 #endif
1460 
1461 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1462 		struct shmem_inode_info *info, pgoff_t index)
1463 {
1464 	/* Create a pseudo vma that just contains the policy */
1465 	vma_init(vma, NULL);
1466 	/* Bias interleave by inode number to distribute better across nodes */
1467 	vma->vm_pgoff = index + info->vfs_inode.i_ino;
1468 	vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1469 }
1470 
1471 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1472 {
1473 	/* Drop reference taken by mpol_shared_policy_lookup() */
1474 	mpol_cond_put(vma->vm_policy);
1475 }
1476 
1477 static struct folio *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1478 			struct shmem_inode_info *info, pgoff_t index)
1479 {
1480 	struct vm_area_struct pvma;
1481 	struct page *page;
1482 	struct vm_fault vmf = {
1483 		.vma = &pvma,
1484 	};
1485 
1486 	shmem_pseudo_vma_init(&pvma, info, index);
1487 	page = swap_cluster_readahead(swap, gfp, &vmf);
1488 	shmem_pseudo_vma_destroy(&pvma);
1489 
1490 	if (!page)
1491 		return NULL;
1492 	return page_folio(page);
1493 }
1494 
1495 /*
1496  * Make sure huge_gfp is always more limited than limit_gfp.
1497  * Some of the flags set permissions, while others set limitations.
1498  */
1499 static gfp_t limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp)
1500 {
1501 	gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM;
1502 	gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY;
1503 	gfp_t zoneflags = limit_gfp & GFP_ZONEMASK;
1504 	gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
1505 
1506 	/* Allow allocations only from the originally specified zones. */
1507 	result |= zoneflags;
1508 
1509 	/*
1510 	 * Minimize the result gfp by taking the union with the deny flags,
1511 	 * and the intersection of the allow flags.
1512 	 */
1513 	result |= (limit_gfp & denyflags);
1514 	result |= (huge_gfp & limit_gfp) & allowflags;
1515 
1516 	return result;
1517 }
1518 
1519 static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1520 		struct shmem_inode_info *info, pgoff_t index)
1521 {
1522 	struct vm_area_struct pvma;
1523 	struct address_space *mapping = info->vfs_inode.i_mapping;
1524 	pgoff_t hindex;
1525 	struct folio *folio;
1526 
1527 	hindex = round_down(index, HPAGE_PMD_NR);
1528 	if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1529 								XA_PRESENT))
1530 		return NULL;
1531 
1532 	shmem_pseudo_vma_init(&pvma, info, hindex);
1533 	folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1534 	shmem_pseudo_vma_destroy(&pvma);
1535 	if (!folio)
1536 		count_vm_event(THP_FILE_FALLBACK);
1537 	return folio;
1538 }
1539 
1540 static struct folio *shmem_alloc_folio(gfp_t gfp,
1541 			struct shmem_inode_info *info, pgoff_t index)
1542 {
1543 	struct vm_area_struct pvma;
1544 	struct folio *folio;
1545 
1546 	shmem_pseudo_vma_init(&pvma, info, index);
1547 	folio = vma_alloc_folio(gfp, 0, &pvma, 0, false);
1548 	shmem_pseudo_vma_destroy(&pvma);
1549 
1550 	return folio;
1551 }
1552 
1553 static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
1554 		pgoff_t index, bool huge)
1555 {
1556 	struct shmem_inode_info *info = SHMEM_I(inode);
1557 	struct folio *folio;
1558 	int nr;
1559 	int err = -ENOSPC;
1560 
1561 	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1562 		huge = false;
1563 	nr = huge ? HPAGE_PMD_NR : 1;
1564 
1565 	if (!shmem_inode_acct_block(inode, nr))
1566 		goto failed;
1567 
1568 	if (huge)
1569 		folio = shmem_alloc_hugefolio(gfp, info, index);
1570 	else
1571 		folio = shmem_alloc_folio(gfp, info, index);
1572 	if (folio) {
1573 		__folio_set_locked(folio);
1574 		__folio_set_swapbacked(folio);
1575 		return folio;
1576 	}
1577 
1578 	err = -ENOMEM;
1579 	shmem_inode_unacct_blocks(inode, nr);
1580 failed:
1581 	return ERR_PTR(err);
1582 }
1583 
1584 /*
1585  * When a page is moved from swapcache to shmem filecache (either by the
1586  * usual swapin of shmem_get_folio_gfp(), or by the less common swapoff of
1587  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1588  * ignorance of the mapping it belongs to.  If that mapping has special
1589  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1590  * we may need to copy to a suitable page before moving to filecache.
1591  *
1592  * In a future release, this may well be extended to respect cpuset and
1593  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1594  * but for now it is a simple matter of zone.
1595  */
1596 static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1597 {
1598 	return folio_zonenum(folio) > gfp_zone(gfp);
1599 }
1600 
1601 static int shmem_replace_folio(struct folio **foliop, gfp_t gfp,
1602 				struct shmem_inode_info *info, pgoff_t index)
1603 {
1604 	struct folio *old, *new;
1605 	struct address_space *swap_mapping;
1606 	swp_entry_t entry;
1607 	pgoff_t swap_index;
1608 	int error;
1609 
1610 	old = *foliop;
1611 	entry = folio_swap_entry(old);
1612 	swap_index = swp_offset(entry);
1613 	swap_mapping = swap_address_space(entry);
1614 
1615 	/*
1616 	 * We have arrived here because our zones are constrained, so don't
1617 	 * limit chance of success by further cpuset and node constraints.
1618 	 */
1619 	gfp &= ~GFP_CONSTRAINT_MASK;
1620 	VM_BUG_ON_FOLIO(folio_test_large(old), old);
1621 	new = shmem_alloc_folio(gfp, info, index);
1622 	if (!new)
1623 		return -ENOMEM;
1624 
1625 	folio_get(new);
1626 	folio_copy(new, old);
1627 	flush_dcache_folio(new);
1628 
1629 	__folio_set_locked(new);
1630 	__folio_set_swapbacked(new);
1631 	folio_mark_uptodate(new);
1632 	folio_set_swap_entry(new, entry);
1633 	folio_set_swapcache(new);
1634 
1635 	/*
1636 	 * Our caller will very soon move newpage out of swapcache, but it's
1637 	 * a nice clean interface for us to replace oldpage by newpage there.
1638 	 */
1639 	xa_lock_irq(&swap_mapping->i_pages);
1640 	error = shmem_replace_entry(swap_mapping, swap_index, old, new);
1641 	if (!error) {
1642 		mem_cgroup_migrate(old, new);
1643 		__lruvec_stat_mod_folio(new, NR_FILE_PAGES, 1);
1644 		__lruvec_stat_mod_folio(new, NR_SHMEM, 1);
1645 		__lruvec_stat_mod_folio(old, NR_FILE_PAGES, -1);
1646 		__lruvec_stat_mod_folio(old, NR_SHMEM, -1);
1647 	}
1648 	xa_unlock_irq(&swap_mapping->i_pages);
1649 
1650 	if (unlikely(error)) {
1651 		/*
1652 		 * Is this possible?  I think not, now that our callers check
1653 		 * both PageSwapCache and page_private after getting page lock;
1654 		 * but be defensive.  Reverse old to newpage for clear and free.
1655 		 */
1656 		old = new;
1657 	} else {
1658 		folio_add_lru(new);
1659 		*foliop = new;
1660 	}
1661 
1662 	folio_clear_swapcache(old);
1663 	old->private = NULL;
1664 
1665 	folio_unlock(old);
1666 	folio_put_refs(old, 2);
1667 	return error;
1668 }
1669 
1670 static void shmem_set_folio_swapin_error(struct inode *inode, pgoff_t index,
1671 					 struct folio *folio, swp_entry_t swap)
1672 {
1673 	struct address_space *mapping = inode->i_mapping;
1674 	struct shmem_inode_info *info = SHMEM_I(inode);
1675 	swp_entry_t swapin_error;
1676 	void *old;
1677 
1678 	swapin_error = make_swapin_error_entry(&folio->page);
1679 	old = xa_cmpxchg_irq(&mapping->i_pages, index,
1680 			     swp_to_radix_entry(swap),
1681 			     swp_to_radix_entry(swapin_error), 0);
1682 	if (old != swp_to_radix_entry(swap))
1683 		return;
1684 
1685 	folio_wait_writeback(folio);
1686 	delete_from_swap_cache(folio);
1687 	spin_lock_irq(&info->lock);
1688 	/*
1689 	 * Don't treat swapin error folio as alloced. Otherwise inode->i_blocks won't
1690 	 * be 0 when inode is released and thus trigger WARN_ON(inode->i_blocks) in
1691 	 * shmem_evict_inode.
1692 	 */
1693 	info->alloced--;
1694 	info->swapped--;
1695 	shmem_recalc_inode(inode);
1696 	spin_unlock_irq(&info->lock);
1697 	swap_free(swap);
1698 }
1699 
1700 /*
1701  * Swap in the folio pointed to by *foliop.
1702  * Caller has to make sure that *foliop contains a valid swapped folio.
1703  * Returns 0 and the folio in foliop if success. On failure, returns the
1704  * error code and NULL in *foliop.
1705  */
1706 static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1707 			     struct folio **foliop, enum sgp_type sgp,
1708 			     gfp_t gfp, struct vm_area_struct *vma,
1709 			     vm_fault_t *fault_type)
1710 {
1711 	struct address_space *mapping = inode->i_mapping;
1712 	struct shmem_inode_info *info = SHMEM_I(inode);
1713 	struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1714 	struct folio *folio = NULL;
1715 	swp_entry_t swap;
1716 	int error;
1717 
1718 	VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1719 	swap = radix_to_swp_entry(*foliop);
1720 	*foliop = NULL;
1721 
1722 	if (is_swapin_error_entry(swap))
1723 		return -EIO;
1724 
1725 	/* Look it up and read it in.. */
1726 	folio = swap_cache_get_folio(swap, NULL, 0);
1727 	if (!folio) {
1728 		/* Or update major stats only when swapin succeeds?? */
1729 		if (fault_type) {
1730 			*fault_type |= VM_FAULT_MAJOR;
1731 			count_vm_event(PGMAJFAULT);
1732 			count_memcg_event_mm(charge_mm, PGMAJFAULT);
1733 		}
1734 		/* Here we actually start the io */
1735 		folio = shmem_swapin(swap, gfp, info, index);
1736 		if (!folio) {
1737 			error = -ENOMEM;
1738 			goto failed;
1739 		}
1740 	}
1741 
1742 	/* We have to do this with folio locked to prevent races */
1743 	folio_lock(folio);
1744 	if (!folio_test_swapcache(folio) ||
1745 	    folio_swap_entry(folio).val != swap.val ||
1746 	    !shmem_confirm_swap(mapping, index, swap)) {
1747 		error = -EEXIST;
1748 		goto unlock;
1749 	}
1750 	if (!folio_test_uptodate(folio)) {
1751 		error = -EIO;
1752 		goto failed;
1753 	}
1754 	folio_wait_writeback(folio);
1755 
1756 	/*
1757 	 * Some architectures may have to restore extra metadata to the
1758 	 * folio after reading from swap.
1759 	 */
1760 	arch_swap_restore(swap, folio);
1761 
1762 	if (shmem_should_replace_folio(folio, gfp)) {
1763 		error = shmem_replace_folio(&folio, gfp, info, index);
1764 		if (error)
1765 			goto failed;
1766 	}
1767 
1768 	error = shmem_add_to_page_cache(folio, mapping, index,
1769 					swp_to_radix_entry(swap), gfp,
1770 					charge_mm);
1771 	if (error)
1772 		goto failed;
1773 
1774 	spin_lock_irq(&info->lock);
1775 	info->swapped--;
1776 	shmem_recalc_inode(inode);
1777 	spin_unlock_irq(&info->lock);
1778 
1779 	if (sgp == SGP_WRITE)
1780 		folio_mark_accessed(folio);
1781 
1782 	delete_from_swap_cache(folio);
1783 	folio_mark_dirty(folio);
1784 	swap_free(swap);
1785 
1786 	*foliop = folio;
1787 	return 0;
1788 failed:
1789 	if (!shmem_confirm_swap(mapping, index, swap))
1790 		error = -EEXIST;
1791 	if (error == -EIO)
1792 		shmem_set_folio_swapin_error(inode, index, folio, swap);
1793 unlock:
1794 	if (folio) {
1795 		folio_unlock(folio);
1796 		folio_put(folio);
1797 	}
1798 
1799 	return error;
1800 }
1801 
1802 /*
1803  * shmem_get_folio_gfp - find page in cache, or get from swap, or allocate
1804  *
1805  * If we allocate a new one we do not mark it dirty. That's up to the
1806  * vm. If we swap it in we mark it dirty since we also free the swap
1807  * entry since a page cannot live in both the swap and page cache.
1808  *
1809  * vma, vmf, and fault_type are only supplied by shmem_fault:
1810  * otherwise they are NULL.
1811  */
1812 static int shmem_get_folio_gfp(struct inode *inode, pgoff_t index,
1813 		struct folio **foliop, enum sgp_type sgp, gfp_t gfp,
1814 		struct vm_area_struct *vma, struct vm_fault *vmf,
1815 		vm_fault_t *fault_type)
1816 {
1817 	struct address_space *mapping = inode->i_mapping;
1818 	struct shmem_inode_info *info = SHMEM_I(inode);
1819 	struct shmem_sb_info *sbinfo;
1820 	struct mm_struct *charge_mm;
1821 	struct folio *folio;
1822 	pgoff_t hindex = index;
1823 	gfp_t huge_gfp;
1824 	int error;
1825 	int once = 0;
1826 	int alloced = 0;
1827 
1828 	if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1829 		return -EFBIG;
1830 repeat:
1831 	if (sgp <= SGP_CACHE &&
1832 	    ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1833 		return -EINVAL;
1834 	}
1835 
1836 	sbinfo = SHMEM_SB(inode->i_sb);
1837 	charge_mm = vma ? vma->vm_mm : NULL;
1838 
1839 	folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0);
1840 	if (folio && vma && userfaultfd_minor(vma)) {
1841 		if (!xa_is_value(folio)) {
1842 			folio_unlock(folio);
1843 			folio_put(folio);
1844 		}
1845 		*fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1846 		return 0;
1847 	}
1848 
1849 	if (xa_is_value(folio)) {
1850 		error = shmem_swapin_folio(inode, index, &folio,
1851 					  sgp, gfp, vma, fault_type);
1852 		if (error == -EEXIST)
1853 			goto repeat;
1854 
1855 		*foliop = folio;
1856 		return error;
1857 	}
1858 
1859 	if (folio) {
1860 		hindex = folio->index;
1861 		if (sgp == SGP_WRITE)
1862 			folio_mark_accessed(folio);
1863 		if (folio_test_uptodate(folio))
1864 			goto out;
1865 		/* fallocated folio */
1866 		if (sgp != SGP_READ)
1867 			goto clear;
1868 		folio_unlock(folio);
1869 		folio_put(folio);
1870 	}
1871 
1872 	/*
1873 	 * SGP_READ: succeed on hole, with NULL folio, letting caller zero.
1874 	 * SGP_NOALLOC: fail on hole, with NULL folio, letting caller fail.
1875 	 */
1876 	*foliop = NULL;
1877 	if (sgp == SGP_READ)
1878 		return 0;
1879 	if (sgp == SGP_NOALLOC)
1880 		return -ENOENT;
1881 
1882 	/*
1883 	 * Fast cache lookup and swap lookup did not find it: allocate.
1884 	 */
1885 
1886 	if (vma && userfaultfd_missing(vma)) {
1887 		*fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1888 		return 0;
1889 	}
1890 
1891 	if (!shmem_is_huge(vma, inode, index))
1892 		goto alloc_nohuge;
1893 
1894 	huge_gfp = vma_thp_gfp_mask(vma);
1895 	huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1896 	folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true);
1897 	if (IS_ERR(folio)) {
1898 alloc_nohuge:
1899 		folio = shmem_alloc_and_acct_folio(gfp, inode, index, false);
1900 	}
1901 	if (IS_ERR(folio)) {
1902 		int retry = 5;
1903 
1904 		error = PTR_ERR(folio);
1905 		folio = NULL;
1906 		if (error != -ENOSPC)
1907 			goto unlock;
1908 		/*
1909 		 * Try to reclaim some space by splitting a large folio
1910 		 * beyond i_size on the filesystem.
1911 		 */
1912 		while (retry--) {
1913 			int ret;
1914 
1915 			ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1916 			if (ret == SHRINK_STOP)
1917 				break;
1918 			if (ret)
1919 				goto alloc_nohuge;
1920 		}
1921 		goto unlock;
1922 	}
1923 
1924 	hindex = round_down(index, folio_nr_pages(folio));
1925 
1926 	if (sgp == SGP_WRITE)
1927 		__folio_set_referenced(folio);
1928 
1929 	error = shmem_add_to_page_cache(folio, mapping, hindex,
1930 					NULL, gfp & GFP_RECLAIM_MASK,
1931 					charge_mm);
1932 	if (error)
1933 		goto unacct;
1934 	folio_add_lru(folio);
1935 
1936 	spin_lock_irq(&info->lock);
1937 	info->alloced += folio_nr_pages(folio);
1938 	inode->i_blocks += (blkcnt_t)BLOCKS_PER_PAGE << folio_order(folio);
1939 	shmem_recalc_inode(inode);
1940 	spin_unlock_irq(&info->lock);
1941 	alloced = true;
1942 
1943 	if (folio_test_pmd_mappable(folio) &&
1944 	    DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1945 					folio_next_index(folio) - 1) {
1946 		/*
1947 		 * Part of the large folio is beyond i_size: subject
1948 		 * to shrink under memory pressure.
1949 		 */
1950 		spin_lock(&sbinfo->shrinklist_lock);
1951 		/*
1952 		 * _careful to defend against unlocked access to
1953 		 * ->shrink_list in shmem_unused_huge_shrink()
1954 		 */
1955 		if (list_empty_careful(&info->shrinklist)) {
1956 			list_add_tail(&info->shrinklist,
1957 				      &sbinfo->shrinklist);
1958 			sbinfo->shrinklist_len++;
1959 		}
1960 		spin_unlock(&sbinfo->shrinklist_lock);
1961 	}
1962 
1963 	/*
1964 	 * Let SGP_FALLOC use the SGP_WRITE optimization on a new folio.
1965 	 */
1966 	if (sgp == SGP_FALLOC)
1967 		sgp = SGP_WRITE;
1968 clear:
1969 	/*
1970 	 * Let SGP_WRITE caller clear ends if write does not fill folio;
1971 	 * but SGP_FALLOC on a folio fallocated earlier must initialize
1972 	 * it now, lest undo on failure cancel our earlier guarantee.
1973 	 */
1974 	if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
1975 		long i, n = folio_nr_pages(folio);
1976 
1977 		for (i = 0; i < n; i++)
1978 			clear_highpage(folio_page(folio, i));
1979 		flush_dcache_folio(folio);
1980 		folio_mark_uptodate(folio);
1981 	}
1982 
1983 	/* Perhaps the file has been truncated since we checked */
1984 	if (sgp <= SGP_CACHE &&
1985 	    ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1986 		if (alloced) {
1987 			folio_clear_dirty(folio);
1988 			filemap_remove_folio(folio);
1989 			spin_lock_irq(&info->lock);
1990 			shmem_recalc_inode(inode);
1991 			spin_unlock_irq(&info->lock);
1992 		}
1993 		error = -EINVAL;
1994 		goto unlock;
1995 	}
1996 out:
1997 	*foliop = folio;
1998 	return 0;
1999 
2000 	/*
2001 	 * Error recovery.
2002 	 */
2003 unacct:
2004 	shmem_inode_unacct_blocks(inode, folio_nr_pages(folio));
2005 
2006 	if (folio_test_large(folio)) {
2007 		folio_unlock(folio);
2008 		folio_put(folio);
2009 		goto alloc_nohuge;
2010 	}
2011 unlock:
2012 	if (folio) {
2013 		folio_unlock(folio);
2014 		folio_put(folio);
2015 	}
2016 	if (error == -ENOSPC && !once++) {
2017 		spin_lock_irq(&info->lock);
2018 		shmem_recalc_inode(inode);
2019 		spin_unlock_irq(&info->lock);
2020 		goto repeat;
2021 	}
2022 	if (error == -EEXIST)
2023 		goto repeat;
2024 	return error;
2025 }
2026 
2027 int shmem_get_folio(struct inode *inode, pgoff_t index, struct folio **foliop,
2028 		enum sgp_type sgp)
2029 {
2030 	return shmem_get_folio_gfp(inode, index, foliop, sgp,
2031 			mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
2032 }
2033 
2034 /*
2035  * This is like autoremove_wake_function, but it removes the wait queue
2036  * entry unconditionally - even if something else had already woken the
2037  * target.
2038  */
2039 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
2040 {
2041 	int ret = default_wake_function(wait, mode, sync, key);
2042 	list_del_init(&wait->entry);
2043 	return ret;
2044 }
2045 
2046 static vm_fault_t shmem_fault(struct vm_fault *vmf)
2047 {
2048 	struct vm_area_struct *vma = vmf->vma;
2049 	struct inode *inode = file_inode(vma->vm_file);
2050 	gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
2051 	struct folio *folio = NULL;
2052 	int err;
2053 	vm_fault_t ret = VM_FAULT_LOCKED;
2054 
2055 	/*
2056 	 * Trinity finds that probing a hole which tmpfs is punching can
2057 	 * prevent the hole-punch from ever completing: which in turn
2058 	 * locks writers out with its hold on i_rwsem.  So refrain from
2059 	 * faulting pages into the hole while it's being punched.  Although
2060 	 * shmem_undo_range() does remove the additions, it may be unable to
2061 	 * keep up, as each new page needs its own unmap_mapping_range() call,
2062 	 * and the i_mmap tree grows ever slower to scan if new vmas are added.
2063 	 *
2064 	 * It does not matter if we sometimes reach this check just before the
2065 	 * hole-punch begins, so that one fault then races with the punch:
2066 	 * we just need to make racing faults a rare case.
2067 	 *
2068 	 * The implementation below would be much simpler if we just used a
2069 	 * standard mutex or completion: but we cannot take i_rwsem in fault,
2070 	 * and bloating every shmem inode for this unlikely case would be sad.
2071 	 */
2072 	if (unlikely(inode->i_private)) {
2073 		struct shmem_falloc *shmem_falloc;
2074 
2075 		spin_lock(&inode->i_lock);
2076 		shmem_falloc = inode->i_private;
2077 		if (shmem_falloc &&
2078 		    shmem_falloc->waitq &&
2079 		    vmf->pgoff >= shmem_falloc->start &&
2080 		    vmf->pgoff < shmem_falloc->next) {
2081 			struct file *fpin;
2082 			wait_queue_head_t *shmem_falloc_waitq;
2083 			DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2084 
2085 			ret = VM_FAULT_NOPAGE;
2086 			fpin = maybe_unlock_mmap_for_io(vmf, NULL);
2087 			if (fpin)
2088 				ret = VM_FAULT_RETRY;
2089 
2090 			shmem_falloc_waitq = shmem_falloc->waitq;
2091 			prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2092 					TASK_UNINTERRUPTIBLE);
2093 			spin_unlock(&inode->i_lock);
2094 			schedule();
2095 
2096 			/*
2097 			 * shmem_falloc_waitq points into the shmem_fallocate()
2098 			 * stack of the hole-punching task: shmem_falloc_waitq
2099 			 * is usually invalid by the time we reach here, but
2100 			 * finish_wait() does not dereference it in that case;
2101 			 * though i_lock needed lest racing with wake_up_all().
2102 			 */
2103 			spin_lock(&inode->i_lock);
2104 			finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2105 			spin_unlock(&inode->i_lock);
2106 
2107 			if (fpin)
2108 				fput(fpin);
2109 			return ret;
2110 		}
2111 		spin_unlock(&inode->i_lock);
2112 	}
2113 
2114 	err = shmem_get_folio_gfp(inode, vmf->pgoff, &folio, SGP_CACHE,
2115 				  gfp, vma, vmf, &ret);
2116 	if (err)
2117 		return vmf_error(err);
2118 	if (folio)
2119 		vmf->page = folio_file_page(folio, vmf->pgoff);
2120 	return ret;
2121 }
2122 
2123 unsigned long shmem_get_unmapped_area(struct file *file,
2124 				      unsigned long uaddr, unsigned long len,
2125 				      unsigned long pgoff, unsigned long flags)
2126 {
2127 	unsigned long (*get_area)(struct file *,
2128 		unsigned long, unsigned long, unsigned long, unsigned long);
2129 	unsigned long addr;
2130 	unsigned long offset;
2131 	unsigned long inflated_len;
2132 	unsigned long inflated_addr;
2133 	unsigned long inflated_offset;
2134 
2135 	if (len > TASK_SIZE)
2136 		return -ENOMEM;
2137 
2138 	get_area = current->mm->get_unmapped_area;
2139 	addr = get_area(file, uaddr, len, pgoff, flags);
2140 
2141 	if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
2142 		return addr;
2143 	if (IS_ERR_VALUE(addr))
2144 		return addr;
2145 	if (addr & ~PAGE_MASK)
2146 		return addr;
2147 	if (addr > TASK_SIZE - len)
2148 		return addr;
2149 
2150 	if (shmem_huge == SHMEM_HUGE_DENY)
2151 		return addr;
2152 	if (len < HPAGE_PMD_SIZE)
2153 		return addr;
2154 	if (flags & MAP_FIXED)
2155 		return addr;
2156 	/*
2157 	 * Our priority is to support MAP_SHARED mapped hugely;
2158 	 * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2159 	 * But if caller specified an address hint and we allocated area there
2160 	 * successfully, respect that as before.
2161 	 */
2162 	if (uaddr == addr)
2163 		return addr;
2164 
2165 	if (shmem_huge != SHMEM_HUGE_FORCE) {
2166 		struct super_block *sb;
2167 
2168 		if (file) {
2169 			VM_BUG_ON(file->f_op != &shmem_file_operations);
2170 			sb = file_inode(file)->i_sb;
2171 		} else {
2172 			/*
2173 			 * Called directly from mm/mmap.c, or drivers/char/mem.c
2174 			 * for "/dev/zero", to create a shared anonymous object.
2175 			 */
2176 			if (IS_ERR(shm_mnt))
2177 				return addr;
2178 			sb = shm_mnt->mnt_sb;
2179 		}
2180 		if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2181 			return addr;
2182 	}
2183 
2184 	offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2185 	if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2186 		return addr;
2187 	if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2188 		return addr;
2189 
2190 	inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2191 	if (inflated_len > TASK_SIZE)
2192 		return addr;
2193 	if (inflated_len < len)
2194 		return addr;
2195 
2196 	inflated_addr = get_area(NULL, uaddr, inflated_len, 0, flags);
2197 	if (IS_ERR_VALUE(inflated_addr))
2198 		return addr;
2199 	if (inflated_addr & ~PAGE_MASK)
2200 		return addr;
2201 
2202 	inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2203 	inflated_addr += offset - inflated_offset;
2204 	if (inflated_offset > offset)
2205 		inflated_addr += HPAGE_PMD_SIZE;
2206 
2207 	if (inflated_addr > TASK_SIZE - len)
2208 		return addr;
2209 	return inflated_addr;
2210 }
2211 
2212 #ifdef CONFIG_NUMA
2213 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2214 {
2215 	struct inode *inode = file_inode(vma->vm_file);
2216 	return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2217 }
2218 
2219 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2220 					  unsigned long addr)
2221 {
2222 	struct inode *inode = file_inode(vma->vm_file);
2223 	pgoff_t index;
2224 
2225 	index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2226 	return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2227 }
2228 #endif
2229 
2230 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
2231 {
2232 	struct inode *inode = file_inode(file);
2233 	struct shmem_inode_info *info = SHMEM_I(inode);
2234 	int retval = -ENOMEM;
2235 
2236 	/*
2237 	 * What serializes the accesses to info->flags?
2238 	 * ipc_lock_object() when called from shmctl_do_lock(),
2239 	 * no serialization needed when called from shm_destroy().
2240 	 */
2241 	if (lock && !(info->flags & VM_LOCKED)) {
2242 		if (!user_shm_lock(inode->i_size, ucounts))
2243 			goto out_nomem;
2244 		info->flags |= VM_LOCKED;
2245 		mapping_set_unevictable(file->f_mapping);
2246 	}
2247 	if (!lock && (info->flags & VM_LOCKED) && ucounts) {
2248 		user_shm_unlock(inode->i_size, ucounts);
2249 		info->flags &= ~VM_LOCKED;
2250 		mapping_clear_unevictable(file->f_mapping);
2251 	}
2252 	retval = 0;
2253 
2254 out_nomem:
2255 	return retval;
2256 }
2257 
2258 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2259 {
2260 	struct shmem_inode_info *info = SHMEM_I(file_inode(file));
2261 	int ret;
2262 
2263 	ret = seal_check_future_write(info->seals, vma);
2264 	if (ret)
2265 		return ret;
2266 
2267 	/* arm64 - allow memory tagging on RAM-based files */
2268 	vma->vm_flags |= VM_MTE_ALLOWED;
2269 
2270 	file_accessed(file);
2271 	vma->vm_ops = &shmem_vm_ops;
2272 	return 0;
2273 }
2274 
2275 #ifdef CONFIG_TMPFS_XATTR
2276 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2277 
2278 /*
2279  * chattr's fsflags are unrelated to extended attributes,
2280  * but tmpfs has chosen to enable them under the same config option.
2281  */
2282 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2283 {
2284 	unsigned int i_flags = 0;
2285 
2286 	if (fsflags & FS_NOATIME_FL)
2287 		i_flags |= S_NOATIME;
2288 	if (fsflags & FS_APPEND_FL)
2289 		i_flags |= S_APPEND;
2290 	if (fsflags & FS_IMMUTABLE_FL)
2291 		i_flags |= S_IMMUTABLE;
2292 	/*
2293 	 * But FS_NODUMP_FL does not require any action in i_flags.
2294 	 */
2295 	inode_set_flags(inode, i_flags, S_NOATIME | S_APPEND | S_IMMUTABLE);
2296 }
2297 #else
2298 static void shmem_set_inode_flags(struct inode *inode, unsigned int fsflags)
2299 {
2300 }
2301 #define shmem_initxattrs NULL
2302 #endif
2303 
2304 static struct inode *shmem_get_inode(struct super_block *sb, struct inode *dir,
2305 				     umode_t mode, dev_t dev, unsigned long flags)
2306 {
2307 	struct inode *inode;
2308 	struct shmem_inode_info *info;
2309 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2310 	ino_t ino;
2311 
2312 	if (shmem_reserve_inode(sb, &ino))
2313 		return NULL;
2314 
2315 	inode = new_inode(sb);
2316 	if (inode) {
2317 		inode->i_ino = ino;
2318 		inode_init_owner(&init_user_ns, inode, dir, mode);
2319 		inode->i_blocks = 0;
2320 		inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2321 		inode->i_generation = prandom_u32();
2322 		info = SHMEM_I(inode);
2323 		memset(info, 0, (char *)inode - (char *)info);
2324 		spin_lock_init(&info->lock);
2325 		atomic_set(&info->stop_eviction, 0);
2326 		info->seals = F_SEAL_SEAL;
2327 		info->flags = flags & VM_NORESERVE;
2328 		info->i_crtime = inode->i_mtime;
2329 		info->fsflags = (dir == NULL) ? 0 :
2330 			SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED;
2331 		if (info->fsflags)
2332 			shmem_set_inode_flags(inode, info->fsflags);
2333 		INIT_LIST_HEAD(&info->shrinklist);
2334 		INIT_LIST_HEAD(&info->swaplist);
2335 		simple_xattrs_init(&info->xattrs);
2336 		cache_no_acl(inode);
2337 		mapping_set_large_folios(inode->i_mapping);
2338 
2339 		switch (mode & S_IFMT) {
2340 		default:
2341 			inode->i_op = &shmem_special_inode_operations;
2342 			init_special_inode(inode, mode, dev);
2343 			break;
2344 		case S_IFREG:
2345 			inode->i_mapping->a_ops = &shmem_aops;
2346 			inode->i_op = &shmem_inode_operations;
2347 			inode->i_fop = &shmem_file_operations;
2348 			mpol_shared_policy_init(&info->policy,
2349 						 shmem_get_sbmpol(sbinfo));
2350 			break;
2351 		case S_IFDIR:
2352 			inc_nlink(inode);
2353 			/* Some things misbehave if size == 0 on a directory */
2354 			inode->i_size = 2 * BOGO_DIRENT_SIZE;
2355 			inode->i_op = &shmem_dir_inode_operations;
2356 			inode->i_fop = &simple_dir_operations;
2357 			break;
2358 		case S_IFLNK:
2359 			/*
2360 			 * Must not load anything in the rbtree,
2361 			 * mpol_free_shared_policy will not be called.
2362 			 */
2363 			mpol_shared_policy_init(&info->policy, NULL);
2364 			break;
2365 		}
2366 
2367 		lockdep_annotate_inode_mutex_key(inode);
2368 	} else
2369 		shmem_free_inode(sb);
2370 	return inode;
2371 }
2372 
2373 #ifdef CONFIG_USERFAULTFD
2374 int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2375 			   pmd_t *dst_pmd,
2376 			   struct vm_area_struct *dst_vma,
2377 			   unsigned long dst_addr,
2378 			   unsigned long src_addr,
2379 			   bool zeropage, bool wp_copy,
2380 			   struct page **pagep)
2381 {
2382 	struct inode *inode = file_inode(dst_vma->vm_file);
2383 	struct shmem_inode_info *info = SHMEM_I(inode);
2384 	struct address_space *mapping = inode->i_mapping;
2385 	gfp_t gfp = mapping_gfp_mask(mapping);
2386 	pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2387 	void *page_kaddr;
2388 	struct folio *folio;
2389 	int ret;
2390 	pgoff_t max_off;
2391 
2392 	if (!shmem_inode_acct_block(inode, 1)) {
2393 		/*
2394 		 * We may have got a page, returned -ENOENT triggering a retry,
2395 		 * and now we find ourselves with -ENOMEM. Release the page, to
2396 		 * avoid a BUG_ON in our caller.
2397 		 */
2398 		if (unlikely(*pagep)) {
2399 			put_page(*pagep);
2400 			*pagep = NULL;
2401 		}
2402 		return -ENOMEM;
2403 	}
2404 
2405 	if (!*pagep) {
2406 		ret = -ENOMEM;
2407 		folio = shmem_alloc_folio(gfp, info, pgoff);
2408 		if (!folio)
2409 			goto out_unacct_blocks;
2410 
2411 		if (!zeropage) {	/* COPY */
2412 			page_kaddr = kmap_local_folio(folio, 0);
2413 			ret = copy_from_user(page_kaddr,
2414 					     (const void __user *)src_addr,
2415 					     PAGE_SIZE);
2416 			kunmap_local(page_kaddr);
2417 
2418 			/* fallback to copy_from_user outside mmap_lock */
2419 			if (unlikely(ret)) {
2420 				*pagep = &folio->page;
2421 				ret = -ENOENT;
2422 				/* don't free the page */
2423 				goto out_unacct_blocks;
2424 			}
2425 
2426 			flush_dcache_folio(folio);
2427 		} else {		/* ZEROPAGE */
2428 			clear_user_highpage(&folio->page, dst_addr);
2429 		}
2430 	} else {
2431 		folio = page_folio(*pagep);
2432 		VM_BUG_ON_FOLIO(folio_test_large(folio), folio);
2433 		*pagep = NULL;
2434 	}
2435 
2436 	VM_BUG_ON(folio_test_locked(folio));
2437 	VM_BUG_ON(folio_test_swapbacked(folio));
2438 	__folio_set_locked(folio);
2439 	__folio_set_swapbacked(folio);
2440 	__folio_mark_uptodate(folio);
2441 
2442 	ret = -EFAULT;
2443 	max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2444 	if (unlikely(pgoff >= max_off))
2445 		goto out_release;
2446 
2447 	ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
2448 				      gfp & GFP_RECLAIM_MASK, dst_mm);
2449 	if (ret)
2450 		goto out_release;
2451 
2452 	ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2453 				       &folio->page, true, wp_copy);
2454 	if (ret)
2455 		goto out_delete_from_cache;
2456 
2457 	spin_lock_irq(&info->lock);
2458 	info->alloced++;
2459 	inode->i_blocks += BLOCKS_PER_PAGE;
2460 	shmem_recalc_inode(inode);
2461 	spin_unlock_irq(&info->lock);
2462 
2463 	folio_unlock(folio);
2464 	return 0;
2465 out_delete_from_cache:
2466 	filemap_remove_folio(folio);
2467 out_release:
2468 	folio_unlock(folio);
2469 	folio_put(folio);
2470 out_unacct_blocks:
2471 	shmem_inode_unacct_blocks(inode, 1);
2472 	return ret;
2473 }
2474 #endif /* CONFIG_USERFAULTFD */
2475 
2476 #ifdef CONFIG_TMPFS
2477 static const struct inode_operations shmem_symlink_inode_operations;
2478 static const struct inode_operations shmem_short_symlink_operations;
2479 
2480 static int
2481 shmem_write_begin(struct file *file, struct address_space *mapping,
2482 			loff_t pos, unsigned len,
2483 			struct page **pagep, void **fsdata)
2484 {
2485 	struct inode *inode = mapping->host;
2486 	struct shmem_inode_info *info = SHMEM_I(inode);
2487 	pgoff_t index = pos >> PAGE_SHIFT;
2488 	struct folio *folio;
2489 	int ret = 0;
2490 
2491 	/* i_rwsem is held by caller */
2492 	if (unlikely(info->seals & (F_SEAL_GROW |
2493 				   F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))) {
2494 		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE))
2495 			return -EPERM;
2496 		if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2497 			return -EPERM;
2498 	}
2499 
2500 	ret = shmem_get_folio(inode, index, &folio, SGP_WRITE);
2501 
2502 	if (ret)
2503 		return ret;
2504 
2505 	*pagep = folio_file_page(folio, index);
2506 	if (PageHWPoison(*pagep)) {
2507 		folio_unlock(folio);
2508 		folio_put(folio);
2509 		*pagep = NULL;
2510 		return -EIO;
2511 	}
2512 
2513 	return 0;
2514 }
2515 
2516 static int
2517 shmem_write_end(struct file *file, struct address_space *mapping,
2518 			loff_t pos, unsigned len, unsigned copied,
2519 			struct page *page, void *fsdata)
2520 {
2521 	struct inode *inode = mapping->host;
2522 
2523 	if (pos + copied > inode->i_size)
2524 		i_size_write(inode, pos + copied);
2525 
2526 	if (!PageUptodate(page)) {
2527 		struct page *head = compound_head(page);
2528 		if (PageTransCompound(page)) {
2529 			int i;
2530 
2531 			for (i = 0; i < HPAGE_PMD_NR; i++) {
2532 				if (head + i == page)
2533 					continue;
2534 				clear_highpage(head + i);
2535 				flush_dcache_page(head + i);
2536 			}
2537 		}
2538 		if (copied < PAGE_SIZE) {
2539 			unsigned from = pos & (PAGE_SIZE - 1);
2540 			zero_user_segments(page, 0, from,
2541 					from + copied, PAGE_SIZE);
2542 		}
2543 		SetPageUptodate(head);
2544 	}
2545 	set_page_dirty(page);
2546 	unlock_page(page);
2547 	put_page(page);
2548 
2549 	return copied;
2550 }
2551 
2552 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2553 {
2554 	struct file *file = iocb->ki_filp;
2555 	struct inode *inode = file_inode(file);
2556 	struct address_space *mapping = inode->i_mapping;
2557 	pgoff_t index;
2558 	unsigned long offset;
2559 	int error = 0;
2560 	ssize_t retval = 0;
2561 	loff_t *ppos = &iocb->ki_pos;
2562 
2563 	index = *ppos >> PAGE_SHIFT;
2564 	offset = *ppos & ~PAGE_MASK;
2565 
2566 	for (;;) {
2567 		struct folio *folio = NULL;
2568 		struct page *page = NULL;
2569 		pgoff_t end_index;
2570 		unsigned long nr, ret;
2571 		loff_t i_size = i_size_read(inode);
2572 
2573 		end_index = i_size >> PAGE_SHIFT;
2574 		if (index > end_index)
2575 			break;
2576 		if (index == end_index) {
2577 			nr = i_size & ~PAGE_MASK;
2578 			if (nr <= offset)
2579 				break;
2580 		}
2581 
2582 		error = shmem_get_folio(inode, index, &folio, SGP_READ);
2583 		if (error) {
2584 			if (error == -EINVAL)
2585 				error = 0;
2586 			break;
2587 		}
2588 		if (folio) {
2589 			folio_unlock(folio);
2590 
2591 			page = folio_file_page(folio, index);
2592 			if (PageHWPoison(page)) {
2593 				folio_put(folio);
2594 				error = -EIO;
2595 				break;
2596 			}
2597 		}
2598 
2599 		/*
2600 		 * We must evaluate after, since reads (unlike writes)
2601 		 * are called without i_rwsem protection against truncate
2602 		 */
2603 		nr = PAGE_SIZE;
2604 		i_size = i_size_read(inode);
2605 		end_index = i_size >> PAGE_SHIFT;
2606 		if (index == end_index) {
2607 			nr = i_size & ~PAGE_MASK;
2608 			if (nr <= offset) {
2609 				if (folio)
2610 					folio_put(folio);
2611 				break;
2612 			}
2613 		}
2614 		nr -= offset;
2615 
2616 		if (folio) {
2617 			/*
2618 			 * If users can be writing to this page using arbitrary
2619 			 * virtual addresses, take care about potential aliasing
2620 			 * before reading the page on the kernel side.
2621 			 */
2622 			if (mapping_writably_mapped(mapping))
2623 				flush_dcache_page(page);
2624 			/*
2625 			 * Mark the page accessed if we read the beginning.
2626 			 */
2627 			if (!offset)
2628 				folio_mark_accessed(folio);
2629 			/*
2630 			 * Ok, we have the page, and it's up-to-date, so
2631 			 * now we can copy it to user space...
2632 			 */
2633 			ret = copy_page_to_iter(page, offset, nr, to);
2634 			folio_put(folio);
2635 
2636 		} else if (user_backed_iter(to)) {
2637 			/*
2638 			 * Copy to user tends to be so well optimized, but
2639 			 * clear_user() not so much, that it is noticeably
2640 			 * faster to copy the zero page instead of clearing.
2641 			 */
2642 			ret = copy_page_to_iter(ZERO_PAGE(0), offset, nr, to);
2643 		} else {
2644 			/*
2645 			 * But submitting the same page twice in a row to
2646 			 * splice() - or others? - can result in confusion:
2647 			 * so don't attempt that optimization on pipes etc.
2648 			 */
2649 			ret = iov_iter_zero(nr, to);
2650 		}
2651 
2652 		retval += ret;
2653 		offset += ret;
2654 		index += offset >> PAGE_SHIFT;
2655 		offset &= ~PAGE_MASK;
2656 
2657 		if (!iov_iter_count(to))
2658 			break;
2659 		if (ret < nr) {
2660 			error = -EFAULT;
2661 			break;
2662 		}
2663 		cond_resched();
2664 	}
2665 
2666 	*ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2667 	file_accessed(file);
2668 	return retval ? retval : error;
2669 }
2670 
2671 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2672 {
2673 	struct address_space *mapping = file->f_mapping;
2674 	struct inode *inode = mapping->host;
2675 
2676 	if (whence != SEEK_DATA && whence != SEEK_HOLE)
2677 		return generic_file_llseek_size(file, offset, whence,
2678 					MAX_LFS_FILESIZE, i_size_read(inode));
2679 	if (offset < 0)
2680 		return -ENXIO;
2681 
2682 	inode_lock(inode);
2683 	/* We're holding i_rwsem so we can access i_size directly */
2684 	offset = mapping_seek_hole_data(mapping, offset, inode->i_size, whence);
2685 	if (offset >= 0)
2686 		offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2687 	inode_unlock(inode);
2688 	return offset;
2689 }
2690 
2691 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2692 							 loff_t len)
2693 {
2694 	struct inode *inode = file_inode(file);
2695 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2696 	struct shmem_inode_info *info = SHMEM_I(inode);
2697 	struct shmem_falloc shmem_falloc;
2698 	pgoff_t start, index, end, undo_fallocend;
2699 	int error;
2700 
2701 	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2702 		return -EOPNOTSUPP;
2703 
2704 	inode_lock(inode);
2705 
2706 	if (mode & FALLOC_FL_PUNCH_HOLE) {
2707 		struct address_space *mapping = file->f_mapping;
2708 		loff_t unmap_start = round_up(offset, PAGE_SIZE);
2709 		loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2710 		DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2711 
2712 		/* protected by i_rwsem */
2713 		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
2714 			error = -EPERM;
2715 			goto out;
2716 		}
2717 
2718 		shmem_falloc.waitq = &shmem_falloc_waitq;
2719 		shmem_falloc.start = (u64)unmap_start >> PAGE_SHIFT;
2720 		shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2721 		spin_lock(&inode->i_lock);
2722 		inode->i_private = &shmem_falloc;
2723 		spin_unlock(&inode->i_lock);
2724 
2725 		if ((u64)unmap_end > (u64)unmap_start)
2726 			unmap_mapping_range(mapping, unmap_start,
2727 					    1 + unmap_end - unmap_start, 0);
2728 		shmem_truncate_range(inode, offset, offset + len - 1);
2729 		/* No need to unmap again: hole-punching leaves COWed pages */
2730 
2731 		spin_lock(&inode->i_lock);
2732 		inode->i_private = NULL;
2733 		wake_up_all(&shmem_falloc_waitq);
2734 		WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2735 		spin_unlock(&inode->i_lock);
2736 		error = 0;
2737 		goto out;
2738 	}
2739 
2740 	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2741 	error = inode_newsize_ok(inode, offset + len);
2742 	if (error)
2743 		goto out;
2744 
2745 	if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2746 		error = -EPERM;
2747 		goto out;
2748 	}
2749 
2750 	start = offset >> PAGE_SHIFT;
2751 	end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2752 	/* Try to avoid a swapstorm if len is impossible to satisfy */
2753 	if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2754 		error = -ENOSPC;
2755 		goto out;
2756 	}
2757 
2758 	shmem_falloc.waitq = NULL;
2759 	shmem_falloc.start = start;
2760 	shmem_falloc.next  = start;
2761 	shmem_falloc.nr_falloced = 0;
2762 	shmem_falloc.nr_unswapped = 0;
2763 	spin_lock(&inode->i_lock);
2764 	inode->i_private = &shmem_falloc;
2765 	spin_unlock(&inode->i_lock);
2766 
2767 	/*
2768 	 * info->fallocend is only relevant when huge pages might be
2769 	 * involved: to prevent split_huge_page() freeing fallocated
2770 	 * pages when FALLOC_FL_KEEP_SIZE committed beyond i_size.
2771 	 */
2772 	undo_fallocend = info->fallocend;
2773 	if (info->fallocend < end)
2774 		info->fallocend = end;
2775 
2776 	for (index = start; index < end; ) {
2777 		struct folio *folio;
2778 
2779 		/*
2780 		 * Good, the fallocate(2) manpage permits EINTR: we may have
2781 		 * been interrupted because we are using up too much memory.
2782 		 */
2783 		if (signal_pending(current))
2784 			error = -EINTR;
2785 		else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2786 			error = -ENOMEM;
2787 		else
2788 			error = shmem_get_folio(inode, index, &folio,
2789 						SGP_FALLOC);
2790 		if (error) {
2791 			info->fallocend = undo_fallocend;
2792 			/* Remove the !uptodate folios we added */
2793 			if (index > start) {
2794 				shmem_undo_range(inode,
2795 				    (loff_t)start << PAGE_SHIFT,
2796 				    ((loff_t)index << PAGE_SHIFT) - 1, true);
2797 			}
2798 			goto undone;
2799 		}
2800 
2801 		/*
2802 		 * Here is a more important optimization than it appears:
2803 		 * a second SGP_FALLOC on the same large folio will clear it,
2804 		 * making it uptodate and un-undoable if we fail later.
2805 		 */
2806 		index = folio_next_index(folio);
2807 		/* Beware 32-bit wraparound */
2808 		if (!index)
2809 			index--;
2810 
2811 		/*
2812 		 * Inform shmem_writepage() how far we have reached.
2813 		 * No need for lock or barrier: we have the page lock.
2814 		 */
2815 		if (!folio_test_uptodate(folio))
2816 			shmem_falloc.nr_falloced += index - shmem_falloc.next;
2817 		shmem_falloc.next = index;
2818 
2819 		/*
2820 		 * If !uptodate, leave it that way so that freeable folios
2821 		 * can be recognized if we need to rollback on error later.
2822 		 * But mark it dirty so that memory pressure will swap rather
2823 		 * than free the folios we are allocating (and SGP_CACHE folios
2824 		 * might still be clean: we now need to mark those dirty too).
2825 		 */
2826 		folio_mark_dirty(folio);
2827 		folio_unlock(folio);
2828 		folio_put(folio);
2829 		cond_resched();
2830 	}
2831 
2832 	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2833 		i_size_write(inode, offset + len);
2834 undone:
2835 	spin_lock(&inode->i_lock);
2836 	inode->i_private = NULL;
2837 	spin_unlock(&inode->i_lock);
2838 out:
2839 	if (!error)
2840 		file_modified(file);
2841 	inode_unlock(inode);
2842 	return error;
2843 }
2844 
2845 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2846 {
2847 	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2848 
2849 	buf->f_type = TMPFS_MAGIC;
2850 	buf->f_bsize = PAGE_SIZE;
2851 	buf->f_namelen = NAME_MAX;
2852 	if (sbinfo->max_blocks) {
2853 		buf->f_blocks = sbinfo->max_blocks;
2854 		buf->f_bavail =
2855 		buf->f_bfree  = sbinfo->max_blocks -
2856 				percpu_counter_sum(&sbinfo->used_blocks);
2857 	}
2858 	if (sbinfo->max_inodes) {
2859 		buf->f_files = sbinfo->max_inodes;
2860 		buf->f_ffree = sbinfo->free_inodes;
2861 	}
2862 	/* else leave those fields 0 like simple_statfs */
2863 
2864 	buf->f_fsid = uuid_to_fsid(dentry->d_sb->s_uuid.b);
2865 
2866 	return 0;
2867 }
2868 
2869 /*
2870  * File creation. Allocate an inode, and we're done..
2871  */
2872 static int
2873 shmem_mknod(struct user_namespace *mnt_userns, struct inode *dir,
2874 	    struct dentry *dentry, umode_t mode, dev_t dev)
2875 {
2876 	struct inode *inode;
2877 	int error = -ENOSPC;
2878 
2879 	inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2880 	if (inode) {
2881 		error = simple_acl_create(dir, inode);
2882 		if (error)
2883 			goto out_iput;
2884 		error = security_inode_init_security(inode, dir,
2885 						     &dentry->d_name,
2886 						     shmem_initxattrs, NULL);
2887 		if (error && error != -EOPNOTSUPP)
2888 			goto out_iput;
2889 
2890 		error = 0;
2891 		dir->i_size += BOGO_DIRENT_SIZE;
2892 		dir->i_ctime = dir->i_mtime = current_time(dir);
2893 		d_instantiate(dentry, inode);
2894 		dget(dentry); /* Extra count - pin the dentry in core */
2895 	}
2896 	return error;
2897 out_iput:
2898 	iput(inode);
2899 	return error;
2900 }
2901 
2902 static int
2903 shmem_tmpfile(struct user_namespace *mnt_userns, struct inode *dir,
2904 	      struct dentry *dentry, umode_t mode)
2905 {
2906 	struct inode *inode;
2907 	int error = -ENOSPC;
2908 
2909 	inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2910 	if (inode) {
2911 		error = security_inode_init_security(inode, dir,
2912 						     NULL,
2913 						     shmem_initxattrs, NULL);
2914 		if (error && error != -EOPNOTSUPP)
2915 			goto out_iput;
2916 		error = simple_acl_create(dir, inode);
2917 		if (error)
2918 			goto out_iput;
2919 		d_tmpfile(dentry, inode);
2920 	}
2921 	return error;
2922 out_iput:
2923 	iput(inode);
2924 	return error;
2925 }
2926 
2927 static int shmem_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
2928 		       struct dentry *dentry, umode_t mode)
2929 {
2930 	int error;
2931 
2932 	if ((error = shmem_mknod(&init_user_ns, dir, dentry,
2933 				 mode | S_IFDIR, 0)))
2934 		return error;
2935 	inc_nlink(dir);
2936 	return 0;
2937 }
2938 
2939 static int shmem_create(struct user_namespace *mnt_userns, struct inode *dir,
2940 			struct dentry *dentry, umode_t mode, bool excl)
2941 {
2942 	return shmem_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
2943 }
2944 
2945 /*
2946  * Link a file..
2947  */
2948 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2949 {
2950 	struct inode *inode = d_inode(old_dentry);
2951 	int ret = 0;
2952 
2953 	/*
2954 	 * No ordinary (disk based) filesystem counts links as inodes;
2955 	 * but each new link needs a new dentry, pinning lowmem, and
2956 	 * tmpfs dentries cannot be pruned until they are unlinked.
2957 	 * But if an O_TMPFILE file is linked into the tmpfs, the
2958 	 * first link must skip that, to get the accounting right.
2959 	 */
2960 	if (inode->i_nlink) {
2961 		ret = shmem_reserve_inode(inode->i_sb, NULL);
2962 		if (ret)
2963 			goto out;
2964 	}
2965 
2966 	dir->i_size += BOGO_DIRENT_SIZE;
2967 	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2968 	inc_nlink(inode);
2969 	ihold(inode);	/* New dentry reference */
2970 	dget(dentry);		/* Extra pinning count for the created dentry */
2971 	d_instantiate(dentry, inode);
2972 out:
2973 	return ret;
2974 }
2975 
2976 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2977 {
2978 	struct inode *inode = d_inode(dentry);
2979 
2980 	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2981 		shmem_free_inode(inode->i_sb);
2982 
2983 	dir->i_size -= BOGO_DIRENT_SIZE;
2984 	inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2985 	drop_nlink(inode);
2986 	dput(dentry);	/* Undo the count from "create" - this does all the work */
2987 	return 0;
2988 }
2989 
2990 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2991 {
2992 	if (!simple_empty(dentry))
2993 		return -ENOTEMPTY;
2994 
2995 	drop_nlink(d_inode(dentry));
2996 	drop_nlink(dir);
2997 	return shmem_unlink(dir, dentry);
2998 }
2999 
3000 static int shmem_whiteout(struct user_namespace *mnt_userns,
3001 			  struct inode *old_dir, struct dentry *old_dentry)
3002 {
3003 	struct dentry *whiteout;
3004 	int error;
3005 
3006 	whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
3007 	if (!whiteout)
3008 		return -ENOMEM;
3009 
3010 	error = shmem_mknod(&init_user_ns, old_dir, whiteout,
3011 			    S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
3012 	dput(whiteout);
3013 	if (error)
3014 		return error;
3015 
3016 	/*
3017 	 * Cheat and hash the whiteout while the old dentry is still in
3018 	 * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3019 	 *
3020 	 * d_lookup() will consistently find one of them at this point,
3021 	 * not sure which one, but that isn't even important.
3022 	 */
3023 	d_rehash(whiteout);
3024 	return 0;
3025 }
3026 
3027 /*
3028  * The VFS layer already does all the dentry stuff for rename,
3029  * we just have to decrement the usage count for the target if
3030  * it exists so that the VFS layer correctly free's it when it
3031  * gets overwritten.
3032  */
3033 static int shmem_rename2(struct user_namespace *mnt_userns,
3034 			 struct inode *old_dir, struct dentry *old_dentry,
3035 			 struct inode *new_dir, struct dentry *new_dentry,
3036 			 unsigned int flags)
3037 {
3038 	struct inode *inode = d_inode(old_dentry);
3039 	int they_are_dirs = S_ISDIR(inode->i_mode);
3040 
3041 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3042 		return -EINVAL;
3043 
3044 	if (flags & RENAME_EXCHANGE)
3045 		return simple_rename_exchange(old_dir, old_dentry, new_dir, new_dentry);
3046 
3047 	if (!simple_empty(new_dentry))
3048 		return -ENOTEMPTY;
3049 
3050 	if (flags & RENAME_WHITEOUT) {
3051 		int error;
3052 
3053 		error = shmem_whiteout(&init_user_ns, old_dir, old_dentry);
3054 		if (error)
3055 			return error;
3056 	}
3057 
3058 	if (d_really_is_positive(new_dentry)) {
3059 		(void) shmem_unlink(new_dir, new_dentry);
3060 		if (they_are_dirs) {
3061 			drop_nlink(d_inode(new_dentry));
3062 			drop_nlink(old_dir);
3063 		}
3064 	} else if (they_are_dirs) {
3065 		drop_nlink(old_dir);
3066 		inc_nlink(new_dir);
3067 	}
3068 
3069 	old_dir->i_size -= BOGO_DIRENT_SIZE;
3070 	new_dir->i_size += BOGO_DIRENT_SIZE;
3071 	old_dir->i_ctime = old_dir->i_mtime =
3072 	new_dir->i_ctime = new_dir->i_mtime =
3073 	inode->i_ctime = current_time(old_dir);
3074 	return 0;
3075 }
3076 
3077 static int shmem_symlink(struct user_namespace *mnt_userns, struct inode *dir,
3078 			 struct dentry *dentry, const char *symname)
3079 {
3080 	int error;
3081 	int len;
3082 	struct inode *inode;
3083 	struct folio *folio;
3084 
3085 	len = strlen(symname) + 1;
3086 	if (len > PAGE_SIZE)
3087 		return -ENAMETOOLONG;
3088 
3089 	inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3090 				VM_NORESERVE);
3091 	if (!inode)
3092 		return -ENOSPC;
3093 
3094 	error = security_inode_init_security(inode, dir, &dentry->d_name,
3095 					     shmem_initxattrs, NULL);
3096 	if (error && error != -EOPNOTSUPP) {
3097 		iput(inode);
3098 		return error;
3099 	}
3100 
3101 	inode->i_size = len-1;
3102 	if (len <= SHORT_SYMLINK_LEN) {
3103 		inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3104 		if (!inode->i_link) {
3105 			iput(inode);
3106 			return -ENOMEM;
3107 		}
3108 		inode->i_op = &shmem_short_symlink_operations;
3109 	} else {
3110 		inode_nohighmem(inode);
3111 		error = shmem_get_folio(inode, 0, &folio, SGP_WRITE);
3112 		if (error) {
3113 			iput(inode);
3114 			return error;
3115 		}
3116 		inode->i_mapping->a_ops = &shmem_aops;
3117 		inode->i_op = &shmem_symlink_inode_operations;
3118 		memcpy(folio_address(folio), symname, len);
3119 		folio_mark_uptodate(folio);
3120 		folio_mark_dirty(folio);
3121 		folio_unlock(folio);
3122 		folio_put(folio);
3123 	}
3124 	dir->i_size += BOGO_DIRENT_SIZE;
3125 	dir->i_ctime = dir->i_mtime = current_time(dir);
3126 	d_instantiate(dentry, inode);
3127 	dget(dentry);
3128 	return 0;
3129 }
3130 
3131 static void shmem_put_link(void *arg)
3132 {
3133 	folio_mark_accessed(arg);
3134 	folio_put(arg);
3135 }
3136 
3137 static const char *shmem_get_link(struct dentry *dentry,
3138 				  struct inode *inode,
3139 				  struct delayed_call *done)
3140 {
3141 	struct folio *folio = NULL;
3142 	int error;
3143 
3144 	if (!dentry) {
3145 		folio = filemap_get_folio(inode->i_mapping, 0);
3146 		if (!folio)
3147 			return ERR_PTR(-ECHILD);
3148 		if (PageHWPoison(folio_page(folio, 0)) ||
3149 		    !folio_test_uptodate(folio)) {
3150 			folio_put(folio);
3151 			return ERR_PTR(-ECHILD);
3152 		}
3153 	} else {
3154 		error = shmem_get_folio(inode, 0, &folio, SGP_READ);
3155 		if (error)
3156 			return ERR_PTR(error);
3157 		if (!folio)
3158 			return ERR_PTR(-ECHILD);
3159 		if (PageHWPoison(folio_page(folio, 0))) {
3160 			folio_unlock(folio);
3161 			folio_put(folio);
3162 			return ERR_PTR(-ECHILD);
3163 		}
3164 		folio_unlock(folio);
3165 	}
3166 	set_delayed_call(done, shmem_put_link, folio);
3167 	return folio_address(folio);
3168 }
3169 
3170 #ifdef CONFIG_TMPFS_XATTR
3171 
3172 static int shmem_fileattr_get(struct dentry *dentry, struct fileattr *fa)
3173 {
3174 	struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3175 
3176 	fileattr_fill_flags(fa, info->fsflags & SHMEM_FL_USER_VISIBLE);
3177 
3178 	return 0;
3179 }
3180 
3181 static int shmem_fileattr_set(struct user_namespace *mnt_userns,
3182 			      struct dentry *dentry, struct fileattr *fa)
3183 {
3184 	struct inode *inode = d_inode(dentry);
3185 	struct shmem_inode_info *info = SHMEM_I(inode);
3186 
3187 	if (fileattr_has_fsx(fa))
3188 		return -EOPNOTSUPP;
3189 	if (fa->flags & ~SHMEM_FL_USER_MODIFIABLE)
3190 		return -EOPNOTSUPP;
3191 
3192 	info->fsflags = (info->fsflags & ~SHMEM_FL_USER_MODIFIABLE) |
3193 		(fa->flags & SHMEM_FL_USER_MODIFIABLE);
3194 
3195 	shmem_set_inode_flags(inode, info->fsflags);
3196 	inode->i_ctime = current_time(inode);
3197 	return 0;
3198 }
3199 
3200 /*
3201  * Superblocks without xattr inode operations may get some security.* xattr
3202  * support from the LSM "for free". As soon as we have any other xattrs
3203  * like ACLs, we also need to implement the security.* handlers at
3204  * filesystem level, though.
3205  */
3206 
3207 /*
3208  * Callback for security_inode_init_security() for acquiring xattrs.
3209  */
3210 static int shmem_initxattrs(struct inode *inode,
3211 			    const struct xattr *xattr_array,
3212 			    void *fs_info)
3213 {
3214 	struct shmem_inode_info *info = SHMEM_I(inode);
3215 	const struct xattr *xattr;
3216 	struct simple_xattr *new_xattr;
3217 	size_t len;
3218 
3219 	for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3220 		new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3221 		if (!new_xattr)
3222 			return -ENOMEM;
3223 
3224 		len = strlen(xattr->name) + 1;
3225 		new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3226 					  GFP_KERNEL);
3227 		if (!new_xattr->name) {
3228 			kvfree(new_xattr);
3229 			return -ENOMEM;
3230 		}
3231 
3232 		memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3233 		       XATTR_SECURITY_PREFIX_LEN);
3234 		memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3235 		       xattr->name, len);
3236 
3237 		simple_xattr_list_add(&info->xattrs, new_xattr);
3238 	}
3239 
3240 	return 0;
3241 }
3242 
3243 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3244 				   struct dentry *unused, struct inode *inode,
3245 				   const char *name, void *buffer, size_t size)
3246 {
3247 	struct shmem_inode_info *info = SHMEM_I(inode);
3248 
3249 	name = xattr_full_name(handler, name);
3250 	return simple_xattr_get(&info->xattrs, name, buffer, size);
3251 }
3252 
3253 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3254 				   struct user_namespace *mnt_userns,
3255 				   struct dentry *unused, struct inode *inode,
3256 				   const char *name, const void *value,
3257 				   size_t size, int flags)
3258 {
3259 	struct shmem_inode_info *info = SHMEM_I(inode);
3260 
3261 	name = xattr_full_name(handler, name);
3262 	return simple_xattr_set(&info->xattrs, name, value, size, flags, NULL);
3263 }
3264 
3265 static const struct xattr_handler shmem_security_xattr_handler = {
3266 	.prefix = XATTR_SECURITY_PREFIX,
3267 	.get = shmem_xattr_handler_get,
3268 	.set = shmem_xattr_handler_set,
3269 };
3270 
3271 static const struct xattr_handler shmem_trusted_xattr_handler = {
3272 	.prefix = XATTR_TRUSTED_PREFIX,
3273 	.get = shmem_xattr_handler_get,
3274 	.set = shmem_xattr_handler_set,
3275 };
3276 
3277 static const struct xattr_handler *shmem_xattr_handlers[] = {
3278 #ifdef CONFIG_TMPFS_POSIX_ACL
3279 	&posix_acl_access_xattr_handler,
3280 	&posix_acl_default_xattr_handler,
3281 #endif
3282 	&shmem_security_xattr_handler,
3283 	&shmem_trusted_xattr_handler,
3284 	NULL
3285 };
3286 
3287 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3288 {
3289 	struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3290 	return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3291 }
3292 #endif /* CONFIG_TMPFS_XATTR */
3293 
3294 static const struct inode_operations shmem_short_symlink_operations = {
3295 	.getattr	= shmem_getattr,
3296 	.get_link	= simple_get_link,
3297 #ifdef CONFIG_TMPFS_XATTR
3298 	.listxattr	= shmem_listxattr,
3299 #endif
3300 };
3301 
3302 static const struct inode_operations shmem_symlink_inode_operations = {
3303 	.getattr	= shmem_getattr,
3304 	.get_link	= shmem_get_link,
3305 #ifdef CONFIG_TMPFS_XATTR
3306 	.listxattr	= shmem_listxattr,
3307 #endif
3308 };
3309 
3310 static struct dentry *shmem_get_parent(struct dentry *child)
3311 {
3312 	return ERR_PTR(-ESTALE);
3313 }
3314 
3315 static int shmem_match(struct inode *ino, void *vfh)
3316 {
3317 	__u32 *fh = vfh;
3318 	__u64 inum = fh[2];
3319 	inum = (inum << 32) | fh[1];
3320 	return ino->i_ino == inum && fh[0] == ino->i_generation;
3321 }
3322 
3323 /* Find any alias of inode, but prefer a hashed alias */
3324 static struct dentry *shmem_find_alias(struct inode *inode)
3325 {
3326 	struct dentry *alias = d_find_alias(inode);
3327 
3328 	return alias ?: d_find_any_alias(inode);
3329 }
3330 
3331 
3332 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3333 		struct fid *fid, int fh_len, int fh_type)
3334 {
3335 	struct inode *inode;
3336 	struct dentry *dentry = NULL;
3337 	u64 inum;
3338 
3339 	if (fh_len < 3)
3340 		return NULL;
3341 
3342 	inum = fid->raw[2];
3343 	inum = (inum << 32) | fid->raw[1];
3344 
3345 	inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3346 			shmem_match, fid->raw);
3347 	if (inode) {
3348 		dentry = shmem_find_alias(inode);
3349 		iput(inode);
3350 	}
3351 
3352 	return dentry;
3353 }
3354 
3355 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,
3356 				struct inode *parent)
3357 {
3358 	if (*len < 3) {
3359 		*len = 3;
3360 		return FILEID_INVALID;
3361 	}
3362 
3363 	if (inode_unhashed(inode)) {
3364 		/* Unfortunately insert_inode_hash is not idempotent,
3365 		 * so as we hash inodes here rather than at creation
3366 		 * time, we need a lock to ensure we only try
3367 		 * to do it once
3368 		 */
3369 		static DEFINE_SPINLOCK(lock);
3370 		spin_lock(&lock);
3371 		if (inode_unhashed(inode))
3372 			__insert_inode_hash(inode,
3373 					    inode->i_ino + inode->i_generation);
3374 		spin_unlock(&lock);
3375 	}
3376 
3377 	fh[0] = inode->i_generation;
3378 	fh[1] = inode->i_ino;
3379 	fh[2] = ((__u64)inode->i_ino) >> 32;
3380 
3381 	*len = 3;
3382 	return 1;
3383 }
3384 
3385 static const struct export_operations shmem_export_ops = {
3386 	.get_parent     = shmem_get_parent,
3387 	.encode_fh      = shmem_encode_fh,
3388 	.fh_to_dentry	= shmem_fh_to_dentry,
3389 };
3390 
3391 enum shmem_param {
3392 	Opt_gid,
3393 	Opt_huge,
3394 	Opt_mode,
3395 	Opt_mpol,
3396 	Opt_nr_blocks,
3397 	Opt_nr_inodes,
3398 	Opt_size,
3399 	Opt_uid,
3400 	Opt_inode32,
3401 	Opt_inode64,
3402 };
3403 
3404 static const struct constant_table shmem_param_enums_huge[] = {
3405 	{"never",	SHMEM_HUGE_NEVER },
3406 	{"always",	SHMEM_HUGE_ALWAYS },
3407 	{"within_size",	SHMEM_HUGE_WITHIN_SIZE },
3408 	{"advise",	SHMEM_HUGE_ADVISE },
3409 	{}
3410 };
3411 
3412 const struct fs_parameter_spec shmem_fs_parameters[] = {
3413 	fsparam_u32   ("gid",		Opt_gid),
3414 	fsparam_enum  ("huge",		Opt_huge,  shmem_param_enums_huge),
3415 	fsparam_u32oct("mode",		Opt_mode),
3416 	fsparam_string("mpol",		Opt_mpol),
3417 	fsparam_string("nr_blocks",	Opt_nr_blocks),
3418 	fsparam_string("nr_inodes",	Opt_nr_inodes),
3419 	fsparam_string("size",		Opt_size),
3420 	fsparam_u32   ("uid",		Opt_uid),
3421 	fsparam_flag  ("inode32",	Opt_inode32),
3422 	fsparam_flag  ("inode64",	Opt_inode64),
3423 	{}
3424 };
3425 
3426 static int shmem_parse_one(struct fs_context *fc, struct fs_parameter *param)
3427 {
3428 	struct shmem_options *ctx = fc->fs_private;
3429 	struct fs_parse_result result;
3430 	unsigned long long size;
3431 	char *rest;
3432 	int opt;
3433 
3434 	opt = fs_parse(fc, shmem_fs_parameters, param, &result);
3435 	if (opt < 0)
3436 		return opt;
3437 
3438 	switch (opt) {
3439 	case Opt_size:
3440 		size = memparse(param->string, &rest);
3441 		if (*rest == '%') {
3442 			size <<= PAGE_SHIFT;
3443 			size *= totalram_pages();
3444 			do_div(size, 100);
3445 			rest++;
3446 		}
3447 		if (*rest)
3448 			goto bad_value;
3449 		ctx->blocks = DIV_ROUND_UP(size, PAGE_SIZE);
3450 		ctx->seen |= SHMEM_SEEN_BLOCKS;
3451 		break;
3452 	case Opt_nr_blocks:
3453 		ctx->blocks = memparse(param->string, &rest);
3454 		if (*rest || ctx->blocks > S64_MAX)
3455 			goto bad_value;
3456 		ctx->seen |= SHMEM_SEEN_BLOCKS;
3457 		break;
3458 	case Opt_nr_inodes:
3459 		ctx->inodes = memparse(param->string, &rest);
3460 		if (*rest)
3461 			goto bad_value;
3462 		ctx->seen |= SHMEM_SEEN_INODES;
3463 		break;
3464 	case Opt_mode:
3465 		ctx->mode = result.uint_32 & 07777;
3466 		break;
3467 	case Opt_uid:
3468 		ctx->uid = make_kuid(current_user_ns(), result.uint_32);
3469 		if (!uid_valid(ctx->uid))
3470 			goto bad_value;
3471 		break;
3472 	case Opt_gid:
3473 		ctx->gid = make_kgid(current_user_ns(), result.uint_32);
3474 		if (!gid_valid(ctx->gid))
3475 			goto bad_value;
3476 		break;
3477 	case Opt_huge:
3478 		ctx->huge = result.uint_32;
3479 		if (ctx->huge != SHMEM_HUGE_NEVER &&
3480 		    !(IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
3481 		      has_transparent_hugepage()))
3482 			goto unsupported_parameter;
3483 		ctx->seen |= SHMEM_SEEN_HUGE;
3484 		break;
3485 	case Opt_mpol:
3486 		if (IS_ENABLED(CONFIG_NUMA)) {
3487 			mpol_put(ctx->mpol);
3488 			ctx->mpol = NULL;
3489 			if (mpol_parse_str(param->string, &ctx->mpol))
3490 				goto bad_value;
3491 			break;
3492 		}
3493 		goto unsupported_parameter;
3494 	case Opt_inode32:
3495 		ctx->full_inums = false;
3496 		ctx->seen |= SHMEM_SEEN_INUMS;
3497 		break;
3498 	case Opt_inode64:
3499 		if (sizeof(ino_t) < 8) {
3500 			return invalfc(fc,
3501 				       "Cannot use inode64 with <64bit inums in kernel\n");
3502 		}
3503 		ctx->full_inums = true;
3504 		ctx->seen |= SHMEM_SEEN_INUMS;
3505 		break;
3506 	}
3507 	return 0;
3508 
3509 unsupported_parameter:
3510 	return invalfc(fc, "Unsupported parameter '%s'", param->key);
3511 bad_value:
3512 	return invalfc(fc, "Bad value for '%s'", param->key);
3513 }
3514 
3515 static int shmem_parse_options(struct fs_context *fc, void *data)
3516 {
3517 	char *options = data;
3518 
3519 	if (options) {
3520 		int err = security_sb_eat_lsm_opts(options, &fc->security);
3521 		if (err)
3522 			return err;
3523 	}
3524 
3525 	while (options != NULL) {
3526 		char *this_char = options;
3527 		for (;;) {
3528 			/*
3529 			 * NUL-terminate this option: unfortunately,
3530 			 * mount options form a comma-separated list,
3531 			 * but mpol's nodelist may also contain commas.
3532 			 */
3533 			options = strchr(options, ',');
3534 			if (options == NULL)
3535 				break;
3536 			options++;
3537 			if (!isdigit(*options)) {
3538 				options[-1] = '\0';
3539 				break;
3540 			}
3541 		}
3542 		if (*this_char) {
3543 			char *value = strchr(this_char, '=');
3544 			size_t len = 0;
3545 			int err;
3546 
3547 			if (value) {
3548 				*value++ = '\0';
3549 				len = strlen(value);
3550 			}
3551 			err = vfs_parse_fs_string(fc, this_char, value, len);
3552 			if (err < 0)
3553 				return err;
3554 		}
3555 	}
3556 	return 0;
3557 }
3558 
3559 /*
3560  * Reconfigure a shmem filesystem.
3561  *
3562  * Note that we disallow change from limited->unlimited blocks/inodes while any
3563  * are in use; but we must separately disallow unlimited->limited, because in
3564  * that case we have no record of how much is already in use.
3565  */
3566 static int shmem_reconfigure(struct fs_context *fc)
3567 {
3568 	struct shmem_options *ctx = fc->fs_private;
3569 	struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3570 	unsigned long inodes;
3571 	struct mempolicy *mpol = NULL;
3572 	const char *err;
3573 
3574 	raw_spin_lock(&sbinfo->stat_lock);
3575 	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3576 
3577 	if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3578 		if (!sbinfo->max_blocks) {
3579 			err = "Cannot retroactively limit size";
3580 			goto out;
3581 		}
3582 		if (percpu_counter_compare(&sbinfo->used_blocks,
3583 					   ctx->blocks) > 0) {
3584 			err = "Too small a size for current use";
3585 			goto out;
3586 		}
3587 	}
3588 	if ((ctx->seen & SHMEM_SEEN_INODES) && ctx->inodes) {
3589 		if (!sbinfo->max_inodes) {
3590 			err = "Cannot retroactively limit inodes";
3591 			goto out;
3592 		}
3593 		if (ctx->inodes < inodes) {
3594 			err = "Too few inodes for current use";
3595 			goto out;
3596 		}
3597 	}
3598 
3599 	if ((ctx->seen & SHMEM_SEEN_INUMS) && !ctx->full_inums &&
3600 	    sbinfo->next_ino > UINT_MAX) {
3601 		err = "Current inum too high to switch to 32-bit inums";
3602 		goto out;
3603 	}
3604 
3605 	if (ctx->seen & SHMEM_SEEN_HUGE)
3606 		sbinfo->huge = ctx->huge;
3607 	if (ctx->seen & SHMEM_SEEN_INUMS)
3608 		sbinfo->full_inums = ctx->full_inums;
3609 	if (ctx->seen & SHMEM_SEEN_BLOCKS)
3610 		sbinfo->max_blocks  = ctx->blocks;
3611 	if (ctx->seen & SHMEM_SEEN_INODES) {
3612 		sbinfo->max_inodes  = ctx->inodes;
3613 		sbinfo->free_inodes = ctx->inodes - inodes;
3614 	}
3615 
3616 	/*
3617 	 * Preserve previous mempolicy unless mpol remount option was specified.
3618 	 */
3619 	if (ctx->mpol) {
3620 		mpol = sbinfo->mpol;
3621 		sbinfo->mpol = ctx->mpol;	/* transfers initial ref */
3622 		ctx->mpol = NULL;
3623 	}
3624 	raw_spin_unlock(&sbinfo->stat_lock);
3625 	mpol_put(mpol);
3626 	return 0;
3627 out:
3628 	raw_spin_unlock(&sbinfo->stat_lock);
3629 	return invalfc(fc, "%s", err);
3630 }
3631 
3632 static int shmem_show_options(struct seq_file *seq, struct dentry *root)
3633 {
3634 	struct shmem_sb_info *sbinfo = SHMEM_SB(root->d_sb);
3635 
3636 	if (sbinfo->max_blocks != shmem_default_max_blocks())
3637 		seq_printf(seq, ",size=%luk",
3638 			sbinfo->max_blocks << (PAGE_SHIFT - 10));
3639 	if (sbinfo->max_inodes != shmem_default_max_inodes())
3640 		seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
3641 	if (sbinfo->mode != (0777 | S_ISVTX))
3642 		seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3643 	if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3644 		seq_printf(seq, ",uid=%u",
3645 				from_kuid_munged(&init_user_ns, sbinfo->uid));
3646 	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3647 		seq_printf(seq, ",gid=%u",
3648 				from_kgid_munged(&init_user_ns, sbinfo->gid));
3649 
3650 	/*
3651 	 * Showing inode{64,32} might be useful even if it's the system default,
3652 	 * since then people don't have to resort to checking both here and
3653 	 * /proc/config.gz to confirm 64-bit inums were successfully applied
3654 	 * (which may not even exist if IKCONFIG_PROC isn't enabled).
3655 	 *
3656 	 * We hide it when inode64 isn't the default and we are using 32-bit
3657 	 * inodes, since that probably just means the feature isn't even under
3658 	 * consideration.
3659 	 *
3660 	 * As such:
3661 	 *
3662 	 *                     +-----------------+-----------------+
3663 	 *                     | TMPFS_INODE64=y | TMPFS_INODE64=n |
3664 	 *  +------------------+-----------------+-----------------+
3665 	 *  | full_inums=true  | show            | show            |
3666 	 *  | full_inums=false | show            | hide            |
3667 	 *  +------------------+-----------------+-----------------+
3668 	 *
3669 	 */
3670 	if (IS_ENABLED(CONFIG_TMPFS_INODE64) || sbinfo->full_inums)
3671 		seq_printf(seq, ",inode%d", (sbinfo->full_inums ? 64 : 32));
3672 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3673 	/* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3674 	if (sbinfo->huge)
3675 		seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3676 #endif
3677 	shmem_show_mpol(seq, sbinfo->mpol);
3678 	return 0;
3679 }
3680 
3681 #endif /* CONFIG_TMPFS */
3682 
3683 static void shmem_put_super(struct super_block *sb)
3684 {
3685 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
3686 
3687 	free_percpu(sbinfo->ino_batch);
3688 	percpu_counter_destroy(&sbinfo->used_blocks);
3689 	mpol_put(sbinfo->mpol);
3690 	kfree(sbinfo);
3691 	sb->s_fs_info = NULL;
3692 }
3693 
3694 static int shmem_fill_super(struct super_block *sb, struct fs_context *fc)
3695 {
3696 	struct shmem_options *ctx = fc->fs_private;
3697 	struct inode *inode;
3698 	struct shmem_sb_info *sbinfo;
3699 
3700 	/* Round up to L1_CACHE_BYTES to resist false sharing */
3701 	sbinfo = kzalloc(max((int)sizeof(struct shmem_sb_info),
3702 				L1_CACHE_BYTES), GFP_KERNEL);
3703 	if (!sbinfo)
3704 		return -ENOMEM;
3705 
3706 	sb->s_fs_info = sbinfo;
3707 
3708 #ifdef CONFIG_TMPFS
3709 	/*
3710 	 * Per default we only allow half of the physical ram per
3711 	 * tmpfs instance, limiting inodes to one per page of lowmem;
3712 	 * but the internal instance is left unlimited.
3713 	 */
3714 	if (!(sb->s_flags & SB_KERNMOUNT)) {
3715 		if (!(ctx->seen & SHMEM_SEEN_BLOCKS))
3716 			ctx->blocks = shmem_default_max_blocks();
3717 		if (!(ctx->seen & SHMEM_SEEN_INODES))
3718 			ctx->inodes = shmem_default_max_inodes();
3719 		if (!(ctx->seen & SHMEM_SEEN_INUMS))
3720 			ctx->full_inums = IS_ENABLED(CONFIG_TMPFS_INODE64);
3721 	} else {
3722 		sb->s_flags |= SB_NOUSER;
3723 	}
3724 	sb->s_export_op = &shmem_export_ops;
3725 	sb->s_flags |= SB_NOSEC;
3726 #else
3727 	sb->s_flags |= SB_NOUSER;
3728 #endif
3729 	sbinfo->max_blocks = ctx->blocks;
3730 	sbinfo->free_inodes = sbinfo->max_inodes = ctx->inodes;
3731 	if (sb->s_flags & SB_KERNMOUNT) {
3732 		sbinfo->ino_batch = alloc_percpu(ino_t);
3733 		if (!sbinfo->ino_batch)
3734 			goto failed;
3735 	}
3736 	sbinfo->uid = ctx->uid;
3737 	sbinfo->gid = ctx->gid;
3738 	sbinfo->full_inums = ctx->full_inums;
3739 	sbinfo->mode = ctx->mode;
3740 	sbinfo->huge = ctx->huge;
3741 	sbinfo->mpol = ctx->mpol;
3742 	ctx->mpol = NULL;
3743 
3744 	raw_spin_lock_init(&sbinfo->stat_lock);
3745 	if (percpu_counter_init(&sbinfo->used_blocks, 0, GFP_KERNEL))
3746 		goto failed;
3747 	spin_lock_init(&sbinfo->shrinklist_lock);
3748 	INIT_LIST_HEAD(&sbinfo->shrinklist);
3749 
3750 	sb->s_maxbytes = MAX_LFS_FILESIZE;
3751 	sb->s_blocksize = PAGE_SIZE;
3752 	sb->s_blocksize_bits = PAGE_SHIFT;
3753 	sb->s_magic = TMPFS_MAGIC;
3754 	sb->s_op = &shmem_ops;
3755 	sb->s_time_gran = 1;
3756 #ifdef CONFIG_TMPFS_XATTR
3757 	sb->s_xattr = shmem_xattr_handlers;
3758 #endif
3759 #ifdef CONFIG_TMPFS_POSIX_ACL
3760 	sb->s_flags |= SB_POSIXACL;
3761 #endif
3762 	uuid_gen(&sb->s_uuid);
3763 
3764 	inode = shmem_get_inode(sb, NULL, S_IFDIR | sbinfo->mode, 0, VM_NORESERVE);
3765 	if (!inode)
3766 		goto failed;
3767 	inode->i_uid = sbinfo->uid;
3768 	inode->i_gid = sbinfo->gid;
3769 	sb->s_root = d_make_root(inode);
3770 	if (!sb->s_root)
3771 		goto failed;
3772 	return 0;
3773 
3774 failed:
3775 	shmem_put_super(sb);
3776 	return -ENOMEM;
3777 }
3778 
3779 static int shmem_get_tree(struct fs_context *fc)
3780 {
3781 	return get_tree_nodev(fc, shmem_fill_super);
3782 }
3783 
3784 static void shmem_free_fc(struct fs_context *fc)
3785 {
3786 	struct shmem_options *ctx = fc->fs_private;
3787 
3788 	if (ctx) {
3789 		mpol_put(ctx->mpol);
3790 		kfree(ctx);
3791 	}
3792 }
3793 
3794 static const struct fs_context_operations shmem_fs_context_ops = {
3795 	.free			= shmem_free_fc,
3796 	.get_tree		= shmem_get_tree,
3797 #ifdef CONFIG_TMPFS
3798 	.parse_monolithic	= shmem_parse_options,
3799 	.parse_param		= shmem_parse_one,
3800 	.reconfigure		= shmem_reconfigure,
3801 #endif
3802 };
3803 
3804 static struct kmem_cache *shmem_inode_cachep;
3805 
3806 static struct inode *shmem_alloc_inode(struct super_block *sb)
3807 {
3808 	struct shmem_inode_info *info;
3809 	info = alloc_inode_sb(sb, shmem_inode_cachep, GFP_KERNEL);
3810 	if (!info)
3811 		return NULL;
3812 	return &info->vfs_inode;
3813 }
3814 
3815 static void shmem_free_in_core_inode(struct inode *inode)
3816 {
3817 	if (S_ISLNK(inode->i_mode))
3818 		kfree(inode->i_link);
3819 	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
3820 }
3821 
3822 static void shmem_destroy_inode(struct inode *inode)
3823 {
3824 	if (S_ISREG(inode->i_mode))
3825 		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
3826 }
3827 
3828 static void shmem_init_inode(void *foo)
3829 {
3830 	struct shmem_inode_info *info = foo;
3831 	inode_init_once(&info->vfs_inode);
3832 }
3833 
3834 static void shmem_init_inodecache(void)
3835 {
3836 	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
3837 				sizeof(struct shmem_inode_info),
3838 				0, SLAB_PANIC|SLAB_ACCOUNT, shmem_init_inode);
3839 }
3840 
3841 static void shmem_destroy_inodecache(void)
3842 {
3843 	kmem_cache_destroy(shmem_inode_cachep);
3844 }
3845 
3846 /* Keep the page in page cache instead of truncating it */
3847 static int shmem_error_remove_page(struct address_space *mapping,
3848 				   struct page *page)
3849 {
3850 	return 0;
3851 }
3852 
3853 const struct address_space_operations shmem_aops = {
3854 	.writepage	= shmem_writepage,
3855 	.dirty_folio	= noop_dirty_folio,
3856 #ifdef CONFIG_TMPFS
3857 	.write_begin	= shmem_write_begin,
3858 	.write_end	= shmem_write_end,
3859 #endif
3860 #ifdef CONFIG_MIGRATION
3861 	.migrate_folio	= migrate_folio,
3862 #endif
3863 	.error_remove_page = shmem_error_remove_page,
3864 };
3865 EXPORT_SYMBOL(shmem_aops);
3866 
3867 static const struct file_operations shmem_file_operations = {
3868 	.mmap		= shmem_mmap,
3869 	.get_unmapped_area = shmem_get_unmapped_area,
3870 #ifdef CONFIG_TMPFS
3871 	.llseek		= shmem_file_llseek,
3872 	.read_iter	= shmem_file_read_iter,
3873 	.write_iter	= generic_file_write_iter,
3874 	.fsync		= noop_fsync,
3875 	.splice_read	= generic_file_splice_read,
3876 	.splice_write	= iter_file_splice_write,
3877 	.fallocate	= shmem_fallocate,
3878 #endif
3879 };
3880 
3881 static const struct inode_operations shmem_inode_operations = {
3882 	.getattr	= shmem_getattr,
3883 	.setattr	= shmem_setattr,
3884 #ifdef CONFIG_TMPFS_XATTR
3885 	.listxattr	= shmem_listxattr,
3886 	.set_acl	= simple_set_acl,
3887 	.fileattr_get	= shmem_fileattr_get,
3888 	.fileattr_set	= shmem_fileattr_set,
3889 #endif
3890 };
3891 
3892 static const struct inode_operations shmem_dir_inode_operations = {
3893 #ifdef CONFIG_TMPFS
3894 	.getattr	= shmem_getattr,
3895 	.create		= shmem_create,
3896 	.lookup		= simple_lookup,
3897 	.link		= shmem_link,
3898 	.unlink		= shmem_unlink,
3899 	.symlink	= shmem_symlink,
3900 	.mkdir		= shmem_mkdir,
3901 	.rmdir		= shmem_rmdir,
3902 	.mknod		= shmem_mknod,
3903 	.rename		= shmem_rename2,
3904 	.tmpfile	= shmem_tmpfile,
3905 #endif
3906 #ifdef CONFIG_TMPFS_XATTR
3907 	.listxattr	= shmem_listxattr,
3908 	.fileattr_get	= shmem_fileattr_get,
3909 	.fileattr_set	= shmem_fileattr_set,
3910 #endif
3911 #ifdef CONFIG_TMPFS_POSIX_ACL
3912 	.setattr	= shmem_setattr,
3913 	.set_acl	= simple_set_acl,
3914 #endif
3915 };
3916 
3917 static const struct inode_operations shmem_special_inode_operations = {
3918 	.getattr	= shmem_getattr,
3919 #ifdef CONFIG_TMPFS_XATTR
3920 	.listxattr	= shmem_listxattr,
3921 #endif
3922 #ifdef CONFIG_TMPFS_POSIX_ACL
3923 	.setattr	= shmem_setattr,
3924 	.set_acl	= simple_set_acl,
3925 #endif
3926 };
3927 
3928 static const struct super_operations shmem_ops = {
3929 	.alloc_inode	= shmem_alloc_inode,
3930 	.free_inode	= shmem_free_in_core_inode,
3931 	.destroy_inode	= shmem_destroy_inode,
3932 #ifdef CONFIG_TMPFS
3933 	.statfs		= shmem_statfs,
3934 	.show_options	= shmem_show_options,
3935 #endif
3936 	.evict_inode	= shmem_evict_inode,
3937 	.drop_inode	= generic_delete_inode,
3938 	.put_super	= shmem_put_super,
3939 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
3940 	.nr_cached_objects	= shmem_unused_huge_count,
3941 	.free_cached_objects	= shmem_unused_huge_scan,
3942 #endif
3943 };
3944 
3945 static const struct vm_operations_struct shmem_vm_ops = {
3946 	.fault		= shmem_fault,
3947 	.map_pages	= filemap_map_pages,
3948 #ifdef CONFIG_NUMA
3949 	.set_policy     = shmem_set_policy,
3950 	.get_policy     = shmem_get_policy,
3951 #endif
3952 };
3953 
3954 int shmem_init_fs_context(struct fs_context *fc)
3955 {
3956 	struct shmem_options *ctx;
3957 
3958 	ctx = kzalloc(sizeof(struct shmem_options), GFP_KERNEL);
3959 	if (!ctx)
3960 		return -ENOMEM;
3961 
3962 	ctx->mode = 0777 | S_ISVTX;
3963 	ctx->uid = current_fsuid();
3964 	ctx->gid = current_fsgid();
3965 
3966 	fc->fs_private = ctx;
3967 	fc->ops = &shmem_fs_context_ops;
3968 	return 0;
3969 }
3970 
3971 static struct file_system_type shmem_fs_type = {
3972 	.owner		= THIS_MODULE,
3973 	.name		= "tmpfs",
3974 	.init_fs_context = shmem_init_fs_context,
3975 #ifdef CONFIG_TMPFS
3976 	.parameters	= shmem_fs_parameters,
3977 #endif
3978 	.kill_sb	= kill_litter_super,
3979 	.fs_flags	= FS_USERNS_MOUNT,
3980 };
3981 
3982 void __init shmem_init(void)
3983 {
3984 	int error;
3985 
3986 	shmem_init_inodecache();
3987 
3988 	error = register_filesystem(&shmem_fs_type);
3989 	if (error) {
3990 		pr_err("Could not register tmpfs\n");
3991 		goto out2;
3992 	}
3993 
3994 	shm_mnt = kern_mount(&shmem_fs_type);
3995 	if (IS_ERR(shm_mnt)) {
3996 		error = PTR_ERR(shm_mnt);
3997 		pr_err("Could not kern_mount tmpfs\n");
3998 		goto out1;
3999 	}
4000 
4001 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
4002 	if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
4003 		SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4004 	else
4005 		shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
4006 #endif
4007 	return;
4008 
4009 out1:
4010 	unregister_filesystem(&shmem_fs_type);
4011 out2:
4012 	shmem_destroy_inodecache();
4013 	shm_mnt = ERR_PTR(error);
4014 }
4015 
4016 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
4017 static ssize_t shmem_enabled_show(struct kobject *kobj,
4018 				  struct kobj_attribute *attr, char *buf)
4019 {
4020 	static const int values[] = {
4021 		SHMEM_HUGE_ALWAYS,
4022 		SHMEM_HUGE_WITHIN_SIZE,
4023 		SHMEM_HUGE_ADVISE,
4024 		SHMEM_HUGE_NEVER,
4025 		SHMEM_HUGE_DENY,
4026 		SHMEM_HUGE_FORCE,
4027 	};
4028 	int len = 0;
4029 	int i;
4030 
4031 	for (i = 0; i < ARRAY_SIZE(values); i++) {
4032 		len += sysfs_emit_at(buf, len,
4033 				     shmem_huge == values[i] ? "%s[%s]" : "%s%s",
4034 				     i ? " " : "",
4035 				     shmem_format_huge(values[i]));
4036 	}
4037 
4038 	len += sysfs_emit_at(buf, len, "\n");
4039 
4040 	return len;
4041 }
4042 
4043 static ssize_t shmem_enabled_store(struct kobject *kobj,
4044 		struct kobj_attribute *attr, const char *buf, size_t count)
4045 {
4046 	char tmp[16];
4047 	int huge;
4048 
4049 	if (count + 1 > sizeof(tmp))
4050 		return -EINVAL;
4051 	memcpy(tmp, buf, count);
4052 	tmp[count] = '\0';
4053 	if (count && tmp[count - 1] == '\n')
4054 		tmp[count - 1] = '\0';
4055 
4056 	huge = shmem_parse_huge(tmp);
4057 	if (huge == -EINVAL)
4058 		return -EINVAL;
4059 	if (!has_transparent_hugepage() &&
4060 			huge != SHMEM_HUGE_NEVER && huge != SHMEM_HUGE_DENY)
4061 		return -EINVAL;
4062 
4063 	shmem_huge = huge;
4064 	if (shmem_huge > SHMEM_HUGE_DENY)
4065 		SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
4066 	return count;
4067 }
4068 
4069 struct kobj_attribute shmem_enabled_attr = __ATTR_RW(shmem_enabled);
4070 #endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
4071 
4072 #else /* !CONFIG_SHMEM */
4073 
4074 /*
4075  * tiny-shmem: simple shmemfs and tmpfs using ramfs code
4076  *
4077  * This is intended for small system where the benefits of the full
4078  * shmem code (swap-backed and resource-limited) are outweighed by
4079  * their complexity. On systems without swap this code should be
4080  * effectively equivalent, but much lighter weight.
4081  */
4082 
4083 static struct file_system_type shmem_fs_type = {
4084 	.name		= "tmpfs",
4085 	.init_fs_context = ramfs_init_fs_context,
4086 	.parameters	= ramfs_fs_parameters,
4087 	.kill_sb	= kill_litter_super,
4088 	.fs_flags	= FS_USERNS_MOUNT,
4089 };
4090 
4091 void __init shmem_init(void)
4092 {
4093 	BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4094 
4095 	shm_mnt = kern_mount(&shmem_fs_type);
4096 	BUG_ON(IS_ERR(shm_mnt));
4097 }
4098 
4099 int shmem_unuse(unsigned int type)
4100 {
4101 	return 0;
4102 }
4103 
4104 int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)
4105 {
4106 	return 0;
4107 }
4108 
4109 void shmem_unlock_mapping(struct address_space *mapping)
4110 {
4111 }
4112 
4113 #ifdef CONFIG_MMU
4114 unsigned long shmem_get_unmapped_area(struct file *file,
4115 				      unsigned long addr, unsigned long len,
4116 				      unsigned long pgoff, unsigned long flags)
4117 {
4118 	return current->mm->get_unmapped_area(file, addr, len, pgoff, flags);
4119 }
4120 #endif
4121 
4122 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
4123 {
4124 	truncate_inode_pages_range(inode->i_mapping, lstart, lend);
4125 }
4126 EXPORT_SYMBOL_GPL(shmem_truncate_range);
4127 
4128 #define shmem_vm_ops				generic_file_vm_ops
4129 #define shmem_file_operations			ramfs_file_operations
4130 #define shmem_get_inode(sb, dir, mode, dev, flags)	ramfs_get_inode(sb, dir, mode, dev)
4131 #define shmem_acct_size(flags, size)		0
4132 #define shmem_unacct_size(flags, size)		do {} while (0)
4133 
4134 #endif /* CONFIG_SHMEM */
4135 
4136 /* common code */
4137 
4138 static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, loff_t size,
4139 				       unsigned long flags, unsigned int i_flags)
4140 {
4141 	struct inode *inode;
4142 	struct file *res;
4143 
4144 	if (IS_ERR(mnt))
4145 		return ERR_CAST(mnt);
4146 
4147 	if (size < 0 || size > MAX_LFS_FILESIZE)
4148 		return ERR_PTR(-EINVAL);
4149 
4150 	if (shmem_acct_size(flags, size))
4151 		return ERR_PTR(-ENOMEM);
4152 
4153 	inode = shmem_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0,
4154 				flags);
4155 	if (unlikely(!inode)) {
4156 		shmem_unacct_size(flags, size);
4157 		return ERR_PTR(-ENOSPC);
4158 	}
4159 	inode->i_flags |= i_flags;
4160 	inode->i_size = size;
4161 	clear_nlink(inode);	/* It is unlinked */
4162 	res = ERR_PTR(ramfs_nommu_expand_for_mapping(inode, size));
4163 	if (!IS_ERR(res))
4164 		res = alloc_file_pseudo(inode, mnt, name, O_RDWR,
4165 				&shmem_file_operations);
4166 	if (IS_ERR(res))
4167 		iput(inode);
4168 	return res;
4169 }
4170 
4171 /**
4172  * shmem_kernel_file_setup - get an unlinked file living in tmpfs which must be
4173  * 	kernel internal.  There will be NO LSM permission checks against the
4174  * 	underlying inode.  So users of this interface must do LSM checks at a
4175  *	higher layer.  The users are the big_key and shm implementations.  LSM
4176  *	checks are provided at the key or shm level rather than the inode.
4177  * @name: name for dentry (to be seen in /proc/<pid>/maps
4178  * @size: size to be set for the file
4179  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4180  */
4181 struct file *shmem_kernel_file_setup(const char *name, loff_t size, unsigned long flags)
4182 {
4183 	return __shmem_file_setup(shm_mnt, name, size, flags, S_PRIVATE);
4184 }
4185 
4186 /**
4187  * shmem_file_setup - get an unlinked file living in tmpfs
4188  * @name: name for dentry (to be seen in /proc/<pid>/maps
4189  * @size: size to be set for the file
4190  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4191  */
4192 struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags)
4193 {
4194 	return __shmem_file_setup(shm_mnt, name, size, flags, 0);
4195 }
4196 EXPORT_SYMBOL_GPL(shmem_file_setup);
4197 
4198 /**
4199  * shmem_file_setup_with_mnt - get an unlinked file living in tmpfs
4200  * @mnt: the tmpfs mount where the file will be created
4201  * @name: name for dentry (to be seen in /proc/<pid>/maps
4202  * @size: size to be set for the file
4203  * @flags: VM_NORESERVE suppresses pre-accounting of the entire object size
4204  */
4205 struct file *shmem_file_setup_with_mnt(struct vfsmount *mnt, const char *name,
4206 				       loff_t size, unsigned long flags)
4207 {
4208 	return __shmem_file_setup(mnt, name, size, flags, 0);
4209 }
4210 EXPORT_SYMBOL_GPL(shmem_file_setup_with_mnt);
4211 
4212 /**
4213  * shmem_zero_setup - setup a shared anonymous mapping
4214  * @vma: the vma to be mmapped is prepared by do_mmap
4215  */
4216 int shmem_zero_setup(struct vm_area_struct *vma)
4217 {
4218 	struct file *file;
4219 	loff_t size = vma->vm_end - vma->vm_start;
4220 
4221 	/*
4222 	 * Cloning a new file under mmap_lock leads to a lock ordering conflict
4223 	 * between XFS directory reading and selinux: since this file is only
4224 	 * accessible to the user through its mapping, use S_PRIVATE flag to
4225 	 * bypass file security, in the same way as shmem_kernel_file_setup().
4226 	 */
4227 	file = shmem_kernel_file_setup("dev/zero", size, vma->vm_flags);
4228 	if (IS_ERR(file))
4229 		return PTR_ERR(file);
4230 
4231 	if (vma->vm_file)
4232 		fput(vma->vm_file);
4233 	vma->vm_file = file;
4234 	vma->vm_ops = &shmem_vm_ops;
4235 
4236 	return 0;
4237 }
4238 
4239 /**
4240  * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4241  * @mapping:	the page's address_space
4242  * @index:	the page index
4243  * @gfp:	the page allocator flags to use if allocating
4244  *
4245  * This behaves as a tmpfs "read_cache_page_gfp(mapping, index, gfp)",
4246  * with any new page allocations done using the specified allocation flags.
4247  * But read_cache_page_gfp() uses the ->read_folio() method: which does not
4248  * suit tmpfs, since it may have pages in swapcache, and needs to find those
4249  * for itself; although drivers/gpu/drm i915 and ttm rely upon this support.
4250  *
4251  * i915_gem_object_get_pages_gtt() mixes __GFP_NORETRY | __GFP_NOWARN in
4252  * with the mapping_gfp_mask(), to avoid OOMing the machine unnecessarily.
4253  */
4254 struct page *shmem_read_mapping_page_gfp(struct address_space *mapping,
4255 					 pgoff_t index, gfp_t gfp)
4256 {
4257 #ifdef CONFIG_SHMEM
4258 	struct inode *inode = mapping->host;
4259 	struct folio *folio;
4260 	struct page *page;
4261 	int error;
4262 
4263 	BUG_ON(!shmem_mapping(mapping));
4264 	error = shmem_get_folio_gfp(inode, index, &folio, SGP_CACHE,
4265 				  gfp, NULL, NULL, NULL);
4266 	if (error)
4267 		return ERR_PTR(error);
4268 
4269 	folio_unlock(folio);
4270 	page = folio_file_page(folio, index);
4271 	if (PageHWPoison(page)) {
4272 		folio_put(folio);
4273 		return ERR_PTR(-EIO);
4274 	}
4275 
4276 	return page;
4277 #else
4278 	/*
4279 	 * The tiny !SHMEM case uses ramfs without swap
4280 	 */
4281 	return read_cache_page_gfp(mapping, index, gfp);
4282 #endif
4283 }
4284 EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4285