xref: /openbmc/linux/mm/swapfile.c (revision 8bdc2a19)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  linux/mm/swapfile.c
4  *
5  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
6  *  Swap reorganised 29.12.95, Stephen Tweedie
7  */
8 
9 #include <linux/blkdev.h>
10 #include <linux/mm.h>
11 #include <linux/sched/mm.h>
12 #include <linux/sched/task.h>
13 #include <linux/hugetlb.h>
14 #include <linux/mman.h>
15 #include <linux/slab.h>
16 #include <linux/kernel_stat.h>
17 #include <linux/swap.h>
18 #include <linux/vmalloc.h>
19 #include <linux/pagemap.h>
20 #include <linux/namei.h>
21 #include <linux/shmem_fs.h>
22 #include <linux/blk-cgroup.h>
23 #include <linux/random.h>
24 #include <linux/writeback.h>
25 #include <linux/proc_fs.h>
26 #include <linux/seq_file.h>
27 #include <linux/init.h>
28 #include <linux/ksm.h>
29 #include <linux/rmap.h>
30 #include <linux/security.h>
31 #include <linux/backing-dev.h>
32 #include <linux/mutex.h>
33 #include <linux/capability.h>
34 #include <linux/syscalls.h>
35 #include <linux/memcontrol.h>
36 #include <linux/poll.h>
37 #include <linux/oom.h>
38 #include <linux/frontswap.h>
39 #include <linux/swapfile.h>
40 #include <linux/export.h>
41 #include <linux/swap_slots.h>
42 #include <linux/sort.h>
43 #include <linux/completion.h>
44 
45 #include <asm/tlbflush.h>
46 #include <linux/swapops.h>
47 #include <linux/swap_cgroup.h>
48 #include "swap.h"
49 
50 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
51 				 unsigned char);
52 static void free_swap_count_continuations(struct swap_info_struct *);
53 
54 static DEFINE_SPINLOCK(swap_lock);
55 static unsigned int nr_swapfiles;
56 atomic_long_t nr_swap_pages;
57 /*
58  * Some modules use swappable objects and may try to swap them out under
59  * memory pressure (via the shrinker). Before doing so, they may wish to
60  * check to see if any swap space is available.
61  */
62 EXPORT_SYMBOL_GPL(nr_swap_pages);
63 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
64 long total_swap_pages;
65 static int least_priority = -1;
66 
67 static const char Bad_file[] = "Bad swap file entry ";
68 static const char Unused_file[] = "Unused swap file entry ";
69 static const char Bad_offset[] = "Bad swap offset entry ";
70 static const char Unused_offset[] = "Unused swap offset entry ";
71 
72 /*
73  * all active swap_info_structs
74  * protected with swap_lock, and ordered by priority.
75  */
76 static PLIST_HEAD(swap_active_head);
77 
78 /*
79  * all available (active, not full) swap_info_structs
80  * protected with swap_avail_lock, ordered by priority.
81  * This is used by folio_alloc_swap() instead of swap_active_head
82  * because swap_active_head includes all swap_info_structs,
83  * but folio_alloc_swap() doesn't need to look at full ones.
84  * This uses its own lock instead of swap_lock because when a
85  * swap_info_struct changes between not-full/full, it needs to
86  * add/remove itself to/from this list, but the swap_info_struct->lock
87  * is held and the locking order requires swap_lock to be taken
88  * before any swap_info_struct->lock.
89  */
90 static struct plist_head *swap_avail_heads;
91 static DEFINE_SPINLOCK(swap_avail_lock);
92 
93 struct swap_info_struct *swap_info[MAX_SWAPFILES];
94 
95 static DEFINE_MUTEX(swapon_mutex);
96 
97 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
98 /* Activity counter to indicate that a swapon or swapoff has occurred */
99 static atomic_t proc_poll_event = ATOMIC_INIT(0);
100 
101 atomic_t nr_rotate_swap = ATOMIC_INIT(0);
102 
103 static struct swap_info_struct *swap_type_to_swap_info(int type)
104 {
105 	if (type >= MAX_SWAPFILES)
106 		return NULL;
107 
108 	return READ_ONCE(swap_info[type]); /* rcu_dereference() */
109 }
110 
111 static inline unsigned char swap_count(unsigned char ent)
112 {
113 	return ent & ~SWAP_HAS_CACHE;	/* may include COUNT_CONTINUED flag */
114 }
115 
116 /* Reclaim the swap entry anyway if possible */
117 #define TTRS_ANYWAY		0x1
118 /*
119  * Reclaim the swap entry if there are no more mappings of the
120  * corresponding page
121  */
122 #define TTRS_UNMAPPED		0x2
123 /* Reclaim the swap entry if swap is getting full*/
124 #define TTRS_FULL		0x4
125 
126 /* returns 1 if swap entry is freed */
127 static int __try_to_reclaim_swap(struct swap_info_struct *si,
128 				 unsigned long offset, unsigned long flags)
129 {
130 	swp_entry_t entry = swp_entry(si->type, offset);
131 	struct page *page;
132 	int ret = 0;
133 
134 	page = find_get_page(swap_address_space(entry), offset);
135 	if (!page)
136 		return 0;
137 	/*
138 	 * When this function is called from scan_swap_map_slots() and it's
139 	 * called by vmscan.c at reclaiming pages. So, we hold a lock on a page,
140 	 * here. We have to use trylock for avoiding deadlock. This is a special
141 	 * case and you should use try_to_free_swap() with explicit lock_page()
142 	 * in usual operations.
143 	 */
144 	if (trylock_page(page)) {
145 		if ((flags & TTRS_ANYWAY) ||
146 		    ((flags & TTRS_UNMAPPED) && !page_mapped(page)) ||
147 		    ((flags & TTRS_FULL) && mem_cgroup_swap_full(page)))
148 			ret = try_to_free_swap(page);
149 		unlock_page(page);
150 	}
151 	put_page(page);
152 	return ret;
153 }
154 
155 static inline struct swap_extent *first_se(struct swap_info_struct *sis)
156 {
157 	struct rb_node *rb = rb_first(&sis->swap_extent_root);
158 	return rb_entry(rb, struct swap_extent, rb_node);
159 }
160 
161 static inline struct swap_extent *next_se(struct swap_extent *se)
162 {
163 	struct rb_node *rb = rb_next(&se->rb_node);
164 	return rb ? rb_entry(rb, struct swap_extent, rb_node) : NULL;
165 }
166 
167 /*
168  * swapon tell device that all the old swap contents can be discarded,
169  * to allow the swap device to optimize its wear-levelling.
170  */
171 static int discard_swap(struct swap_info_struct *si)
172 {
173 	struct swap_extent *se;
174 	sector_t start_block;
175 	sector_t nr_blocks;
176 	int err = 0;
177 
178 	/* Do not discard the swap header page! */
179 	se = first_se(si);
180 	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
181 	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
182 	if (nr_blocks) {
183 		err = blkdev_issue_discard(si->bdev, start_block,
184 				nr_blocks, GFP_KERNEL);
185 		if (err)
186 			return err;
187 		cond_resched();
188 	}
189 
190 	for (se = next_se(se); se; se = next_se(se)) {
191 		start_block = se->start_block << (PAGE_SHIFT - 9);
192 		nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
193 
194 		err = blkdev_issue_discard(si->bdev, start_block,
195 				nr_blocks, GFP_KERNEL);
196 		if (err)
197 			break;
198 
199 		cond_resched();
200 	}
201 	return err;		/* That will often be -EOPNOTSUPP */
202 }
203 
204 static struct swap_extent *
205 offset_to_swap_extent(struct swap_info_struct *sis, unsigned long offset)
206 {
207 	struct swap_extent *se;
208 	struct rb_node *rb;
209 
210 	rb = sis->swap_extent_root.rb_node;
211 	while (rb) {
212 		se = rb_entry(rb, struct swap_extent, rb_node);
213 		if (offset < se->start_page)
214 			rb = rb->rb_left;
215 		else if (offset >= se->start_page + se->nr_pages)
216 			rb = rb->rb_right;
217 		else
218 			return se;
219 	}
220 	/* It *must* be present */
221 	BUG();
222 }
223 
224 sector_t swap_page_sector(struct page *page)
225 {
226 	struct swap_info_struct *sis = page_swap_info(page);
227 	struct swap_extent *se;
228 	sector_t sector;
229 	pgoff_t offset;
230 
231 	offset = __page_file_index(page);
232 	se = offset_to_swap_extent(sis, offset);
233 	sector = se->start_block + (offset - se->start_page);
234 	return sector << (PAGE_SHIFT - 9);
235 }
236 
237 /*
238  * swap allocation tell device that a cluster of swap can now be discarded,
239  * to allow the swap device to optimize its wear-levelling.
240  */
241 static void discard_swap_cluster(struct swap_info_struct *si,
242 				 pgoff_t start_page, pgoff_t nr_pages)
243 {
244 	struct swap_extent *se = offset_to_swap_extent(si, start_page);
245 
246 	while (nr_pages) {
247 		pgoff_t offset = start_page - se->start_page;
248 		sector_t start_block = se->start_block + offset;
249 		sector_t nr_blocks = se->nr_pages - offset;
250 
251 		if (nr_blocks > nr_pages)
252 			nr_blocks = nr_pages;
253 		start_page += nr_blocks;
254 		nr_pages -= nr_blocks;
255 
256 		start_block <<= PAGE_SHIFT - 9;
257 		nr_blocks <<= PAGE_SHIFT - 9;
258 		if (blkdev_issue_discard(si->bdev, start_block,
259 					nr_blocks, GFP_NOIO))
260 			break;
261 
262 		se = next_se(se);
263 	}
264 }
265 
266 #ifdef CONFIG_THP_SWAP
267 #define SWAPFILE_CLUSTER	HPAGE_PMD_NR
268 
269 #define swap_entry_size(size)	(size)
270 #else
271 #define SWAPFILE_CLUSTER	256
272 
273 /*
274  * Define swap_entry_size() as constant to let compiler to optimize
275  * out some code if !CONFIG_THP_SWAP
276  */
277 #define swap_entry_size(size)	1
278 #endif
279 #define LATENCY_LIMIT		256
280 
281 static inline void cluster_set_flag(struct swap_cluster_info *info,
282 	unsigned int flag)
283 {
284 	info->flags = flag;
285 }
286 
287 static inline unsigned int cluster_count(struct swap_cluster_info *info)
288 {
289 	return info->data;
290 }
291 
292 static inline void cluster_set_count(struct swap_cluster_info *info,
293 				     unsigned int c)
294 {
295 	info->data = c;
296 }
297 
298 static inline void cluster_set_count_flag(struct swap_cluster_info *info,
299 					 unsigned int c, unsigned int f)
300 {
301 	info->flags = f;
302 	info->data = c;
303 }
304 
305 static inline unsigned int cluster_next(struct swap_cluster_info *info)
306 {
307 	return info->data;
308 }
309 
310 static inline void cluster_set_next(struct swap_cluster_info *info,
311 				    unsigned int n)
312 {
313 	info->data = n;
314 }
315 
316 static inline void cluster_set_next_flag(struct swap_cluster_info *info,
317 					 unsigned int n, unsigned int f)
318 {
319 	info->flags = f;
320 	info->data = n;
321 }
322 
323 static inline bool cluster_is_free(struct swap_cluster_info *info)
324 {
325 	return info->flags & CLUSTER_FLAG_FREE;
326 }
327 
328 static inline bool cluster_is_null(struct swap_cluster_info *info)
329 {
330 	return info->flags & CLUSTER_FLAG_NEXT_NULL;
331 }
332 
333 static inline void cluster_set_null(struct swap_cluster_info *info)
334 {
335 	info->flags = CLUSTER_FLAG_NEXT_NULL;
336 	info->data = 0;
337 }
338 
339 static inline bool cluster_is_huge(struct swap_cluster_info *info)
340 {
341 	if (IS_ENABLED(CONFIG_THP_SWAP))
342 		return info->flags & CLUSTER_FLAG_HUGE;
343 	return false;
344 }
345 
346 static inline void cluster_clear_huge(struct swap_cluster_info *info)
347 {
348 	info->flags &= ~CLUSTER_FLAG_HUGE;
349 }
350 
351 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si,
352 						     unsigned long offset)
353 {
354 	struct swap_cluster_info *ci;
355 
356 	ci = si->cluster_info;
357 	if (ci) {
358 		ci += offset / SWAPFILE_CLUSTER;
359 		spin_lock(&ci->lock);
360 	}
361 	return ci;
362 }
363 
364 static inline void unlock_cluster(struct swap_cluster_info *ci)
365 {
366 	if (ci)
367 		spin_unlock(&ci->lock);
368 }
369 
370 /*
371  * Determine the locking method in use for this device.  Return
372  * swap_cluster_info if SSD-style cluster-based locking is in place.
373  */
374 static inline struct swap_cluster_info *lock_cluster_or_swap_info(
375 		struct swap_info_struct *si, unsigned long offset)
376 {
377 	struct swap_cluster_info *ci;
378 
379 	/* Try to use fine-grained SSD-style locking if available: */
380 	ci = lock_cluster(si, offset);
381 	/* Otherwise, fall back to traditional, coarse locking: */
382 	if (!ci)
383 		spin_lock(&si->lock);
384 
385 	return ci;
386 }
387 
388 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si,
389 					       struct swap_cluster_info *ci)
390 {
391 	if (ci)
392 		unlock_cluster(ci);
393 	else
394 		spin_unlock(&si->lock);
395 }
396 
397 static inline bool cluster_list_empty(struct swap_cluster_list *list)
398 {
399 	return cluster_is_null(&list->head);
400 }
401 
402 static inline unsigned int cluster_list_first(struct swap_cluster_list *list)
403 {
404 	return cluster_next(&list->head);
405 }
406 
407 static void cluster_list_init(struct swap_cluster_list *list)
408 {
409 	cluster_set_null(&list->head);
410 	cluster_set_null(&list->tail);
411 }
412 
413 static void cluster_list_add_tail(struct swap_cluster_list *list,
414 				  struct swap_cluster_info *ci,
415 				  unsigned int idx)
416 {
417 	if (cluster_list_empty(list)) {
418 		cluster_set_next_flag(&list->head, idx, 0);
419 		cluster_set_next_flag(&list->tail, idx, 0);
420 	} else {
421 		struct swap_cluster_info *ci_tail;
422 		unsigned int tail = cluster_next(&list->tail);
423 
424 		/*
425 		 * Nested cluster lock, but both cluster locks are
426 		 * only acquired when we held swap_info_struct->lock
427 		 */
428 		ci_tail = ci + tail;
429 		spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING);
430 		cluster_set_next(ci_tail, idx);
431 		spin_unlock(&ci_tail->lock);
432 		cluster_set_next_flag(&list->tail, idx, 0);
433 	}
434 }
435 
436 static unsigned int cluster_list_del_first(struct swap_cluster_list *list,
437 					   struct swap_cluster_info *ci)
438 {
439 	unsigned int idx;
440 
441 	idx = cluster_next(&list->head);
442 	if (cluster_next(&list->tail) == idx) {
443 		cluster_set_null(&list->head);
444 		cluster_set_null(&list->tail);
445 	} else
446 		cluster_set_next_flag(&list->head,
447 				      cluster_next(&ci[idx]), 0);
448 
449 	return idx;
450 }
451 
452 /* Add a cluster to discard list and schedule it to do discard */
453 static void swap_cluster_schedule_discard(struct swap_info_struct *si,
454 		unsigned int idx)
455 {
456 	/*
457 	 * If scan_swap_map_slots() can't find a free cluster, it will check
458 	 * si->swap_map directly. To make sure the discarding cluster isn't
459 	 * taken by scan_swap_map_slots(), mark the swap entries bad (occupied).
460 	 * It will be cleared after discard
461 	 */
462 	memset(si->swap_map + idx * SWAPFILE_CLUSTER,
463 			SWAP_MAP_BAD, SWAPFILE_CLUSTER);
464 
465 	cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx);
466 
467 	schedule_work(&si->discard_work);
468 }
469 
470 static void __free_cluster(struct swap_info_struct *si, unsigned long idx)
471 {
472 	struct swap_cluster_info *ci = si->cluster_info;
473 
474 	cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE);
475 	cluster_list_add_tail(&si->free_clusters, ci, idx);
476 }
477 
478 /*
479  * Doing discard actually. After a cluster discard is finished, the cluster
480  * will be added to free cluster list. caller should hold si->lock.
481 */
482 static void swap_do_scheduled_discard(struct swap_info_struct *si)
483 {
484 	struct swap_cluster_info *info, *ci;
485 	unsigned int idx;
486 
487 	info = si->cluster_info;
488 
489 	while (!cluster_list_empty(&si->discard_clusters)) {
490 		idx = cluster_list_del_first(&si->discard_clusters, info);
491 		spin_unlock(&si->lock);
492 
493 		discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
494 				SWAPFILE_CLUSTER);
495 
496 		spin_lock(&si->lock);
497 		ci = lock_cluster(si, idx * SWAPFILE_CLUSTER);
498 		__free_cluster(si, idx);
499 		memset(si->swap_map + idx * SWAPFILE_CLUSTER,
500 				0, SWAPFILE_CLUSTER);
501 		unlock_cluster(ci);
502 	}
503 }
504 
505 static void swap_discard_work(struct work_struct *work)
506 {
507 	struct swap_info_struct *si;
508 
509 	si = container_of(work, struct swap_info_struct, discard_work);
510 
511 	spin_lock(&si->lock);
512 	swap_do_scheduled_discard(si);
513 	spin_unlock(&si->lock);
514 }
515 
516 static void swap_users_ref_free(struct percpu_ref *ref)
517 {
518 	struct swap_info_struct *si;
519 
520 	si = container_of(ref, struct swap_info_struct, users);
521 	complete(&si->comp);
522 }
523 
524 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx)
525 {
526 	struct swap_cluster_info *ci = si->cluster_info;
527 
528 	VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx);
529 	cluster_list_del_first(&si->free_clusters, ci);
530 	cluster_set_count_flag(ci + idx, 0, 0);
531 }
532 
533 static void free_cluster(struct swap_info_struct *si, unsigned long idx)
534 {
535 	struct swap_cluster_info *ci = si->cluster_info + idx;
536 
537 	VM_BUG_ON(cluster_count(ci) != 0);
538 	/*
539 	 * If the swap is discardable, prepare discard the cluster
540 	 * instead of free it immediately. The cluster will be freed
541 	 * after discard.
542 	 */
543 	if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
544 	    (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
545 		swap_cluster_schedule_discard(si, idx);
546 		return;
547 	}
548 
549 	__free_cluster(si, idx);
550 }
551 
552 /*
553  * The cluster corresponding to page_nr will be used. The cluster will be
554  * removed from free cluster list and its usage counter will be increased.
555  */
556 static void inc_cluster_info_page(struct swap_info_struct *p,
557 	struct swap_cluster_info *cluster_info, unsigned long page_nr)
558 {
559 	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
560 
561 	if (!cluster_info)
562 		return;
563 	if (cluster_is_free(&cluster_info[idx]))
564 		alloc_cluster(p, idx);
565 
566 	VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
567 	cluster_set_count(&cluster_info[idx],
568 		cluster_count(&cluster_info[idx]) + 1);
569 }
570 
571 /*
572  * The cluster corresponding to page_nr decreases one usage. If the usage
573  * counter becomes 0, which means no page in the cluster is in using, we can
574  * optionally discard the cluster and add it to free cluster list.
575  */
576 static void dec_cluster_info_page(struct swap_info_struct *p,
577 	struct swap_cluster_info *cluster_info, unsigned long page_nr)
578 {
579 	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
580 
581 	if (!cluster_info)
582 		return;
583 
584 	VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
585 	cluster_set_count(&cluster_info[idx],
586 		cluster_count(&cluster_info[idx]) - 1);
587 
588 	if (cluster_count(&cluster_info[idx]) == 0)
589 		free_cluster(p, idx);
590 }
591 
592 /*
593  * It's possible scan_swap_map_slots() uses a free cluster in the middle of free
594  * cluster list. Avoiding such abuse to avoid list corruption.
595  */
596 static bool
597 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
598 	unsigned long offset)
599 {
600 	struct percpu_cluster *percpu_cluster;
601 	bool conflict;
602 
603 	offset /= SWAPFILE_CLUSTER;
604 	conflict = !cluster_list_empty(&si->free_clusters) &&
605 		offset != cluster_list_first(&si->free_clusters) &&
606 		cluster_is_free(&si->cluster_info[offset]);
607 
608 	if (!conflict)
609 		return false;
610 
611 	percpu_cluster = this_cpu_ptr(si->percpu_cluster);
612 	cluster_set_null(&percpu_cluster->index);
613 	return true;
614 }
615 
616 /*
617  * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
618  * might involve allocating a new cluster for current CPU too.
619  */
620 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
621 	unsigned long *offset, unsigned long *scan_base)
622 {
623 	struct percpu_cluster *cluster;
624 	struct swap_cluster_info *ci;
625 	unsigned long tmp, max;
626 
627 new_cluster:
628 	cluster = this_cpu_ptr(si->percpu_cluster);
629 	if (cluster_is_null(&cluster->index)) {
630 		if (!cluster_list_empty(&si->free_clusters)) {
631 			cluster->index = si->free_clusters.head;
632 			cluster->next = cluster_next(&cluster->index) *
633 					SWAPFILE_CLUSTER;
634 		} else if (!cluster_list_empty(&si->discard_clusters)) {
635 			/*
636 			 * we don't have free cluster but have some clusters in
637 			 * discarding, do discard now and reclaim them, then
638 			 * reread cluster_next_cpu since we dropped si->lock
639 			 */
640 			swap_do_scheduled_discard(si);
641 			*scan_base = this_cpu_read(*si->cluster_next_cpu);
642 			*offset = *scan_base;
643 			goto new_cluster;
644 		} else
645 			return false;
646 	}
647 
648 	/*
649 	 * Other CPUs can use our cluster if they can't find a free cluster,
650 	 * check if there is still free entry in the cluster
651 	 */
652 	tmp = cluster->next;
653 	max = min_t(unsigned long, si->max,
654 		    (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER);
655 	if (tmp < max) {
656 		ci = lock_cluster(si, tmp);
657 		while (tmp < max) {
658 			if (!si->swap_map[tmp])
659 				break;
660 			tmp++;
661 		}
662 		unlock_cluster(ci);
663 	}
664 	if (tmp >= max) {
665 		cluster_set_null(&cluster->index);
666 		goto new_cluster;
667 	}
668 	cluster->next = tmp + 1;
669 	*offset = tmp;
670 	*scan_base = tmp;
671 	return true;
672 }
673 
674 static void __del_from_avail_list(struct swap_info_struct *p)
675 {
676 	int nid;
677 
678 	for_each_node(nid)
679 		plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]);
680 }
681 
682 static void del_from_avail_list(struct swap_info_struct *p)
683 {
684 	spin_lock(&swap_avail_lock);
685 	__del_from_avail_list(p);
686 	spin_unlock(&swap_avail_lock);
687 }
688 
689 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset,
690 			     unsigned int nr_entries)
691 {
692 	unsigned int end = offset + nr_entries - 1;
693 
694 	if (offset == si->lowest_bit)
695 		si->lowest_bit += nr_entries;
696 	if (end == si->highest_bit)
697 		WRITE_ONCE(si->highest_bit, si->highest_bit - nr_entries);
698 	si->inuse_pages += nr_entries;
699 	if (si->inuse_pages == si->pages) {
700 		si->lowest_bit = si->max;
701 		si->highest_bit = 0;
702 		del_from_avail_list(si);
703 	}
704 }
705 
706 static void add_to_avail_list(struct swap_info_struct *p)
707 {
708 	int nid;
709 
710 	spin_lock(&swap_avail_lock);
711 	for_each_node(nid) {
712 		WARN_ON(!plist_node_empty(&p->avail_lists[nid]));
713 		plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]);
714 	}
715 	spin_unlock(&swap_avail_lock);
716 }
717 
718 static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
719 			    unsigned int nr_entries)
720 {
721 	unsigned long begin = offset;
722 	unsigned long end = offset + nr_entries - 1;
723 	void (*swap_slot_free_notify)(struct block_device *, unsigned long);
724 
725 	if (offset < si->lowest_bit)
726 		si->lowest_bit = offset;
727 	if (end > si->highest_bit) {
728 		bool was_full = !si->highest_bit;
729 
730 		WRITE_ONCE(si->highest_bit, end);
731 		if (was_full && (si->flags & SWP_WRITEOK))
732 			add_to_avail_list(si);
733 	}
734 	atomic_long_add(nr_entries, &nr_swap_pages);
735 	si->inuse_pages -= nr_entries;
736 	if (si->flags & SWP_BLKDEV)
737 		swap_slot_free_notify =
738 			si->bdev->bd_disk->fops->swap_slot_free_notify;
739 	else
740 		swap_slot_free_notify = NULL;
741 	while (offset <= end) {
742 		arch_swap_invalidate_page(si->type, offset);
743 		frontswap_invalidate_page(si->type, offset);
744 		if (swap_slot_free_notify)
745 			swap_slot_free_notify(si->bdev, offset);
746 		offset++;
747 	}
748 	clear_shadow_from_swap_cache(si->type, begin, end);
749 }
750 
751 static void set_cluster_next(struct swap_info_struct *si, unsigned long next)
752 {
753 	unsigned long prev;
754 
755 	if (!(si->flags & SWP_SOLIDSTATE)) {
756 		si->cluster_next = next;
757 		return;
758 	}
759 
760 	prev = this_cpu_read(*si->cluster_next_cpu);
761 	/*
762 	 * Cross the swap address space size aligned trunk, choose
763 	 * another trunk randomly to avoid lock contention on swap
764 	 * address space if possible.
765 	 */
766 	if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) !=
767 	    (next >> SWAP_ADDRESS_SPACE_SHIFT)) {
768 		/* No free swap slots available */
769 		if (si->highest_bit <= si->lowest_bit)
770 			return;
771 		next = si->lowest_bit +
772 			prandom_u32_max(si->highest_bit - si->lowest_bit + 1);
773 		next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES);
774 		next = max_t(unsigned int, next, si->lowest_bit);
775 	}
776 	this_cpu_write(*si->cluster_next_cpu, next);
777 }
778 
779 static bool swap_offset_available_and_locked(struct swap_info_struct *si,
780 					     unsigned long offset)
781 {
782 	if (data_race(!si->swap_map[offset])) {
783 		spin_lock(&si->lock);
784 		return true;
785 	}
786 
787 	if (vm_swap_full() && READ_ONCE(si->swap_map[offset]) == SWAP_HAS_CACHE) {
788 		spin_lock(&si->lock);
789 		return true;
790 	}
791 
792 	return false;
793 }
794 
795 static int scan_swap_map_slots(struct swap_info_struct *si,
796 			       unsigned char usage, int nr,
797 			       swp_entry_t slots[])
798 {
799 	struct swap_cluster_info *ci;
800 	unsigned long offset;
801 	unsigned long scan_base;
802 	unsigned long last_in_cluster = 0;
803 	int latency_ration = LATENCY_LIMIT;
804 	int n_ret = 0;
805 	bool scanned_many = false;
806 
807 	/*
808 	 * We try to cluster swap pages by allocating them sequentially
809 	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
810 	 * way, however, we resort to first-free allocation, starting
811 	 * a new cluster.  This prevents us from scattering swap pages
812 	 * all over the entire swap partition, so that we reduce
813 	 * overall disk seek times between swap pages.  -- sct
814 	 * But we do now try to find an empty cluster.  -Andrea
815 	 * And we let swap pages go all over an SSD partition.  Hugh
816 	 */
817 
818 	si->flags += SWP_SCANNING;
819 	/*
820 	 * Use percpu scan base for SSD to reduce lock contention on
821 	 * cluster and swap cache.  For HDD, sequential access is more
822 	 * important.
823 	 */
824 	if (si->flags & SWP_SOLIDSTATE)
825 		scan_base = this_cpu_read(*si->cluster_next_cpu);
826 	else
827 		scan_base = si->cluster_next;
828 	offset = scan_base;
829 
830 	/* SSD algorithm */
831 	if (si->cluster_info) {
832 		if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
833 			goto scan;
834 	} else if (unlikely(!si->cluster_nr--)) {
835 		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
836 			si->cluster_nr = SWAPFILE_CLUSTER - 1;
837 			goto checks;
838 		}
839 
840 		spin_unlock(&si->lock);
841 
842 		/*
843 		 * If seek is expensive, start searching for new cluster from
844 		 * start of partition, to minimize the span of allocated swap.
845 		 * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
846 		 * case, just handled by scan_swap_map_try_ssd_cluster() above.
847 		 */
848 		scan_base = offset = si->lowest_bit;
849 		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
850 
851 		/* Locate the first empty (unaligned) cluster */
852 		for (; last_in_cluster <= si->highest_bit; offset++) {
853 			if (si->swap_map[offset])
854 				last_in_cluster = offset + SWAPFILE_CLUSTER;
855 			else if (offset == last_in_cluster) {
856 				spin_lock(&si->lock);
857 				offset -= SWAPFILE_CLUSTER - 1;
858 				si->cluster_next = offset;
859 				si->cluster_nr = SWAPFILE_CLUSTER - 1;
860 				goto checks;
861 			}
862 			if (unlikely(--latency_ration < 0)) {
863 				cond_resched();
864 				latency_ration = LATENCY_LIMIT;
865 			}
866 		}
867 
868 		offset = scan_base;
869 		spin_lock(&si->lock);
870 		si->cluster_nr = SWAPFILE_CLUSTER - 1;
871 	}
872 
873 checks:
874 	if (si->cluster_info) {
875 		while (scan_swap_map_ssd_cluster_conflict(si, offset)) {
876 		/* take a break if we already got some slots */
877 			if (n_ret)
878 				goto done;
879 			if (!scan_swap_map_try_ssd_cluster(si, &offset,
880 							&scan_base))
881 				goto scan;
882 		}
883 	}
884 	if (!(si->flags & SWP_WRITEOK))
885 		goto no_page;
886 	if (!si->highest_bit)
887 		goto no_page;
888 	if (offset > si->highest_bit)
889 		scan_base = offset = si->lowest_bit;
890 
891 	ci = lock_cluster(si, offset);
892 	/* reuse swap entry of cache-only swap if not busy. */
893 	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
894 		int swap_was_freed;
895 		unlock_cluster(ci);
896 		spin_unlock(&si->lock);
897 		swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
898 		spin_lock(&si->lock);
899 		/* entry was freed successfully, try to use this again */
900 		if (swap_was_freed)
901 			goto checks;
902 		goto scan; /* check next one */
903 	}
904 
905 	if (si->swap_map[offset]) {
906 		unlock_cluster(ci);
907 		if (!n_ret)
908 			goto scan;
909 		else
910 			goto done;
911 	}
912 	WRITE_ONCE(si->swap_map[offset], usage);
913 	inc_cluster_info_page(si, si->cluster_info, offset);
914 	unlock_cluster(ci);
915 
916 	swap_range_alloc(si, offset, 1);
917 	slots[n_ret++] = swp_entry(si->type, offset);
918 
919 	/* got enough slots or reach max slots? */
920 	if ((n_ret == nr) || (offset >= si->highest_bit))
921 		goto done;
922 
923 	/* search for next available slot */
924 
925 	/* time to take a break? */
926 	if (unlikely(--latency_ration < 0)) {
927 		if (n_ret)
928 			goto done;
929 		spin_unlock(&si->lock);
930 		cond_resched();
931 		spin_lock(&si->lock);
932 		latency_ration = LATENCY_LIMIT;
933 	}
934 
935 	/* try to get more slots in cluster */
936 	if (si->cluster_info) {
937 		if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
938 			goto checks;
939 	} else if (si->cluster_nr && !si->swap_map[++offset]) {
940 		/* non-ssd case, still more slots in cluster? */
941 		--si->cluster_nr;
942 		goto checks;
943 	}
944 
945 	/*
946 	 * Even if there's no free clusters available (fragmented),
947 	 * try to scan a little more quickly with lock held unless we
948 	 * have scanned too many slots already.
949 	 */
950 	if (!scanned_many) {
951 		unsigned long scan_limit;
952 
953 		if (offset < scan_base)
954 			scan_limit = scan_base;
955 		else
956 			scan_limit = si->highest_bit;
957 		for (; offset <= scan_limit && --latency_ration > 0;
958 		     offset++) {
959 			if (!si->swap_map[offset])
960 				goto checks;
961 		}
962 	}
963 
964 done:
965 	set_cluster_next(si, offset + 1);
966 	si->flags -= SWP_SCANNING;
967 	return n_ret;
968 
969 scan:
970 	spin_unlock(&si->lock);
971 	while (++offset <= READ_ONCE(si->highest_bit)) {
972 		if (swap_offset_available_and_locked(si, offset))
973 			goto checks;
974 		if (unlikely(--latency_ration < 0)) {
975 			cond_resched();
976 			latency_ration = LATENCY_LIMIT;
977 			scanned_many = true;
978 		}
979 	}
980 	offset = si->lowest_bit;
981 	while (offset < scan_base) {
982 		if (swap_offset_available_and_locked(si, offset))
983 			goto checks;
984 		if (unlikely(--latency_ration < 0)) {
985 			cond_resched();
986 			latency_ration = LATENCY_LIMIT;
987 			scanned_many = true;
988 		}
989 		offset++;
990 	}
991 	spin_lock(&si->lock);
992 
993 no_page:
994 	si->flags -= SWP_SCANNING;
995 	return n_ret;
996 }
997 
998 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot)
999 {
1000 	unsigned long idx;
1001 	struct swap_cluster_info *ci;
1002 	unsigned long offset;
1003 
1004 	/*
1005 	 * Should not even be attempting cluster allocations when huge
1006 	 * page swap is disabled.  Warn and fail the allocation.
1007 	 */
1008 	if (!IS_ENABLED(CONFIG_THP_SWAP)) {
1009 		VM_WARN_ON_ONCE(1);
1010 		return 0;
1011 	}
1012 
1013 	if (cluster_list_empty(&si->free_clusters))
1014 		return 0;
1015 
1016 	idx = cluster_list_first(&si->free_clusters);
1017 	offset = idx * SWAPFILE_CLUSTER;
1018 	ci = lock_cluster(si, offset);
1019 	alloc_cluster(si, idx);
1020 	cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE);
1021 
1022 	memset(si->swap_map + offset, SWAP_HAS_CACHE, SWAPFILE_CLUSTER);
1023 	unlock_cluster(ci);
1024 	swap_range_alloc(si, offset, SWAPFILE_CLUSTER);
1025 	*slot = swp_entry(si->type, offset);
1026 
1027 	return 1;
1028 }
1029 
1030 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx)
1031 {
1032 	unsigned long offset = idx * SWAPFILE_CLUSTER;
1033 	struct swap_cluster_info *ci;
1034 
1035 	ci = lock_cluster(si, offset);
1036 	memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER);
1037 	cluster_set_count_flag(ci, 0, 0);
1038 	free_cluster(si, idx);
1039 	unlock_cluster(ci);
1040 	swap_range_free(si, offset, SWAPFILE_CLUSTER);
1041 }
1042 
1043 int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size)
1044 {
1045 	unsigned long size = swap_entry_size(entry_size);
1046 	struct swap_info_struct *si, *next;
1047 	long avail_pgs;
1048 	int n_ret = 0;
1049 	int node;
1050 
1051 	/* Only single cluster request supported */
1052 	WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER);
1053 
1054 	spin_lock(&swap_avail_lock);
1055 
1056 	avail_pgs = atomic_long_read(&nr_swap_pages) / size;
1057 	if (avail_pgs <= 0) {
1058 		spin_unlock(&swap_avail_lock);
1059 		goto noswap;
1060 	}
1061 
1062 	n_goal = min3((long)n_goal, (long)SWAP_BATCH, avail_pgs);
1063 
1064 	atomic_long_sub(n_goal * size, &nr_swap_pages);
1065 
1066 start_over:
1067 	node = numa_node_id();
1068 	plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) {
1069 		/* requeue si to after same-priority siblings */
1070 		plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]);
1071 		spin_unlock(&swap_avail_lock);
1072 		spin_lock(&si->lock);
1073 		if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
1074 			spin_lock(&swap_avail_lock);
1075 			if (plist_node_empty(&si->avail_lists[node])) {
1076 				spin_unlock(&si->lock);
1077 				goto nextsi;
1078 			}
1079 			WARN(!si->highest_bit,
1080 			     "swap_info %d in list but !highest_bit\n",
1081 			     si->type);
1082 			WARN(!(si->flags & SWP_WRITEOK),
1083 			     "swap_info %d in list but !SWP_WRITEOK\n",
1084 			     si->type);
1085 			__del_from_avail_list(si);
1086 			spin_unlock(&si->lock);
1087 			goto nextsi;
1088 		}
1089 		if (size == SWAPFILE_CLUSTER) {
1090 			if (si->flags & SWP_BLKDEV)
1091 				n_ret = swap_alloc_cluster(si, swp_entries);
1092 		} else
1093 			n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE,
1094 						    n_goal, swp_entries);
1095 		spin_unlock(&si->lock);
1096 		if (n_ret || size == SWAPFILE_CLUSTER)
1097 			goto check_out;
1098 		pr_debug("scan_swap_map of si %d failed to find offset\n",
1099 			si->type);
1100 
1101 		spin_lock(&swap_avail_lock);
1102 nextsi:
1103 		/*
1104 		 * if we got here, it's likely that si was almost full before,
1105 		 * and since scan_swap_map_slots() can drop the si->lock,
1106 		 * multiple callers probably all tried to get a page from the
1107 		 * same si and it filled up before we could get one; or, the si
1108 		 * filled up between us dropping swap_avail_lock and taking
1109 		 * si->lock. Since we dropped the swap_avail_lock, the
1110 		 * swap_avail_head list may have been modified; so if next is
1111 		 * still in the swap_avail_head list then try it, otherwise
1112 		 * start over if we have not gotten any slots.
1113 		 */
1114 		if (plist_node_empty(&next->avail_lists[node]))
1115 			goto start_over;
1116 	}
1117 
1118 	spin_unlock(&swap_avail_lock);
1119 
1120 check_out:
1121 	if (n_ret < n_goal)
1122 		atomic_long_add((long)(n_goal - n_ret) * size,
1123 				&nr_swap_pages);
1124 noswap:
1125 	return n_ret;
1126 }
1127 
1128 static struct swap_info_struct *_swap_info_get(swp_entry_t entry)
1129 {
1130 	struct swap_info_struct *p;
1131 	unsigned long offset;
1132 
1133 	if (!entry.val)
1134 		goto out;
1135 	p = swp_swap_info(entry);
1136 	if (!p)
1137 		goto bad_nofile;
1138 	if (data_race(!(p->flags & SWP_USED)))
1139 		goto bad_device;
1140 	offset = swp_offset(entry);
1141 	if (offset >= p->max)
1142 		goto bad_offset;
1143 	if (data_race(!p->swap_map[swp_offset(entry)]))
1144 		goto bad_free;
1145 	return p;
1146 
1147 bad_free:
1148 	pr_err("%s: %s%08lx\n", __func__, Unused_offset, entry.val);
1149 	goto out;
1150 bad_offset:
1151 	pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1152 	goto out;
1153 bad_device:
1154 	pr_err("%s: %s%08lx\n", __func__, Unused_file, entry.val);
1155 	goto out;
1156 bad_nofile:
1157 	pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1158 out:
1159 	return NULL;
1160 }
1161 
1162 static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry,
1163 					struct swap_info_struct *q)
1164 {
1165 	struct swap_info_struct *p;
1166 
1167 	p = _swap_info_get(entry);
1168 
1169 	if (p != q) {
1170 		if (q != NULL)
1171 			spin_unlock(&q->lock);
1172 		if (p != NULL)
1173 			spin_lock(&p->lock);
1174 	}
1175 	return p;
1176 }
1177 
1178 static unsigned char __swap_entry_free_locked(struct swap_info_struct *p,
1179 					      unsigned long offset,
1180 					      unsigned char usage)
1181 {
1182 	unsigned char count;
1183 	unsigned char has_cache;
1184 
1185 	count = p->swap_map[offset];
1186 
1187 	has_cache = count & SWAP_HAS_CACHE;
1188 	count &= ~SWAP_HAS_CACHE;
1189 
1190 	if (usage == SWAP_HAS_CACHE) {
1191 		VM_BUG_ON(!has_cache);
1192 		has_cache = 0;
1193 	} else if (count == SWAP_MAP_SHMEM) {
1194 		/*
1195 		 * Or we could insist on shmem.c using a special
1196 		 * swap_shmem_free() and free_shmem_swap_and_cache()...
1197 		 */
1198 		count = 0;
1199 	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
1200 		if (count == COUNT_CONTINUED) {
1201 			if (swap_count_continued(p, offset, count))
1202 				count = SWAP_MAP_MAX | COUNT_CONTINUED;
1203 			else
1204 				count = SWAP_MAP_MAX;
1205 		} else
1206 			count--;
1207 	}
1208 
1209 	usage = count | has_cache;
1210 	if (usage)
1211 		WRITE_ONCE(p->swap_map[offset], usage);
1212 	else
1213 		WRITE_ONCE(p->swap_map[offset], SWAP_HAS_CACHE);
1214 
1215 	return usage;
1216 }
1217 
1218 /*
1219  * Check whether swap entry is valid in the swap device.  If so,
1220  * return pointer to swap_info_struct, and keep the swap entry valid
1221  * via preventing the swap device from being swapoff, until
1222  * put_swap_device() is called.  Otherwise return NULL.
1223  *
1224  * Notice that swapoff or swapoff+swapon can still happen before the
1225  * percpu_ref_tryget_live() in get_swap_device() or after the
1226  * percpu_ref_put() in put_swap_device() if there isn't any other way
1227  * to prevent swapoff, such as page lock, page table lock, etc.  The
1228  * caller must be prepared for that.  For example, the following
1229  * situation is possible.
1230  *
1231  *   CPU1				CPU2
1232  *   do_swap_page()
1233  *     ...				swapoff+swapon
1234  *     __read_swap_cache_async()
1235  *       swapcache_prepare()
1236  *         __swap_duplicate()
1237  *           // check swap_map
1238  *     // verify PTE not changed
1239  *
1240  * In __swap_duplicate(), the swap_map need to be checked before
1241  * changing partly because the specified swap entry may be for another
1242  * swap device which has been swapoff.  And in do_swap_page(), after
1243  * the page is read from the swap device, the PTE is verified not
1244  * changed with the page table locked to check whether the swap device
1245  * has been swapoff or swapoff+swapon.
1246  */
1247 struct swap_info_struct *get_swap_device(swp_entry_t entry)
1248 {
1249 	struct swap_info_struct *si;
1250 	unsigned long offset;
1251 
1252 	if (!entry.val)
1253 		goto out;
1254 	si = swp_swap_info(entry);
1255 	if (!si)
1256 		goto bad_nofile;
1257 	if (!percpu_ref_tryget_live(&si->users))
1258 		goto out;
1259 	/*
1260 	 * Guarantee the si->users are checked before accessing other
1261 	 * fields of swap_info_struct.
1262 	 *
1263 	 * Paired with the spin_unlock() after setup_swap_info() in
1264 	 * enable_swap_info().
1265 	 */
1266 	smp_rmb();
1267 	offset = swp_offset(entry);
1268 	if (offset >= si->max)
1269 		goto put_out;
1270 
1271 	return si;
1272 bad_nofile:
1273 	pr_err("%s: %s%08lx\n", __func__, Bad_file, entry.val);
1274 out:
1275 	return NULL;
1276 put_out:
1277 	pr_err("%s: %s%08lx\n", __func__, Bad_offset, entry.val);
1278 	percpu_ref_put(&si->users);
1279 	return NULL;
1280 }
1281 
1282 static unsigned char __swap_entry_free(struct swap_info_struct *p,
1283 				       swp_entry_t entry)
1284 {
1285 	struct swap_cluster_info *ci;
1286 	unsigned long offset = swp_offset(entry);
1287 	unsigned char usage;
1288 
1289 	ci = lock_cluster_or_swap_info(p, offset);
1290 	usage = __swap_entry_free_locked(p, offset, 1);
1291 	unlock_cluster_or_swap_info(p, ci);
1292 	if (!usage)
1293 		free_swap_slot(entry);
1294 
1295 	return usage;
1296 }
1297 
1298 static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry)
1299 {
1300 	struct swap_cluster_info *ci;
1301 	unsigned long offset = swp_offset(entry);
1302 	unsigned char count;
1303 
1304 	ci = lock_cluster(p, offset);
1305 	count = p->swap_map[offset];
1306 	VM_BUG_ON(count != SWAP_HAS_CACHE);
1307 	p->swap_map[offset] = 0;
1308 	dec_cluster_info_page(p, p->cluster_info, offset);
1309 	unlock_cluster(ci);
1310 
1311 	mem_cgroup_uncharge_swap(entry, 1);
1312 	swap_range_free(p, offset, 1);
1313 }
1314 
1315 /*
1316  * Caller has made sure that the swap device corresponding to entry
1317  * is still around or has not been recycled.
1318  */
1319 void swap_free(swp_entry_t entry)
1320 {
1321 	struct swap_info_struct *p;
1322 
1323 	p = _swap_info_get(entry);
1324 	if (p)
1325 		__swap_entry_free(p, entry);
1326 }
1327 
1328 /*
1329  * Called after dropping swapcache to decrease refcnt to swap entries.
1330  */
1331 void put_swap_page(struct page *page, swp_entry_t entry)
1332 {
1333 	unsigned long offset = swp_offset(entry);
1334 	unsigned long idx = offset / SWAPFILE_CLUSTER;
1335 	struct swap_cluster_info *ci;
1336 	struct swap_info_struct *si;
1337 	unsigned char *map;
1338 	unsigned int i, free_entries = 0;
1339 	unsigned char val;
1340 	int size = swap_entry_size(thp_nr_pages(page));
1341 
1342 	si = _swap_info_get(entry);
1343 	if (!si)
1344 		return;
1345 
1346 	ci = lock_cluster_or_swap_info(si, offset);
1347 	if (size == SWAPFILE_CLUSTER) {
1348 		VM_BUG_ON(!cluster_is_huge(ci));
1349 		map = si->swap_map + offset;
1350 		for (i = 0; i < SWAPFILE_CLUSTER; i++) {
1351 			val = map[i];
1352 			VM_BUG_ON(!(val & SWAP_HAS_CACHE));
1353 			if (val == SWAP_HAS_CACHE)
1354 				free_entries++;
1355 		}
1356 		cluster_clear_huge(ci);
1357 		if (free_entries == SWAPFILE_CLUSTER) {
1358 			unlock_cluster_or_swap_info(si, ci);
1359 			spin_lock(&si->lock);
1360 			mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER);
1361 			swap_free_cluster(si, idx);
1362 			spin_unlock(&si->lock);
1363 			return;
1364 		}
1365 	}
1366 	for (i = 0; i < size; i++, entry.val++) {
1367 		if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) {
1368 			unlock_cluster_or_swap_info(si, ci);
1369 			free_swap_slot(entry);
1370 			if (i == size - 1)
1371 				return;
1372 			lock_cluster_or_swap_info(si, offset);
1373 		}
1374 	}
1375 	unlock_cluster_or_swap_info(si, ci);
1376 }
1377 
1378 #ifdef CONFIG_THP_SWAP
1379 int split_swap_cluster(swp_entry_t entry)
1380 {
1381 	struct swap_info_struct *si;
1382 	struct swap_cluster_info *ci;
1383 	unsigned long offset = swp_offset(entry);
1384 
1385 	si = _swap_info_get(entry);
1386 	if (!si)
1387 		return -EBUSY;
1388 	ci = lock_cluster(si, offset);
1389 	cluster_clear_huge(ci);
1390 	unlock_cluster(ci);
1391 	return 0;
1392 }
1393 #endif
1394 
1395 static int swp_entry_cmp(const void *ent1, const void *ent2)
1396 {
1397 	const swp_entry_t *e1 = ent1, *e2 = ent2;
1398 
1399 	return (int)swp_type(*e1) - (int)swp_type(*e2);
1400 }
1401 
1402 void swapcache_free_entries(swp_entry_t *entries, int n)
1403 {
1404 	struct swap_info_struct *p, *prev;
1405 	int i;
1406 
1407 	if (n <= 0)
1408 		return;
1409 
1410 	prev = NULL;
1411 	p = NULL;
1412 
1413 	/*
1414 	 * Sort swap entries by swap device, so each lock is only taken once.
1415 	 * nr_swapfiles isn't absolutely correct, but the overhead of sort() is
1416 	 * so low that it isn't necessary to optimize further.
1417 	 */
1418 	if (nr_swapfiles > 1)
1419 		sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL);
1420 	for (i = 0; i < n; ++i) {
1421 		p = swap_info_get_cont(entries[i], prev);
1422 		if (p)
1423 			swap_entry_free(p, entries[i]);
1424 		prev = p;
1425 	}
1426 	if (p)
1427 		spin_unlock(&p->lock);
1428 }
1429 
1430 /*
1431  * How many references to page are currently swapped out?
1432  * This does not give an exact answer when swap count is continued,
1433  * but does include the high COUNT_CONTINUED flag to allow for that.
1434  */
1435 static int page_swapcount(struct page *page)
1436 {
1437 	int count = 0;
1438 	struct swap_info_struct *p;
1439 	struct swap_cluster_info *ci;
1440 	swp_entry_t entry;
1441 	unsigned long offset;
1442 
1443 	entry.val = page_private(page);
1444 	p = _swap_info_get(entry);
1445 	if (p) {
1446 		offset = swp_offset(entry);
1447 		ci = lock_cluster_or_swap_info(p, offset);
1448 		count = swap_count(p->swap_map[offset]);
1449 		unlock_cluster_or_swap_info(p, ci);
1450 	}
1451 	return count;
1452 }
1453 
1454 int __swap_count(swp_entry_t entry)
1455 {
1456 	struct swap_info_struct *si;
1457 	pgoff_t offset = swp_offset(entry);
1458 	int count = 0;
1459 
1460 	si = get_swap_device(entry);
1461 	if (si) {
1462 		count = swap_count(si->swap_map[offset]);
1463 		put_swap_device(si);
1464 	}
1465 	return count;
1466 }
1467 
1468 static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
1469 {
1470 	int count = 0;
1471 	pgoff_t offset = swp_offset(entry);
1472 	struct swap_cluster_info *ci;
1473 
1474 	ci = lock_cluster_or_swap_info(si, offset);
1475 	count = swap_count(si->swap_map[offset]);
1476 	unlock_cluster_or_swap_info(si, ci);
1477 	return count;
1478 }
1479 
1480 /*
1481  * How many references to @entry are currently swapped out?
1482  * This does not give an exact answer when swap count is continued,
1483  * but does include the high COUNT_CONTINUED flag to allow for that.
1484  */
1485 int __swp_swapcount(swp_entry_t entry)
1486 {
1487 	int count = 0;
1488 	struct swap_info_struct *si;
1489 
1490 	si = get_swap_device(entry);
1491 	if (si) {
1492 		count = swap_swapcount(si, entry);
1493 		put_swap_device(si);
1494 	}
1495 	return count;
1496 }
1497 
1498 /*
1499  * How many references to @entry are currently swapped out?
1500  * This considers COUNT_CONTINUED so it returns exact answer.
1501  */
1502 int swp_swapcount(swp_entry_t entry)
1503 {
1504 	int count, tmp_count, n;
1505 	struct swap_info_struct *p;
1506 	struct swap_cluster_info *ci;
1507 	struct page *page;
1508 	pgoff_t offset;
1509 	unsigned char *map;
1510 
1511 	p = _swap_info_get(entry);
1512 	if (!p)
1513 		return 0;
1514 
1515 	offset = swp_offset(entry);
1516 
1517 	ci = lock_cluster_or_swap_info(p, offset);
1518 
1519 	count = swap_count(p->swap_map[offset]);
1520 	if (!(count & COUNT_CONTINUED))
1521 		goto out;
1522 
1523 	count &= ~COUNT_CONTINUED;
1524 	n = SWAP_MAP_MAX + 1;
1525 
1526 	page = vmalloc_to_page(p->swap_map + offset);
1527 	offset &= ~PAGE_MASK;
1528 	VM_BUG_ON(page_private(page) != SWP_CONTINUED);
1529 
1530 	do {
1531 		page = list_next_entry(page, lru);
1532 		map = kmap_atomic(page);
1533 		tmp_count = map[offset];
1534 		kunmap_atomic(map);
1535 
1536 		count += (tmp_count & ~COUNT_CONTINUED) * n;
1537 		n *= (SWAP_CONT_MAX + 1);
1538 	} while (tmp_count & COUNT_CONTINUED);
1539 out:
1540 	unlock_cluster_or_swap_info(p, ci);
1541 	return count;
1542 }
1543 
1544 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
1545 					 swp_entry_t entry)
1546 {
1547 	struct swap_cluster_info *ci;
1548 	unsigned char *map = si->swap_map;
1549 	unsigned long roffset = swp_offset(entry);
1550 	unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER);
1551 	int i;
1552 	bool ret = false;
1553 
1554 	ci = lock_cluster_or_swap_info(si, offset);
1555 	if (!ci || !cluster_is_huge(ci)) {
1556 		if (swap_count(map[roffset]))
1557 			ret = true;
1558 		goto unlock_out;
1559 	}
1560 	for (i = 0; i < SWAPFILE_CLUSTER; i++) {
1561 		if (swap_count(map[offset + i])) {
1562 			ret = true;
1563 			break;
1564 		}
1565 	}
1566 unlock_out:
1567 	unlock_cluster_or_swap_info(si, ci);
1568 	return ret;
1569 }
1570 
1571 static bool page_swapped(struct page *page)
1572 {
1573 	swp_entry_t entry;
1574 	struct swap_info_struct *si;
1575 
1576 	if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page)))
1577 		return page_swapcount(page) != 0;
1578 
1579 	page = compound_head(page);
1580 	entry.val = page_private(page);
1581 	si = _swap_info_get(entry);
1582 	if (si)
1583 		return swap_page_trans_huge_swapped(si, entry);
1584 	return false;
1585 }
1586 
1587 /*
1588  * If swap is getting full, or if there are no more mappings of this page,
1589  * then try_to_free_swap is called to free its swap space.
1590  */
1591 int try_to_free_swap(struct page *page)
1592 {
1593 	VM_BUG_ON_PAGE(!PageLocked(page), page);
1594 
1595 	if (!PageSwapCache(page))
1596 		return 0;
1597 	if (PageWriteback(page))
1598 		return 0;
1599 	if (page_swapped(page))
1600 		return 0;
1601 
1602 	/*
1603 	 * Once hibernation has begun to create its image of memory,
1604 	 * there's a danger that one of the calls to try_to_free_swap()
1605 	 * - most probably a call from __try_to_reclaim_swap() while
1606 	 * hibernation is allocating its own swap pages for the image,
1607 	 * but conceivably even a call from memory reclaim - will free
1608 	 * the swap from a page which has already been recorded in the
1609 	 * image as a clean swapcache page, and then reuse its swap for
1610 	 * another page of the image.  On waking from hibernation, the
1611 	 * original page might be freed under memory pressure, then
1612 	 * later read back in from swap, now with the wrong data.
1613 	 *
1614 	 * Hibernation suspends storage while it is writing the image
1615 	 * to disk so check that here.
1616 	 */
1617 	if (pm_suspended_storage())
1618 		return 0;
1619 
1620 	page = compound_head(page);
1621 	delete_from_swap_cache(page);
1622 	SetPageDirty(page);
1623 	return 1;
1624 }
1625 
1626 /*
1627  * Free the swap entry like above, but also try to
1628  * free the page cache entry if it is the last user.
1629  */
1630 int free_swap_and_cache(swp_entry_t entry)
1631 {
1632 	struct swap_info_struct *p;
1633 	unsigned char count;
1634 
1635 	if (non_swap_entry(entry))
1636 		return 1;
1637 
1638 	p = _swap_info_get(entry);
1639 	if (p) {
1640 		count = __swap_entry_free(p, entry);
1641 		if (count == SWAP_HAS_CACHE &&
1642 		    !swap_page_trans_huge_swapped(p, entry))
1643 			__try_to_reclaim_swap(p, swp_offset(entry),
1644 					      TTRS_UNMAPPED | TTRS_FULL);
1645 	}
1646 	return p != NULL;
1647 }
1648 
1649 #ifdef CONFIG_HIBERNATION
1650 
1651 swp_entry_t get_swap_page_of_type(int type)
1652 {
1653 	struct swap_info_struct *si = swap_type_to_swap_info(type);
1654 	swp_entry_t entry = {0};
1655 
1656 	if (!si)
1657 		goto fail;
1658 
1659 	/* This is called for allocating swap entry, not cache */
1660 	spin_lock(&si->lock);
1661 	if ((si->flags & SWP_WRITEOK) && scan_swap_map_slots(si, 1, 1, &entry))
1662 		atomic_long_dec(&nr_swap_pages);
1663 	spin_unlock(&si->lock);
1664 fail:
1665 	return entry;
1666 }
1667 
1668 /*
1669  * Find the swap type that corresponds to given device (if any).
1670  *
1671  * @offset - number of the PAGE_SIZE-sized block of the device, starting
1672  * from 0, in which the swap header is expected to be located.
1673  *
1674  * This is needed for the suspend to disk (aka swsusp).
1675  */
1676 int swap_type_of(dev_t device, sector_t offset)
1677 {
1678 	int type;
1679 
1680 	if (!device)
1681 		return -1;
1682 
1683 	spin_lock(&swap_lock);
1684 	for (type = 0; type < nr_swapfiles; type++) {
1685 		struct swap_info_struct *sis = swap_info[type];
1686 
1687 		if (!(sis->flags & SWP_WRITEOK))
1688 			continue;
1689 
1690 		if (device == sis->bdev->bd_dev) {
1691 			struct swap_extent *se = first_se(sis);
1692 
1693 			if (se->start_block == offset) {
1694 				spin_unlock(&swap_lock);
1695 				return type;
1696 			}
1697 		}
1698 	}
1699 	spin_unlock(&swap_lock);
1700 	return -ENODEV;
1701 }
1702 
1703 int find_first_swap(dev_t *device)
1704 {
1705 	int type;
1706 
1707 	spin_lock(&swap_lock);
1708 	for (type = 0; type < nr_swapfiles; type++) {
1709 		struct swap_info_struct *sis = swap_info[type];
1710 
1711 		if (!(sis->flags & SWP_WRITEOK))
1712 			continue;
1713 		*device = sis->bdev->bd_dev;
1714 		spin_unlock(&swap_lock);
1715 		return type;
1716 	}
1717 	spin_unlock(&swap_lock);
1718 	return -ENODEV;
1719 }
1720 
1721 /*
1722  * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1723  * corresponding to given index in swap_info (swap type).
1724  */
1725 sector_t swapdev_block(int type, pgoff_t offset)
1726 {
1727 	struct swap_info_struct *si = swap_type_to_swap_info(type);
1728 	struct swap_extent *se;
1729 
1730 	if (!si || !(si->flags & SWP_WRITEOK))
1731 		return 0;
1732 	se = offset_to_swap_extent(si, offset);
1733 	return se->start_block + (offset - se->start_page);
1734 }
1735 
1736 /*
1737  * Return either the total number of swap pages of given type, or the number
1738  * of free pages of that type (depending on @free)
1739  *
1740  * This is needed for software suspend
1741  */
1742 unsigned int count_swap_pages(int type, int free)
1743 {
1744 	unsigned int n = 0;
1745 
1746 	spin_lock(&swap_lock);
1747 	if ((unsigned int)type < nr_swapfiles) {
1748 		struct swap_info_struct *sis = swap_info[type];
1749 
1750 		spin_lock(&sis->lock);
1751 		if (sis->flags & SWP_WRITEOK) {
1752 			n = sis->pages;
1753 			if (free)
1754 				n -= sis->inuse_pages;
1755 		}
1756 		spin_unlock(&sis->lock);
1757 	}
1758 	spin_unlock(&swap_lock);
1759 	return n;
1760 }
1761 #endif /* CONFIG_HIBERNATION */
1762 
1763 static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
1764 {
1765 	return pte_same(pte_swp_clear_flags(pte), swp_pte);
1766 }
1767 
1768 /*
1769  * No need to decide whether this PTE shares the swap entry with others,
1770  * just let do_wp_page work it out if a write is requested later - to
1771  * force COW, vm_page_prot omits write permission from any private vma.
1772  */
1773 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1774 		unsigned long addr, swp_entry_t entry, struct page *page)
1775 {
1776 	struct page *swapcache;
1777 	spinlock_t *ptl;
1778 	pte_t *pte;
1779 	int ret = 1;
1780 
1781 	swapcache = page;
1782 	page = ksm_might_need_to_copy(page, vma, addr);
1783 	if (unlikely(!page))
1784 		return -ENOMEM;
1785 
1786 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1787 	if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) {
1788 		ret = 0;
1789 		goto out;
1790 	}
1791 
1792 	/* See do_swap_page() */
1793 	BUG_ON(!PageAnon(page) && PageMappedToDisk(page));
1794 	BUG_ON(PageAnon(page) && PageAnonExclusive(page));
1795 
1796 	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1797 	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1798 	get_page(page);
1799 	if (page == swapcache) {
1800 		rmap_t rmap_flags = RMAP_NONE;
1801 
1802 		/*
1803 		 * See do_swap_page(): PageWriteback() would be problematic.
1804 		 * However, we do a wait_on_page_writeback() just before this
1805 		 * call and have the page locked.
1806 		 */
1807 		VM_BUG_ON_PAGE(PageWriteback(page), page);
1808 		if (pte_swp_exclusive(*pte))
1809 			rmap_flags |= RMAP_EXCLUSIVE;
1810 
1811 		page_add_anon_rmap(page, vma, addr, rmap_flags);
1812 	} else { /* ksm created a completely new copy */
1813 		page_add_new_anon_rmap(page, vma, addr);
1814 		lru_cache_add_inactive_or_unevictable(page, vma);
1815 	}
1816 	set_pte_at(vma->vm_mm, addr, pte,
1817 		   pte_mkold(mk_pte(page, vma->vm_page_prot)));
1818 	swap_free(entry);
1819 out:
1820 	pte_unmap_unlock(pte, ptl);
1821 	if (page != swapcache) {
1822 		unlock_page(page);
1823 		put_page(page);
1824 	}
1825 	return ret;
1826 }
1827 
1828 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1829 			unsigned long addr, unsigned long end,
1830 			unsigned int type)
1831 {
1832 	struct page *page;
1833 	swp_entry_t entry;
1834 	pte_t *pte;
1835 	struct swap_info_struct *si;
1836 	unsigned long offset;
1837 	int ret = 0;
1838 	volatile unsigned char *swap_map;
1839 
1840 	si = swap_info[type];
1841 	pte = pte_offset_map(pmd, addr);
1842 	do {
1843 		if (!is_swap_pte(*pte))
1844 			continue;
1845 
1846 		entry = pte_to_swp_entry(*pte);
1847 		if (swp_type(entry) != type)
1848 			continue;
1849 
1850 		offset = swp_offset(entry);
1851 		pte_unmap(pte);
1852 		swap_map = &si->swap_map[offset];
1853 		page = lookup_swap_cache(entry, vma, addr);
1854 		if (!page) {
1855 			struct vm_fault vmf = {
1856 				.vma = vma,
1857 				.address = addr,
1858 				.real_address = addr,
1859 				.pmd = pmd,
1860 			};
1861 
1862 			page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE,
1863 						&vmf);
1864 		}
1865 		if (!page) {
1866 			if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD)
1867 				goto try_next;
1868 			return -ENOMEM;
1869 		}
1870 
1871 		lock_page(page);
1872 		wait_on_page_writeback(page);
1873 		ret = unuse_pte(vma, pmd, addr, entry, page);
1874 		if (ret < 0) {
1875 			unlock_page(page);
1876 			put_page(page);
1877 			goto out;
1878 		}
1879 
1880 		try_to_free_swap(page);
1881 		unlock_page(page);
1882 		put_page(page);
1883 try_next:
1884 		pte = pte_offset_map(pmd, addr);
1885 	} while (pte++, addr += PAGE_SIZE, addr != end);
1886 	pte_unmap(pte - 1);
1887 
1888 	ret = 0;
1889 out:
1890 	return ret;
1891 }
1892 
1893 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1894 				unsigned long addr, unsigned long end,
1895 				unsigned int type)
1896 {
1897 	pmd_t *pmd;
1898 	unsigned long next;
1899 	int ret;
1900 
1901 	pmd = pmd_offset(pud, addr);
1902 	do {
1903 		cond_resched();
1904 		next = pmd_addr_end(addr, end);
1905 		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1906 			continue;
1907 		ret = unuse_pte_range(vma, pmd, addr, next, type);
1908 		if (ret)
1909 			return ret;
1910 	} while (pmd++, addr = next, addr != end);
1911 	return 0;
1912 }
1913 
1914 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
1915 				unsigned long addr, unsigned long end,
1916 				unsigned int type)
1917 {
1918 	pud_t *pud;
1919 	unsigned long next;
1920 	int ret;
1921 
1922 	pud = pud_offset(p4d, addr);
1923 	do {
1924 		next = pud_addr_end(addr, end);
1925 		if (pud_none_or_clear_bad(pud))
1926 			continue;
1927 		ret = unuse_pmd_range(vma, pud, addr, next, type);
1928 		if (ret)
1929 			return ret;
1930 	} while (pud++, addr = next, addr != end);
1931 	return 0;
1932 }
1933 
1934 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
1935 				unsigned long addr, unsigned long end,
1936 				unsigned int type)
1937 {
1938 	p4d_t *p4d;
1939 	unsigned long next;
1940 	int ret;
1941 
1942 	p4d = p4d_offset(pgd, addr);
1943 	do {
1944 		next = p4d_addr_end(addr, end);
1945 		if (p4d_none_or_clear_bad(p4d))
1946 			continue;
1947 		ret = unuse_pud_range(vma, p4d, addr, next, type);
1948 		if (ret)
1949 			return ret;
1950 	} while (p4d++, addr = next, addr != end);
1951 	return 0;
1952 }
1953 
1954 static int unuse_vma(struct vm_area_struct *vma, unsigned int type)
1955 {
1956 	pgd_t *pgd;
1957 	unsigned long addr, end, next;
1958 	int ret;
1959 
1960 	addr = vma->vm_start;
1961 	end = vma->vm_end;
1962 
1963 	pgd = pgd_offset(vma->vm_mm, addr);
1964 	do {
1965 		next = pgd_addr_end(addr, end);
1966 		if (pgd_none_or_clear_bad(pgd))
1967 			continue;
1968 		ret = unuse_p4d_range(vma, pgd, addr, next, type);
1969 		if (ret)
1970 			return ret;
1971 	} while (pgd++, addr = next, addr != end);
1972 	return 0;
1973 }
1974 
1975 static int unuse_mm(struct mm_struct *mm, unsigned int type)
1976 {
1977 	struct vm_area_struct *vma;
1978 	int ret = 0;
1979 
1980 	mmap_read_lock(mm);
1981 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1982 		if (vma->anon_vma) {
1983 			ret = unuse_vma(vma, type);
1984 			if (ret)
1985 				break;
1986 		}
1987 		cond_resched();
1988 	}
1989 	mmap_read_unlock(mm);
1990 	return ret;
1991 }
1992 
1993 /*
1994  * Scan swap_map from current position to next entry still in use.
1995  * Return 0 if there are no inuse entries after prev till end of
1996  * the map.
1997  */
1998 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1999 					unsigned int prev)
2000 {
2001 	unsigned int i;
2002 	unsigned char count;
2003 
2004 	/*
2005 	 * No need for swap_lock here: we're just looking
2006 	 * for whether an entry is in use, not modifying it; false
2007 	 * hits are okay, and sys_swapoff() has already prevented new
2008 	 * allocations from this area (while holding swap_lock).
2009 	 */
2010 	for (i = prev + 1; i < si->max; i++) {
2011 		count = READ_ONCE(si->swap_map[i]);
2012 		if (count && swap_count(count) != SWAP_MAP_BAD)
2013 			break;
2014 		if ((i % LATENCY_LIMIT) == 0)
2015 			cond_resched();
2016 	}
2017 
2018 	if (i == si->max)
2019 		i = 0;
2020 
2021 	return i;
2022 }
2023 
2024 static int try_to_unuse(unsigned int type)
2025 {
2026 	struct mm_struct *prev_mm;
2027 	struct mm_struct *mm;
2028 	struct list_head *p;
2029 	int retval = 0;
2030 	struct swap_info_struct *si = swap_info[type];
2031 	struct page *page;
2032 	swp_entry_t entry;
2033 	unsigned int i;
2034 
2035 	if (!READ_ONCE(si->inuse_pages))
2036 		return 0;
2037 
2038 retry:
2039 	retval = shmem_unuse(type);
2040 	if (retval)
2041 		return retval;
2042 
2043 	prev_mm = &init_mm;
2044 	mmget(prev_mm);
2045 
2046 	spin_lock(&mmlist_lock);
2047 	p = &init_mm.mmlist;
2048 	while (READ_ONCE(si->inuse_pages) &&
2049 	       !signal_pending(current) &&
2050 	       (p = p->next) != &init_mm.mmlist) {
2051 
2052 		mm = list_entry(p, struct mm_struct, mmlist);
2053 		if (!mmget_not_zero(mm))
2054 			continue;
2055 		spin_unlock(&mmlist_lock);
2056 		mmput(prev_mm);
2057 		prev_mm = mm;
2058 		retval = unuse_mm(mm, type);
2059 		if (retval) {
2060 			mmput(prev_mm);
2061 			return retval;
2062 		}
2063 
2064 		/*
2065 		 * Make sure that we aren't completely killing
2066 		 * interactive performance.
2067 		 */
2068 		cond_resched();
2069 		spin_lock(&mmlist_lock);
2070 	}
2071 	spin_unlock(&mmlist_lock);
2072 
2073 	mmput(prev_mm);
2074 
2075 	i = 0;
2076 	while (READ_ONCE(si->inuse_pages) &&
2077 	       !signal_pending(current) &&
2078 	       (i = find_next_to_unuse(si, i)) != 0) {
2079 
2080 		entry = swp_entry(type, i);
2081 		page = find_get_page(swap_address_space(entry), i);
2082 		if (!page)
2083 			continue;
2084 
2085 		/*
2086 		 * It is conceivable that a racing task removed this page from
2087 		 * swap cache just before we acquired the page lock. The page
2088 		 * might even be back in swap cache on another swap area. But
2089 		 * that is okay, try_to_free_swap() only removes stale pages.
2090 		 */
2091 		lock_page(page);
2092 		wait_on_page_writeback(page);
2093 		try_to_free_swap(page);
2094 		unlock_page(page);
2095 		put_page(page);
2096 	}
2097 
2098 	/*
2099 	 * Lets check again to see if there are still swap entries in the map.
2100 	 * If yes, we would need to do retry the unuse logic again.
2101 	 * Under global memory pressure, swap entries can be reinserted back
2102 	 * into process space after the mmlist loop above passes over them.
2103 	 *
2104 	 * Limit the number of retries? No: when mmget_not_zero()
2105 	 * above fails, that mm is likely to be freeing swap from
2106 	 * exit_mmap(), which proceeds at its own independent pace;
2107 	 * and even shmem_writepage() could have been preempted after
2108 	 * folio_alloc_swap(), temporarily hiding that swap.  It's easy
2109 	 * and robust (though cpu-intensive) just to keep retrying.
2110 	 */
2111 	if (READ_ONCE(si->inuse_pages)) {
2112 		if (!signal_pending(current))
2113 			goto retry;
2114 		return -EINTR;
2115 	}
2116 
2117 	return 0;
2118 }
2119 
2120 /*
2121  * After a successful try_to_unuse, if no swap is now in use, we know
2122  * we can empty the mmlist.  swap_lock must be held on entry and exit.
2123  * Note that mmlist_lock nests inside swap_lock, and an mm must be
2124  * added to the mmlist just after page_duplicate - before would be racy.
2125  */
2126 static void drain_mmlist(void)
2127 {
2128 	struct list_head *p, *next;
2129 	unsigned int type;
2130 
2131 	for (type = 0; type < nr_swapfiles; type++)
2132 		if (swap_info[type]->inuse_pages)
2133 			return;
2134 	spin_lock(&mmlist_lock);
2135 	list_for_each_safe(p, next, &init_mm.mmlist)
2136 		list_del_init(p);
2137 	spin_unlock(&mmlist_lock);
2138 }
2139 
2140 /*
2141  * Free all of a swapdev's extent information
2142  */
2143 static void destroy_swap_extents(struct swap_info_struct *sis)
2144 {
2145 	while (!RB_EMPTY_ROOT(&sis->swap_extent_root)) {
2146 		struct rb_node *rb = sis->swap_extent_root.rb_node;
2147 		struct swap_extent *se = rb_entry(rb, struct swap_extent, rb_node);
2148 
2149 		rb_erase(rb, &sis->swap_extent_root);
2150 		kfree(se);
2151 	}
2152 
2153 	if (sis->flags & SWP_ACTIVATED) {
2154 		struct file *swap_file = sis->swap_file;
2155 		struct address_space *mapping = swap_file->f_mapping;
2156 
2157 		sis->flags &= ~SWP_ACTIVATED;
2158 		if (mapping->a_ops->swap_deactivate)
2159 			mapping->a_ops->swap_deactivate(swap_file);
2160 	}
2161 }
2162 
2163 /*
2164  * Add a block range (and the corresponding page range) into this swapdev's
2165  * extent tree.
2166  *
2167  * This function rather assumes that it is called in ascending page order.
2168  */
2169 int
2170 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
2171 		unsigned long nr_pages, sector_t start_block)
2172 {
2173 	struct rb_node **link = &sis->swap_extent_root.rb_node, *parent = NULL;
2174 	struct swap_extent *se;
2175 	struct swap_extent *new_se;
2176 
2177 	/*
2178 	 * place the new node at the right most since the
2179 	 * function is called in ascending page order.
2180 	 */
2181 	while (*link) {
2182 		parent = *link;
2183 		link = &parent->rb_right;
2184 	}
2185 
2186 	if (parent) {
2187 		se = rb_entry(parent, struct swap_extent, rb_node);
2188 		BUG_ON(se->start_page + se->nr_pages != start_page);
2189 		if (se->start_block + se->nr_pages == start_block) {
2190 			/* Merge it */
2191 			se->nr_pages += nr_pages;
2192 			return 0;
2193 		}
2194 	}
2195 
2196 	/* No merge, insert a new extent. */
2197 	new_se = kmalloc(sizeof(*se), GFP_KERNEL);
2198 	if (new_se == NULL)
2199 		return -ENOMEM;
2200 	new_se->start_page = start_page;
2201 	new_se->nr_pages = nr_pages;
2202 	new_se->start_block = start_block;
2203 
2204 	rb_link_node(&new_se->rb_node, parent, link);
2205 	rb_insert_color(&new_se->rb_node, &sis->swap_extent_root);
2206 	return 1;
2207 }
2208 EXPORT_SYMBOL_GPL(add_swap_extent);
2209 
2210 /*
2211  * A `swap extent' is a simple thing which maps a contiguous range of pages
2212  * onto a contiguous range of disk blocks.  A rbtree of swap extents is
2213  * built at swapon time and is then used at swap_writepage/swap_readpage
2214  * time for locating where on disk a page belongs.
2215  *
2216  * If the swapfile is an S_ISBLK block device, a single extent is installed.
2217  * This is done so that the main operating code can treat S_ISBLK and S_ISREG
2218  * swap files identically.
2219  *
2220  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2221  * extent rbtree operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
2222  * swapfiles are handled *identically* after swapon time.
2223  *
2224  * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
2225  * and will parse them into a rbtree, in PAGE_SIZE chunks.  If some stray
2226  * blocks are found which do not fall within the PAGE_SIZE alignment
2227  * requirements, they are simply tossed out - we will never use those blocks
2228  * for swapping.
2229  *
2230  * For all swap devices we set S_SWAPFILE across the life of the swapon.  This
2231  * prevents users from writing to the swap device, which will corrupt memory.
2232  *
2233  * The amount of disk space which a single swap extent represents varies.
2234  * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
2235  * extents in the rbtree. - akpm.
2236  */
2237 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
2238 {
2239 	struct file *swap_file = sis->swap_file;
2240 	struct address_space *mapping = swap_file->f_mapping;
2241 	struct inode *inode = mapping->host;
2242 	int ret;
2243 
2244 	if (S_ISBLK(inode->i_mode)) {
2245 		ret = add_swap_extent(sis, 0, sis->max, 0);
2246 		*span = sis->pages;
2247 		return ret;
2248 	}
2249 
2250 	if (mapping->a_ops->swap_activate) {
2251 		ret = mapping->a_ops->swap_activate(sis, swap_file, span);
2252 		if (ret < 0)
2253 			return ret;
2254 		sis->flags |= SWP_ACTIVATED;
2255 		if ((sis->flags & SWP_FS_OPS) &&
2256 		    sio_pool_init() != 0) {
2257 			destroy_swap_extents(sis);
2258 			return -ENOMEM;
2259 		}
2260 		return ret;
2261 	}
2262 
2263 	return generic_swapfile_activate(sis, swap_file, span);
2264 }
2265 
2266 static int swap_node(struct swap_info_struct *p)
2267 {
2268 	struct block_device *bdev;
2269 
2270 	if (p->bdev)
2271 		bdev = p->bdev;
2272 	else
2273 		bdev = p->swap_file->f_inode->i_sb->s_bdev;
2274 
2275 	return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE;
2276 }
2277 
2278 static void setup_swap_info(struct swap_info_struct *p, int prio,
2279 			    unsigned char *swap_map,
2280 			    struct swap_cluster_info *cluster_info)
2281 {
2282 	int i;
2283 
2284 	if (prio >= 0)
2285 		p->prio = prio;
2286 	else
2287 		p->prio = --least_priority;
2288 	/*
2289 	 * the plist prio is negated because plist ordering is
2290 	 * low-to-high, while swap ordering is high-to-low
2291 	 */
2292 	p->list.prio = -p->prio;
2293 	for_each_node(i) {
2294 		if (p->prio >= 0)
2295 			p->avail_lists[i].prio = -p->prio;
2296 		else {
2297 			if (swap_node(p) == i)
2298 				p->avail_lists[i].prio = 1;
2299 			else
2300 				p->avail_lists[i].prio = -p->prio;
2301 		}
2302 	}
2303 	p->swap_map = swap_map;
2304 	p->cluster_info = cluster_info;
2305 }
2306 
2307 static void _enable_swap_info(struct swap_info_struct *p)
2308 {
2309 	p->flags |= SWP_WRITEOK;
2310 	atomic_long_add(p->pages, &nr_swap_pages);
2311 	total_swap_pages += p->pages;
2312 
2313 	assert_spin_locked(&swap_lock);
2314 	/*
2315 	 * both lists are plists, and thus priority ordered.
2316 	 * swap_active_head needs to be priority ordered for swapoff(),
2317 	 * which on removal of any swap_info_struct with an auto-assigned
2318 	 * (i.e. negative) priority increments the auto-assigned priority
2319 	 * of any lower-priority swap_info_structs.
2320 	 * swap_avail_head needs to be priority ordered for folio_alloc_swap(),
2321 	 * which allocates swap pages from the highest available priority
2322 	 * swap_info_struct.
2323 	 */
2324 	plist_add(&p->list, &swap_active_head);
2325 	add_to_avail_list(p);
2326 }
2327 
2328 static void enable_swap_info(struct swap_info_struct *p, int prio,
2329 				unsigned char *swap_map,
2330 				struct swap_cluster_info *cluster_info,
2331 				unsigned long *frontswap_map)
2332 {
2333 	if (IS_ENABLED(CONFIG_FRONTSWAP))
2334 		frontswap_init(p->type, frontswap_map);
2335 	spin_lock(&swap_lock);
2336 	spin_lock(&p->lock);
2337 	setup_swap_info(p, prio, swap_map, cluster_info);
2338 	spin_unlock(&p->lock);
2339 	spin_unlock(&swap_lock);
2340 	/*
2341 	 * Finished initializing swap device, now it's safe to reference it.
2342 	 */
2343 	percpu_ref_resurrect(&p->users);
2344 	spin_lock(&swap_lock);
2345 	spin_lock(&p->lock);
2346 	_enable_swap_info(p);
2347 	spin_unlock(&p->lock);
2348 	spin_unlock(&swap_lock);
2349 }
2350 
2351 static void reinsert_swap_info(struct swap_info_struct *p)
2352 {
2353 	spin_lock(&swap_lock);
2354 	spin_lock(&p->lock);
2355 	setup_swap_info(p, p->prio, p->swap_map, p->cluster_info);
2356 	_enable_swap_info(p);
2357 	spin_unlock(&p->lock);
2358 	spin_unlock(&swap_lock);
2359 }
2360 
2361 bool has_usable_swap(void)
2362 {
2363 	bool ret = true;
2364 
2365 	spin_lock(&swap_lock);
2366 	if (plist_head_empty(&swap_active_head))
2367 		ret = false;
2368 	spin_unlock(&swap_lock);
2369 	return ret;
2370 }
2371 
2372 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
2373 {
2374 	struct swap_info_struct *p = NULL;
2375 	unsigned char *swap_map;
2376 	struct swap_cluster_info *cluster_info;
2377 	unsigned long *frontswap_map;
2378 	struct file *swap_file, *victim;
2379 	struct address_space *mapping;
2380 	struct inode *inode;
2381 	struct filename *pathname;
2382 	int err, found = 0;
2383 	unsigned int old_block_size;
2384 
2385 	if (!capable(CAP_SYS_ADMIN))
2386 		return -EPERM;
2387 
2388 	BUG_ON(!current->mm);
2389 
2390 	pathname = getname(specialfile);
2391 	if (IS_ERR(pathname))
2392 		return PTR_ERR(pathname);
2393 
2394 	victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
2395 	err = PTR_ERR(victim);
2396 	if (IS_ERR(victim))
2397 		goto out;
2398 
2399 	mapping = victim->f_mapping;
2400 	spin_lock(&swap_lock);
2401 	plist_for_each_entry(p, &swap_active_head, list) {
2402 		if (p->flags & SWP_WRITEOK) {
2403 			if (p->swap_file->f_mapping == mapping) {
2404 				found = 1;
2405 				break;
2406 			}
2407 		}
2408 	}
2409 	if (!found) {
2410 		err = -EINVAL;
2411 		spin_unlock(&swap_lock);
2412 		goto out_dput;
2413 	}
2414 	if (!security_vm_enough_memory_mm(current->mm, p->pages))
2415 		vm_unacct_memory(p->pages);
2416 	else {
2417 		err = -ENOMEM;
2418 		spin_unlock(&swap_lock);
2419 		goto out_dput;
2420 	}
2421 	del_from_avail_list(p);
2422 	spin_lock(&p->lock);
2423 	if (p->prio < 0) {
2424 		struct swap_info_struct *si = p;
2425 		int nid;
2426 
2427 		plist_for_each_entry_continue(si, &swap_active_head, list) {
2428 			si->prio++;
2429 			si->list.prio--;
2430 			for_each_node(nid) {
2431 				if (si->avail_lists[nid].prio != 1)
2432 					si->avail_lists[nid].prio--;
2433 			}
2434 		}
2435 		least_priority++;
2436 	}
2437 	plist_del(&p->list, &swap_active_head);
2438 	atomic_long_sub(p->pages, &nr_swap_pages);
2439 	total_swap_pages -= p->pages;
2440 	p->flags &= ~SWP_WRITEOK;
2441 	spin_unlock(&p->lock);
2442 	spin_unlock(&swap_lock);
2443 
2444 	disable_swap_slots_cache_lock();
2445 
2446 	set_current_oom_origin();
2447 	err = try_to_unuse(p->type);
2448 	clear_current_oom_origin();
2449 
2450 	if (err) {
2451 		/* re-insert swap space back into swap_list */
2452 		reinsert_swap_info(p);
2453 		reenable_swap_slots_cache_unlock();
2454 		goto out_dput;
2455 	}
2456 
2457 	reenable_swap_slots_cache_unlock();
2458 
2459 	/*
2460 	 * Wait for swap operations protected by get/put_swap_device()
2461 	 * to complete.
2462 	 *
2463 	 * We need synchronize_rcu() here to protect the accessing to
2464 	 * the swap cache data structure.
2465 	 */
2466 	percpu_ref_kill(&p->users);
2467 	synchronize_rcu();
2468 	wait_for_completion(&p->comp);
2469 
2470 	flush_work(&p->discard_work);
2471 
2472 	destroy_swap_extents(p);
2473 	if (p->flags & SWP_CONTINUED)
2474 		free_swap_count_continuations(p);
2475 
2476 	if (!p->bdev || !bdev_nonrot(p->bdev))
2477 		atomic_dec(&nr_rotate_swap);
2478 
2479 	mutex_lock(&swapon_mutex);
2480 	spin_lock(&swap_lock);
2481 	spin_lock(&p->lock);
2482 	drain_mmlist();
2483 
2484 	/* wait for anyone still in scan_swap_map_slots */
2485 	p->highest_bit = 0;		/* cuts scans short */
2486 	while (p->flags >= SWP_SCANNING) {
2487 		spin_unlock(&p->lock);
2488 		spin_unlock(&swap_lock);
2489 		schedule_timeout_uninterruptible(1);
2490 		spin_lock(&swap_lock);
2491 		spin_lock(&p->lock);
2492 	}
2493 
2494 	swap_file = p->swap_file;
2495 	old_block_size = p->old_block_size;
2496 	p->swap_file = NULL;
2497 	p->max = 0;
2498 	swap_map = p->swap_map;
2499 	p->swap_map = NULL;
2500 	cluster_info = p->cluster_info;
2501 	p->cluster_info = NULL;
2502 	frontswap_map = frontswap_map_get(p);
2503 	spin_unlock(&p->lock);
2504 	spin_unlock(&swap_lock);
2505 	arch_swap_invalidate_area(p->type);
2506 	frontswap_invalidate_area(p->type);
2507 	frontswap_map_set(p, NULL);
2508 	mutex_unlock(&swapon_mutex);
2509 	free_percpu(p->percpu_cluster);
2510 	p->percpu_cluster = NULL;
2511 	free_percpu(p->cluster_next_cpu);
2512 	p->cluster_next_cpu = NULL;
2513 	vfree(swap_map);
2514 	kvfree(cluster_info);
2515 	kvfree(frontswap_map);
2516 	/* Destroy swap account information */
2517 	swap_cgroup_swapoff(p->type);
2518 	exit_swap_address_space(p->type);
2519 
2520 	inode = mapping->host;
2521 	if (S_ISBLK(inode->i_mode)) {
2522 		struct block_device *bdev = I_BDEV(inode);
2523 
2524 		set_blocksize(bdev, old_block_size);
2525 		blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2526 	}
2527 
2528 	inode_lock(inode);
2529 	inode->i_flags &= ~S_SWAPFILE;
2530 	inode_unlock(inode);
2531 	filp_close(swap_file, NULL);
2532 
2533 	/*
2534 	 * Clear the SWP_USED flag after all resources are freed so that swapon
2535 	 * can reuse this swap_info in alloc_swap_info() safely.  It is ok to
2536 	 * not hold p->lock after we cleared its SWP_WRITEOK.
2537 	 */
2538 	spin_lock(&swap_lock);
2539 	p->flags = 0;
2540 	spin_unlock(&swap_lock);
2541 
2542 	err = 0;
2543 	atomic_inc(&proc_poll_event);
2544 	wake_up_interruptible(&proc_poll_wait);
2545 
2546 out_dput:
2547 	filp_close(victim, NULL);
2548 out:
2549 	putname(pathname);
2550 	return err;
2551 }
2552 
2553 #ifdef CONFIG_PROC_FS
2554 static __poll_t swaps_poll(struct file *file, poll_table *wait)
2555 {
2556 	struct seq_file *seq = file->private_data;
2557 
2558 	poll_wait(file, &proc_poll_wait, wait);
2559 
2560 	if (seq->poll_event != atomic_read(&proc_poll_event)) {
2561 		seq->poll_event = atomic_read(&proc_poll_event);
2562 		return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI;
2563 	}
2564 
2565 	return EPOLLIN | EPOLLRDNORM;
2566 }
2567 
2568 /* iterator */
2569 static void *swap_start(struct seq_file *swap, loff_t *pos)
2570 {
2571 	struct swap_info_struct *si;
2572 	int type;
2573 	loff_t l = *pos;
2574 
2575 	mutex_lock(&swapon_mutex);
2576 
2577 	if (!l)
2578 		return SEQ_START_TOKEN;
2579 
2580 	for (type = 0; (si = swap_type_to_swap_info(type)); type++) {
2581 		if (!(si->flags & SWP_USED) || !si->swap_map)
2582 			continue;
2583 		if (!--l)
2584 			return si;
2585 	}
2586 
2587 	return NULL;
2588 }
2589 
2590 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
2591 {
2592 	struct swap_info_struct *si = v;
2593 	int type;
2594 
2595 	if (v == SEQ_START_TOKEN)
2596 		type = 0;
2597 	else
2598 		type = si->type + 1;
2599 
2600 	++(*pos);
2601 	for (; (si = swap_type_to_swap_info(type)); type++) {
2602 		if (!(si->flags & SWP_USED) || !si->swap_map)
2603 			continue;
2604 		return si;
2605 	}
2606 
2607 	return NULL;
2608 }
2609 
2610 static void swap_stop(struct seq_file *swap, void *v)
2611 {
2612 	mutex_unlock(&swapon_mutex);
2613 }
2614 
2615 static int swap_show(struct seq_file *swap, void *v)
2616 {
2617 	struct swap_info_struct *si = v;
2618 	struct file *file;
2619 	int len;
2620 	unsigned long bytes, inuse;
2621 
2622 	if (si == SEQ_START_TOKEN) {
2623 		seq_puts(swap, "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");
2624 		return 0;
2625 	}
2626 
2627 	bytes = si->pages << (PAGE_SHIFT - 10);
2628 	inuse = si->inuse_pages << (PAGE_SHIFT - 10);
2629 
2630 	file = si->swap_file;
2631 	len = seq_file_path(swap, file, " \t\n\\");
2632 	seq_printf(swap, "%*s%s\t%lu\t%s%lu\t%s%d\n",
2633 			len < 40 ? 40 - len : 1, " ",
2634 			S_ISBLK(file_inode(file)->i_mode) ?
2635 				"partition" : "file\t",
2636 			bytes, bytes < 10000000 ? "\t" : "",
2637 			inuse, inuse < 10000000 ? "\t" : "",
2638 			si->prio);
2639 	return 0;
2640 }
2641 
2642 static const struct seq_operations swaps_op = {
2643 	.start =	swap_start,
2644 	.next =		swap_next,
2645 	.stop =		swap_stop,
2646 	.show =		swap_show
2647 };
2648 
2649 static int swaps_open(struct inode *inode, struct file *file)
2650 {
2651 	struct seq_file *seq;
2652 	int ret;
2653 
2654 	ret = seq_open(file, &swaps_op);
2655 	if (ret)
2656 		return ret;
2657 
2658 	seq = file->private_data;
2659 	seq->poll_event = atomic_read(&proc_poll_event);
2660 	return 0;
2661 }
2662 
2663 static const struct proc_ops swaps_proc_ops = {
2664 	.proc_flags	= PROC_ENTRY_PERMANENT,
2665 	.proc_open	= swaps_open,
2666 	.proc_read	= seq_read,
2667 	.proc_lseek	= seq_lseek,
2668 	.proc_release	= seq_release,
2669 	.proc_poll	= swaps_poll,
2670 };
2671 
2672 static int __init procswaps_init(void)
2673 {
2674 	proc_create("swaps", 0, NULL, &swaps_proc_ops);
2675 	return 0;
2676 }
2677 __initcall(procswaps_init);
2678 #endif /* CONFIG_PROC_FS */
2679 
2680 #ifdef MAX_SWAPFILES_CHECK
2681 static int __init max_swapfiles_check(void)
2682 {
2683 	MAX_SWAPFILES_CHECK();
2684 	return 0;
2685 }
2686 late_initcall(max_swapfiles_check);
2687 #endif
2688 
2689 static struct swap_info_struct *alloc_swap_info(void)
2690 {
2691 	struct swap_info_struct *p;
2692 	struct swap_info_struct *defer = NULL;
2693 	unsigned int type;
2694 	int i;
2695 
2696 	p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL);
2697 	if (!p)
2698 		return ERR_PTR(-ENOMEM);
2699 
2700 	if (percpu_ref_init(&p->users, swap_users_ref_free,
2701 			    PERCPU_REF_INIT_DEAD, GFP_KERNEL)) {
2702 		kvfree(p);
2703 		return ERR_PTR(-ENOMEM);
2704 	}
2705 
2706 	spin_lock(&swap_lock);
2707 	for (type = 0; type < nr_swapfiles; type++) {
2708 		if (!(swap_info[type]->flags & SWP_USED))
2709 			break;
2710 	}
2711 	if (type >= MAX_SWAPFILES) {
2712 		spin_unlock(&swap_lock);
2713 		percpu_ref_exit(&p->users);
2714 		kvfree(p);
2715 		return ERR_PTR(-EPERM);
2716 	}
2717 	if (type >= nr_swapfiles) {
2718 		p->type = type;
2719 		/*
2720 		 * Publish the swap_info_struct after initializing it.
2721 		 * Note that kvzalloc() above zeroes all its fields.
2722 		 */
2723 		smp_store_release(&swap_info[type], p); /* rcu_assign_pointer() */
2724 		nr_swapfiles++;
2725 	} else {
2726 		defer = p;
2727 		p = swap_info[type];
2728 		/*
2729 		 * Do not memset this entry: a racing procfs swap_next()
2730 		 * would be relying on p->type to remain valid.
2731 		 */
2732 	}
2733 	p->swap_extent_root = RB_ROOT;
2734 	plist_node_init(&p->list, 0);
2735 	for_each_node(i)
2736 		plist_node_init(&p->avail_lists[i], 0);
2737 	p->flags = SWP_USED;
2738 	spin_unlock(&swap_lock);
2739 	if (defer) {
2740 		percpu_ref_exit(&defer->users);
2741 		kvfree(defer);
2742 	}
2743 	spin_lock_init(&p->lock);
2744 	spin_lock_init(&p->cont_lock);
2745 	init_completion(&p->comp);
2746 
2747 	return p;
2748 }
2749 
2750 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2751 {
2752 	int error;
2753 
2754 	if (S_ISBLK(inode->i_mode)) {
2755 		p->bdev = blkdev_get_by_dev(inode->i_rdev,
2756 				   FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2757 		if (IS_ERR(p->bdev)) {
2758 			error = PTR_ERR(p->bdev);
2759 			p->bdev = NULL;
2760 			return error;
2761 		}
2762 		p->old_block_size = block_size(p->bdev);
2763 		error = set_blocksize(p->bdev, PAGE_SIZE);
2764 		if (error < 0)
2765 			return error;
2766 		/*
2767 		 * Zoned block devices contain zones that have a sequential
2768 		 * write only restriction.  Hence zoned block devices are not
2769 		 * suitable for swapping.  Disallow them here.
2770 		 */
2771 		if (bdev_is_zoned(p->bdev))
2772 			return -EINVAL;
2773 		p->flags |= SWP_BLKDEV;
2774 	} else if (S_ISREG(inode->i_mode)) {
2775 		p->bdev = inode->i_sb->s_bdev;
2776 	}
2777 
2778 	return 0;
2779 }
2780 
2781 
2782 /*
2783  * Find out how many pages are allowed for a single swap device. There
2784  * are two limiting factors:
2785  * 1) the number of bits for the swap offset in the swp_entry_t type, and
2786  * 2) the number of bits in the swap pte, as defined by the different
2787  * architectures.
2788  *
2789  * In order to find the largest possible bit mask, a swap entry with
2790  * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
2791  * decoded to a swp_entry_t again, and finally the swap offset is
2792  * extracted.
2793  *
2794  * This will mask all the bits from the initial ~0UL mask that can't
2795  * be encoded in either the swp_entry_t or the architecture definition
2796  * of a swap pte.
2797  */
2798 unsigned long generic_max_swapfile_size(void)
2799 {
2800 	return swp_offset(pte_to_swp_entry(
2801 			swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2802 }
2803 
2804 /* Can be overridden by an architecture for additional checks. */
2805 __weak unsigned long max_swapfile_size(void)
2806 {
2807 	return generic_max_swapfile_size();
2808 }
2809 
2810 static unsigned long read_swap_header(struct swap_info_struct *p,
2811 					union swap_header *swap_header,
2812 					struct inode *inode)
2813 {
2814 	int i;
2815 	unsigned long maxpages;
2816 	unsigned long swapfilepages;
2817 	unsigned long last_page;
2818 
2819 	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2820 		pr_err("Unable to find swap-space signature\n");
2821 		return 0;
2822 	}
2823 
2824 	/* swap partition endianness hack... */
2825 	if (swab32(swap_header->info.version) == 1) {
2826 		swab32s(&swap_header->info.version);
2827 		swab32s(&swap_header->info.last_page);
2828 		swab32s(&swap_header->info.nr_badpages);
2829 		if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2830 			return 0;
2831 		for (i = 0; i < swap_header->info.nr_badpages; i++)
2832 			swab32s(&swap_header->info.badpages[i]);
2833 	}
2834 	/* Check the swap header's sub-version */
2835 	if (swap_header->info.version != 1) {
2836 		pr_warn("Unable to handle swap header version %d\n",
2837 			swap_header->info.version);
2838 		return 0;
2839 	}
2840 
2841 	p->lowest_bit  = 1;
2842 	p->cluster_next = 1;
2843 	p->cluster_nr = 0;
2844 
2845 	maxpages = max_swapfile_size();
2846 	last_page = swap_header->info.last_page;
2847 	if (!last_page) {
2848 		pr_warn("Empty swap-file\n");
2849 		return 0;
2850 	}
2851 	if (last_page > maxpages) {
2852 		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2853 			maxpages << (PAGE_SHIFT - 10),
2854 			last_page << (PAGE_SHIFT - 10));
2855 	}
2856 	if (maxpages > last_page) {
2857 		maxpages = last_page + 1;
2858 		/* p->max is an unsigned int: don't overflow it */
2859 		if ((unsigned int)maxpages == 0)
2860 			maxpages = UINT_MAX;
2861 	}
2862 	p->highest_bit = maxpages - 1;
2863 
2864 	if (!maxpages)
2865 		return 0;
2866 	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2867 	if (swapfilepages && maxpages > swapfilepages) {
2868 		pr_warn("Swap area shorter than signature indicates\n");
2869 		return 0;
2870 	}
2871 	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2872 		return 0;
2873 	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2874 		return 0;
2875 
2876 	return maxpages;
2877 }
2878 
2879 #define SWAP_CLUSTER_INFO_COLS						\
2880 	DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info))
2881 #define SWAP_CLUSTER_SPACE_COLS						\
2882 	DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER)
2883 #define SWAP_CLUSTER_COLS						\
2884 	max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS)
2885 
2886 static int setup_swap_map_and_extents(struct swap_info_struct *p,
2887 					union swap_header *swap_header,
2888 					unsigned char *swap_map,
2889 					struct swap_cluster_info *cluster_info,
2890 					unsigned long maxpages,
2891 					sector_t *span)
2892 {
2893 	unsigned int j, k;
2894 	unsigned int nr_good_pages;
2895 	int nr_extents;
2896 	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2897 	unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS;
2898 	unsigned long i, idx;
2899 
2900 	nr_good_pages = maxpages - 1;	/* omit header page */
2901 
2902 	cluster_list_init(&p->free_clusters);
2903 	cluster_list_init(&p->discard_clusters);
2904 
2905 	for (i = 0; i < swap_header->info.nr_badpages; i++) {
2906 		unsigned int page_nr = swap_header->info.badpages[i];
2907 		if (page_nr == 0 || page_nr > swap_header->info.last_page)
2908 			return -EINVAL;
2909 		if (page_nr < maxpages) {
2910 			swap_map[page_nr] = SWAP_MAP_BAD;
2911 			nr_good_pages--;
2912 			/*
2913 			 * Haven't marked the cluster free yet, no list
2914 			 * operation involved
2915 			 */
2916 			inc_cluster_info_page(p, cluster_info, page_nr);
2917 		}
2918 	}
2919 
2920 	/* Haven't marked the cluster free yet, no list operation involved */
2921 	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2922 		inc_cluster_info_page(p, cluster_info, i);
2923 
2924 	if (nr_good_pages) {
2925 		swap_map[0] = SWAP_MAP_BAD;
2926 		/*
2927 		 * Not mark the cluster free yet, no list
2928 		 * operation involved
2929 		 */
2930 		inc_cluster_info_page(p, cluster_info, 0);
2931 		p->max = maxpages;
2932 		p->pages = nr_good_pages;
2933 		nr_extents = setup_swap_extents(p, span);
2934 		if (nr_extents < 0)
2935 			return nr_extents;
2936 		nr_good_pages = p->pages;
2937 	}
2938 	if (!nr_good_pages) {
2939 		pr_warn("Empty swap-file\n");
2940 		return -EINVAL;
2941 	}
2942 
2943 	if (!cluster_info)
2944 		return nr_extents;
2945 
2946 
2947 	/*
2948 	 * Reduce false cache line sharing between cluster_info and
2949 	 * sharing same address space.
2950 	 */
2951 	for (k = 0; k < SWAP_CLUSTER_COLS; k++) {
2952 		j = (k + col) % SWAP_CLUSTER_COLS;
2953 		for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) {
2954 			idx = i * SWAP_CLUSTER_COLS + j;
2955 			if (idx >= nr_clusters)
2956 				continue;
2957 			if (cluster_count(&cluster_info[idx]))
2958 				continue;
2959 			cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2960 			cluster_list_add_tail(&p->free_clusters, cluster_info,
2961 					      idx);
2962 		}
2963 	}
2964 	return nr_extents;
2965 }
2966 
2967 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2968 {
2969 	struct swap_info_struct *p;
2970 	struct filename *name;
2971 	struct file *swap_file = NULL;
2972 	struct address_space *mapping;
2973 	struct dentry *dentry;
2974 	int prio;
2975 	int error;
2976 	union swap_header *swap_header;
2977 	int nr_extents;
2978 	sector_t span;
2979 	unsigned long maxpages;
2980 	unsigned char *swap_map = NULL;
2981 	struct swap_cluster_info *cluster_info = NULL;
2982 	unsigned long *frontswap_map = NULL;
2983 	struct page *page = NULL;
2984 	struct inode *inode = NULL;
2985 	bool inced_nr_rotate_swap = false;
2986 
2987 	if (swap_flags & ~SWAP_FLAGS_VALID)
2988 		return -EINVAL;
2989 
2990 	if (!capable(CAP_SYS_ADMIN))
2991 		return -EPERM;
2992 
2993 	if (!swap_avail_heads)
2994 		return -ENOMEM;
2995 
2996 	p = alloc_swap_info();
2997 	if (IS_ERR(p))
2998 		return PTR_ERR(p);
2999 
3000 	INIT_WORK(&p->discard_work, swap_discard_work);
3001 
3002 	name = getname(specialfile);
3003 	if (IS_ERR(name)) {
3004 		error = PTR_ERR(name);
3005 		name = NULL;
3006 		goto bad_swap;
3007 	}
3008 	swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
3009 	if (IS_ERR(swap_file)) {
3010 		error = PTR_ERR(swap_file);
3011 		swap_file = NULL;
3012 		goto bad_swap;
3013 	}
3014 
3015 	p->swap_file = swap_file;
3016 	mapping = swap_file->f_mapping;
3017 	dentry = swap_file->f_path.dentry;
3018 	inode = mapping->host;
3019 
3020 	error = claim_swapfile(p, inode);
3021 	if (unlikely(error))
3022 		goto bad_swap;
3023 
3024 	inode_lock(inode);
3025 	if (d_unlinked(dentry) || cant_mount(dentry)) {
3026 		error = -ENOENT;
3027 		goto bad_swap_unlock_inode;
3028 	}
3029 	if (IS_SWAPFILE(inode)) {
3030 		error = -EBUSY;
3031 		goto bad_swap_unlock_inode;
3032 	}
3033 
3034 	/*
3035 	 * Read the swap header.
3036 	 */
3037 	if (!mapping->a_ops->read_folio) {
3038 		error = -EINVAL;
3039 		goto bad_swap_unlock_inode;
3040 	}
3041 	page = read_mapping_page(mapping, 0, swap_file);
3042 	if (IS_ERR(page)) {
3043 		error = PTR_ERR(page);
3044 		goto bad_swap_unlock_inode;
3045 	}
3046 	swap_header = kmap(page);
3047 
3048 	maxpages = read_swap_header(p, swap_header, inode);
3049 	if (unlikely(!maxpages)) {
3050 		error = -EINVAL;
3051 		goto bad_swap_unlock_inode;
3052 	}
3053 
3054 	/* OK, set up the swap map and apply the bad block list */
3055 	swap_map = vzalloc(maxpages);
3056 	if (!swap_map) {
3057 		error = -ENOMEM;
3058 		goto bad_swap_unlock_inode;
3059 	}
3060 
3061 	if (p->bdev && bdev_stable_writes(p->bdev))
3062 		p->flags |= SWP_STABLE_WRITES;
3063 
3064 	if (p->bdev && p->bdev->bd_disk->fops->rw_page)
3065 		p->flags |= SWP_SYNCHRONOUS_IO;
3066 
3067 	if (p->bdev && bdev_nonrot(p->bdev)) {
3068 		int cpu;
3069 		unsigned long ci, nr_cluster;
3070 
3071 		p->flags |= SWP_SOLIDSTATE;
3072 		p->cluster_next_cpu = alloc_percpu(unsigned int);
3073 		if (!p->cluster_next_cpu) {
3074 			error = -ENOMEM;
3075 			goto bad_swap_unlock_inode;
3076 		}
3077 		/*
3078 		 * select a random position to start with to help wear leveling
3079 		 * SSD
3080 		 */
3081 		for_each_possible_cpu(cpu) {
3082 			per_cpu(*p->cluster_next_cpu, cpu) =
3083 				1 + prandom_u32_max(p->highest_bit);
3084 		}
3085 		nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3086 
3087 		cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info),
3088 					GFP_KERNEL);
3089 		if (!cluster_info) {
3090 			error = -ENOMEM;
3091 			goto bad_swap_unlock_inode;
3092 		}
3093 
3094 		for (ci = 0; ci < nr_cluster; ci++)
3095 			spin_lock_init(&((cluster_info + ci)->lock));
3096 
3097 		p->percpu_cluster = alloc_percpu(struct percpu_cluster);
3098 		if (!p->percpu_cluster) {
3099 			error = -ENOMEM;
3100 			goto bad_swap_unlock_inode;
3101 		}
3102 		for_each_possible_cpu(cpu) {
3103 			struct percpu_cluster *cluster;
3104 			cluster = per_cpu_ptr(p->percpu_cluster, cpu);
3105 			cluster_set_null(&cluster->index);
3106 		}
3107 	} else {
3108 		atomic_inc(&nr_rotate_swap);
3109 		inced_nr_rotate_swap = true;
3110 	}
3111 
3112 	error = swap_cgroup_swapon(p->type, maxpages);
3113 	if (error)
3114 		goto bad_swap_unlock_inode;
3115 
3116 	nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
3117 		cluster_info, maxpages, &span);
3118 	if (unlikely(nr_extents < 0)) {
3119 		error = nr_extents;
3120 		goto bad_swap_unlock_inode;
3121 	}
3122 	/* frontswap enabled? set up bit-per-page map for frontswap */
3123 	if (IS_ENABLED(CONFIG_FRONTSWAP))
3124 		frontswap_map = kvcalloc(BITS_TO_LONGS(maxpages),
3125 					 sizeof(long),
3126 					 GFP_KERNEL);
3127 
3128 	if ((swap_flags & SWAP_FLAG_DISCARD) &&
3129 	    p->bdev && bdev_max_discard_sectors(p->bdev)) {
3130 		/*
3131 		 * When discard is enabled for swap with no particular
3132 		 * policy flagged, we set all swap discard flags here in
3133 		 * order to sustain backward compatibility with older
3134 		 * swapon(8) releases.
3135 		 */
3136 		p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
3137 			     SWP_PAGE_DISCARD);
3138 
3139 		/*
3140 		 * By flagging sys_swapon, a sysadmin can tell us to
3141 		 * either do single-time area discards only, or to just
3142 		 * perform discards for released swap page-clusters.
3143 		 * Now it's time to adjust the p->flags accordingly.
3144 		 */
3145 		if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
3146 			p->flags &= ~SWP_PAGE_DISCARD;
3147 		else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
3148 			p->flags &= ~SWP_AREA_DISCARD;
3149 
3150 		/* issue a swapon-time discard if it's still required */
3151 		if (p->flags & SWP_AREA_DISCARD) {
3152 			int err = discard_swap(p);
3153 			if (unlikely(err))
3154 				pr_err("swapon: discard_swap(%p): %d\n",
3155 					p, err);
3156 		}
3157 	}
3158 
3159 	error = init_swap_address_space(p->type, maxpages);
3160 	if (error)
3161 		goto bad_swap_unlock_inode;
3162 
3163 	/*
3164 	 * Flush any pending IO and dirty mappings before we start using this
3165 	 * swap device.
3166 	 */
3167 	inode->i_flags |= S_SWAPFILE;
3168 	error = inode_drain_writes(inode);
3169 	if (error) {
3170 		inode->i_flags &= ~S_SWAPFILE;
3171 		goto free_swap_address_space;
3172 	}
3173 
3174 	mutex_lock(&swapon_mutex);
3175 	prio = -1;
3176 	if (swap_flags & SWAP_FLAG_PREFER)
3177 		prio =
3178 		  (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
3179 	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
3180 
3181 	pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
3182 		p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
3183 		nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
3184 		(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
3185 		(p->flags & SWP_DISCARDABLE) ? "D" : "",
3186 		(p->flags & SWP_AREA_DISCARD) ? "s" : "",
3187 		(p->flags & SWP_PAGE_DISCARD) ? "c" : "",
3188 		(frontswap_map) ? "FS" : "");
3189 
3190 	mutex_unlock(&swapon_mutex);
3191 	atomic_inc(&proc_poll_event);
3192 	wake_up_interruptible(&proc_poll_wait);
3193 
3194 	error = 0;
3195 	goto out;
3196 free_swap_address_space:
3197 	exit_swap_address_space(p->type);
3198 bad_swap_unlock_inode:
3199 	inode_unlock(inode);
3200 bad_swap:
3201 	free_percpu(p->percpu_cluster);
3202 	p->percpu_cluster = NULL;
3203 	free_percpu(p->cluster_next_cpu);
3204 	p->cluster_next_cpu = NULL;
3205 	if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
3206 		set_blocksize(p->bdev, p->old_block_size);
3207 		blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
3208 	}
3209 	inode = NULL;
3210 	destroy_swap_extents(p);
3211 	swap_cgroup_swapoff(p->type);
3212 	spin_lock(&swap_lock);
3213 	p->swap_file = NULL;
3214 	p->flags = 0;
3215 	spin_unlock(&swap_lock);
3216 	vfree(swap_map);
3217 	kvfree(cluster_info);
3218 	kvfree(frontswap_map);
3219 	if (inced_nr_rotate_swap)
3220 		atomic_dec(&nr_rotate_swap);
3221 	if (swap_file)
3222 		filp_close(swap_file, NULL);
3223 out:
3224 	if (page && !IS_ERR(page)) {
3225 		kunmap(page);
3226 		put_page(page);
3227 	}
3228 	if (name)
3229 		putname(name);
3230 	if (inode)
3231 		inode_unlock(inode);
3232 	if (!error)
3233 		enable_swap_slots_cache();
3234 	return error;
3235 }
3236 
3237 void si_swapinfo(struct sysinfo *val)
3238 {
3239 	unsigned int type;
3240 	unsigned long nr_to_be_unused = 0;
3241 
3242 	spin_lock(&swap_lock);
3243 	for (type = 0; type < nr_swapfiles; type++) {
3244 		struct swap_info_struct *si = swap_info[type];
3245 
3246 		if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
3247 			nr_to_be_unused += si->inuse_pages;
3248 	}
3249 	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
3250 	val->totalswap = total_swap_pages + nr_to_be_unused;
3251 	spin_unlock(&swap_lock);
3252 }
3253 
3254 /*
3255  * Verify that a swap entry is valid and increment its swap map count.
3256  *
3257  * Returns error code in following case.
3258  * - success -> 0
3259  * - swp_entry is invalid -> EINVAL
3260  * - swp_entry is migration entry -> EINVAL
3261  * - swap-cache reference is requested but there is already one. -> EEXIST
3262  * - swap-cache reference is requested but the entry is not used. -> ENOENT
3263  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
3264  */
3265 static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
3266 {
3267 	struct swap_info_struct *p;
3268 	struct swap_cluster_info *ci;
3269 	unsigned long offset;
3270 	unsigned char count;
3271 	unsigned char has_cache;
3272 	int err;
3273 
3274 	p = get_swap_device(entry);
3275 	if (!p)
3276 		return -EINVAL;
3277 
3278 	offset = swp_offset(entry);
3279 	ci = lock_cluster_or_swap_info(p, offset);
3280 
3281 	count = p->swap_map[offset];
3282 
3283 	/*
3284 	 * swapin_readahead() doesn't check if a swap entry is valid, so the
3285 	 * swap entry could be SWAP_MAP_BAD. Check here with lock held.
3286 	 */
3287 	if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
3288 		err = -ENOENT;
3289 		goto unlock_out;
3290 	}
3291 
3292 	has_cache = count & SWAP_HAS_CACHE;
3293 	count &= ~SWAP_HAS_CACHE;
3294 	err = 0;
3295 
3296 	if (usage == SWAP_HAS_CACHE) {
3297 
3298 		/* set SWAP_HAS_CACHE if there is no cache and entry is used */
3299 		if (!has_cache && count)
3300 			has_cache = SWAP_HAS_CACHE;
3301 		else if (has_cache)		/* someone else added cache */
3302 			err = -EEXIST;
3303 		else				/* no users remaining */
3304 			err = -ENOENT;
3305 
3306 	} else if (count || has_cache) {
3307 
3308 		if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
3309 			count += usage;
3310 		else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
3311 			err = -EINVAL;
3312 		else if (swap_count_continued(p, offset, count))
3313 			count = COUNT_CONTINUED;
3314 		else
3315 			err = -ENOMEM;
3316 	} else
3317 		err = -ENOENT;			/* unused swap entry */
3318 
3319 	WRITE_ONCE(p->swap_map[offset], count | has_cache);
3320 
3321 unlock_out:
3322 	unlock_cluster_or_swap_info(p, ci);
3323 	put_swap_device(p);
3324 	return err;
3325 }
3326 
3327 /*
3328  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
3329  * (in which case its reference count is never incremented).
3330  */
3331 void swap_shmem_alloc(swp_entry_t entry)
3332 {
3333 	__swap_duplicate(entry, SWAP_MAP_SHMEM);
3334 }
3335 
3336 /*
3337  * Increase reference count of swap entry by 1.
3338  * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
3339  * but could not be atomically allocated.  Returns 0, just as if it succeeded,
3340  * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
3341  * might occur if a page table entry has got corrupted.
3342  */
3343 int swap_duplicate(swp_entry_t entry)
3344 {
3345 	int err = 0;
3346 
3347 	while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
3348 		err = add_swap_count_continuation(entry, GFP_ATOMIC);
3349 	return err;
3350 }
3351 
3352 /*
3353  * @entry: swap entry for which we allocate swap cache.
3354  *
3355  * Called when allocating swap cache for existing swap entry,
3356  * This can return error codes. Returns 0 at success.
3357  * -EEXIST means there is a swap cache.
3358  * Note: return code is different from swap_duplicate().
3359  */
3360 int swapcache_prepare(swp_entry_t entry)
3361 {
3362 	return __swap_duplicate(entry, SWAP_HAS_CACHE);
3363 }
3364 
3365 struct swap_info_struct *swp_swap_info(swp_entry_t entry)
3366 {
3367 	return swap_type_to_swap_info(swp_type(entry));
3368 }
3369 
3370 struct swap_info_struct *page_swap_info(struct page *page)
3371 {
3372 	swp_entry_t entry = { .val = page_private(page) };
3373 	return swp_swap_info(entry);
3374 }
3375 
3376 /*
3377  * out-of-line methods to avoid include hell.
3378  */
3379 struct address_space *swapcache_mapping(struct folio *folio)
3380 {
3381 	return page_swap_info(&folio->page)->swap_file->f_mapping;
3382 }
3383 EXPORT_SYMBOL_GPL(swapcache_mapping);
3384 
3385 pgoff_t __page_file_index(struct page *page)
3386 {
3387 	swp_entry_t swap = { .val = page_private(page) };
3388 	return swp_offset(swap);
3389 }
3390 EXPORT_SYMBOL_GPL(__page_file_index);
3391 
3392 /*
3393  * add_swap_count_continuation - called when a swap count is duplicated
3394  * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
3395  * page of the original vmalloc'ed swap_map, to hold the continuation count
3396  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
3397  * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
3398  *
3399  * These continuation pages are seldom referenced: the common paths all work
3400  * on the original swap_map, only referring to a continuation page when the
3401  * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
3402  *
3403  * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
3404  * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
3405  * can be called after dropping locks.
3406  */
3407 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
3408 {
3409 	struct swap_info_struct *si;
3410 	struct swap_cluster_info *ci;
3411 	struct page *head;
3412 	struct page *page;
3413 	struct page *list_page;
3414 	pgoff_t offset;
3415 	unsigned char count;
3416 	int ret = 0;
3417 
3418 	/*
3419 	 * When debugging, it's easier to use __GFP_ZERO here; but it's better
3420 	 * for latency not to zero a page while GFP_ATOMIC and holding locks.
3421 	 */
3422 	page = alloc_page(gfp_mask | __GFP_HIGHMEM);
3423 
3424 	si = get_swap_device(entry);
3425 	if (!si) {
3426 		/*
3427 		 * An acceptable race has occurred since the failing
3428 		 * __swap_duplicate(): the swap device may be swapoff
3429 		 */
3430 		goto outer;
3431 	}
3432 	spin_lock(&si->lock);
3433 
3434 	offset = swp_offset(entry);
3435 
3436 	ci = lock_cluster(si, offset);
3437 
3438 	count = swap_count(si->swap_map[offset]);
3439 
3440 	if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
3441 		/*
3442 		 * The higher the swap count, the more likely it is that tasks
3443 		 * will race to add swap count continuation: we need to avoid
3444 		 * over-provisioning.
3445 		 */
3446 		goto out;
3447 	}
3448 
3449 	if (!page) {
3450 		ret = -ENOMEM;
3451 		goto out;
3452 	}
3453 
3454 	/*
3455 	 * We are fortunate that although vmalloc_to_page uses pte_offset_map,
3456 	 * no architecture is using highmem pages for kernel page tables: so it
3457 	 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps.
3458 	 */
3459 	head = vmalloc_to_page(si->swap_map + offset);
3460 	offset &= ~PAGE_MASK;
3461 
3462 	spin_lock(&si->cont_lock);
3463 	/*
3464 	 * Page allocation does not initialize the page's lru field,
3465 	 * but it does always reset its private field.
3466 	 */
3467 	if (!page_private(head)) {
3468 		BUG_ON(count & COUNT_CONTINUED);
3469 		INIT_LIST_HEAD(&head->lru);
3470 		set_page_private(head, SWP_CONTINUED);
3471 		si->flags |= SWP_CONTINUED;
3472 	}
3473 
3474 	list_for_each_entry(list_page, &head->lru, lru) {
3475 		unsigned char *map;
3476 
3477 		/*
3478 		 * If the previous map said no continuation, but we've found
3479 		 * a continuation page, free our allocation and use this one.
3480 		 */
3481 		if (!(count & COUNT_CONTINUED))
3482 			goto out_unlock_cont;
3483 
3484 		map = kmap_atomic(list_page) + offset;
3485 		count = *map;
3486 		kunmap_atomic(map);
3487 
3488 		/*
3489 		 * If this continuation count now has some space in it,
3490 		 * free our allocation and use this one.
3491 		 */
3492 		if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
3493 			goto out_unlock_cont;
3494 	}
3495 
3496 	list_add_tail(&page->lru, &head->lru);
3497 	page = NULL;			/* now it's attached, don't free it */
3498 out_unlock_cont:
3499 	spin_unlock(&si->cont_lock);
3500 out:
3501 	unlock_cluster(ci);
3502 	spin_unlock(&si->lock);
3503 	put_swap_device(si);
3504 outer:
3505 	if (page)
3506 		__free_page(page);
3507 	return ret;
3508 }
3509 
3510 /*
3511  * swap_count_continued - when the original swap_map count is incremented
3512  * from SWAP_MAP_MAX, check if there is already a continuation page to carry
3513  * into, carry if so, or else fail until a new continuation page is allocated;
3514  * when the original swap_map count is decremented from 0 with continuation,
3515  * borrow from the continuation and report whether it still holds more.
3516  * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster
3517  * lock.
3518  */
3519 static bool swap_count_continued(struct swap_info_struct *si,
3520 				 pgoff_t offset, unsigned char count)
3521 {
3522 	struct page *head;
3523 	struct page *page;
3524 	unsigned char *map;
3525 	bool ret;
3526 
3527 	head = vmalloc_to_page(si->swap_map + offset);
3528 	if (page_private(head) != SWP_CONTINUED) {
3529 		BUG_ON(count & COUNT_CONTINUED);
3530 		return false;		/* need to add count continuation */
3531 	}
3532 
3533 	spin_lock(&si->cont_lock);
3534 	offset &= ~PAGE_MASK;
3535 	page = list_next_entry(head, lru);
3536 	map = kmap_atomic(page) + offset;
3537 
3538 	if (count == SWAP_MAP_MAX)	/* initial increment from swap_map */
3539 		goto init_map;		/* jump over SWAP_CONT_MAX checks */
3540 
3541 	if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
3542 		/*
3543 		 * Think of how you add 1 to 999
3544 		 */
3545 		while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
3546 			kunmap_atomic(map);
3547 			page = list_next_entry(page, lru);
3548 			BUG_ON(page == head);
3549 			map = kmap_atomic(page) + offset;
3550 		}
3551 		if (*map == SWAP_CONT_MAX) {
3552 			kunmap_atomic(map);
3553 			page = list_next_entry(page, lru);
3554 			if (page == head) {
3555 				ret = false;	/* add count continuation */
3556 				goto out;
3557 			}
3558 			map = kmap_atomic(page) + offset;
3559 init_map:		*map = 0;		/* we didn't zero the page */
3560 		}
3561 		*map += 1;
3562 		kunmap_atomic(map);
3563 		while ((page = list_prev_entry(page, lru)) != head) {
3564 			map = kmap_atomic(page) + offset;
3565 			*map = COUNT_CONTINUED;
3566 			kunmap_atomic(map);
3567 		}
3568 		ret = true;			/* incremented */
3569 
3570 	} else {				/* decrementing */
3571 		/*
3572 		 * Think of how you subtract 1 from 1000
3573 		 */
3574 		BUG_ON(count != COUNT_CONTINUED);
3575 		while (*map == COUNT_CONTINUED) {
3576 			kunmap_atomic(map);
3577 			page = list_next_entry(page, lru);
3578 			BUG_ON(page == head);
3579 			map = kmap_atomic(page) + offset;
3580 		}
3581 		BUG_ON(*map == 0);
3582 		*map -= 1;
3583 		if (*map == 0)
3584 			count = 0;
3585 		kunmap_atomic(map);
3586 		while ((page = list_prev_entry(page, lru)) != head) {
3587 			map = kmap_atomic(page) + offset;
3588 			*map = SWAP_CONT_MAX | count;
3589 			count = COUNT_CONTINUED;
3590 			kunmap_atomic(map);
3591 		}
3592 		ret = count == COUNT_CONTINUED;
3593 	}
3594 out:
3595 	spin_unlock(&si->cont_lock);
3596 	return ret;
3597 }
3598 
3599 /*
3600  * free_swap_count_continuations - swapoff free all the continuation pages
3601  * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
3602  */
3603 static void free_swap_count_continuations(struct swap_info_struct *si)
3604 {
3605 	pgoff_t offset;
3606 
3607 	for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
3608 		struct page *head;
3609 		head = vmalloc_to_page(si->swap_map + offset);
3610 		if (page_private(head)) {
3611 			struct page *page, *next;
3612 
3613 			list_for_each_entry_safe(page, next, &head->lru, lru) {
3614 				list_del(&page->lru);
3615 				__free_page(page);
3616 			}
3617 		}
3618 	}
3619 }
3620 
3621 #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
3622 void __cgroup_throttle_swaprate(struct page *page, gfp_t gfp_mask)
3623 {
3624 	struct swap_info_struct *si, *next;
3625 	int nid = page_to_nid(page);
3626 
3627 	if (!(gfp_mask & __GFP_IO))
3628 		return;
3629 
3630 	if (!blk_cgroup_congested())
3631 		return;
3632 
3633 	/*
3634 	 * We've already scheduled a throttle, avoid taking the global swap
3635 	 * lock.
3636 	 */
3637 	if (current->throttle_queue)
3638 		return;
3639 
3640 	spin_lock(&swap_avail_lock);
3641 	plist_for_each_entry_safe(si, next, &swap_avail_heads[nid],
3642 				  avail_lists[nid]) {
3643 		if (si->bdev) {
3644 			blkcg_schedule_throttle(bdev_get_queue(si->bdev), true);
3645 			break;
3646 		}
3647 	}
3648 	spin_unlock(&swap_avail_lock);
3649 }
3650 #endif
3651 
3652 static int __init swapfile_init(void)
3653 {
3654 	int nid;
3655 
3656 	swap_avail_heads = kmalloc_array(nr_node_ids, sizeof(struct plist_head),
3657 					 GFP_KERNEL);
3658 	if (!swap_avail_heads) {
3659 		pr_emerg("Not enough memory for swap heads, swap is disabled\n");
3660 		return -ENOMEM;
3661 	}
3662 
3663 	for_each_node(nid)
3664 		plist_head_init(&swap_avail_heads[nid]);
3665 
3666 	return 0;
3667 }
3668 subsys_initcall(swapfile_init);
3669