swapfile.c (09fe06ce0bf5abe53b77a9515d7fb7579edec9c0) swapfile.c (490705888107c3edf8c264ec930909107f76a984)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/swapfile.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 * Swap reorganised 29.12.95, Stephen Tweedie
7 */
8

--- 599 unchanged lines hidden (view full) ---

608 if (cluster_is_null(&cluster->index)) {
609 if (!cluster_list_empty(&si->free_clusters)) {
610 cluster->index = si->free_clusters.head;
611 cluster->next = cluster_next(&cluster->index) *
612 SWAPFILE_CLUSTER;
613 } else if (!cluster_list_empty(&si->discard_clusters)) {
614 /*
615 * we don't have free cluster but have some clusters in
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/swapfile.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 * Swap reorganised 29.12.95, Stephen Tweedie
7 */
8

--- 599 unchanged lines hidden (view full) ---

608 if (cluster_is_null(&cluster->index)) {
609 if (!cluster_list_empty(&si->free_clusters)) {
610 cluster->index = si->free_clusters.head;
611 cluster->next = cluster_next(&cluster->index) *
612 SWAPFILE_CLUSTER;
613 } else if (!cluster_list_empty(&si->discard_clusters)) {
614 /*
615 * we don't have free cluster but have some clusters in
616 * discarding, do discard now and reclaim them
616 * discarding, do discard now and reclaim them, then
617 * reread cluster_next_cpu since we dropped si->lock
617 */
618 swap_do_scheduled_discard(si);
618 */
619 swap_do_scheduled_discard(si);
619 *scan_base = *offset = si->cluster_next;
620 *scan_base = this_cpu_read(*si->cluster_next_cpu);
621 *offset = *scan_base;
620 goto new_cluster;
621 } else
622 return false;
623 }
624
625 /*
626 * Other CPUs can use our cluster if they can't find a free cluster,
627 * check if there is still free entry in the cluster

--- 89 unchanged lines hidden (view full) ---

717 while (offset <= end) {
718 frontswap_invalidate_page(si->type, offset);
719 if (swap_slot_free_notify)
720 swap_slot_free_notify(si->bdev, offset);
721 offset++;
722 }
723}
724
622 goto new_cluster;
623 } else
624 return false;
625 }
626
627 /*
628 * Other CPUs can use our cluster if they can't find a free cluster,
629 * check if there is still free entry in the cluster

--- 89 unchanged lines hidden (view full) ---

719 while (offset <= end) {
720 frontswap_invalidate_page(si->type, offset);
721 if (swap_slot_free_notify)
722 swap_slot_free_notify(si->bdev, offset);
723 offset++;
724 }
725}
726
727static void set_cluster_next(struct swap_info_struct *si, unsigned long next)
728{
729 unsigned long prev;
730
731 if (!(si->flags & SWP_SOLIDSTATE)) {
732 si->cluster_next = next;
733 return;
734 }
735
736 prev = this_cpu_read(*si->cluster_next_cpu);
737 /*
738 * Cross the swap address space size aligned trunk, choose
739 * another trunk randomly to avoid lock contention on swap
740 * address space if possible.
741 */
742 if ((prev >> SWAP_ADDRESS_SPACE_SHIFT) !=
743 (next >> SWAP_ADDRESS_SPACE_SHIFT)) {
744 /* No free swap slots available */
745 if (si->highest_bit <= si->lowest_bit)
746 return;
747 next = si->lowest_bit +
748 prandom_u32_max(si->highest_bit - si->lowest_bit + 1);
749 next = ALIGN_DOWN(next, SWAP_ADDRESS_SPACE_PAGES);
750 next = max_t(unsigned int, next, si->lowest_bit);
751 }
752 this_cpu_write(*si->cluster_next_cpu, next);
753}
754
725static int scan_swap_map_slots(struct swap_info_struct *si,
726 unsigned char usage, int nr,
727 swp_entry_t slots[])
728{
729 struct swap_cluster_info *ci;
730 unsigned long offset;
731 unsigned long scan_base;
732 unsigned long last_in_cluster = 0;

--- 8 unchanged lines hidden (view full) ---

741 * a new cluster. This prevents us from scattering swap pages
742 * all over the entire swap partition, so that we reduce
743 * overall disk seek times between swap pages. -- sct
744 * But we do now try to find an empty cluster. -Andrea
745 * And we let swap pages go all over an SSD partition. Hugh
746 */
747
748 si->flags += SWP_SCANNING;
755static int scan_swap_map_slots(struct swap_info_struct *si,
756 unsigned char usage, int nr,
757 swp_entry_t slots[])
758{
759 struct swap_cluster_info *ci;
760 unsigned long offset;
761 unsigned long scan_base;
762 unsigned long last_in_cluster = 0;

--- 8 unchanged lines hidden (view full) ---

771 * a new cluster. This prevents us from scattering swap pages
772 * all over the entire swap partition, so that we reduce
773 * overall disk seek times between swap pages. -- sct
774 * But we do now try to find an empty cluster. -Andrea
775 * And we let swap pages go all over an SSD partition. Hugh
776 */
777
778 si->flags += SWP_SCANNING;
749 scan_base = offset = si->cluster_next;
779 /*
780 * Use percpu scan base for SSD to reduce lock contention on
781 * cluster and swap cache. For HDD, sequential access is more
782 * important.
783 */
784 if (si->flags & SWP_SOLIDSTATE)
785 scan_base = this_cpu_read(*si->cluster_next_cpu);
786 else
787 scan_base = si->cluster_next;
788 offset = scan_base;
750
751 /* SSD algorithm */
752 if (si->cluster_info) {
753 if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
754 goto scan;
755 } else if (unlikely(!si->cluster_nr--)) {
756 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
757 si->cluster_nr = SWAPFILE_CLUSTER - 1;

--- 72 unchanged lines hidden (view full) ---

830 else
831 goto done;
832 }
833 si->swap_map[offset] = usage;
834 inc_cluster_info_page(si, si->cluster_info, offset);
835 unlock_cluster(ci);
836
837 swap_range_alloc(si, offset, 1);
789
790 /* SSD algorithm */
791 if (si->cluster_info) {
792 if (!scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
793 goto scan;
794 } else if (unlikely(!si->cluster_nr--)) {
795 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
796 si->cluster_nr = SWAPFILE_CLUSTER - 1;

--- 72 unchanged lines hidden (view full) ---

869 else
870 goto done;
871 }
872 si->swap_map[offset] = usage;
873 inc_cluster_info_page(si, si->cluster_info, offset);
874 unlock_cluster(ci);
875
876 swap_range_alloc(si, offset, 1);
838 si->cluster_next = offset + 1;
839 slots[n_ret++] = swp_entry(si->type, offset);
840
841 /* got enough slots or reach max slots? */
842 if ((n_ret == nr) || (offset >= si->highest_bit))
843 goto done;
844
845 /* search for next available slot */
846

--- 32 unchanged lines hidden (view full) ---

879 for (; offset <= scan_limit && --latency_ration > 0;
880 offset++) {
881 if (!si->swap_map[offset])
882 goto checks;
883 }
884 }
885
886done:
877 slots[n_ret++] = swp_entry(si->type, offset);
878
879 /* got enough slots or reach max slots? */
880 if ((n_ret == nr) || (offset >= si->highest_bit))
881 goto done;
882
883 /* search for next available slot */
884

--- 32 unchanged lines hidden (view full) ---

917 for (; offset <= scan_limit && --latency_ration > 0;
918 offset++) {
919 if (!si->swap_map[offset])
920 goto checks;
921 }
922 }
923
924done:
925 set_cluster_next(si, offset + 1);
887 si->flags -= SWP_SCANNING;
888 return n_ret;
889
890scan:
891 spin_unlock(&si->lock);
892 while (++offset <= si->highest_bit) {
893 if (!si->swap_map[offset]) {
894 spin_lock(&si->lock);

--- 1753 unchanged lines hidden (view full) ---

2648 frontswap_map = frontswap_map_get(p);
2649 spin_unlock(&p->lock);
2650 spin_unlock(&swap_lock);
2651 frontswap_invalidate_area(p->type);
2652 frontswap_map_set(p, NULL);
2653 mutex_unlock(&swapon_mutex);
2654 free_percpu(p->percpu_cluster);
2655 p->percpu_cluster = NULL;
926 si->flags -= SWP_SCANNING;
927 return n_ret;
928
929scan:
930 spin_unlock(&si->lock);
931 while (++offset <= si->highest_bit) {
932 if (!si->swap_map[offset]) {
933 spin_lock(&si->lock);

--- 1753 unchanged lines hidden (view full) ---

2687 frontswap_map = frontswap_map_get(p);
2688 spin_unlock(&p->lock);
2689 spin_unlock(&swap_lock);
2690 frontswap_invalidate_area(p->type);
2691 frontswap_map_set(p, NULL);
2692 mutex_unlock(&swapon_mutex);
2693 free_percpu(p->percpu_cluster);
2694 p->percpu_cluster = NULL;
2695 free_percpu(p->cluster_next_cpu);
2696 p->cluster_next_cpu = NULL;
2656 vfree(swap_map);
2657 kvfree(cluster_info);
2658 kvfree(frontswap_map);
2659 /* Destroy swap account information */
2660 swap_cgroup_swapoff(p->type);
2661 exit_swap_address_space(p->type);
2662
2663 inode = mapping->host;

--- 536 unchanged lines hidden (view full) ---

3200 if (bdi_cap_synchronous_io(inode_to_bdi(inode)))
3201 p->flags |= SWP_SYNCHRONOUS_IO;
3202
3203 if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
3204 int cpu;
3205 unsigned long ci, nr_cluster;
3206
3207 p->flags |= SWP_SOLIDSTATE;
2697 vfree(swap_map);
2698 kvfree(cluster_info);
2699 kvfree(frontswap_map);
2700 /* Destroy swap account information */
2701 swap_cgroup_swapoff(p->type);
2702 exit_swap_address_space(p->type);
2703
2704 inode = mapping->host;

--- 536 unchanged lines hidden (view full) ---

3241 if (bdi_cap_synchronous_io(inode_to_bdi(inode)))
3242 p->flags |= SWP_SYNCHRONOUS_IO;
3243
3244 if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
3245 int cpu;
3246 unsigned long ci, nr_cluster;
3247
3248 p->flags |= SWP_SOLIDSTATE;
3249 p->cluster_next_cpu = alloc_percpu(unsigned int);
3250 if (!p->cluster_next_cpu) {
3251 error = -ENOMEM;
3252 goto bad_swap_unlock_inode;
3253 }
3208 /*
3209 * select a random position to start with to help wear leveling
3210 * SSD
3211 */
3254 /*
3255 * select a random position to start with to help wear leveling
3256 * SSD
3257 */
3212 p->cluster_next = 1 + prandom_u32_max(p->highest_bit);
3258 for_each_possible_cpu(cpu) {
3259 per_cpu(*p->cluster_next_cpu, cpu) =
3260 1 + prandom_u32_max(p->highest_bit);
3261 }
3213 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3214
3215 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info),
3216 GFP_KERNEL);
3217 if (!cluster_info) {
3218 error = -ENOMEM;
3219 goto bad_swap_unlock_inode;
3220 }

--- 99 unchanged lines hidden (view full) ---

3320
3321 error = 0;
3322 goto out;
3323bad_swap_unlock_inode:
3324 inode_unlock(inode);
3325bad_swap:
3326 free_percpu(p->percpu_cluster);
3327 p->percpu_cluster = NULL;
3262 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3263
3264 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info),
3265 GFP_KERNEL);
3266 if (!cluster_info) {
3267 error = -ENOMEM;
3268 goto bad_swap_unlock_inode;
3269 }

--- 99 unchanged lines hidden (view full) ---

3369
3370 error = 0;
3371 goto out;
3372bad_swap_unlock_inode:
3373 inode_unlock(inode);
3374bad_swap:
3375 free_percpu(p->percpu_cluster);
3376 p->percpu_cluster = NULL;
3377 free_percpu(p->cluster_next_cpu);
3378 p->cluster_next_cpu = NULL;
3328 if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
3329 set_blocksize(p->bdev, p->old_block_size);
3330 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
3331 }
3332 inode = NULL;
3333 destroy_swap_extents(p);
3334 swap_cgroup_swapoff(p->type);
3335 spin_lock(&swap_lock);

--- 458 unchanged lines hidden --- 		3379 if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
3380 set_blocksize(p->bdev, p->old_block_size);
3381 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
3382 }
3383 inode = NULL;
3384 destroy_swap_extents(p);
3385 swap_cgroup_swapoff(p->type);
3386 spin_lock(&swap_lock);

--- 458 unchanged lines hidden ---