swapfile.c (97acb6a8fcc4e5c2cdc2693a35acdc5a7461aaa3) swapfile.c (14d01ee9fcb901c9e020f2dcd71c500f10c3bd03)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/swapfile.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 * Swap reorganised 29.12.95, Stephen Tweedie
7 */
8

--- 49 unchanged lines hidden (view full) ---

58 * Some modules use swappable objects and may try to swap them out under
59 * memory pressure (via the shrinker). Before doing so, they may wish to
60 * check to see if any swap space is available.
61 */
62EXPORT_SYMBOL_GPL(nr_swap_pages);
63/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
64long total_swap_pages;
65static int least_priority = -1;
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/swapfile.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 * Swap reorganised 29.12.95, Stephen Tweedie
7 */
8

--- 49 unchanged lines hidden (view full) ---

58 * Some modules use swappable objects and may try to swap them out under
59 * memory pressure (via the shrinker). Before doing so, they may wish to
60 * check to see if any swap space is available.
61 */
62EXPORT_SYMBOL_GPL(nr_swap_pages);
63/* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
64long total_swap_pages;
65static int least_priority = -1;
66unsigned long swapfile_maximum_size;
67#ifdef CONFIG_MIGRATION
68bool swap_migration_ad_supported;
69#endif /* CONFIG_MIGRATION */
66
67static const char Bad_file[] = "Bad swap file entry ";
68static const char Unused_file[] = "Unused swap file entry ";
69static const char Bad_offset[] = "Bad swap offset entry ";
70static const char Unused_offset[] = "Unused swap offset entry ";
71
72/*
73 * all active swap_info_structs

--- 1348 unchanged lines hidden (view full) ---

1422 if (p)
1423 swap_entry_free(p, entries[i]);
1424 prev = p;
1425 }
1426 if (p)
1427 spin_unlock(&p->lock);
1428}
1429
70
71static const char Bad_file[] = "Bad swap file entry ";
72static const char Unused_file[] = "Unused swap file entry ";
73static const char Bad_offset[] = "Bad swap offset entry ";
74static const char Unused_offset[] = "Unused swap offset entry ";
75
76/*
77 * all active swap_info_structs

--- 1348 unchanged lines hidden (view full) ---

1426 if (p)
1427 swap_entry_free(p, entries[i]);
1428 prev = p;
1429 }
1430 if (p)
1431 spin_unlock(&p->lock);
1432}
1433
1430/*
1431 * How many references to page are currently swapped out?
1432 * This does not give an exact answer when swap count is continued,
1433 * but does include the high COUNT_CONTINUED flag to allow for that.
1434 */
1435static int page_swapcount(struct page *page)
1436{
1437 int count = 0;
1438 struct swap_info_struct *p;
1439 struct swap_cluster_info *ci;
1440 swp_entry_t entry;
1441 unsigned long offset;
1442
1443 entry.val = page_private(page);
1444 p = _swap_info_get(entry);
1445 if (p) {
1446 offset = swp_offset(entry);
1447 ci = lock_cluster_or_swap_info(p, offset);
1448 count = swap_count(p->swap_map[offset]);
1449 unlock_cluster_or_swap_info(p, ci);
1450 }
1451 return count;
1452}
1453
1454int __swap_count(swp_entry_t entry)
1455{
1456 struct swap_info_struct *si;
1457 pgoff_t offset = swp_offset(entry);
1458 int count = 0;
1459
1460 si = get_swap_device(entry);
1461 if (si) {
1462 count = swap_count(si->swap_map[offset]);
1463 put_swap_device(si);
1464 }
1465 return count;
1466}
1467
1434int __swap_count(swp_entry_t entry)
1435{
1436 struct swap_info_struct *si;
1437 pgoff_t offset = swp_offset(entry);
1438 int count = 0;
1439
1440 si = get_swap_device(entry);
1441 if (si) {
1442 count = swap_count(si->swap_map[offset]);
1443 put_swap_device(si);
1444 }
1445 return count;
1446}
1447
1448/*
1449 * How many references to @entry are currently swapped out?
1450 * This does not give an exact answer when swap count is continued,
1451 * but does include the high COUNT_CONTINUED flag to allow for that.
1452 */
1468static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
1469{
1453static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
1454{
1470 int count = 0;
1471 pgoff_t offset = swp_offset(entry);
1472 struct swap_cluster_info *ci;
1455 pgoff_t offset = swp_offset(entry);
1456 struct swap_cluster_info *ci;
1457 int count;
1473
1474 ci = lock_cluster_or_swap_info(si, offset);
1475 count = swap_count(si->swap_map[offset]);
1476 unlock_cluster_or_swap_info(si, ci);
1477 return count;
1478}
1479
1480/*

--- 84 unchanged lines hidden (view full) ---

1565 }
1566unlock_out:
1567 unlock_cluster_or_swap_info(si, ci);
1568 return ret;
1569}
1570
1571static bool folio_swapped(struct folio *folio)
1572{
1458
1459 ci = lock_cluster_or_swap_info(si, offset);
1460 count = swap_count(si->swap_map[offset]);
1461 unlock_cluster_or_swap_info(si, ci);
1462 return count;
1463}
1464
1465/*

--- 84 unchanged lines hidden (view full) ---

1550 }
1551unlock_out:
1552 unlock_cluster_or_swap_info(si, ci);
1553 return ret;
1554}
1555
1556static bool folio_swapped(struct folio *folio)
1557{
1573 swp_entry_t entry;
1574 struct swap_info_struct *si;
1558 swp_entry_t entry = folio_swap_entry(folio);
1559 struct swap_info_struct *si = _swap_info_get(entry);
1575
1560
1561 if (!si)
1562 return false;
1563
1576 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio)))
1564 if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!folio_test_large(folio)))
1577 return page_swapcount(&folio->page) != 0;
1565 return swap_swapcount(si, entry) != 0;
1578
1566
1579 entry = folio_swap_entry(folio);
1580 si = _swap_info_get(entry);
1581 if (si)
1582 return swap_page_trans_huge_swapped(si, entry);
1583 return false;
1567 return swap_page_trans_huge_swapped(si, entry);
1584}
1585
1586/*
1587 * If swap is getting full, or if there are no more mappings of this page,
1588 * then try_to_free_swap is called to free its swap space.
1589 */
1590int try_to_free_swap(struct page *page)
1591{

--- 393 unchanged lines hidden (view full) ---

1985 } while (pgd++, addr = next, addr != end);
1986 return 0;
1987}
1988
1989static int unuse_mm(struct mm_struct *mm, unsigned int type)
1990{
1991 struct vm_area_struct *vma;
1992 int ret = 0;
1568}
1569
1570/*
1571 * If swap is getting full, or if there are no more mappings of this page,
1572 * then try_to_free_swap is called to free its swap space.
1573 */
1574int try_to_free_swap(struct page *page)
1575{

--- 393 unchanged lines hidden (view full) ---

1969 } while (pgd++, addr = next, addr != end);
1970 return 0;
1971}
1972
1973static int unuse_mm(struct mm_struct *mm, unsigned int type)
1974{
1975 struct vm_area_struct *vma;
1976 int ret = 0;
1977 VMA_ITERATOR(vmi, mm, 0);
1993
1994 mmap_read_lock(mm);
1978
1979 mmap_read_lock(mm);
1995 for (vma = mm->mmap; vma; vma = vma->vm_next) {
1980 for_each_vma(vmi, vma) {
1996 if (vma->anon_vma) {
1997 ret = unuse_vma(vma, type);
1998 if (ret)
1999 break;
2000 }
1981 if (vma->anon_vma) {
1982 ret = unuse_vma(vma, type);
1983 if (ret)
1984 break;
1985 }
1986
2001 cond_resched();
2002 }
2003 mmap_read_unlock(mm);
2004 return ret;
2005}
2006
2007/*
2008 * Scan swap_map from current position to next entry still in use.

--- 802 unchanged lines hidden (view full) ---

2811 */
2812unsigned long generic_max_swapfile_size(void)
2813{
2814 return swp_offset(pte_to_swp_entry(
2815 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2816}
2817
2818/* Can be overridden by an architecture for additional checks. */
1987 cond_resched();
1988 }
1989 mmap_read_unlock(mm);
1990 return ret;
1991}
1992
1993/*
1994 * Scan swap_map from current position to next entry still in use.

--- 802 unchanged lines hidden (view full) ---

2797 */
2798unsigned long generic_max_swapfile_size(void)
2799{
2800 return swp_offset(pte_to_swp_entry(
2801 swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2802}
2803
2804/* Can be overridden by an architecture for additional checks. */
2819__weak unsigned long max_swapfile_size(void)
2805__weak unsigned long arch_max_swapfile_size(void)
2820{
2821 return generic_max_swapfile_size();
2822}
2823
2824static unsigned long read_swap_header(struct swap_info_struct *p,
2825 union swap_header *swap_header,
2826 struct inode *inode)
2827{

--- 23 unchanged lines hidden (view full) ---

2851 swap_header->info.version);
2852 return 0;
2853 }
2854
2855 p->lowest_bit = 1;
2856 p->cluster_next = 1;
2857 p->cluster_nr = 0;
2858
2806{
2807 return generic_max_swapfile_size();
2808}
2809
2810static unsigned long read_swap_header(struct swap_info_struct *p,
2811 union swap_header *swap_header,
2812 struct inode *inode)
2813{

--- 23 unchanged lines hidden (view full) ---

2837 swap_header->info.version);
2838 return 0;
2839 }
2840
2841 p->lowest_bit = 1;
2842 p->cluster_next = 1;
2843 p->cluster_nr = 0;
2844
2859 maxpages = max_swapfile_size();
2845 maxpages = swapfile_maximum_size;
2860 last_page = swap_header->info.last_page;
2861 if (!last_page) {
2862 pr_warn("Empty swap-file\n");
2863 return 0;
2864 }
2865 if (last_page > maxpages) {
2866 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2867 maxpages << (PAGE_SHIFT - 10),

--- 804 unchanged lines hidden (view full) ---

3672 if (!swap_avail_heads) {
3673 pr_emerg("Not enough memory for swap heads, swap is disabled\n");
3674 return -ENOMEM;
3675 }
3676
3677 for_each_node(nid)
3678 plist_head_init(&swap_avail_heads[nid]);
3679
2846 last_page = swap_header->info.last_page;
2847 if (!last_page) {
2848 pr_warn("Empty swap-file\n");
2849 return 0;
2850 }
2851 if (last_page > maxpages) {
2852 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2853 maxpages << (PAGE_SHIFT - 10),

--- 804 unchanged lines hidden (view full) ---

3658 if (!swap_avail_heads) {
3659 pr_emerg("Not enough memory for swap heads, swap is disabled\n");
3660 return -ENOMEM;
3661 }
3662
3663 for_each_node(nid)
3664 plist_head_init(&swap_avail_heads[nid]);
3665
3666 swapfile_maximum_size = arch_max_swapfile_size();
3667
3668#ifdef CONFIG_MIGRATION
3669 if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))
3670 swap_migration_ad_supported = true;
3671#endif /* CONFIG_MIGRATION */
3672
3680 return 0;
3681}
3682subsys_initcall(swapfile_init);
3673 return 0;
3674}
3675subsys_initcall(swapfile_init);