shmem.c (47e8eec83262083c7da220446551eaad614218ea) shmem.c (e384200e70664cd690cdadb72b2a1bc9dfcdec1a)
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.

--- 20 unchanged lines hidden (view full) ---

29#include <linux/pagemap.h>
30#include <linux/file.h>
31#include <linux/mm.h>
32#include <linux/random.h>
33#include <linux/sched/signal.h>
34#include <linux/export.h>
35#include <linux/swap.h>
36#include <linux/uio.h>
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.

--- 20 unchanged lines hidden (view full) ---

29#include <linux/pagemap.h>
30#include <linux/file.h>
31#include <linux/mm.h>
32#include <linux/random.h>
33#include <linux/sched/signal.h>
34#include <linux/export.h>
35#include <linux/swap.h>
36#include <linux/uio.h>
37#include <linux/khugepaged.h>
38#include <linux/hugetlb.h>
39#include <linux/fs_parser.h>
40#include <linux/swapfile.h>
37#include <linux/hugetlb.h>
38#include <linux/fs_parser.h>
39#include <linux/swapfile.h>
40#include "swap.h"
41
42static struct vfsmount *shm_mnt;
43
44#ifdef CONFIG_SHMEM
45/*
46 * This virtual memory filesystem is heavily based on the ramfs. It
47 * extends ramfs by the ability to use swap and honor resource limits
48 * which makes it a completely usable filesystem.

--- 80 unchanged lines hidden (view full) ---

129static unsigned long shmem_default_max_inodes(void)
130{
131 unsigned long nr_pages = totalram_pages();
132
133 return min(nr_pages - totalhigh_pages(), nr_pages / 2);
134}
135#endif
136
41
42static struct vfsmount *shm_mnt;
43
44#ifdef CONFIG_SHMEM
45/*
46 * This virtual memory filesystem is heavily based on the ramfs. It
47 * extends ramfs by the ability to use swap and honor resource limits
48 * which makes it a completely usable filesystem.

--- 80 unchanged lines hidden (view full) ---

129static unsigned long shmem_default_max_inodes(void)
130{
131 unsigned long nr_pages = totalram_pages();
132
133 return min(nr_pages - totalhigh_pages(), nr_pages / 2);
134}
135#endif
136
137static int shmem_swapin_page(struct inode *inode, pgoff_t index,
138 struct page **pagep, enum sgp_type sgp,
137static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
138 struct folio **foliop, enum sgp_type sgp,
139 gfp_t gfp, struct vm_area_struct *vma,
140 vm_fault_t *fault_type);
141static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
142 struct page **pagep, enum sgp_type sgp,
143 gfp_t gfp, struct vm_area_struct *vma,
144 struct vm_fault *vmf, vm_fault_t *fault_type);
145
146int shmem_getpage(struct inode *inode, pgoff_t index,

--- 401 unchanged lines hidden (view full) ---

548
549static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
550 struct shrink_control *sc, unsigned long nr_to_split)
551{
552 LIST_HEAD(list), *pos, *next;
553 LIST_HEAD(to_remove);
554 struct inode *inode;
555 struct shmem_inode_info *info;
139 gfp_t gfp, struct vm_area_struct *vma,
140 vm_fault_t *fault_type);
141static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
142 struct page **pagep, enum sgp_type sgp,
143 gfp_t gfp, struct vm_area_struct *vma,
144 struct vm_fault *vmf, vm_fault_t *fault_type);
145
146int shmem_getpage(struct inode *inode, pgoff_t index,

--- 401 unchanged lines hidden (view full) ---

548
549static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
550 struct shrink_control *sc, unsigned long nr_to_split)
551{
552 LIST_HEAD(list), *pos, *next;
553 LIST_HEAD(to_remove);
554 struct inode *inode;
555 struct shmem_inode_info *info;
556 struct page *page;
556 struct folio *folio;
557 unsigned long batch = sc ? sc->nr_to_scan : 128;
558 int split = 0;
559
560 if (list_empty(&sbinfo->shrinklist))
561 return SHRINK_STOP;
562
563 spin_lock(&sbinfo->shrinklist_lock);
564 list_for_each_safe(pos, next, &sbinfo->shrinklist) {

--- 27 unchanged lines hidden (view full) ---

592 info = list_entry(pos, struct shmem_inode_info, shrinklist);
593 inode = &info->vfs_inode;
594 list_del_init(&info->shrinklist);
595 iput(inode);
596 }
597
598 list_for_each_safe(pos, next, &list) {
599 int ret;
557 unsigned long batch = sc ? sc->nr_to_scan : 128;
558 int split = 0;
559
560 if (list_empty(&sbinfo->shrinklist))
561 return SHRINK_STOP;
562
563 spin_lock(&sbinfo->shrinklist_lock);
564 list_for_each_safe(pos, next, &sbinfo->shrinklist) {

--- 27 unchanged lines hidden (view full) ---

592 info = list_entry(pos, struct shmem_inode_info, shrinklist);
593 inode = &info->vfs_inode;
594 list_del_init(&info->shrinklist);
595 iput(inode);
596 }
597
598 list_for_each_safe(pos, next, &list) {
599 int ret;
600 pgoff_t index;
600
601 info = list_entry(pos, struct shmem_inode_info, shrinklist);
602 inode = &info->vfs_inode;
603
604 if (nr_to_split && split >= nr_to_split)
605 goto move_back;
606
601
602 info = list_entry(pos, struct shmem_inode_info, shrinklist);
603 inode = &info->vfs_inode;
604
605 if (nr_to_split && split >= nr_to_split)
606 goto move_back;
607
607 page = find_get_page(inode->i_mapping,
608 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
609 if (!page)
608 index = (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT;
609 folio = filemap_get_folio(inode->i_mapping, index);
610 if (!folio)
610 goto drop;
611
612 /* No huge page at the end of the file: nothing to split */
611 goto drop;
612
613 /* No huge page at the end of the file: nothing to split */
613 if (!PageTransHuge(page)) {
614 put_page(page);
614 if (!folio_test_large(folio)) {
615 folio_put(folio);
615 goto drop;
616 }
617
618 /*
619 * Move the inode on the list back to shrinklist if we failed
620 * to lock the page at this time.
621 *
622 * Waiting for the lock may lead to deadlock in the
623 * reclaim path.
624 */
616 goto drop;
617 }
618
619 /*
620 * Move the inode on the list back to shrinklist if we failed
621 * to lock the page at this time.
622 *
623 * Waiting for the lock may lead to deadlock in the
624 * reclaim path.
625 */
625 if (!trylock_page(page)) {
626 put_page(page);
626 if (!folio_trylock(folio)) {
627 folio_put(folio);
627 goto move_back;
628 }
629
628 goto move_back;
629 }
630
630 ret = split_huge_page(page);
631 unlock_page(page);
632 put_page(page);
631 ret = split_huge_page(&folio->page);
632 folio_unlock(folio);
633 folio_put(folio);
633
634 /* If split failed move the inode on the list back to shrinklist */
635 if (ret)
636 goto move_back;
637
638 split++;
639drop:
640 list_del_init(&info->shrinklist);

--- 48 unchanged lines hidden (view full) ---

689{
690 return 0;
691}
692#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
693
694/*
695 * Like add_to_page_cache_locked, but error if expected item has gone.
696 */
634
635 /* If split failed move the inode on the list back to shrinklist */
636 if (ret)
637 goto move_back;
638
639 split++;
640drop:
641 list_del_init(&info->shrinklist);

--- 48 unchanged lines hidden (view full) ---

690{
691 return 0;
692}
693#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
694
695/*
696 * Like add_to_page_cache_locked, but error if expected item has gone.
697 */
697static int shmem_add_to_page_cache(struct page *page,
698static int shmem_add_to_page_cache(struct folio *folio,
698 struct address_space *mapping,
699 pgoff_t index, void *expected, gfp_t gfp,
700 struct mm_struct *charge_mm)
701{
699 struct address_space *mapping,
700 pgoff_t index, void *expected, gfp_t gfp,
701 struct mm_struct *charge_mm)
702{
702 XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
703 unsigned long nr = compound_nr(page);
703 XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio));
704 long nr = folio_nr_pages(folio);
704 int error;
705
705 int error;
706
706 VM_BUG_ON_PAGE(PageTail(page), page);
707 VM_BUG_ON_PAGE(index != round_down(index, nr), page);
708 VM_BUG_ON_PAGE(!PageLocked(page), page);
709 VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
710 VM_BUG_ON(expected && PageTransHuge(page));
707 VM_BUG_ON_FOLIO(index != round_down(index, nr), folio);
708 VM_BUG_ON_FOLIO(!folio_test_locked(folio), folio);
709 VM_BUG_ON_FOLIO(!folio_test_swapbacked(folio), folio);
710 VM_BUG_ON(expected && folio_test_large(folio));
711
711
712 page_ref_add(page, nr);
713 page->mapping = mapping;
714 page->index = index;
712 folio_ref_add(folio, nr);
713 folio->mapping = mapping;
714 folio->index = index;
715
715
716 if (!PageSwapCache(page)) {
717 error = mem_cgroup_charge(page_folio(page), charge_mm, gfp);
716 if (!folio_test_swapcache(folio)) {
717 error = mem_cgroup_charge(folio, charge_mm, gfp);
718 if (error) {
718 if (error) {
719 if (PageTransHuge(page)) {
719 if (folio_test_pmd_mappable(folio)) {
720 count_vm_event(THP_FILE_FALLBACK);
721 count_vm_event(THP_FILE_FALLBACK_CHARGE);
722 }
723 goto error;
724 }
725 }
720 count_vm_event(THP_FILE_FALLBACK);
721 count_vm_event(THP_FILE_FALLBACK_CHARGE);
722 }
723 goto error;
724 }
725 }
726 cgroup_throttle_swaprate(page, gfp);
726 folio_throttle_swaprate(folio, gfp);
727
728 do {
729 xas_lock_irq(&xas);
730 if (expected != xas_find_conflict(&xas)) {
731 xas_set_err(&xas, -EEXIST);
732 goto unlock;
733 }
734 if (expected && xas_find_conflict(&xas)) {
735 xas_set_err(&xas, -EEXIST);
736 goto unlock;
737 }
727
728 do {
729 xas_lock_irq(&xas);
730 if (expected != xas_find_conflict(&xas)) {
731 xas_set_err(&xas, -EEXIST);
732 goto unlock;
733 }
734 if (expected && xas_find_conflict(&xas)) {
735 xas_set_err(&xas, -EEXIST);
736 goto unlock;
737 }
738 xas_store(&xas, page);
738 xas_store(&xas, folio);
739 if (xas_error(&xas))
740 goto unlock;
739 if (xas_error(&xas))
740 goto unlock;
741 if (PageTransHuge(page)) {
741 if (folio_test_pmd_mappable(folio)) {
742 count_vm_event(THP_FILE_ALLOC);
742 count_vm_event(THP_FILE_ALLOC);
743 __mod_lruvec_page_state(page, NR_SHMEM_THPS, nr);
743 __lruvec_stat_mod_folio(folio, NR_SHMEM_THPS, nr);
744 }
745 mapping->nrpages += nr;
744 }
745 mapping->nrpages += nr;
746 __mod_lruvec_page_state(page, NR_FILE_PAGES, nr);
747 __mod_lruvec_page_state(page, NR_SHMEM, nr);
746 __lruvec_stat_mod_folio(folio, NR_FILE_PAGES, nr);
747 __lruvec_stat_mod_folio(folio, NR_SHMEM, nr);
748unlock:
749 xas_unlock_irq(&xas);
750 } while (xas_nomem(&xas, gfp));
751
752 if (xas_error(&xas)) {
753 error = xas_error(&xas);
754 goto error;
755 }
756
757 return 0;
758error:
748unlock:
749 xas_unlock_irq(&xas);
750 } while (xas_nomem(&xas, gfp));
751
752 if (xas_error(&xas)) {
753 error = xas_error(&xas);
754 goto error;
755 }
756
757 return 0;
758error:
759 page->mapping = NULL;
760 page_ref_sub(page, nr);
759 folio->mapping = NULL;
760 folio_ref_sub(folio, nr);
761 return error;
762}
763
764/*
765 * Like delete_from_page_cache, but substitutes swap for page.
766 */
767static void shmem_delete_from_page_cache(struct page *page, void *radswap)
768{

--- 384 unchanged lines hidden (view full) ---

1153
1154 simple_xattrs_free(&info->xattrs);
1155 WARN_ON(inode->i_blocks);
1156 shmem_free_inode(inode->i_sb);
1157 clear_inode(inode);
1158}
1159
1160static int shmem_find_swap_entries(struct address_space *mapping,
761 return error;
762}
763
764/*
765 * Like delete_from_page_cache, but substitutes swap for page.
766 */
767static void shmem_delete_from_page_cache(struct page *page, void *radswap)
768{

--- 384 unchanged lines hidden (view full) ---

1153
1154 simple_xattrs_free(&info->xattrs);
1155 WARN_ON(inode->i_blocks);
1156 shmem_free_inode(inode->i_sb);
1157 clear_inode(inode);
1158}
1159
1160static int shmem_find_swap_entries(struct address_space *mapping,
1161 pgoff_t start, unsigned int nr_entries,
1162 struct page **entries, pgoff_t *indices,
1163 unsigned int type)
1161 pgoff_t start, struct folio_batch *fbatch,
1162 pgoff_t *indices, unsigned int type)
1164{
1165 XA_STATE(xas, &mapping->i_pages, start);
1163{
1164 XA_STATE(xas, &mapping->i_pages, start);
1166 struct page *page;
1165 struct folio *folio;
1167 swp_entry_t entry;
1166 swp_entry_t entry;
1168 unsigned int ret = 0;
1169
1167
1170 if (!nr_entries)
1171 return 0;
1172
1173 rcu_read_lock();
1168 rcu_read_lock();
1174 xas_for_each(&xas, page, ULONG_MAX) {
1175 if (xas_retry(&xas, page))
1169 xas_for_each(&xas, folio, ULONG_MAX) {
1170 if (xas_retry(&xas, folio))
1176 continue;
1177
1171 continue;
1172
1178 if (!xa_is_value(page))
1173 if (!xa_is_value(folio))
1179 continue;
1180
1174 continue;
1175
1181 entry = radix_to_swp_entry(page);
1176 entry = radix_to_swp_entry(folio);
1182 if (swp_type(entry) != type)
1183 continue;
1184
1177 if (swp_type(entry) != type)
1178 continue;
1179
1185 indices[ret] = xas.xa_index;
1186 entries[ret] = page;
1180 indices[folio_batch_count(fbatch)] = xas.xa_index;
1181 if (!folio_batch_add(fbatch, folio))
1182 break;
1187
1188 if (need_resched()) {
1189 xas_pause(&xas);
1190 cond_resched_rcu();
1191 }
1183
1184 if (need_resched()) {
1185 xas_pause(&xas);
1186 cond_resched_rcu();
1187 }
1192 if (++ret == nr_entries)
1193 break;
1194 }
1195 rcu_read_unlock();
1196
1188 }
1189 rcu_read_unlock();
1190
1197 return ret;
1191 return xas.xa_index;
1198}
1199
1200/*
1201 * Move the swapped pages for an inode to page cache. Returns the count
1202 * of pages swapped in, or the error in case of failure.
1203 */
1192}
1193
1194/*
1195 * Move the swapped pages for an inode to page cache. Returns the count
1196 * of pages swapped in, or the error in case of failure.
1197 */
1204static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
1205 pgoff_t *indices)
1198static int shmem_unuse_swap_entries(struct inode *inode,
1199 struct folio_batch *fbatch, pgoff_t *indices)
1206{
1207 int i = 0;
1208 int ret = 0;
1209 int error = 0;
1210 struct address_space *mapping = inode->i_mapping;
1211
1200{
1201 int i = 0;
1202 int ret = 0;
1203 int error = 0;
1204 struct address_space *mapping = inode->i_mapping;
1205
1212 for (i = 0; i < pvec.nr; i++) {
1213 struct page *page = pvec.pages[i];
1206 for (i = 0; i < folio_batch_count(fbatch); i++) {
1207 struct folio *folio = fbatch->folios[i];
1214
1208
1215 if (!xa_is_value(page))
1209 if (!xa_is_value(folio))
1216 continue;
1210 continue;
1217 error = shmem_swapin_page(inode, indices[i],
1218 &page, SGP_CACHE,
1211 error = shmem_swapin_folio(inode, indices[i],
1212 &folio, SGP_CACHE,
1219 mapping_gfp_mask(mapping),
1220 NULL, NULL);
1221 if (error == 0) {
1213 mapping_gfp_mask(mapping),
1214 NULL, NULL);
1215 if (error == 0) {
1222 unlock_page(page);
1223 put_page(page);
1216 folio_unlock(folio);
1217 folio_put(folio);
1224 ret++;
1225 }
1226 if (error == -ENOMEM)
1227 break;
1228 error = 0;
1229 }
1230 return error ? error : ret;
1231}
1232
1233/*
1234 * If swap found in inode, free it and move page from swapcache to filecache.
1235 */
1236static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1237{
1238 struct address_space *mapping = inode->i_mapping;
1239 pgoff_t start = 0;
1218 ret++;
1219 }
1220 if (error == -ENOMEM)
1221 break;
1222 error = 0;
1223 }
1224 return error ? error : ret;
1225}
1226
1227/*
1228 * If swap found in inode, free it and move page from swapcache to filecache.
1229 */
1230static int shmem_unuse_inode(struct inode *inode, unsigned int type)
1231{
1232 struct address_space *mapping = inode->i_mapping;
1233 pgoff_t start = 0;
1240 struct pagevec pvec;
1234 struct folio_batch fbatch;
1241 pgoff_t indices[PAGEVEC_SIZE];
1242 int ret = 0;
1243
1235 pgoff_t indices[PAGEVEC_SIZE];
1236 int ret = 0;
1237
1244 pagevec_init(&pvec);
1245 do {
1238 do {
1246 unsigned int nr_entries = PAGEVEC_SIZE;
1247
1248 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
1249 pvec.pages, indices, type);
1250 if (pvec.nr == 0) {
1239 folio_batch_init(&fbatch);
1240 shmem_find_swap_entries(mapping, start, &fbatch, indices, type);
1241 if (folio_batch_count(&fbatch) == 0) {
1251 ret = 0;
1252 break;
1253 }
1254
1242 ret = 0;
1243 break;
1244 }
1245
1255 ret = shmem_unuse_swap_entries(inode, pvec, indices);
1246 ret = shmem_unuse_swap_entries(inode, &fbatch, indices);
1256 if (ret < 0)
1257 break;
1258
1247 if (ret < 0)
1248 break;
1249
1259 start = indices[pvec.nr - 1];
1250 start = indices[folio_batch_count(&fbatch) - 1];
1260 } while (true);
1261
1262 return ret;
1263}
1264
1265/*
1266 * Read all the shared memory data that resides in the swap
1267 * device 'type' back into memory, so the swap device can be

--- 39 unchanged lines hidden (view full) ---

1307 return error;
1308}
1309
1310/*
1311 * Move the page from the page cache to the swap cache.
1312 */
1313static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1314{
1251 } while (true);
1252
1253 return ret;
1254}
1255
1256/*
1257 * Read all the shared memory data that resides in the swap
1258 * device 'type' back into memory, so the swap device can be

--- 39 unchanged lines hidden (view full) ---

1298 return error;
1299}
1300
1301/*
1302 * Move the page from the page cache to the swap cache.
1303 */
1304static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1305{
1306 struct folio *folio = page_folio(page);
1315 struct shmem_inode_info *info;
1316 struct address_space *mapping;
1317 struct inode *inode;
1318 swp_entry_t swap;
1319 pgoff_t index;
1320
1321 /*
1322 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or

--- 57 unchanged lines hidden (view full) ---

1380 if (shmem_falloc)
1381 goto redirty;
1382 }
1383 clear_highpage(page);
1384 flush_dcache_page(page);
1385 SetPageUptodate(page);
1386 }
1387
1307 struct shmem_inode_info *info;
1308 struct address_space *mapping;
1309 struct inode *inode;
1310 swp_entry_t swap;
1311 pgoff_t index;
1312
1313 /*
1314 * If /sys/kernel/mm/transparent_hugepage/shmem_enabled is "always" or

--- 57 unchanged lines hidden (view full) ---

1372 if (shmem_falloc)
1373 goto redirty;
1374 }
1375 clear_highpage(page);
1376 flush_dcache_page(page);
1377 SetPageUptodate(page);
1378 }
1379
1388 swap = get_swap_page(page);
1380 swap = folio_alloc_swap(folio);
1389 if (!swap.val)
1390 goto redirty;
1391
1392 /*
1393 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1394 * if it's not already there. Do it now before the page is
1395 * moved to swap cache, when its pagelock no longer protects
1396 * the inode from eviction. But don't unlock the mutex until

--- 119 unchanged lines hidden (view full) ---

1516 * and the intersection of the allow flags.
1517 */
1518 result |= (limit_gfp & denyflags);
1519 result |= (huge_gfp & limit_gfp) & allowflags;
1520
1521 return result;
1522}
1523
1381 if (!swap.val)
1382 goto redirty;
1383
1384 /*
1385 * Add inode to shmem_unuse()'s list of swapped-out inodes,
1386 * if it's not already there. Do it now before the page is
1387 * moved to swap cache, when its pagelock no longer protects
1388 * the inode from eviction. But don't unlock the mutex until

--- 119 unchanged lines hidden (view full) ---

1508 * and the intersection of the allow flags.
1509 */
1510 result |= (limit_gfp & denyflags);
1511 result |= (huge_gfp & limit_gfp) & allowflags;
1512
1513 return result;
1514}
1515
1524static struct page *shmem_alloc_hugepage(gfp_t gfp,
1516static struct folio *shmem_alloc_hugefolio(gfp_t gfp,
1525 struct shmem_inode_info *info, pgoff_t index)
1526{
1527 struct vm_area_struct pvma;
1528 struct address_space *mapping = info->vfs_inode.i_mapping;
1529 pgoff_t hindex;
1517 struct shmem_inode_info *info, pgoff_t index)
1518{
1519 struct vm_area_struct pvma;
1520 struct address_space *mapping = info->vfs_inode.i_mapping;
1521 pgoff_t hindex;
1530 struct page *page;
1522 struct folio *folio;
1531
1532 hindex = round_down(index, HPAGE_PMD_NR);
1533 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1534 XA_PRESENT))
1535 return NULL;
1536
1537 shmem_pseudo_vma_init(&pvma, info, hindex);
1523
1524 hindex = round_down(index, HPAGE_PMD_NR);
1525 if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1526 XA_PRESENT))
1527 return NULL;
1528
1529 shmem_pseudo_vma_init(&pvma, info, hindex);
1538 page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1530 folio = vma_alloc_folio(gfp, HPAGE_PMD_ORDER, &pvma, 0, true);
1539 shmem_pseudo_vma_destroy(&pvma);
1531 shmem_pseudo_vma_destroy(&pvma);
1540 if (page)
1541 prep_transhuge_page(page);
1542 else
1532 if (!folio)
1543 count_vm_event(THP_FILE_FALLBACK);
1533 count_vm_event(THP_FILE_FALLBACK);
1544 return page;
1534 return folio;
1545}
1546
1535}
1536
1547static struct page *shmem_alloc_page(gfp_t gfp,
1537static struct folio *shmem_alloc_folio(gfp_t gfp,
1548 struct shmem_inode_info *info, pgoff_t index)
1549{
1550 struct vm_area_struct pvma;
1538 struct shmem_inode_info *info, pgoff_t index)
1539{
1540 struct vm_area_struct pvma;
1551 struct page *page;
1541 struct folio *folio;
1552
1553 shmem_pseudo_vma_init(&pvma, info, index);
1542
1543 shmem_pseudo_vma_init(&pvma, info, index);
1554 page = alloc_page_vma(gfp, &pvma, 0);
1544 folio = vma_alloc_folio(gfp, 0, &pvma, 0, false);
1555 shmem_pseudo_vma_destroy(&pvma);
1556
1545 shmem_pseudo_vma_destroy(&pvma);
1546
1557 return page;
1547 return folio;
1558}
1559
1548}
1549
1560static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1561 struct inode *inode,
1550static struct page *shmem_alloc_page(gfp_t gfp,
1551 struct shmem_inode_info *info, pgoff_t index)
1552{
1553 return &shmem_alloc_folio(gfp, info, index)->page;
1554}
1555
1556static struct folio *shmem_alloc_and_acct_folio(gfp_t gfp, struct inode *inode,
1562 pgoff_t index, bool huge)
1563{
1564 struct shmem_inode_info *info = SHMEM_I(inode);
1557 pgoff_t index, bool huge)
1558{
1559 struct shmem_inode_info *info = SHMEM_I(inode);
1565 struct page *page;
1560 struct folio *folio;
1566 int nr;
1567 int err = -ENOSPC;
1568
1569 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1570 huge = false;
1571 nr = huge ? HPAGE_PMD_NR : 1;
1572
1573 if (!shmem_inode_acct_block(inode, nr))
1574 goto failed;
1575
1576 if (huge)
1561 int nr;
1562 int err = -ENOSPC;
1563
1564 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1565 huge = false;
1566 nr = huge ? HPAGE_PMD_NR : 1;
1567
1568 if (!shmem_inode_acct_block(inode, nr))
1569 goto failed;
1570
1571 if (huge)
1577 page = shmem_alloc_hugepage(gfp, info, index);
1572 folio = shmem_alloc_hugefolio(gfp, info, index);
1578 else
1573 else
1579 page = shmem_alloc_page(gfp, info, index);
1580 if (page) {
1581 __SetPageLocked(page);
1582 __SetPageSwapBacked(page);
1583 return page;
1574 folio = shmem_alloc_folio(gfp, info, index);
1575 if (folio) {
1576 __folio_set_locked(folio);
1577 __folio_set_swapbacked(folio);
1578 return folio;
1584 }
1585
1586 err = -ENOMEM;
1587 shmem_inode_unacct_blocks(inode, nr);
1588failed:
1589 return ERR_PTR(err);
1590}
1591

--- 4 unchanged lines hidden (view full) ---

1596 * ignorance of the mapping it belongs to. If that mapping has special
1597 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1598 * we may need to copy to a suitable page before moving to filecache.
1599 *
1600 * In a future release, this may well be extended to respect cpuset and
1601 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1602 * but for now it is a simple matter of zone.
1603 */
1579 }
1580
1581 err = -ENOMEM;
1582 shmem_inode_unacct_blocks(inode, nr);
1583failed:
1584 return ERR_PTR(err);
1585}
1586

--- 4 unchanged lines hidden (view full) ---

1591 * ignorance of the mapping it belongs to. If that mapping has special
1592 * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1593 * we may need to copy to a suitable page before moving to filecache.
1594 *
1595 * In a future release, this may well be extended to respect cpuset and
1596 * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1597 * but for now it is a simple matter of zone.
1598 */
1604static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1599static bool shmem_should_replace_folio(struct folio *folio, gfp_t gfp)
1605{
1600{
1606 return page_zonenum(page) > gfp_zone(gfp);
1601 return folio_zonenum(folio) > gfp_zone(gfp);
1607}
1608
1609static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1610 struct shmem_inode_info *info, pgoff_t index)
1611{
1612 struct page *oldpage, *newpage;
1613 struct folio *old, *new;
1614 struct address_space *swap_mapping;

--- 62 unchanged lines hidden (view full) ---

1677}
1678
1679/*
1680 * Swap in the page pointed to by *pagep.
1681 * Caller has to make sure that *pagep contains a valid swapped page.
1682 * Returns 0 and the page in pagep if success. On failure, returns the
1683 * error code and NULL in *pagep.
1684 */
1602}
1603
1604static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1605 struct shmem_inode_info *info, pgoff_t index)
1606{
1607 struct page *oldpage, *newpage;
1608 struct folio *old, *new;
1609 struct address_space *swap_mapping;

--- 62 unchanged lines hidden (view full) ---

1672}
1673
1674/*
1675 * Swap in the page pointed to by *pagep.
1676 * Caller has to make sure that *pagep contains a valid swapped page.
1677 * Returns 0 and the page in pagep if success. On failure, returns the
1678 * error code and NULL in *pagep.
1679 */
1685static int shmem_swapin_page(struct inode *inode, pgoff_t index,
1686 struct page **pagep, enum sgp_type sgp,
1680static int shmem_swapin_folio(struct inode *inode, pgoff_t index,
1681 struct folio **foliop, enum sgp_type sgp,
1687 gfp_t gfp, struct vm_area_struct *vma,
1688 vm_fault_t *fault_type)
1689{
1690 struct address_space *mapping = inode->i_mapping;
1691 struct shmem_inode_info *info = SHMEM_I(inode);
1692 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1693 struct page *page;
1682 gfp_t gfp, struct vm_area_struct *vma,
1683 vm_fault_t *fault_type)
1684{
1685 struct address_space *mapping = inode->i_mapping;
1686 struct shmem_inode_info *info = SHMEM_I(inode);
1687 struct mm_struct *charge_mm = vma ? vma->vm_mm : NULL;
1688 struct page *page;
1689 struct folio *folio = NULL;
1694 swp_entry_t swap;
1695 int error;
1696
1690 swp_entry_t swap;
1691 int error;
1692
1697 VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
1698 swap = radix_to_swp_entry(*pagep);
1699 *pagep = NULL;
1693 VM_BUG_ON(!*foliop || !xa_is_value(*foliop));
1694 swap = radix_to_swp_entry(*foliop);
1695 *foliop = NULL;
1700
1701 /* Look it up and read it in.. */
1702 page = lookup_swap_cache(swap, NULL, 0);
1703 if (!page) {
1704 /* Or update major stats only when swapin succeeds?? */
1705 if (fault_type) {
1706 *fault_type |= VM_FAULT_MAJOR;
1707 count_vm_event(PGMAJFAULT);
1708 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1709 }
1710 /* Here we actually start the io */
1711 page = shmem_swapin(swap, gfp, info, index);
1712 if (!page) {
1713 error = -ENOMEM;
1714 goto failed;
1715 }
1716 }
1696
1697 /* Look it up and read it in.. */
1698 page = lookup_swap_cache(swap, NULL, 0);
1699 if (!page) {
1700 /* Or update major stats only when swapin succeeds?? */
1701 if (fault_type) {
1702 *fault_type |= VM_FAULT_MAJOR;
1703 count_vm_event(PGMAJFAULT);
1704 count_memcg_event_mm(charge_mm, PGMAJFAULT);
1705 }
1706 /* Here we actually start the io */
1707 page = shmem_swapin(swap, gfp, info, index);
1708 if (!page) {
1709 error = -ENOMEM;
1710 goto failed;
1711 }
1712 }
1713 folio = page_folio(page);
1717
1718 /* We have to do this with page locked to prevent races */
1714
1715 /* We have to do this with page locked to prevent races */
1719 lock_page(page);
1720 if (!PageSwapCache(page) || page_private(page) != swap.val ||
1716 folio_lock(folio);
1717 if (!folio_test_swapcache(folio) ||
1718 folio_swap_entry(folio).val != swap.val ||
1721 !shmem_confirm_swap(mapping, index, swap)) {
1722 error = -EEXIST;
1723 goto unlock;
1724 }
1719 !shmem_confirm_swap(mapping, index, swap)) {
1720 error = -EEXIST;
1721 goto unlock;
1722 }
1725 if (!PageUptodate(page)) {
1723 if (!folio_test_uptodate(folio)) {
1726 error = -EIO;
1727 goto failed;
1728 }
1724 error = -EIO;
1725 goto failed;
1726 }
1729 wait_on_page_writeback(page);
1727 folio_wait_writeback(folio);
1730
1731 /*
1732 * Some architectures may have to restore extra metadata to the
1728
1729 /*
1730 * Some architectures may have to restore extra metadata to the
1733 * physical page after reading from swap.
1731 * folio after reading from swap.
1734 */
1732 */
1735 arch_swap_restore(swap, page);
1733 arch_swap_restore(swap, folio);
1736
1734
1737 if (shmem_should_replace_page(page, gfp)) {
1735 if (shmem_should_replace_folio(folio, gfp)) {
1738 error = shmem_replace_page(&page, gfp, info, index);
1739 if (error)
1740 goto failed;
1741 }
1742
1736 error = shmem_replace_page(&page, gfp, info, index);
1737 if (error)
1738 goto failed;
1739 }
1740
1743 error = shmem_add_to_page_cache(page, mapping, index,
1741 error = shmem_add_to_page_cache(folio, mapping, index,
1744 swp_to_radix_entry(swap), gfp,
1745 charge_mm);
1746 if (error)
1747 goto failed;
1748
1749 spin_lock_irq(&info->lock);
1750 info->swapped--;
1751 shmem_recalc_inode(inode);
1752 spin_unlock_irq(&info->lock);
1753
1754 if (sgp == SGP_WRITE)
1742 swp_to_radix_entry(swap), gfp,
1743 charge_mm);
1744 if (error)
1745 goto failed;
1746
1747 spin_lock_irq(&info->lock);
1748 info->swapped--;
1749 shmem_recalc_inode(inode);
1750 spin_unlock_irq(&info->lock);
1751
1752 if (sgp == SGP_WRITE)
1755 mark_page_accessed(page);
1753 folio_mark_accessed(folio);
1756
1754
1757 delete_from_swap_cache(page);
1758 set_page_dirty(page);
1755 delete_from_swap_cache(&folio->page);
1756 folio_mark_dirty(folio);
1759 swap_free(swap);
1760
1757 swap_free(swap);
1758
1761 *pagep = page;
1759 *foliop = folio;
1762 return 0;
1763failed:
1764 if (!shmem_confirm_swap(mapping, index, swap))
1765 error = -EEXIST;
1766unlock:
1760 return 0;
1761failed:
1762 if (!shmem_confirm_swap(mapping, index, swap))
1763 error = -EEXIST;
1764unlock:
1767 if (page) {
1768 unlock_page(page);
1769 put_page(page);
1765 if (folio) {
1766 folio_unlock(folio);
1767 folio_put(folio);
1770 }
1771
1772 return error;
1773}
1774
1775/*
1776 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1777 *

--- 8 unchanged lines hidden (view full) ---

1786 struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1787 struct vm_area_struct *vma, struct vm_fault *vmf,
1788 vm_fault_t *fault_type)
1789{
1790 struct address_space *mapping = inode->i_mapping;
1791 struct shmem_inode_info *info = SHMEM_I(inode);
1792 struct shmem_sb_info *sbinfo;
1793 struct mm_struct *charge_mm;
1768 }
1769
1770 return error;
1771}
1772
1773/*
1774 * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1775 *

--- 8 unchanged lines hidden (view full) ---

1784 struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1785 struct vm_area_struct *vma, struct vm_fault *vmf,
1786 vm_fault_t *fault_type)
1787{
1788 struct address_space *mapping = inode->i_mapping;
1789 struct shmem_inode_info *info = SHMEM_I(inode);
1790 struct shmem_sb_info *sbinfo;
1791 struct mm_struct *charge_mm;
1794 struct page *page;
1792 struct folio *folio;
1795 pgoff_t hindex = index;
1796 gfp_t huge_gfp;
1797 int error;
1798 int once = 0;
1799 int alloced = 0;
1800
1801 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1802 return -EFBIG;
1803repeat:
1804 if (sgp <= SGP_CACHE &&
1805 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1806 return -EINVAL;
1807 }
1808
1809 sbinfo = SHMEM_SB(inode->i_sb);
1810 charge_mm = vma ? vma->vm_mm : NULL;
1811
1793 pgoff_t hindex = index;
1794 gfp_t huge_gfp;
1795 int error;
1796 int once = 0;
1797 int alloced = 0;
1798
1799 if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1800 return -EFBIG;
1801repeat:
1802 if (sgp <= SGP_CACHE &&
1803 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1804 return -EINVAL;
1805 }
1806
1807 sbinfo = SHMEM_SB(inode->i_sb);
1808 charge_mm = vma ? vma->vm_mm : NULL;
1809
1812 page = pagecache_get_page(mapping, index,
1813 FGP_ENTRY | FGP_HEAD | FGP_LOCK, 0);
1814
1815 if (page && vma && userfaultfd_minor(vma)) {
1816 if (!xa_is_value(page)) {
1817 unlock_page(page);
1818 put_page(page);
1810 folio = __filemap_get_folio(mapping, index, FGP_ENTRY | FGP_LOCK, 0);
1811 if (folio && vma && userfaultfd_minor(vma)) {
1812 if (!xa_is_value(folio)) {
1813 folio_unlock(folio);
1814 folio_put(folio);
1819 }
1820 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1821 return 0;
1822 }
1823
1815 }
1816 *fault_type = handle_userfault(vmf, VM_UFFD_MINOR);
1817 return 0;
1818 }
1819
1824 if (xa_is_value(page)) {
1825 error = shmem_swapin_page(inode, index, &page,
1820 if (xa_is_value(folio)) {
1821 error = shmem_swapin_folio(inode, index, &folio,
1826 sgp, gfp, vma, fault_type);
1827 if (error == -EEXIST)
1828 goto repeat;
1829
1822 sgp, gfp, vma, fault_type);
1823 if (error == -EEXIST)
1824 goto repeat;
1825
1830 *pagep = page;
1826 *pagep = &folio->page;
1831 return error;
1832 }
1833
1827 return error;
1828 }
1829
1834 if (page) {
1835 hindex = page->index;
1830 if (folio) {
1831 hindex = folio->index;
1836 if (sgp == SGP_WRITE)
1832 if (sgp == SGP_WRITE)
1837 mark_page_accessed(page);
1838 if (PageUptodate(page))
1833 folio_mark_accessed(folio);
1834 if (folio_test_uptodate(folio))
1839 goto out;
1840 /* fallocated page */
1841 if (sgp != SGP_READ)
1842 goto clear;
1835 goto out;
1836 /* fallocated page */
1837 if (sgp != SGP_READ)
1838 goto clear;
1843 unlock_page(page);
1844 put_page(page);
1839 folio_unlock(folio);
1840 folio_put(folio);
1845 }
1846
1847 /*
1848 * SGP_READ: succeed on hole, with NULL page, letting caller zero.
1849 * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail.
1850 */
1851 *pagep = NULL;
1852 if (sgp == SGP_READ)

--- 10 unchanged lines hidden (view full) ---

1863 return 0;
1864 }
1865
1866 if (!shmem_is_huge(vma, inode, index))
1867 goto alloc_nohuge;
1868
1869 huge_gfp = vma_thp_gfp_mask(vma);
1870 huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1841 }
1842
1843 /*
1844 * SGP_READ: succeed on hole, with NULL page, letting caller zero.
1845 * SGP_NOALLOC: fail on hole, with NULL page, letting caller fail.
1846 */
1847 *pagep = NULL;
1848 if (sgp == SGP_READ)

--- 10 unchanged lines hidden (view full) ---

1859 return 0;
1860 }
1861
1862 if (!shmem_is_huge(vma, inode, index))
1863 goto alloc_nohuge;
1864
1865 huge_gfp = vma_thp_gfp_mask(vma);
1866 huge_gfp = limit_gfp_mask(huge_gfp, gfp);
1871 page = shmem_alloc_and_acct_page(huge_gfp, inode, index, true);
1872 if (IS_ERR(page)) {
1867 folio = shmem_alloc_and_acct_folio(huge_gfp, inode, index, true);
1868 if (IS_ERR(folio)) {
1873alloc_nohuge:
1869alloc_nohuge:
1874 page = shmem_alloc_and_acct_page(gfp, inode,
1875 index, false);
1870 folio = shmem_alloc_and_acct_folio(gfp, inode, index, false);
1876 }
1871 }
1877 if (IS_ERR(page)) {
1872 if (IS_ERR(folio)) {
1878 int retry = 5;
1879
1873 int retry = 5;
1874
1880 error = PTR_ERR(page);
1881 page = NULL;
1875 error = PTR_ERR(folio);
1876 folio = NULL;
1882 if (error != -ENOSPC)
1883 goto unlock;
1884 /*
1885 * Try to reclaim some space by splitting a huge page
1886 * beyond i_size on the filesystem.
1887 */
1888 while (retry--) {
1889 int ret;
1890
1891 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1892 if (ret == SHRINK_STOP)
1893 break;
1894 if (ret)
1895 goto alloc_nohuge;
1896 }
1897 goto unlock;
1898 }
1899
1877 if (error != -ENOSPC)
1878 goto unlock;
1879 /*
1880 * Try to reclaim some space by splitting a huge page
1881 * beyond i_size on the filesystem.
1882 */
1883 while (retry--) {
1884 int ret;
1885
1886 ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1887 if (ret == SHRINK_STOP)
1888 break;
1889 if (ret)
1890 goto alloc_nohuge;
1891 }
1892 goto unlock;
1893 }
1894
1900 if (PageTransHuge(page))
1901 hindex = round_down(index, HPAGE_PMD_NR);
1902 else
1903 hindex = index;
1895 hindex = round_down(index, folio_nr_pages(folio));
1904
1905 if (sgp == SGP_WRITE)
1896
1897 if (sgp == SGP_WRITE)
1906 __SetPageReferenced(page);
1898 __folio_set_referenced(folio);
1907
1899
1908 error = shmem_add_to_page_cache(page, mapping, hindex,
1900 error = shmem_add_to_page_cache(folio, mapping, hindex,
1909 NULL, gfp & GFP_RECLAIM_MASK,
1910 charge_mm);
1911 if (error)
1912 goto unacct;
1901 NULL, gfp & GFP_RECLAIM_MASK,
1902 charge_mm);
1903 if (error)
1904 goto unacct;
1913 lru_cache_add(page);
1905 folio_add_lru(folio);
1914
1915 spin_lock_irq(&info->lock);
1906
1907 spin_lock_irq(&info->lock);
1916 info->alloced += compound_nr(page);
1917 inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1908 info->alloced += folio_nr_pages(folio);
1909 inode->i_blocks += BLOCKS_PER_PAGE << folio_order(folio);
1918 shmem_recalc_inode(inode);
1919 spin_unlock_irq(&info->lock);
1920 alloced = true;
1921
1910 shmem_recalc_inode(inode);
1911 spin_unlock_irq(&info->lock);
1912 alloced = true;
1913
1922 if (PageTransHuge(page) &&
1914 if (folio_test_pmd_mappable(folio) &&
1923 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1924 hindex + HPAGE_PMD_NR - 1) {
1925 /*
1926 * Part of the huge page is beyond i_size: subject
1927 * to shrink under memory pressure.
1928 */
1929 spin_lock(&sbinfo->shrinklist_lock);
1930 /*

--- 14 unchanged lines hidden (view full) ---

1945 if (sgp == SGP_FALLOC)
1946 sgp = SGP_WRITE;
1947clear:
1948 /*
1949 * Let SGP_WRITE caller clear ends if write does not fill page;
1950 * but SGP_FALLOC on a page fallocated earlier must initialize
1951 * it now, lest undo on failure cancel our earlier guarantee.
1952 */
1915 DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1916 hindex + HPAGE_PMD_NR - 1) {
1917 /*
1918 * Part of the huge page is beyond i_size: subject
1919 * to shrink under memory pressure.
1920 */
1921 spin_lock(&sbinfo->shrinklist_lock);
1922 /*

--- 14 unchanged lines hidden (view full) ---

1937 if (sgp == SGP_FALLOC)
1938 sgp = SGP_WRITE;
1939clear:
1940 /*
1941 * Let SGP_WRITE caller clear ends if write does not fill page;
1942 * but SGP_FALLOC on a page fallocated earlier must initialize
1943 * it now, lest undo on failure cancel our earlier guarantee.
1944 */
1953 if (sgp != SGP_WRITE && !PageUptodate(page)) {
1954 int i;
1945 if (sgp != SGP_WRITE && !folio_test_uptodate(folio)) {
1946 long i, n = folio_nr_pages(folio);
1955
1947
1956 for (i = 0; i < compound_nr(page); i++) {
1957 clear_highpage(page + i);
1958 flush_dcache_page(page + i);
1959 }
1960 SetPageUptodate(page);
1948 for (i = 0; i < n; i++)
1949 clear_highpage(folio_page(folio, i));
1950 flush_dcache_folio(folio);
1951 folio_mark_uptodate(folio);
1961 }
1962
1963 /* Perhaps the file has been truncated since we checked */
1964 if (sgp <= SGP_CACHE &&
1965 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1966 if (alloced) {
1952 }
1953
1954 /* Perhaps the file has been truncated since we checked */
1955 if (sgp <= SGP_CACHE &&
1956 ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1957 if (alloced) {
1967 ClearPageDirty(page);
1968 delete_from_page_cache(page);
1958 folio_clear_dirty(folio);
1959 filemap_remove_folio(folio);
1969 spin_lock_irq(&info->lock);
1970 shmem_recalc_inode(inode);
1971 spin_unlock_irq(&info->lock);
1972 }
1973 error = -EINVAL;
1974 goto unlock;
1975 }
1976out:
1960 spin_lock_irq(&info->lock);
1961 shmem_recalc_inode(inode);
1962 spin_unlock_irq(&info->lock);
1963 }
1964 error = -EINVAL;
1965 goto unlock;
1966 }
1967out:
1977 *pagep = page + index - hindex;
1968 *pagep = folio_page(folio, index - hindex);
1978 return 0;
1979
1980 /*
1981 * Error recovery.
1982 */
1983unacct:
1969 return 0;
1970
1971 /*
1972 * Error recovery.
1973 */
1974unacct:
1984 shmem_inode_unacct_blocks(inode, compound_nr(page));
1975 shmem_inode_unacct_blocks(inode, folio_nr_pages(folio));
1985
1976
1986 if (PageTransHuge(page)) {
1987 unlock_page(page);
1988 put_page(page);
1977 if (folio_test_large(folio)) {
1978 folio_unlock(folio);
1979 folio_put(folio);
1989 goto alloc_nohuge;
1990 }
1991unlock:
1980 goto alloc_nohuge;
1981 }
1982unlock:
1992 if (page) {
1993 unlock_page(page);
1994 put_page(page);
1983 if (folio) {
1984 folio_unlock(folio);
1985 folio_put(folio);
1995 }
1996 if (error == -ENOSPC && !once++) {
1997 spin_lock_irq(&info->lock);
1998 shmem_recalc_inode(inode);
1999 spin_unlock_irq(&info->lock);
2000 goto repeat;
2001 }
2002 if (error == -EEXIST)

--- 231 unchanged lines hidden (view full) ---

2234 if (ret)
2235 return ret;
2236
2237 /* arm64 - allow memory tagging on RAM-based files */
2238 vma->vm_flags |= VM_MTE_ALLOWED;
2239
2240 file_accessed(file);
2241 vma->vm_ops = &shmem_vm_ops;
1986 }
1987 if (error == -ENOSPC && !once++) {
1988 spin_lock_irq(&info->lock);
1989 shmem_recalc_inode(inode);
1990 spin_unlock_irq(&info->lock);
1991 goto repeat;
1992 }
1993 if (error == -EEXIST)

--- 231 unchanged lines hidden (view full) ---

2225 if (ret)
2226 return ret;
2227
2228 /* arm64 - allow memory tagging on RAM-based files */
2229 vma->vm_flags |= VM_MTE_ALLOWED;
2230
2231 file_accessed(file);
2232 vma->vm_ops = &shmem_vm_ops;
2242 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
2243 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2244 (vma->vm_end & HPAGE_PMD_MASK)) {
2245 khugepaged_enter(vma, vma->vm_flags);
2246 }
2247 return 0;
2248}
2249
2250static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2251 umode_t mode, dev_t dev, unsigned long flags)
2252{
2253 struct inode *inode;
2254 struct shmem_inode_info *info;

--- 58 unchanged lines hidden (view full) ---

2313}
2314
2315#ifdef CONFIG_USERFAULTFD
2316int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2317 pmd_t *dst_pmd,
2318 struct vm_area_struct *dst_vma,
2319 unsigned long dst_addr,
2320 unsigned long src_addr,
2233 return 0;
2234}
2235
2236static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2237 umode_t mode, dev_t dev, unsigned long flags)
2238{
2239 struct inode *inode;
2240 struct shmem_inode_info *info;

--- 58 unchanged lines hidden (view full) ---

2299}
2300
2301#ifdef CONFIG_USERFAULTFD
2302int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2303 pmd_t *dst_pmd,
2304 struct vm_area_struct *dst_vma,
2305 unsigned long dst_addr,
2306 unsigned long src_addr,
2321 bool zeropage,
2307 bool zeropage, bool wp_copy,
2322 struct page **pagep)
2323{
2324 struct inode *inode = file_inode(dst_vma->vm_file);
2325 struct shmem_inode_info *info = SHMEM_I(inode);
2326 struct address_space *mapping = inode->i_mapping;
2327 gfp_t gfp = mapping_gfp_mask(mapping);
2328 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2329 void *page_kaddr;
2308 struct page **pagep)
2309{
2310 struct inode *inode = file_inode(dst_vma->vm_file);
2311 struct shmem_inode_info *info = SHMEM_I(inode);
2312 struct address_space *mapping = inode->i_mapping;
2313 gfp_t gfp = mapping_gfp_mask(mapping);
2314 pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2315 void *page_kaddr;
2316 struct folio *folio;
2330 struct page *page;
2331 int ret;
2332 pgoff_t max_off;
2333
2334 if (!shmem_inode_acct_block(inode, 1)) {
2335 /*
2336 * We may have got a page, returned -ENOENT triggering a retry,
2337 * and now we find ourselves with -ENOMEM. Release the page, to

--- 42 unchanged lines hidden (view full) ---

2380 __SetPageSwapBacked(page);
2381 __SetPageUptodate(page);
2382
2383 ret = -EFAULT;
2384 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2385 if (unlikely(pgoff >= max_off))
2386 goto out_release;
2387
2317 struct page *page;
2318 int ret;
2319 pgoff_t max_off;
2320
2321 if (!shmem_inode_acct_block(inode, 1)) {
2322 /*
2323 * We may have got a page, returned -ENOENT triggering a retry,
2324 * and now we find ourselves with -ENOMEM. Release the page, to

--- 42 unchanged lines hidden (view full) ---

2367 __SetPageSwapBacked(page);
2368 __SetPageUptodate(page);
2369
2370 ret = -EFAULT;
2371 max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2372 if (unlikely(pgoff >= max_off))
2373 goto out_release;
2374
2388 ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2375 folio = page_folio(page);
2376 ret = shmem_add_to_page_cache(folio, mapping, pgoff, NULL,
2389 gfp & GFP_RECLAIM_MASK, dst_mm);
2390 if (ret)
2391 goto out_release;
2392
2393 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2377 gfp & GFP_RECLAIM_MASK, dst_mm);
2378 if (ret)
2379 goto out_release;
2380
2381 ret = mfill_atomic_install_pte(dst_mm, dst_pmd, dst_vma, dst_addr,
2394 page, true, false);
2382 page, true, wp_copy);
2395 if (ret)
2396 goto out_delete_from_cache;
2397
2398 spin_lock_irq(&info->lock);
2399 info->alloced++;
2400 inode->i_blocks += BLOCKS_PER_PAGE;
2401 shmem_recalc_inode(inode);
2402 spin_unlock_irq(&info->lock);

--- 1079 unchanged lines hidden (view full) ---

3482 struct shmem_options *ctx = fc->fs_private;
3483 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3484 unsigned long inodes;
3485 struct mempolicy *mpol = NULL;
3486 const char *err;
3487
3488 raw_spin_lock(&sbinfo->stat_lock);
3489 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2383 if (ret)
2384 goto out_delete_from_cache;
2385
2386 spin_lock_irq(&info->lock);
2387 info->alloced++;
2388 inode->i_blocks += BLOCKS_PER_PAGE;
2389 shmem_recalc_inode(inode);
2390 spin_unlock_irq(&info->lock);

--- 1079 unchanged lines hidden (view full) ---

3470 struct shmem_options *ctx = fc->fs_private;
3471 struct shmem_sb_info *sbinfo = SHMEM_SB(fc->root->d_sb);
3472 unsigned long inodes;
3473 struct mempolicy *mpol = NULL;
3474 const char *err;
3475
3476 raw_spin_lock(&sbinfo->stat_lock);
3477 inodes = sbinfo->max_inodes - sbinfo->free_inodes;
3478 if (ctx->blocks > S64_MAX) {
3479 err = "Number of blocks too large";
3480 goto out;
3481 }
3490 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3491 if (!sbinfo->max_blocks) {
3492 err = "Cannot retroactively limit size";
3493 goto out;
3494 }
3495 if (percpu_counter_compare(&sbinfo->used_blocks,
3496 ctx->blocks) > 0) {
3497 err = "Too small a size for current use";

--- 385 unchanged lines hidden (view full) ---

3883 .init_fs_context = shmem_init_fs_context,
3884#ifdef CONFIG_TMPFS
3885 .parameters = shmem_fs_parameters,
3886#endif
3887 .kill_sb = kill_litter_super,
3888 .fs_flags = FS_USERNS_MOUNT,
3889};
3890
3482 if ((ctx->seen & SHMEM_SEEN_BLOCKS) && ctx->blocks) {
3483 if (!sbinfo->max_blocks) {
3484 err = "Cannot retroactively limit size";
3485 goto out;
3486 }
3487 if (percpu_counter_compare(&sbinfo->used_blocks,
3488 ctx->blocks) > 0) {
3489 err = "Too small a size for current use";

--- 385 unchanged lines hidden (view full) ---

3875 .init_fs_context = shmem_init_fs_context,
3876#ifdef CONFIG_TMPFS
3877 .parameters = shmem_fs_parameters,
3878#endif
3879 .kill_sb = kill_litter_super,
3880 .fs_flags = FS_USERNS_MOUNT,
3881};
3882
3891int __init shmem_init(void)
3883void __init shmem_init(void)
3892{
3893 int error;
3894
3895 shmem_init_inodecache();
3896
3897 error = register_filesystem(&shmem_fs_type);
3898 if (error) {
3899 pr_err("Could not register tmpfs\n");

--- 8 unchanged lines hidden (view full) ---

3908 }
3909
3910#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3911 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
3912 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3913 else
3914 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
3915#endif
3884{
3885 int error;
3886
3887 shmem_init_inodecache();
3888
3889 error = register_filesystem(&shmem_fs_type);
3890 if (error) {
3891 pr_err("Could not register tmpfs\n");

--- 8 unchanged lines hidden (view full) ---

3900 }
3901
3902#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3903 if (has_transparent_hugepage() && shmem_huge > SHMEM_HUGE_DENY)
3904 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3905 else
3906 shmem_huge = SHMEM_HUGE_NEVER; /* just in case it was patched */
3907#endif
3916 return 0;
3908 return;
3917
3918out1:
3919 unregister_filesystem(&shmem_fs_type);
3920out2:
3921 shmem_destroy_inodecache();
3922 shm_mnt = ERR_PTR(error);
3909
3910out1:
3911 unregister_filesystem(&shmem_fs_type);
3912out2:
3913 shmem_destroy_inodecache();
3914 shm_mnt = ERR_PTR(error);
3923 return error;
3924}
3925
3926#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
3927static ssize_t shmem_enabled_show(struct kobject *kobj,
3928 struct kobj_attribute *attr, char *buf)
3929{
3930 static const int values[] = {
3931 SHMEM_HUGE_ALWAYS,

--- 61 unchanged lines hidden (view full) ---

3993static struct file_system_type shmem_fs_type = {
3994 .name = "tmpfs",
3995 .init_fs_context = ramfs_init_fs_context,
3996 .parameters = ramfs_fs_parameters,
3997 .kill_sb = kill_litter_super,
3998 .fs_flags = FS_USERNS_MOUNT,
3999};
4000
3915}
3916
3917#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
3918static ssize_t shmem_enabled_show(struct kobject *kobj,
3919 struct kobj_attribute *attr, char *buf)
3920{
3921 static const int values[] = {
3922 SHMEM_HUGE_ALWAYS,

--- 61 unchanged lines hidden (view full) ---

3984static struct file_system_type shmem_fs_type = {
3985 .name = "tmpfs",
3986 .init_fs_context = ramfs_init_fs_context,
3987 .parameters = ramfs_fs_parameters,
3988 .kill_sb = kill_litter_super,
3989 .fs_flags = FS_USERNS_MOUNT,
3990};
3991
4001int __init shmem_init(void)
3992void __init shmem_init(void)
4002{
4003 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
4004
4005 shm_mnt = kern_mount(&shmem_fs_type);
4006 BUG_ON(IS_ERR(shm_mnt));
3993{
3994 BUG_ON(register_filesystem(&shmem_fs_type) != 0);
3995
3996 shm_mnt = kern_mount(&shmem_fs_type);
3997 BUG_ON(IS_ERR(shm_mnt));
4007
4008 return 0;
4009}
4010
4011int shmem_unuse(unsigned int type)
4012{
4013 return 0;
4014}
4015
4016int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)

--- 123 unchanged lines hidden (view full) ---

4140 if (IS_ERR(file))
4141 return PTR_ERR(file);
4142
4143 if (vma->vm_file)
4144 fput(vma->vm_file);
4145 vma->vm_file = file;
4146 vma->vm_ops = &shmem_vm_ops;
4147
3998}
3999
4000int shmem_unuse(unsigned int type)
4001{
4002 return 0;
4003}
4004
4005int shmem_lock(struct file *file, int lock, struct ucounts *ucounts)

--- 123 unchanged lines hidden (view full) ---

4129 if (IS_ERR(file))
4130 return PTR_ERR(file);
4131
4132 if (vma->vm_file)
4133 fput(vma->vm_file);
4134 vma->vm_file = file;
4135 vma->vm_ops = &shmem_vm_ops;
4136
4148 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4149 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4150 (vma->vm_end & HPAGE_PMD_MASK)) {
4151 khugepaged_enter(vma, vma->vm_flags);
4152 }
4153
4154 return 0;
4155}
4156
4157/**
4158 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4159 * @mapping: the page's address_space
4160 * @index: the page index
4161 * @gfp: the page allocator flags to use if allocating

--- 39 unchanged lines hidden --- 		4137 return 0;
4138}
4139
4140/**
4141 * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
4142 * @mapping: the page's address_space
4143 * @index: the page index
4144 * @gfp: the page allocator flags to use if allocating

--- 39 unchanged lines hidden ---