rmap.c (6f84981772535e670e4e2df051a672af229b6694) rmap.c (7d4a8be0c4b2b7ffb367929d2b352651f083806b)
1/*
2 * mm/rmap.c - physical to virtual reverse mappings
3 *
4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License (GPL).
6 *
7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as possible.

--- 809 unchanged lines hidden (view full) ---

818 /* Restore the mlock which got missed */
819 mlock_vma_folio(folio, vma, !pvmw.pte);
820 page_vma_mapped_walk_done(&pvmw);
821 pra->vm_flags |= VM_LOCKED;
822 return false; /* To break the loop */
823 }
824
825 if (pvmw.pte) {
1/*
2 * mm/rmap.c - physical to virtual reverse mappings
3 *
4 * Copyright 2001, Rik van Riel <riel@conectiva.com.br>
5 * Released under the General Public License (GPL).
6 *
7 * Simple, low overhead reverse mapping scheme.
8 * Please try to keep this thing as modular as possible.

--- 809 unchanged lines hidden (view full) ---

818 /* Restore the mlock which got missed */
819 mlock_vma_folio(folio, vma, !pvmw.pte);
820 page_vma_mapped_walk_done(&pvmw);
821 pra->vm_flags |= VM_LOCKED;
822 return false; /* To break the loop */
823 }
824
825 if (pvmw.pte) {
826 if (lru_gen_enabled() && pte_young(*pvmw.pte) &&
827 !(vma->vm_flags & (VM_SEQ_READ | VM_RAND_READ))) {
826 if (lru_gen_enabled() && pte_young(*pvmw.pte)) {
828 lru_gen_look_around(&pvmw);
829 referenced++;
830 }
831
832 if (ptep_clear_flush_young_notify(vma, address,
827 lru_gen_look_around(&pvmw);
828 referenced++;
829 }
830
831 if (ptep_clear_flush_young_notify(vma, address,
833 pvmw.pte)) {
834 /*
835 * Don't treat a reference through
836 * a sequentially read mapping as such.
837 * If the folio has been used in another mapping,
838 * we will catch it; if this other mapping is
839 * already gone, the unmap path will have set
840 * the referenced flag or activated the folio.
841 */
842 if (likely(!(vma->vm_flags & VM_SEQ_READ)))
843 referenced++;
844 }
832 pvmw.pte))
833 referenced++;
845 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
846 if (pmdp_clear_flush_young_notify(vma, address,
847 pvmw.pmd))
848 referenced++;
849 } else {
850 /* unexpected pmd-mapped folio? */
851 WARN_ON_ONCE(1);
852 }

--- 17 unchanged lines hidden (view full) ---

870 return true;
871}
872
873static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
874{
875 struct folio_referenced_arg *pra = arg;
876 struct mem_cgroup *memcg = pra->memcg;
877
834 } else if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE)) {
835 if (pmdp_clear_flush_young_notify(vma, address,
836 pvmw.pmd))
837 referenced++;
838 } else {
839 /* unexpected pmd-mapped folio? */
840 WARN_ON_ONCE(1);
841 }

--- 17 unchanged lines hidden (view full) ---

859 return true;
860}
861
862static bool invalid_folio_referenced_vma(struct vm_area_struct *vma, void *arg)
863{
864 struct folio_referenced_arg *pra = arg;
865 struct mem_cgroup *memcg = pra->memcg;
866
878 if (!mm_match_cgroup(vma->vm_mm, memcg))
867 /*
868 * Ignore references from this mapping if it has no recency. If the
869 * folio has been used in another mapping, we will catch it; if this
870 * other mapping is already gone, the unmap path will have set the
871 * referenced flag or activated the folio in zap_pte_range().
872 */
873 if (!vma_has_recency(vma))
879 return true;
880
874 return true;
875
876 /*
877 * If we are reclaiming on behalf of a cgroup, skip counting on behalf
878 * of references from different cgroups.
879 */
880 if (memcg && !mm_match_cgroup(vma->vm_mm, memcg))
881 return true;
882
881 return false;
882}
883
884/**
885 * folio_referenced() - Test if the folio was referenced.
886 * @folio: The folio to test.
887 * @is_locked: Caller holds lock on the folio.
888 * @memcg: target memory cgroup

--- 12 unchanged lines hidden (view full) ---

901 .mapcount = folio_mapcount(folio),
902 .memcg = memcg,
903 };
904 struct rmap_walk_control rwc = {
905 .rmap_one = folio_referenced_one,
906 .arg = (void *)&pra,
907 .anon_lock = folio_lock_anon_vma_read,
908 .try_lock = true,
883 return false;
884}
885
886/**
887 * folio_referenced() - Test if the folio was referenced.
888 * @folio: The folio to test.
889 * @is_locked: Caller holds lock on the folio.
890 * @memcg: target memory cgroup

--- 12 unchanged lines hidden (view full) ---

903 .mapcount = folio_mapcount(folio),
904 .memcg = memcg,
905 };
906 struct rmap_walk_control rwc = {
907 .rmap_one = folio_referenced_one,
908 .arg = (void *)&pra,
909 .anon_lock = folio_lock_anon_vma_read,
910 .try_lock = true,
911 .invalid_vma = invalid_folio_referenced_vma,
909 };
910
911 *vm_flags = 0;
912 if (!pra.mapcount)
913 return 0;
914
915 if (!folio_raw_mapping(folio))
916 return 0;
917
918 if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) {
919 we_locked = folio_trylock(folio);
920 if (!we_locked)
921 return 1;
922 }
923
912 };
913
914 *vm_flags = 0;
915 if (!pra.mapcount)
916 return 0;
917
918 if (!folio_raw_mapping(folio))
919 return 0;
920
921 if (!is_locked && (!folio_test_anon(folio) || folio_test_ksm(folio))) {
922 we_locked = folio_trylock(folio);
923 if (!we_locked)
924 return 1;
925 }
926
924 /*
925 * If we are reclaiming on behalf of a cgroup, skip
926 * counting on behalf of references from different
927 * cgroups
928 */
929 if (memcg) {
930 rwc.invalid_vma = invalid_folio_referenced_vma;
931 }
932
933 rmap_walk(folio, &rwc);
934 *vm_flags = pra.vm_flags;
935
936 if (we_locked)
937 folio_unlock(folio);
938
939 return rwc.contended ? -1 : pra.referenced;
940}

--- 4 unchanged lines hidden (view full) ---

945 struct vm_area_struct *vma = pvmw->vma;
946 struct mmu_notifier_range range;
947 unsigned long address = pvmw->address;
948
949 /*
950 * We have to assume the worse case ie pmd for invalidation. Note that
951 * the folio can not be freed from this function.
952 */
927 rmap_walk(folio, &rwc);
928 *vm_flags = pra.vm_flags;
929
930 if (we_locked)
931 folio_unlock(folio);
932
933 return rwc.contended ? -1 : pra.referenced;
934}

--- 4 unchanged lines hidden (view full) ---

939 struct vm_area_struct *vma = pvmw->vma;
940 struct mmu_notifier_range range;
941 unsigned long address = pvmw->address;
942
943 /*
944 * We have to assume the worse case ie pmd for invalidation. Note that
945 * the folio can not be freed from this function.
946 */
953 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE,
954 0, vma, vma->vm_mm, address,
955 vma_address_end(pvmw));
947 mmu_notifier_range_init(&range, MMU_NOTIFY_PROTECTION_PAGE, 0,
948 vma->vm_mm, address, vma_address_end(pvmw));
956 mmu_notifier_invalidate_range_start(&range);
957
958 while (page_vma_mapped_walk(pvmw)) {
959 int ret = 0;
960
961 address = pvmw->address;
962 if (pvmw->pte) {
963 pte_t entry;

--- 253 unchanged lines hidden (view full) ---

1217void page_add_anon_rmap(struct page *page,
1218 struct vm_area_struct *vma, unsigned long address, rmap_t flags)
1219{
1220 atomic_t *mapped;
1221 int nr = 0, nr_pmdmapped = 0;
1222 bool compound = flags & RMAP_COMPOUND;
1223 bool first = true;
1224
949 mmu_notifier_invalidate_range_start(&range);
950
951 while (page_vma_mapped_walk(pvmw)) {
952 int ret = 0;
953
954 address = pvmw->address;
955 if (pvmw->pte) {
956 pte_t entry;

--- 253 unchanged lines hidden (view full) ---

1210void page_add_anon_rmap(struct page *page,
1211 struct vm_area_struct *vma, unsigned long address, rmap_t flags)
1212{
1213 atomic_t *mapped;
1214 int nr = 0, nr_pmdmapped = 0;
1215 bool compound = flags & RMAP_COMPOUND;
1216 bool first = true;
1217
1225 if (unlikely(PageKsm(page)))
1226 lock_page_memcg(page);
1227
1228 /* Is page being mapped by PTE? Is this its first map to be added? */
1229 if (likely(!compound)) {
1230 first = atomic_inc_and_test(&page->_mapcount);
1231 nr = first;
1232 if (first && PageCompound(page)) {
1233 mapped = subpages_mapcount_ptr(compound_head(page));
1234 nr = atomic_inc_return_relaxed(mapped);
1235 nr = (nr < COMPOUND_MAPPED);

--- 21 unchanged lines hidden (view full) ---

1257 VM_BUG_ON_PAGE(!first && (flags & RMAP_EXCLUSIVE), page);
1258 VM_BUG_ON_PAGE(!first && PageAnonExclusive(page), page);
1259
1260 if (nr_pmdmapped)
1261 __mod_lruvec_page_state(page, NR_ANON_THPS, nr_pmdmapped);
1262 if (nr)
1263 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);
1264
1218 /* Is page being mapped by PTE? Is this its first map to be added? */
1219 if (likely(!compound)) {
1220 first = atomic_inc_and_test(&page->_mapcount);
1221 nr = first;
1222 if (first && PageCompound(page)) {
1223 mapped = subpages_mapcount_ptr(compound_head(page));
1224 nr = atomic_inc_return_relaxed(mapped);
1225 nr = (nr < COMPOUND_MAPPED);

--- 21 unchanged lines hidden (view full) ---

1247 VM_BUG_ON_PAGE(!first && (flags & RMAP_EXCLUSIVE), page);
1248 VM_BUG_ON_PAGE(!first && PageAnonExclusive(page), page);
1249
1250 if (nr_pmdmapped)
1251 __mod_lruvec_page_state(page, NR_ANON_THPS, nr_pmdmapped);
1252 if (nr)
1253 __mod_lruvec_page_state(page, NR_ANON_MAPPED, nr);
1254
1265 if (unlikely(PageKsm(page)))
1266 unlock_page_memcg(page);
1255 if (likely(!PageKsm(page))) {
1256 /* address might be in next vma when migration races vma_adjust */
1257 if (first)
1258 __page_set_anon_rmap(page, vma, address,
1259 !!(flags & RMAP_EXCLUSIVE));
1260 else
1261 __page_check_anon_rmap(page, vma, address);
1262 }
1267
1263
1268 /* address might be in next vma when migration races vma_adjust */
1269 else if (first)
1270 __page_set_anon_rmap(page, vma, address,
1271 !!(flags & RMAP_EXCLUSIVE));
1272 else
1273 __page_check_anon_rmap(page, vma, address);
1274
1275 mlock_vma_page(page, vma, compound);
1276}
1277
1278/**
1279 * page_add_new_anon_rmap - add mapping to a new anonymous page
1280 * @page: the page to add the mapping to
1281 * @vma: the vm area in which the mapping is added
1282 * @address: the user virtual address mapped

--- 41 unchanged lines hidden (view full) ---

1324void page_add_file_rmap(struct page *page,
1325 struct vm_area_struct *vma, bool compound)
1326{
1327 atomic_t *mapped;
1328 int nr = 0, nr_pmdmapped = 0;
1329 bool first;
1330
1331 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
1264 mlock_vma_page(page, vma, compound);
1265}
1266
1267/**
1268 * page_add_new_anon_rmap - add mapping to a new anonymous page
1269 * @page: the page to add the mapping to
1270 * @vma: the vm area in which the mapping is added
1271 * @address: the user virtual address mapped

--- 41 unchanged lines hidden (view full) ---

1313void page_add_file_rmap(struct page *page,
1314 struct vm_area_struct *vma, bool compound)
1315{
1316 atomic_t *mapped;
1317 int nr = 0, nr_pmdmapped = 0;
1318 bool first;
1319
1320 VM_BUG_ON_PAGE(compound && !PageTransHuge(page), page);
1332 lock_page_memcg(page);
1333
1334 /* Is page being mapped by PTE? Is this its first map to be added? */
1335 if (likely(!compound)) {
1336 first = atomic_inc_and_test(&page->_mapcount);
1337 nr = first;
1338 if (first && PageCompound(page)) {
1339 mapped = subpages_mapcount_ptr(compound_head(page));
1340 nr = atomic_inc_return_relaxed(mapped);

--- 19 unchanged lines hidden (view full) ---

1360 }
1361 }
1362
1363 if (nr_pmdmapped)
1364 __mod_lruvec_page_state(page, PageSwapBacked(page) ?
1365 NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped);
1366 if (nr)
1367 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);
1321
1322 /* Is page being mapped by PTE? Is this its first map to be added? */
1323 if (likely(!compound)) {
1324 first = atomic_inc_and_test(&page->_mapcount);
1325 nr = first;
1326 if (first && PageCompound(page)) {
1327 mapped = subpages_mapcount_ptr(compound_head(page));
1328 nr = atomic_inc_return_relaxed(mapped);

--- 19 unchanged lines hidden (view full) ---

1348 }
1349 }
1350
1351 if (nr_pmdmapped)
1352 __mod_lruvec_page_state(page, PageSwapBacked(page) ?
1353 NR_SHMEM_PMDMAPPED : NR_FILE_PMDMAPPED, nr_pmdmapped);
1354 if (nr)
1355 __mod_lruvec_page_state(page, NR_FILE_MAPPED, nr);
1368 unlock_page_memcg(page);
1369
1370 mlock_vma_page(page, vma, compound);
1371}
1372
1373/**
1374 * page_remove_rmap - take down pte mapping from a page
1375 * @page: page to remove mapping from
1376 * @vma: the vm area from which the mapping is removed

--- 12 unchanged lines hidden (view full) ---

1389
1390 /* Hugetlb pages are not counted in NR_*MAPPED */
1391 if (unlikely(PageHuge(page))) {
1392 /* hugetlb pages are always mapped with pmds */
1393 atomic_dec(compound_mapcount_ptr(page));
1394 return;
1395 }
1396
1356
1357 mlock_vma_page(page, vma, compound);
1358}
1359
1360/**
1361 * page_remove_rmap - take down pte mapping from a page
1362 * @page: page to remove mapping from
1363 * @vma: the vm area from which the mapping is removed

--- 12 unchanged lines hidden (view full) ---

1376
1377 /* Hugetlb pages are not counted in NR_*MAPPED */
1378 if (unlikely(PageHuge(page))) {
1379 /* hugetlb pages are always mapped with pmds */
1380 atomic_dec(compound_mapcount_ptr(page));
1381 return;
1382 }
1383
1397 lock_page_memcg(page);
1398
1399 /* Is page being unmapped by PTE? Is this its last map to be removed? */
1400 if (likely(!compound)) {
1401 last = atomic_add_negative(-1, &page->_mapcount);
1402 nr = last;
1403 if (last && PageCompound(page)) {
1404 mapped = subpages_mapcount_ptr(compound_head(page));
1405 nr = atomic_dec_return_relaxed(mapped);
1406 nr = (nr < COMPOUND_MAPPED);

--- 39 unchanged lines hidden (view full) ---

1446 /*
1447 * It would be tidy to reset PageAnon mapping when fully unmapped,
1448 * but that might overwrite a racing page_add_anon_rmap
1449 * which increments mapcount after us but sets mapping
1450 * before us: so leave the reset to free_pages_prepare,
1451 * and remember that it's only reliable while mapped.
1452 */
1453
1384 /* Is page being unmapped by PTE? Is this its last map to be removed? */
1385 if (likely(!compound)) {
1386 last = atomic_add_negative(-1, &page->_mapcount);
1387 nr = last;
1388 if (last && PageCompound(page)) {
1389 mapped = subpages_mapcount_ptr(compound_head(page));
1390 nr = atomic_dec_return_relaxed(mapped);
1391 nr = (nr < COMPOUND_MAPPED);

--- 39 unchanged lines hidden (view full) ---

1431 /*
1432 * It would be tidy to reset PageAnon mapping when fully unmapped,
1433 * but that might overwrite a racing page_add_anon_rmap
1434 * which increments mapcount after us but sets mapping
1435 * before us: so leave the reset to free_pages_prepare,
1436 * and remember that it's only reliable while mapped.
1437 */
1438
1454 unlock_page_memcg(page);
1455
1456 munlock_vma_page(page, vma, compound);
1457}
1458
1459/*
1460 * @arg: enum ttu_flags will be passed to this argument
1461 */
1462static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
1463 unsigned long address, void *arg)

--- 22 unchanged lines hidden (view full) ---

1486 * For THP, we have to assume the worse case ie pmd for invalidation.
1487 * For hugetlb, it could be much worse if we need to do pud
1488 * invalidation in the case of pmd sharing.
1489 *
1490 * Note that the folio can not be freed in this function as call of
1491 * try_to_unmap() must hold a reference on the folio.
1492 */
1493 range.end = vma_address_end(&pvmw);
1439 munlock_vma_page(page, vma, compound);
1440}
1441
1442/*
1443 * @arg: enum ttu_flags will be passed to this argument
1444 */
1445static bool try_to_unmap_one(struct folio *folio, struct vm_area_struct *vma,
1446 unsigned long address, void *arg)

--- 22 unchanged lines hidden (view full) ---

1469 * For THP, we have to assume the worse case ie pmd for invalidation.
1470 * For hugetlb, it could be much worse if we need to do pud
1471 * invalidation in the case of pmd sharing.
1472 *
1473 * Note that the folio can not be freed in this function as call of
1474 * try_to_unmap() must hold a reference on the folio.
1475 */
1476 range.end = vma_address_end(&pvmw);
1494 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1477 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1495 address, range.end);
1496 if (folio_test_hugetlb(folio)) {
1497 /*
1498 * If sharing is possible, start and end will be adjusted
1499 * accordingly.
1500 */
1501 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1502 &range.end);

--- 358 unchanged lines hidden (view full) ---

1861 * For THP, we have to assume the worse case ie pmd for invalidation.
1862 * For hugetlb, it could be much worse if we need to do pud
1863 * invalidation in the case of pmd sharing.
1864 *
1865 * Note that the page can not be free in this function as call of
1866 * try_to_unmap() must hold a reference on the page.
1867 */
1868 range.end = vma_address_end(&pvmw);
1478 address, range.end);
1479 if (folio_test_hugetlb(folio)) {
1480 /*
1481 * If sharing is possible, start and end will be adjusted
1482 * accordingly.
1483 */
1484 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1485 &range.end);

--- 358 unchanged lines hidden (view full) ---

1844 * For THP, we have to assume the worse case ie pmd for invalidation.
1845 * For hugetlb, it could be much worse if we need to do pud
1846 * invalidation in the case of pmd sharing.
1847 *
1848 * Note that the page can not be free in this function as call of
1849 * try_to_unmap() must hold a reference on the page.
1850 */
1851 range.end = vma_address_end(&pvmw);
1869 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, vma->vm_mm,
1852 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma->vm_mm,
1870 address, range.end);
1871 if (folio_test_hugetlb(folio)) {
1872 /*
1873 * If sharing is possible, start and end will be adjusted
1874 * accordingly.
1875 */
1876 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1877 &range.end);

--- 313 unchanged lines hidden (view full) ---

2191 struct make_exclusive_args *args = priv;
2192 pte_t pteval;
2193 struct page *subpage;
2194 bool ret = true;
2195 struct mmu_notifier_range range;
2196 swp_entry_t entry;
2197 pte_t swp_pte;
2198
1853 address, range.end);
1854 if (folio_test_hugetlb(folio)) {
1855 /*
1856 * If sharing is possible, start and end will be adjusted
1857 * accordingly.
1858 */
1859 adjust_range_if_pmd_sharing_possible(vma, &range.start,
1860 &range.end);

--- 313 unchanged lines hidden (view full) ---

2174 struct make_exclusive_args *args = priv;
2175 pte_t pteval;
2176 struct page *subpage;
2177 bool ret = true;
2178 struct mmu_notifier_range range;
2179 swp_entry_t entry;
2180 pte_t swp_pte;
2181
2199 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0, vma,
2182 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
2200 vma->vm_mm, address, min(vma->vm_end,
2201 address + folio_size(folio)),
2202 args->owner);
2203 mmu_notifier_invalidate_range_start(&range);
2204
2205 while (page_vma_mapped_walk(&pvmw)) {
2206 /* Unexpected PMD-mapped THP? */
2207 VM_BUG_ON_FOLIO(!pvmw.pte, folio);

--- 362 unchanged lines hidden --- 		2183 vma->vm_mm, address, min(vma->vm_end,
2184 address + folio_size(folio)),
2185 args->owner);
2186 mmu_notifier_invalidate_range_start(&range);
2187
2188 while (page_vma_mapped_walk(&pvmw)) {
2189 /* Unexpected PMD-mapped THP? */
2190 VM_BUG_ON_FOLIO(!pvmw.pte, folio);

--- 362 unchanged lines hidden ---