memory.c (7c7b962938ddda6a9cd095de557ee5250706ea88) memory.c (2bad466cc9d9b4c3b4b16eb9c03c919b59561316)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/memory.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 */
7
8/*

--- 90 unchanged lines hidden (view full) ---

99unsigned long max_mapnr;
100EXPORT_SYMBOL(max_mapnr);
101
102struct page *mem_map;
103EXPORT_SYMBOL(mem_map);
104#endif
105
106static vm_fault_t do_fault(struct vm_fault *vmf);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * linux/mm/memory.c
4 *
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
6 */
7
8/*

--- 90 unchanged lines hidden (view full) ---

99unsigned long max_mapnr;
100EXPORT_SYMBOL(max_mapnr);
101
102struct page *mem_map;
103EXPORT_SYMBOL(mem_map);
104#endif
105
106static vm_fault_t do_fault(struct vm_fault *vmf);
107static vm_fault_t do_anonymous_page(struct vm_fault *vmf);
108static bool vmf_pte_changed(struct vm_fault *vmf);
107
108/*
109
110/*
111 * Return true if the original pte was a uffd-wp pte marker (so the pte was
112 * wr-protected).
113 */
114static bool vmf_orig_pte_uffd_wp(struct vm_fault *vmf)
115{
116 if (!(vmf->flags & FAULT_FLAG_ORIG_PTE_VALID))
117 return false;
118
119 return pte_marker_uffd_wp(vmf->orig_pte);
120}
121
122/*
109 * A number of key systems in x86 including ioremap() rely on the assumption
110 * that high_memory defines the upper bound on direct map memory, then end
111 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
112 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
113 * and ZONE_HIGHMEM.
114 */
115void *high_memory;
116EXPORT_SYMBOL(high_memory);

--- 848 unchanged lines hidden (view full) ---

965 new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr, false);
966 if (!new_folio)
967 return NULL;
968
969 if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
970 folio_put(new_folio);
971 return NULL;
972 }
123 * A number of key systems in x86 including ioremap() rely on the assumption
124 * that high_memory defines the upper bound on direct map memory, then end
125 * of ZONE_NORMAL. Under CONFIG_DISCONTIG this means that max_low_pfn and
126 * highstart_pfn must be the same; there must be no gap between ZONE_NORMAL
127 * and ZONE_HIGHMEM.
128 */
129void *high_memory;
130EXPORT_SYMBOL(high_memory);

--- 848 unchanged lines hidden (view full) ---

979 new_folio = vma_alloc_folio(GFP_HIGHUSER_MOVABLE, 0, vma, addr, false);
980 if (!new_folio)
981 return NULL;
982
983 if (mem_cgroup_charge(new_folio, src_mm, GFP_KERNEL)) {
984 folio_put(new_folio);
985 return NULL;
986 }
973 cgroup_throttle_swaprate(&new_folio->page, GFP_KERNEL);
987 folio_throttle_swaprate(new_folio, GFP_KERNEL);
974
975 return new_folio;
976}
977
978static int
979copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
980 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
981 unsigned long end)

--- 303 unchanged lines hidden (view full) ---

1285 dst_pgd = pgd_offset(dst_mm, addr);
1286 src_pgd = pgd_offset(src_mm, addr);
1287 do {
1288 next = pgd_addr_end(addr, end);
1289 if (pgd_none_or_clear_bad(src_pgd))
1290 continue;
1291 if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1292 addr, next))) {
988
989 return new_folio;
990}
991
992static int
993copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
994 pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr,
995 unsigned long end)

--- 303 unchanged lines hidden (view full) ---

1299 dst_pgd = pgd_offset(dst_mm, addr);
1300 src_pgd = pgd_offset(src_mm, addr);
1301 do {
1302 next = pgd_addr_end(addr, end);
1303 if (pgd_none_or_clear_bad(src_pgd))
1304 continue;
1305 if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,
1306 addr, next))) {
1307 untrack_pfn_clear(dst_vma);
1293 ret = -ENOMEM;
1294 break;
1295 }
1296 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
1297
1298 if (is_cow) {
1299 raw_write_seqcount_end(&src_mm->write_protect_seq);
1300 mmu_notifier_invalidate_range_end(&range);

--- 39 unchanged lines hidden (view full) ---

1340 * This function makes sure that we'll replace the none pte with an uffd-wp
1341 * swap special pte marker when necessary. Must be with the pgtable lock held.
1342 */
1343static inline void
1344zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
1345 unsigned long addr, pte_t *pte,
1346 struct zap_details *details, pte_t pteval)
1347{
1308 ret = -ENOMEM;
1309 break;
1310 }
1311 } while (dst_pgd++, src_pgd++, addr = next, addr != end);
1312
1313 if (is_cow) {
1314 raw_write_seqcount_end(&src_mm->write_protect_seq);
1315 mmu_notifier_invalidate_range_end(&range);

--- 39 unchanged lines hidden (view full) ---

1355 * This function makes sure that we'll replace the none pte with an uffd-wp
1356 * swap special pte marker when necessary. Must be with the pgtable lock held.
1357 */
1358static inline void
1359zap_install_uffd_wp_if_needed(struct vm_area_struct *vma,
1360 unsigned long addr, pte_t *pte,
1361 struct zap_details *details, pte_t pteval)
1362{
1363 /* Zap on anonymous always means dropping everything */
1364 if (vma_is_anonymous(vma))
1365 return;
1366
1348 if (zap_drop_file_uffd_wp(details))
1349 return;
1350
1351 pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
1352}
1353
1354static unsigned long zap_pte_range(struct mmu_gather *tlb,
1355 struct vm_area_struct *vma, pmd_t *pmd,

--- 90 unchanged lines hidden (view full) ---

1446 if (unlikely(!free_swap_and_cache(entry)))
1447 print_bad_pte(vma, addr, ptent, NULL);
1448 } else if (is_migration_entry(entry)) {
1449 page = pfn_swap_entry_to_page(entry);
1450 if (!should_zap_page(details, page))
1451 continue;
1452 rss[mm_counter(page)]--;
1453 } else if (pte_marker_entry_uffd_wp(entry)) {
1367 if (zap_drop_file_uffd_wp(details))
1368 return;
1369
1370 pte_install_uffd_wp_if_needed(vma, addr, pte, pteval);
1371}
1372
1373static unsigned long zap_pte_range(struct mmu_gather *tlb,
1374 struct vm_area_struct *vma, pmd_t *pmd,

--- 90 unchanged lines hidden (view full) ---

1465 if (unlikely(!free_swap_and_cache(entry)))
1466 print_bad_pte(vma, addr, ptent, NULL);
1467 } else if (is_migration_entry(entry)) {
1468 page = pfn_swap_entry_to_page(entry);
1469 if (!should_zap_page(details, page))
1470 continue;
1471 rss[mm_counter(page)]--;
1472 } else if (pte_marker_entry_uffd_wp(entry)) {
1454 /* Only drop the uffd-wp marker if explicitly requested */
1455 if (!zap_drop_file_uffd_wp(details))
1473 /*
1474 * For anon: always drop the marker; for file: only
1475 * drop the marker if explicitly requested.
1476 */
1477 if (!vma_is_anonymous(vma) &&
1478 !zap_drop_file_uffd_wp(details))
1456 continue;
1457 } else if (is_hwpoison_entry(entry) ||
1458 is_swapin_error_entry(entry)) {
1459 if (!should_zap_cows(details))
1460 continue;
1461 } else {
1462 /* We should have covered all the swap entry types */
1463 WARN_ON_ONCE(1);

--- 1622 unchanged lines hidden (view full) ---

3086 delayacct_wpcopy_end();
3087 return ret == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
3088 }
3089 kmsan_copy_page_meta(&new_folio->page, vmf->page);
3090 }
3091
3092 if (mem_cgroup_charge(new_folio, mm, GFP_KERNEL))
3093 goto oom_free_new;
1479 continue;
1480 } else if (is_hwpoison_entry(entry) ||
1481 is_swapin_error_entry(entry)) {
1482 if (!should_zap_cows(details))
1483 continue;
1484 } else {
1485 /* We should have covered all the swap entry types */
1486 WARN_ON_ONCE(1);

--- 1622 unchanged lines hidden (view full) ---

3109 delayacct_wpcopy_end();
3110 return ret == -EHWPOISON ? VM_FAULT_HWPOISON : 0;
3111 }
3112 kmsan_copy_page_meta(&new_folio->page, vmf->page);
3113 }
3114
3115 if (mem_cgroup_charge(new_folio, mm, GFP_KERNEL))
3116 goto oom_free_new;
3094 cgroup_throttle_swaprate(&new_folio->page, GFP_KERNEL);
3117 folio_throttle_swaprate(new_folio, GFP_KERNEL);
3095
3096 __folio_mark_uptodate(new_folio);
3097
3098 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
3099 vmf->address & PAGE_MASK,
3100 (vmf->address & PAGE_MASK) + PAGE_SIZE);
3101 mmu_notifier_invalidate_range_start(&range);
3102

--- 455 unchanged lines hidden (view full) ---

3558 * Restore a potential device exclusive pte to a working pte entry
3559 */
3560static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
3561{
3562 struct folio *folio = page_folio(vmf->page);
3563 struct vm_area_struct *vma = vmf->vma;
3564 struct mmu_notifier_range range;
3565
3118
3119 __folio_mark_uptodate(new_folio);
3120
3121 mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, mm,
3122 vmf->address & PAGE_MASK,
3123 (vmf->address & PAGE_MASK) + PAGE_SIZE);
3124 mmu_notifier_invalidate_range_start(&range);
3125

--- 455 unchanged lines hidden (view full) ---

3581 * Restore a potential device exclusive pte to a working pte entry
3582 */
3583static vm_fault_t remove_device_exclusive_entry(struct vm_fault *vmf)
3584{
3585 struct folio *folio = page_folio(vmf->page);
3586 struct vm_area_struct *vma = vmf->vma;
3587 struct mmu_notifier_range range;
3588
3566 /*
3567 * We need a reference to lock the folio because we don't hold
3568 * the PTL so a racing thread can remove the device-exclusive
3569 * entry and unmap it. If the folio is free the entry must
3570 * have been removed already. If it happens to have already
3571 * been re-allocated after being freed all we do is lock and
3572 * unlock it.
3573 */
3574 if (!folio_try_get(folio))
3575 return 0;
3576
3577 if (!folio_lock_or_retry(folio, vma->vm_mm, vmf->flags)) {
3578 folio_put(folio);
3589 if (!folio_lock_or_retry(folio, vma->vm_mm, vmf->flags))
3579 return VM_FAULT_RETRY;
3590 return VM_FAULT_RETRY;
3580 }
3581 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
3582 vma->vm_mm, vmf->address & PAGE_MASK,
3583 (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
3584 mmu_notifier_invalidate_range_start(&range);
3585
3586 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3587 &vmf->ptl);
3588 if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
3589 restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
3590
3591 pte_unmap_unlock(vmf->pte, vmf->ptl);
3592 folio_unlock(folio);
3591 mmu_notifier_range_init_owner(&range, MMU_NOTIFY_EXCLUSIVE, 0,
3592 vma->vm_mm, vmf->address & PAGE_MASK,
3593 (vmf->address & PAGE_MASK) + PAGE_SIZE, NULL);
3594 mmu_notifier_invalidate_range_start(&range);
3595
3596 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3597 &vmf->ptl);
3598 if (likely(pte_same(*vmf->pte, vmf->orig_pte)))
3599 restore_exclusive_pte(vma, vmf->page, vmf->address, vmf->pte);
3600
3601 pte_unmap_unlock(vmf->pte, vmf->ptl);
3602 folio_unlock(folio);
3593 folio_put(folio);
3594
3595 mmu_notifier_invalidate_range_end(&range);
3596 return 0;
3597}
3598
3599static inline bool should_try_to_free_swap(struct folio *folio,
3600 struct vm_area_struct *vma,
3601 unsigned int fault_flags)

--- 26 unchanged lines hidden (view full) ---

3628 * So is_pte_marker() check is not enough to safely drop the pte.
3629 */
3630 if (pte_same(vmf->orig_pte, *vmf->pte))
3631 pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
3632 pte_unmap_unlock(vmf->pte, vmf->ptl);
3633 return 0;
3634}
3635
3603
3604 mmu_notifier_invalidate_range_end(&range);
3605 return 0;
3606}
3607
3608static inline bool should_try_to_free_swap(struct folio *folio,
3609 struct vm_area_struct *vma,
3610 unsigned int fault_flags)

--- 26 unchanged lines hidden (view full) ---

3637 * So is_pte_marker() check is not enough to safely drop the pte.
3638 */
3639 if (pte_same(vmf->orig_pte, *vmf->pte))
3640 pte_clear(vmf->vma->vm_mm, vmf->address, vmf->pte);
3641 pte_unmap_unlock(vmf->pte, vmf->ptl);
3642 return 0;
3643}
3644
3645static vm_fault_t do_pte_missing(struct vm_fault *vmf)
3646{
3647 if (vma_is_anonymous(vmf->vma))
3648 return do_anonymous_page(vmf);
3649 else
3650 return do_fault(vmf);
3651}
3652
3636/*
3637 * This is actually a page-missing access, but with uffd-wp special pte
3638 * installed. It means this pte was wr-protected before being unmapped.
3639 */
3640static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
3641{
3642 /*
3643 * Just in case there're leftover special ptes even after the region
3644 * got unregistered - we can simply clear them.
3645 */
3653/*
3654 * This is actually a page-missing access, but with uffd-wp special pte
3655 * installed. It means this pte was wr-protected before being unmapped.
3656 */
3657static vm_fault_t pte_marker_handle_uffd_wp(struct vm_fault *vmf)
3658{
3659 /*
3660 * Just in case there're leftover special ptes even after the region
3661 * got unregistered - we can simply clear them.
3662 */
3646 if (unlikely(!userfaultfd_wp(vmf->vma) || vma_is_anonymous(vmf->vma)))
3663 if (unlikely(!userfaultfd_wp(vmf->vma)))
3647 return pte_marker_clear(vmf);
3648
3664 return pte_marker_clear(vmf);
3665
3649 /* do_fault() can handle pte markers too like none pte */
3650 return do_fault(vmf);
3666 return do_pte_missing(vmf);
3651}
3652
3653static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
3654{
3655 swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
3656 unsigned long marker = pte_marker_get(entry);
3657
3658 /*

--- 188 unchanged lines hidden (view full) ---

3847 * owner. Try removing the extra reference from the local LRU
3848 * pagevecs if required.
3849 */
3850 if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
3851 !folio_test_ksm(folio) && !folio_test_lru(folio))
3852 lru_add_drain();
3853 }
3854
3667}
3668
3669static vm_fault_t handle_pte_marker(struct vm_fault *vmf)
3670{
3671 swp_entry_t entry = pte_to_swp_entry(vmf->orig_pte);
3672 unsigned long marker = pte_marker_get(entry);
3673
3674 /*

--- 188 unchanged lines hidden (view full) ---

3863 * owner. Try removing the extra reference from the local LRU
3864 * pagevecs if required.
3865 */
3866 if ((vmf->flags & FAULT_FLAG_WRITE) && folio == swapcache &&
3867 !folio_test_ksm(folio) && !folio_test_lru(folio))
3868 lru_add_drain();
3869 }
3870
3855 cgroup_throttle_swaprate(page, GFP_KERNEL);
3871 folio_throttle_swaprate(folio, GFP_KERNEL);
3856
3857 /*
3858 * Back out if somebody else already faulted in this pte.
3859 */
3860 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3861 &vmf->ptl);
3862 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
3863 goto out_nomap;

--- 143 unchanged lines hidden (view full) ---

4007
4008/*
4009 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4010 * but allow concurrent faults), and pte mapped but not yet locked.
4011 * We return with mmap_lock still held, but pte unmapped and unlocked.
4012 */
4013static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4014{
3872
3873 /*
3874 * Back out if somebody else already faulted in this pte.
3875 */
3876 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
3877 &vmf->ptl);
3878 if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte)))
3879 goto out_nomap;

--- 143 unchanged lines hidden (view full) ---

4023
4024/*
4025 * We enter with non-exclusive mmap_lock (to exclude vma changes,
4026 * but allow concurrent faults), and pte mapped but not yet locked.
4027 * We return with mmap_lock still held, but pte unmapped and unlocked.
4028 */
4029static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
4030{
4031 bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4015 struct vm_area_struct *vma = vmf->vma;
4016 struct folio *folio;
4017 vm_fault_t ret = 0;
4018 pte_t entry;
4019
4020 /* File mapping without ->vm_ops ? */
4021 if (vma->vm_flags & VM_SHARED)
4022 return VM_FAULT_SIGBUS;

--- 17 unchanged lines hidden (view full) ---

4040
4041 /* Use the zero-page for reads */
4042 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4043 !mm_forbids_zeropage(vma->vm_mm)) {
4044 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4045 vma->vm_page_prot));
4046 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4047 vmf->address, &vmf->ptl);
4032 struct vm_area_struct *vma = vmf->vma;
4033 struct folio *folio;
4034 vm_fault_t ret = 0;
4035 pte_t entry;
4036
4037 /* File mapping without ->vm_ops ? */
4038 if (vma->vm_flags & VM_SHARED)
4039 return VM_FAULT_SIGBUS;

--- 17 unchanged lines hidden (view full) ---

4057
4058 /* Use the zero-page for reads */
4059 if (!(vmf->flags & FAULT_FLAG_WRITE) &&
4060 !mm_forbids_zeropage(vma->vm_mm)) {
4061 entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
4062 vma->vm_page_prot));
4063 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
4064 vmf->address, &vmf->ptl);
4048 if (!pte_none(*vmf->pte)) {
4065 if (vmf_pte_changed(vmf)) {
4049 update_mmu_tlb(vma, vmf->address, vmf->pte);
4050 goto unlock;
4051 }
4052 ret = check_stable_address_space(vma->vm_mm);
4053 if (ret)
4054 goto unlock;
4055 /* Deliver the page fault to userland, check inside PT lock */
4056 if (userfaultfd_missing(vma)) {

--- 7 unchanged lines hidden (view full) ---

4064 if (unlikely(anon_vma_prepare(vma)))
4065 goto oom;
4066 folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
4067 if (!folio)
4068 goto oom;
4069
4070 if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL))
4071 goto oom_free_page;
4066 update_mmu_tlb(vma, vmf->address, vmf->pte);
4067 goto unlock;
4068 }
4069 ret = check_stable_address_space(vma->vm_mm);
4070 if (ret)
4071 goto unlock;
4072 /* Deliver the page fault to userland, check inside PT lock */
4073 if (userfaultfd_missing(vma)) {

--- 7 unchanged lines hidden (view full) ---

4081 if (unlikely(anon_vma_prepare(vma)))
4082 goto oom;
4083 folio = vma_alloc_zeroed_movable_folio(vma, vmf->address);
4084 if (!folio)
4085 goto oom;
4086
4087 if (mem_cgroup_charge(folio, vma->vm_mm, GFP_KERNEL))
4088 goto oom_free_page;
4072 cgroup_throttle_swaprate(&folio->page, GFP_KERNEL);
4089 folio_throttle_swaprate(folio, GFP_KERNEL);
4073
4074 /*
4075 * The memory barrier inside __folio_mark_uptodate makes sure that
4076 * preceding stores to the page contents become visible before
4077 * the set_pte_at() write.
4078 */
4079 __folio_mark_uptodate(folio);
4080
4081 entry = mk_pte(&folio->page, vma->vm_page_prot);
4082 entry = pte_sw_mkyoung(entry);
4083 if (vma->vm_flags & VM_WRITE)
4084 entry = pte_mkwrite(pte_mkdirty(entry));
4085
4086 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4087 &vmf->ptl);
4090
4091 /*
4092 * The memory barrier inside __folio_mark_uptodate makes sure that
4093 * preceding stores to the page contents become visible before
4094 * the set_pte_at() write.
4095 */
4096 __folio_mark_uptodate(folio);
4097
4098 entry = mk_pte(&folio->page, vma->vm_page_prot);
4099 entry = pte_sw_mkyoung(entry);
4100 if (vma->vm_flags & VM_WRITE)
4101 entry = pte_mkwrite(pte_mkdirty(entry));
4102
4103 vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
4104 &vmf->ptl);
4088 if (!pte_none(*vmf->pte)) {
4105 if (vmf_pte_changed(vmf)) {
4089 update_mmu_tlb(vma, vmf->address, vmf->pte);
4090 goto release;
4091 }
4092
4093 ret = check_stable_address_space(vma->vm_mm);
4094 if (ret)
4095 goto release;
4096
4097 /* Deliver the page fault to userland, check inside PT lock */
4098 if (userfaultfd_missing(vma)) {
4099 pte_unmap_unlock(vmf->pte, vmf->ptl);
4100 folio_put(folio);
4101 return handle_userfault(vmf, VM_UFFD_MISSING);
4102 }
4103
4104 inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4105 folio_add_new_anon_rmap(folio, vma, vmf->address);
4106 folio_add_lru_vma(folio, vma);
4107setpte:
4106 update_mmu_tlb(vma, vmf->address, vmf->pte);
4107 goto release;
4108 }
4109
4110 ret = check_stable_address_space(vma->vm_mm);
4111 if (ret)
4112 goto release;
4113
4114 /* Deliver the page fault to userland, check inside PT lock */
4115 if (userfaultfd_missing(vma)) {
4116 pte_unmap_unlock(vmf->pte, vmf->ptl);
4117 folio_put(folio);
4118 return handle_userfault(vmf, VM_UFFD_MISSING);
4119 }
4120
4121 inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
4122 folio_add_new_anon_rmap(folio, vma, vmf->address);
4123 folio_add_lru_vma(folio, vma);
4124setpte:
4125 if (uffd_wp)
4126 entry = pte_mkuffd_wp(entry);
4108 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
4109
4110 /* No need to invalidate - it was non-present before */
4111 update_mmu_cache(vma, vmf->address, vmf->pte);
4112unlock:
4113 pte_unmap_unlock(vmf->pte, vmf->ptl);
4114 return ret;
4115release:

--- 151 unchanged lines hidden (view full) ---

4267{
4268 return VM_FAULT_FALLBACK;
4269}
4270#endif
4271
4272void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr)
4273{
4274 struct vm_area_struct *vma = vmf->vma;
4127 set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
4128
4129 /* No need to invalidate - it was non-present before */
4130 update_mmu_cache(vma, vmf->address, vmf->pte);
4131unlock:
4132 pte_unmap_unlock(vmf->pte, vmf->ptl);
4133 return ret;
4134release:

--- 151 unchanged lines hidden (view full) ---

4286{
4287 return VM_FAULT_FALLBACK;
4288}
4289#endif
4290
4291void do_set_pte(struct vm_fault *vmf, struct page *page, unsigned long addr)
4292{
4293 struct vm_area_struct *vma = vmf->vma;
4275 bool uffd_wp = pte_marker_uffd_wp(vmf->orig_pte);
4294 bool uffd_wp = vmf_orig_pte_uffd_wp(vmf);
4276 bool write = vmf->flags & FAULT_FLAG_WRITE;
4277 bool prefault = vmf->address != addr;
4278 pte_t entry;
4279
4280 flush_icache_page(vma, page);
4281 entry = mk_pte(page, vma->vm_page_prot);
4282
4283 if (prefault && arch_wants_old_prefaulted_pte())

--- 242 unchanged lines hidden (view full) ---

4526 if (!vmf->cow_page)
4527 return VM_FAULT_OOM;
4528
4529 if (mem_cgroup_charge(page_folio(vmf->cow_page), vma->vm_mm,
4530 GFP_KERNEL)) {
4531 put_page(vmf->cow_page);
4532 return VM_FAULT_OOM;
4533 }
4295 bool write = vmf->flags & FAULT_FLAG_WRITE;
4296 bool prefault = vmf->address != addr;
4297 pte_t entry;
4298
4299 flush_icache_page(vma, page);
4300 entry = mk_pte(page, vma->vm_page_prot);
4301
4302 if (prefault && arch_wants_old_prefaulted_pte())

--- 242 unchanged lines hidden (view full) ---

4545 if (!vmf->cow_page)
4546 return VM_FAULT_OOM;
4547
4548 if (mem_cgroup_charge(page_folio(vmf->cow_page), vma->vm_mm,
4549 GFP_KERNEL)) {
4550 put_page(vmf->cow_page);
4551 return VM_FAULT_OOM;
4552 }
4534 cgroup_throttle_swaprate(vmf->cow_page, GFP_KERNEL);
4553 folio_throttle_swaprate(page_folio(vmf->cow_page), GFP_KERNEL);
4535
4536 ret = __do_fault(vmf);
4537 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4538 goto uncharge_out;
4539 if (ret & VM_FAULT_DONE_COW)
4540 return ret;
4541
4542 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);

--- 368 unchanged lines hidden (view full) ---

4911 */
4912 barrier();
4913 if (pte_none(vmf->orig_pte)) {
4914 pte_unmap(vmf->pte);
4915 vmf->pte = NULL;
4916 }
4917 }
4918
4554
4555 ret = __do_fault(vmf);
4556 if (unlikely(ret & (VM_FAULT_ERROR | VM_FAULT_NOPAGE | VM_FAULT_RETRY)))
4557 goto uncharge_out;
4558 if (ret & VM_FAULT_DONE_COW)
4559 return ret;
4560
4561 copy_user_highpage(vmf->cow_page, vmf->page, vmf->address, vma);

--- 368 unchanged lines hidden (view full) ---

4930 */
4931 barrier();
4932 if (pte_none(vmf->orig_pte)) {
4933 pte_unmap(vmf->pte);
4934 vmf->pte = NULL;
4935 }
4936 }
4937
4919 if (!vmf->pte) {
4920 if (vma_is_anonymous(vmf->vma))
4921 return do_anonymous_page(vmf);
4922 else
4923 return do_fault(vmf);
4924 }
4938 if (!vmf->pte)
4939 return do_pte_missing(vmf);
4925
4926 if (!pte_present(vmf->orig_pte))
4927 return do_swap_page(vmf);
4928
4929 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
4930 return do_numa_page(vmf);
4931
4932 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);

--- 19 unchanged lines hidden (view full) ---

4952 goto unlock;
4953 /*
4954 * This is needed only for protection faults but the arch code
4955 * is not yet telling us if this is a protection fault or not.
4956 * This still avoids useless tlb flushes for .text page faults
4957 * with threads.
4958 */
4959 if (vmf->flags & FAULT_FLAG_WRITE)
4940
4941 if (!pte_present(vmf->orig_pte))
4942 return do_swap_page(vmf);
4943
4944 if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
4945 return do_numa_page(vmf);
4946
4947 vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);

--- 19 unchanged lines hidden (view full) ---

4967 goto unlock;
4968 /*
4969 * This is needed only for protection faults but the arch code
4970 * is not yet telling us if this is a protection fault or not.
4971 * This still avoids useless tlb flushes for .text page faults
4972 * with threads.
4973 */
4974 if (vmf->flags & FAULT_FLAG_WRITE)
4960 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address);
4975 flush_tlb_fix_spurious_fault(vmf->vma, vmf->address,
4976 vmf->pte);
4961 }
4962unlock:
4963 pte_unmap_unlock(vmf->pte, vmf->ptl);
4964 return 0;
4965}
4966
4967/*
4968 * By the time we get here, we already hold the mm semaphore

--- 878 unchanged lines hidden --- 		4977 }
4978unlock:
4979 pte_unmap_unlock(vmf->pte, vmf->ptl);
4980 return 0;
4981}
4982
4983/*
4984 * By the time we get here, we already hold the mm semaphore

--- 878 unchanged lines hidden ---