linux/mm/memory.c

768 		struct vm_area_struct *src_vma, unsigned long addr, int *rss)
830 		BUG_ON(page_try_dup_anon_rmap(page, false, src_vma));
855 		VM_BUG_ON(!is_cow_mapping(src_vma->vm_flags));
856 		if (try_restore_exclusive_pte(src_pte, src_vma, addr))
886 copy_present_page(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
902 	copy_user_highpage(&new_folio->page, page, addr, src_vma);
923 copy_present_pte(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
927 	struct mm_struct *src_mm = src_vma->vm_mm;
928 	unsigned long vm_flags = src_vma->vm_flags;
933 	page = vm_normal_page(src_vma, addr, pte);
944 		if (unlikely(page_try_dup_anon_rmap(page, false, src_vma))) {
947 			return copy_present_page(dst_vma, src_vma, dst_pte, src_pte,
1001 copy_pte_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1006 	struct mm_struct *src_mm = src_vma->vm_mm;
1063 						  dst_vma, src_vma,
1082 		ret = copy_present_pte(dst_vma, src_vma, dst_pte, src_pte,
1119 		prealloc = page_copy_prealloc(src_mm, src_vma, addr);
1138 copy_pmd_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1143 	struct mm_struct *src_mm = src_vma->vm_mm;
1156 			VM_BUG_ON_VMA(next-addr != HPAGE_PMD_SIZE, src_vma);
1158 					    addr, dst_vma, src_vma);
1167 		if (copy_pte_range(dst_vma, src_vma, dst_pmd, src_pmd,
1175 copy_pud_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1180 	struct mm_struct *src_mm = src_vma->vm_mm;
1193 			VM_BUG_ON_VMA(next-addr != HPAGE_PUD_SIZE, src_vma);
1195 					    dst_pud, src_pud, addr, src_vma);
1204 		if (copy_pmd_range(dst_vma, src_vma, dst_pud, src_pud,
1212 copy_p4d_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma,
1228 		if (copy_pud_range(dst_vma, src_vma, dst_p4d, src_p4d,
1241 vma_needs_copy(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1252 	if (src_vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
1255 	if (src_vma->anon_vma)
1268 copy_page_range(struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma)
1272 	unsigned long addr = src_vma->vm_start;
1273 	unsigned long end = src_vma->vm_end;
1275 	struct mm_struct *src_mm = src_vma->vm_mm;
1280 	if (!vma_needs_copy(dst_vma, src_vma))
1283 	if (is_vm_hugetlb_page(src_vma))
1284 		return copy_hugetlb_page_range(dst_mm, src_mm, dst_vma, src_vma);
1286 	if (unlikely(src_vma->vm_flags & VM_PFNMAP)) {
1291 		ret = track_pfn_copy(src_vma);
1302 	is_cow = is_cow_mapping(src_vma->vm_flags);
1315 		vma_assert_write_locked(src_vma);
1326 		if (unlikely(copy_p4d_range(dst_vma, src_vma, dst_pgd, src_pgd,