huge_memory.c (ac79f78dab892fcdc11fda8af5cc5e80d09dca8a) huge_memory.c (19deb7695e072deaff025e03de40c61b525bd57e)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2009 Red Hat, Inc.
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/mm.h>

--- 631 unchanged lines hidden (view full) ---

640 * always: directly stall for all thp allocations
641 * defer: wake kswapd and fail if not immediately available
642 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
643 * fail if not immediately available
644 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
645 * available
646 * never: never stall for any thp allocation
647 */
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2009 Red Hat, Inc.
4 */
5
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7
8#include <linux/mm.h>

--- 631 unchanged lines hidden (view full) ---

640 * always: directly stall for all thp allocations
641 * defer: wake kswapd and fail if not immediately available
642 * defer+madvise: wake kswapd and directly stall for MADV_HUGEPAGE, otherwise
643 * fail if not immediately available
644 * madvise: directly stall for MADV_HUGEPAGE, otherwise fail if not immediately
645 * available
646 * never: never stall for any thp allocation
647 */
648static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma, unsigned long addr)
648static inline gfp_t alloc_hugepage_direct_gfpmask(struct vm_area_struct *vma)
649{
650 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
649{
650 const bool vma_madvised = !!(vma->vm_flags & VM_HUGEPAGE);
651 const gfp_t gfp_mask = GFP_TRANSHUGE_LIGHT | __GFP_THISNODE;
652
653 /* Always do synchronous compaction */
654 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
651
652 /* Always do synchronous compaction */
653 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, &transparent_hugepage_flags))
655 return GFP_TRANSHUGE | __GFP_THISNODE |
656 (vma_madvised ? 0 : __GFP_NORETRY);
654 return GFP_TRANSHUGE | (vma_madvised ? 0 : __GFP_NORETRY);
657
658 /* Kick kcompactd and fail quickly */
659 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
655
656 /* Kick kcompactd and fail quickly */
657 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, &transparent_hugepage_flags))
660 return gfp_mask | __GFP_KSWAPD_RECLAIM;
658 return GFP_TRANSHUGE_LIGHT | __GFP_KSWAPD_RECLAIM;
661
662 /* Synchronous compaction if madvised, otherwise kick kcompactd */
663 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
659
660 /* Synchronous compaction if madvised, otherwise kick kcompactd */
661 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, &transparent_hugepage_flags))
664 return gfp_mask | (vma_madvised ? __GFP_DIRECT_RECLAIM :
665 __GFP_KSWAPD_RECLAIM);
662 return GFP_TRANSHUGE_LIGHT |
663 (vma_madvised ? __GFP_DIRECT_RECLAIM :
664 __GFP_KSWAPD_RECLAIM);
666
667 /* Only do synchronous compaction if madvised */
668 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
665
666 /* Only do synchronous compaction if madvised */
667 if (test_bit(TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, &transparent_hugepage_flags))
669 return gfp_mask | (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
668 return GFP_TRANSHUGE_LIGHT |
669 (vma_madvised ? __GFP_DIRECT_RECLAIM : 0);
670
670
671 return gfp_mask;
671 return GFP_TRANSHUGE_LIGHT;
672}
673
674/* Caller must hold page table lock. */
675static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
676 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
677 struct page *zero_page)
678{
679 pmd_t entry;

--- 55 unchanged lines hidden (view full) ---

735 set = true;
736 }
737 } else
738 spin_unlock(vmf->ptl);
739 if (!set)
740 pte_free(vma->vm_mm, pgtable);
741 return ret;
742 }
672}
673
674/* Caller must hold page table lock. */
675static bool set_huge_zero_page(pgtable_t pgtable, struct mm_struct *mm,
676 struct vm_area_struct *vma, unsigned long haddr, pmd_t *pmd,
677 struct page *zero_page)
678{
679 pmd_t entry;

--- 55 unchanged lines hidden (view full) ---

735 set = true;
736 }
737 } else
738 spin_unlock(vmf->ptl);
739 if (!set)
740 pte_free(vma->vm_mm, pgtable);
741 return ret;
742 }
743 gfp = alloc_hugepage_direct_gfpmask(vma, haddr);
744 page = alloc_pages_vma(gfp, HPAGE_PMD_ORDER, vma, haddr, numa_node_id());
743 gfp = alloc_hugepage_direct_gfpmask(vma);
744 page = alloc_hugepage_vma(gfp, vma, haddr, HPAGE_PMD_ORDER);
745 if (unlikely(!page)) {
746 count_vm_event(THP_FAULT_FALLBACK);
747 return VM_FAULT_FALLBACK;
748 }
749 prep_transhuge_page(page);
750 return __do_huge_pmd_anonymous_page(vmf, page, gfp);
751}
752

--- 590 unchanged lines hidden (view full) ---

1343 goto out_unlock;
1344 }
1345 unlock_page(page);
1346 get_page(page);
1347 spin_unlock(vmf->ptl);
1348alloc:
1349 if (__transparent_hugepage_enabled(vma) &&
1350 !transparent_hugepage_debug_cow()) {
745 if (unlikely(!page)) {
746 count_vm_event(THP_FAULT_FALLBACK);
747 return VM_FAULT_FALLBACK;
748 }
749 prep_transhuge_page(page);
750 return __do_huge_pmd_anonymous_page(vmf, page, gfp);
751}
752

--- 590 unchanged lines hidden (view full) ---

1343 goto out_unlock;
1344 }
1345 unlock_page(page);
1346 get_page(page);
1347 spin_unlock(vmf->ptl);
1348alloc:
1349 if (__transparent_hugepage_enabled(vma) &&
1350 !transparent_hugepage_debug_cow()) {
1351 huge_gfp = alloc_hugepage_direct_gfpmask(vma, haddr);
1352 new_page = alloc_pages_vma(huge_gfp, HPAGE_PMD_ORDER, vma,
1353 haddr, numa_node_id());
1351 huge_gfp = alloc_hugepage_direct_gfpmask(vma);
1352 new_page = alloc_hugepage_vma(huge_gfp, vma, haddr, HPAGE_PMD_ORDER);
1354 } else
1355 new_page = NULL;
1356
1357 if (likely(new_page)) {
1358 prep_transhuge_page(new_page);
1359 } else {
1360 if (!page) {
1361 split_huge_pmd(vma, vmf->pmd, vmf->address);

--- 1644 unchanged lines hidden --- 		1353 } else
1354 new_page = NULL;
1355
1356 if (likely(new_page)) {
1357 prep_transhuge_page(new_page);
1358 } else {
1359 if (!page) {
1360 split_huge_pmd(vma, vmf->pmd, vmf->address);

--- 1644 unchanged lines hidden ---