khugepaged.c (f3f0e1d2150b2b99da2cbdfaad000089efe9bf30) khugepaged.c (e496cf3d782135c1cca0d154d4b924517ff58de0)
1#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2
3#include <linux/mm.h>
4#include <linux/sched.h>
5#include <linux/mmu_notifier.h>
6#include <linux/rmap.h>
7#include <linux/swap.h>
8#include <linux/mm_inline.h>

--- 805 unchanged lines hidden (view full) ---

814#endif
815
816static bool hugepage_vma_check(struct vm_area_struct *vma)
817{
818 if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
819 (vma->vm_flags & VM_NOHUGEPAGE))
820 return false;
821 if (shmem_file(vma->vm_file)) {
1#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
2
3#include <linux/mm.h>
4#include <linux/sched.h>
5#include <linux/mmu_notifier.h>
6#include <linux/rmap.h>
7#include <linux/swap.h>
8#include <linux/mm_inline.h>

--- 805 unchanged lines hidden (view full) ---

814#endif
815
816static bool hugepage_vma_check(struct vm_area_struct *vma)
817{
818 if ((!(vma->vm_flags & VM_HUGEPAGE) && !khugepaged_always()) ||
819 (vma->vm_flags & VM_NOHUGEPAGE))
820 return false;
821 if (shmem_file(vma->vm_file)) {
822 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
823 return false;
822 return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
823 HPAGE_PMD_NR);
824 }
825 if (!vma->anon_vma || vma->vm_ops)
826 return false;
827 if (is_vma_temporary_stack(vma))
828 return false;
829 return !(vma->vm_flags & VM_NO_KHUGEPAGED);

--- 387 unchanged lines hidden (view full) ---

1217 */
1218
1219 /* khugepaged_mm_lock actually not necessary for the below */
1220 free_mm_slot(mm_slot);
1221 mmdrop(mm);
1222 }
1223}
1224
824 return IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff,
825 HPAGE_PMD_NR);
826 }
827 if (!vma->anon_vma || vma->vm_ops)
828 return false;
829 if (is_vma_temporary_stack(vma))
830 return false;
831 return !(vma->vm_flags & VM_NO_KHUGEPAGED);

--- 387 unchanged lines hidden (view full) ---

1219 */
1220
1221 /* khugepaged_mm_lock actually not necessary for the below */
1222 free_mm_slot(mm_slot);
1223 mmdrop(mm);
1224 }
1225}
1226
1225#ifdef CONFIG_SHMEM
1227#if defined(CONFIG_SHMEM) && defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE)
1226static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
1227{
1228 struct vm_area_struct *vma;
1229 unsigned long addr;
1230 pmd_t *pmd, _pmd;
1231
1232 i_mmap_lock_write(mapping);
1233 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {

--- 442 unchanged lines hidden (view full) ---

1676 hend = vma->vm_end & HPAGE_PMD_MASK;
1677 if (hstart >= hend)
1678 goto skip;
1679 if (khugepaged_scan.address > hend)
1680 goto skip;
1681 if (khugepaged_scan.address < hstart)
1682 khugepaged_scan.address = hstart;
1683 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
1228static void retract_page_tables(struct address_space *mapping, pgoff_t pgoff)
1229{
1230 struct vm_area_struct *vma;
1231 unsigned long addr;
1232 pmd_t *pmd, _pmd;
1233
1234 i_mmap_lock_write(mapping);
1235 vma_interval_tree_foreach(vma, &mapping->i_mmap, pgoff, pgoff) {

--- 442 unchanged lines hidden (view full) ---

1678 hend = vma->vm_end & HPAGE_PMD_MASK;
1679 if (hstart >= hend)
1680 goto skip;
1681 if (khugepaged_scan.address > hend)
1682 goto skip;
1683 if (khugepaged_scan.address < hstart)
1684 khugepaged_scan.address = hstart;
1685 VM_BUG_ON(khugepaged_scan.address & ~HPAGE_PMD_MASK);
1684 if (shmem_file(vma->vm_file) && !shmem_huge_enabled(vma))
1685 goto skip;
1686
1687 while (khugepaged_scan.address < hend) {
1688 int ret;
1689 cond_resched();
1690 if (unlikely(khugepaged_test_exit(mm)))
1691 goto breakouterloop;
1692
1693 VM_BUG_ON(khugepaged_scan.address < hstart ||
1694 khugepaged_scan.address + HPAGE_PMD_SIZE >
1695 hend);
1696 if (shmem_file(vma->vm_file)) {
1686
1687 while (khugepaged_scan.address < hend) {
1688 int ret;
1689 cond_resched();
1690 if (unlikely(khugepaged_test_exit(mm)))
1691 goto breakouterloop;
1692
1693 VM_BUG_ON(khugepaged_scan.address < hstart ||
1694 khugepaged_scan.address + HPAGE_PMD_SIZE >
1695 hend);
1696 if (shmem_file(vma->vm_file)) {
1697 struct file *file = get_file(vma->vm_file);
1697 struct file *file;
1698 pgoff_t pgoff = linear_page_index(vma,
1699 khugepaged_scan.address);
1698 pgoff_t pgoff = linear_page_index(vma,
1699 khugepaged_scan.address);
1700 if (!shmem_huge_enabled(vma))
1701 goto skip;
1702 file = get_file(vma->vm_file);
1700 up_read(&mm->mmap_sem);
1701 ret = 1;
1702 khugepaged_scan_shmem(mm, file->f_mapping,
1703 pgoff, hpage);
1704 fput(file);
1705 } else {
1706 ret = khugepaged_scan_pmd(mm, vma,
1707 khugepaged_scan.address,

--- 203 unchanged lines hidden --- 		1703 up_read(&mm->mmap_sem);
1704 ret = 1;
1705 khugepaged_scan_shmem(mm, file->f_mapping,
1706 pgoff, hpage);
1707 fput(file);
1708 } else {
1709 ret = khugepaged_scan_pmd(mm, vma,
1710 khugepaged_scan.address,

--- 203 unchanged lines hidden ---