huge_memory.c (19ee151e140daa5183c4984981801e542e0544fb) huge_memory.c (5c4b4be3b6b937256103a5ae49177e0c3a17cb8f)
1/*
2 * Copyright (C) 2009 Red Hat, Inc.
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
6 */
7
8#include <linux/mm.h>

--- 636 unchanged lines hidden (view full) ---

645
646static inline gfp_t alloc_hugepage_gfpmask(int defrag)
647{
648 return GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT);
649}
650
651static inline struct page *alloc_hugepage_vma(int defrag,
652 struct vm_area_struct *vma,
1/*
2 * Copyright (C) 2009 Red Hat, Inc.
3 *
4 * This work is licensed under the terms of the GNU GPL, version 2. See
5 * the COPYING file in the top-level directory.
6 */
7
8#include <linux/mm.h>

--- 636 unchanged lines hidden (view full) ---

645
646static inline gfp_t alloc_hugepage_gfpmask(int defrag)
647{
648 return GFP_TRANSHUGE & ~(defrag ? 0 : __GFP_WAIT);
649}
650
651static inline struct page *alloc_hugepage_vma(int defrag,
652 struct vm_area_struct *vma,
653 unsigned long haddr)
653 unsigned long haddr, int nd)
654{
655 return alloc_pages_vma(alloc_hugepage_gfpmask(defrag),
654{
655 return alloc_pages_vma(alloc_hugepage_gfpmask(defrag),
656 HPAGE_PMD_ORDER, vma, haddr, numa_node_id());
656 HPAGE_PMD_ORDER, vma, haddr, nd);
657}
658
659#ifndef CONFIG_NUMA
660static inline struct page *alloc_hugepage(int defrag)
661{
662 return alloc_pages(alloc_hugepage_gfpmask(defrag),
663 HPAGE_PMD_ORDER);
664}

--- 8 unchanged lines hidden (view full) ---

673 pte_t *pte;
674
675 if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) {
676 if (unlikely(anon_vma_prepare(vma)))
677 return VM_FAULT_OOM;
678 if (unlikely(khugepaged_enter(vma)))
679 return VM_FAULT_OOM;
680 page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
657}
658
659#ifndef CONFIG_NUMA
660static inline struct page *alloc_hugepage(int defrag)
661{
662 return alloc_pages(alloc_hugepage_gfpmask(defrag),
663 HPAGE_PMD_ORDER);
664}

--- 8 unchanged lines hidden (view full) ---

673 pte_t *pte;
674
675 if (haddr >= vma->vm_start && haddr + HPAGE_PMD_SIZE <= vma->vm_end) {
676 if (unlikely(anon_vma_prepare(vma)))
677 return VM_FAULT_OOM;
678 if (unlikely(khugepaged_enter(vma)))
679 return VM_FAULT_OOM;
680 page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
681 vma, haddr);
681 vma, haddr, numa_node_id());
682 if (unlikely(!page))
683 goto out;
684 if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
685 put_page(page);
686 goto out;
687 }
688
689 return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page);

--- 207 unchanged lines hidden (view full) ---

897 goto out_unlock;
898 }
899 get_page(page);
900 spin_unlock(&mm->page_table_lock);
901
902 if (transparent_hugepage_enabled(vma) &&
903 !transparent_hugepage_debug_cow())
904 new_page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
682 if (unlikely(!page))
683 goto out;
684 if (unlikely(mem_cgroup_newpage_charge(page, mm, GFP_KERNEL))) {
685 put_page(page);
686 goto out;
687 }
688
689 return __do_huge_pmd_anonymous_page(mm, vma, haddr, pmd, page);

--- 207 unchanged lines hidden (view full) ---

897 goto out_unlock;
898 }
899 get_page(page);
900 spin_unlock(&mm->page_table_lock);
901
902 if (transparent_hugepage_enabled(vma) &&
903 !transparent_hugepage_debug_cow())
904 new_page = alloc_hugepage_vma(transparent_hugepage_defrag(vma),
905 vma, haddr);
905 vma, haddr, numa_node_id());
906 else
907 new_page = NULL;
908
909 if (unlikely(!new_page)) {
910 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
911 pmd, orig_pmd, page, haddr);
912 put_page(page);
913 goto out;

--- 826 unchanged lines hidden (view full) ---

1740 address += PAGE_SIZE;
1741 page++;
1742 }
1743}
1744
1745static void collapse_huge_page(struct mm_struct *mm,
1746 unsigned long address,
1747 struct page **hpage,
906 else
907 new_page = NULL;
908
909 if (unlikely(!new_page)) {
910 ret = do_huge_pmd_wp_page_fallback(mm, vma, address,
911 pmd, orig_pmd, page, haddr);
912 put_page(page);
913 goto out;

--- 826 unchanged lines hidden (view full) ---

1740 address += PAGE_SIZE;
1741 page++;
1742 }
1743}
1744
1745static void collapse_huge_page(struct mm_struct *mm,
1746 unsigned long address,
1747 struct page **hpage,
1748 struct vm_area_struct *vma)
1748 struct vm_area_struct *vma,
1749 int node)
1749{
1750 pgd_t *pgd;
1751 pud_t *pud;
1752 pmd_t *pmd, _pmd;
1753 pte_t *pte;
1754 pgtable_t pgtable;
1755 struct page *new_page;
1756 spinlock_t *ptl;

--- 11 unchanged lines hidden (view full) ---

1768 * the mmap_sem read mode so there is no memory allocation
1769 * later when we take the mmap_sem in write mode. This is more
1770 * friendly behavior (OTOH it may actually hide bugs) to
1771 * filesystems in userland with daemons allocating memory in
1772 * the userland I/O paths. Allocating memory with the
1773 * mmap_sem in read mode is good idea also to allow greater
1774 * scalability.
1775 */
1750{
1751 pgd_t *pgd;
1752 pud_t *pud;
1753 pmd_t *pmd, _pmd;
1754 pte_t *pte;
1755 pgtable_t pgtable;
1756 struct page *new_page;
1757 spinlock_t *ptl;

--- 11 unchanged lines hidden (view full) ---

1769 * the mmap_sem read mode so there is no memory allocation
1770 * later when we take the mmap_sem in write mode. This is more
1771 * friendly behavior (OTOH it may actually hide bugs) to
1772 * filesystems in userland with daemons allocating memory in
1773 * the userland I/O paths. Allocating memory with the
1774 * mmap_sem in read mode is good idea also to allow greater
1775 * scalability.
1776 */
1776 new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address);
1777 new_page = alloc_hugepage_vma(khugepaged_defrag(), vma, address,
1778 node);
1777 if (unlikely(!new_page)) {
1778 up_read(&mm->mmap_sem);
1779 *hpage = ERR_PTR(-ENOMEM);
1780 return;
1781 }
1782#endif
1783 if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
1784 up_read(&mm->mmap_sem);

--- 129 unchanged lines hidden (view full) ---

1914 pgd_t *pgd;
1915 pud_t *pud;
1916 pmd_t *pmd;
1917 pte_t *pte, *_pte;
1918 int ret = 0, referenced = 0, none = 0;
1919 struct page *page;
1920 unsigned long _address;
1921 spinlock_t *ptl;
1779 if (unlikely(!new_page)) {
1780 up_read(&mm->mmap_sem);
1781 *hpage = ERR_PTR(-ENOMEM);
1782 return;
1783 }
1784#endif
1785 if (unlikely(mem_cgroup_newpage_charge(new_page, mm, GFP_KERNEL))) {
1786 up_read(&mm->mmap_sem);

--- 129 unchanged lines hidden (view full) ---

1916 pgd_t *pgd;
1917 pud_t *pud;
1918 pmd_t *pmd;
1919 pte_t *pte, *_pte;
1920 int ret = 0, referenced = 0, none = 0;
1921 struct page *page;
1922 unsigned long _address;
1923 spinlock_t *ptl;
1924 int node = -1;
1922
1923 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1924
1925 pgd = pgd_offset(mm, address);
1926 if (!pgd_present(*pgd))
1927 goto out;
1928
1929 pud = pud_offset(pgd, address);

--- 14 unchanged lines hidden (view full) ---

1944 else
1945 goto out_unmap;
1946 }
1947 if (!pte_present(pteval) || !pte_write(pteval))
1948 goto out_unmap;
1949 page = vm_normal_page(vma, _address, pteval);
1950 if (unlikely(!page))
1951 goto out_unmap;
1925
1926 VM_BUG_ON(address & ~HPAGE_PMD_MASK);
1927
1928 pgd = pgd_offset(mm, address);
1929 if (!pgd_present(*pgd))
1930 goto out;
1931
1932 pud = pud_offset(pgd, address);

--- 14 unchanged lines hidden (view full) ---

1947 else
1948 goto out_unmap;
1949 }
1950 if (!pte_present(pteval) || !pte_write(pteval))
1951 goto out_unmap;
1952 page = vm_normal_page(vma, _address, pteval);
1953 if (unlikely(!page))
1954 goto out_unmap;
1955 /*
1956 * Chose the node of the first page. This could
1957 * be more sophisticated and look at more pages,
1958 * but isn't for now.
1959 */
1960 if (node == -1)
1961 node = page_to_nid(page);
1952 VM_BUG_ON(PageCompound(page));
1953 if (!PageLRU(page) || PageLocked(page) || !PageAnon(page))
1954 goto out_unmap;
1955 /* cannot use mapcount: can't collapse if there's a gup pin */
1956 if (page_count(page) != 1)
1957 goto out_unmap;
1958 if (pte_young(pteval) || PageReferenced(page) ||
1959 mmu_notifier_test_young(vma->vm_mm, address))
1960 referenced = 1;
1961 }
1962 if (referenced)
1963 ret = 1;
1964out_unmap:
1965 pte_unmap_unlock(pte, ptl);
1966 if (ret)
1967 /* collapse_huge_page will return with the mmap_sem released */
1962 VM_BUG_ON(PageCompound(page));
1963 if (!PageLRU(page) || PageLocked(page) || !PageAnon(page))
1964 goto out_unmap;
1965 /* cannot use mapcount: can't collapse if there's a gup pin */
1966 if (page_count(page) != 1)
1967 goto out_unmap;
1968 if (pte_young(pteval) || PageReferenced(page) ||
1969 mmu_notifier_test_young(vma->vm_mm, address))
1970 referenced = 1;
1971 }
1972 if (referenced)
1973 ret = 1;
1974out_unmap:
1975 pte_unmap_unlock(pte, ptl);
1976 if (ret)
1977 /* collapse_huge_page will return with the mmap_sem released */
1968 collapse_huge_page(mm, address, hpage, vma);
1978 collapse_huge_page(mm, address, hpage, vma, node);
1969out:
1970 return ret;
1971}
1972
1973static void collect_mm_slot(struct mm_slot *mm_slot)
1974{
1975 struct mm_struct *mm = mm_slot->mm;
1976

--- 375 unchanged lines hidden --- 		1979out:
1980 return ret;
1981}
1982
1983static void collect_mm_slot(struct mm_slot *mm_slot)
1984{
1985 struct mm_struct *mm = mm_slot->mm;
1986

--- 375 unchanged lines hidden ---