hugetlb.c (9bf9d47a29afbf7a43eae74a988a4aefe88ccbfd) hugetlb.c (106c992a5ebef28193cf5958e49ceff5e4aebb04)
1/*
2 * Generic hugetlb support.
3 * (C) Nadia Yvette Chambers, April 2004
4 */
5#include <linux/list.h>
6#include <linux/init.h>
7#include <linux/module.h>
8#include <linux/mm.h>

--- 2233 unchanged lines hidden (view full) ---

2242};
2243
2244static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2245 int writable)
2246{
2247 pte_t entry;
2248
2249 if (writable) {
1/*
2 * Generic hugetlb support.
3 * (C) Nadia Yvette Chambers, April 2004
4 */
5#include <linux/list.h>
6#include <linux/init.h>
7#include <linux/module.h>
8#include <linux/mm.h>

--- 2233 unchanged lines hidden (view full) ---

2242};
2243
2244static pte_t make_huge_pte(struct vm_area_struct *vma, struct page *page,
2245 int writable)
2246{
2247 pte_t entry;
2248
2249 if (writable) {
2250 entry =
2251 pte_mkwrite(pte_mkdirty(mk_pte(page, vma->vm_page_prot)));
2250 entry = huge_pte_mkwrite(huge_pte_mkdirty(mk_huge_pte(page,
2251 vma->vm_page_prot)));
2252 } else {
2252 } else {
2253 entry = huge_pte_wrprotect(mk_pte(page, vma->vm_page_prot));
2253 entry = huge_pte_wrprotect(mk_huge_pte(page,
2254 vma->vm_page_prot));
2254 }
2255 entry = pte_mkyoung(entry);
2256 entry = pte_mkhuge(entry);
2257 entry = arch_make_huge_pte(entry, vma, page, writable);
2258
2259 return entry;
2260}
2261
2262static void set_huge_ptep_writable(struct vm_area_struct *vma,
2263 unsigned long address, pte_t *ptep)
2264{
2265 pte_t entry;
2266
2255 }
2256 entry = pte_mkyoung(entry);
2257 entry = pte_mkhuge(entry);
2258 entry = arch_make_huge_pte(entry, vma, page, writable);
2259
2260 return entry;
2261}
2262
2263static void set_huge_ptep_writable(struct vm_area_struct *vma,
2264 unsigned long address, pte_t *ptep)
2265{
2266 pte_t entry;
2267
2267 entry = pte_mkwrite(pte_mkdirty(huge_ptep_get(ptep)));
2268 entry = huge_pte_mkwrite(huge_pte_mkdirty(huge_ptep_get(ptep)));
2268 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2269 update_mmu_cache(vma, address, ptep);
2270}
2271
2272
2273int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2274 struct vm_area_struct *vma)
2275{

--- 98 unchanged lines hidden (view full) ---

2374 pte = huge_ptep_get(ptep);
2375 if (huge_pte_none(pte))
2376 continue;
2377
2378 /*
2379 * HWPoisoned hugepage is already unmapped and dropped reference
2380 */
2381 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2269 if (huge_ptep_set_access_flags(vma, address, ptep, entry, 1))
2270 update_mmu_cache(vma, address, ptep);
2271}
2272
2273
2274int copy_hugetlb_page_range(struct mm_struct *dst, struct mm_struct *src,
2275 struct vm_area_struct *vma)
2276{

--- 98 unchanged lines hidden (view full) ---

2375 pte = huge_ptep_get(ptep);
2376 if (huge_pte_none(pte))
2377 continue;
2378
2379 /*
2380 * HWPoisoned hugepage is already unmapped and dropped reference
2381 */
2382 if (unlikely(is_hugetlb_entry_hwpoisoned(pte))) {
2382 pte_clear(mm, address, ptep);
2383 huge_pte_clear(mm, address, ptep);
2383 continue;
2384 }
2385
2386 page = pte_page(pte);
2387 /*
2388 * If a reference page is supplied, it is because a specific
2389 * page is being unmapped, not a range. Ensure the page we
2390 * are about to unmap is the actual page of interest.

--- 7 unchanged lines hidden (view full) ---

2398 * future faults in this VMA will fail rather than
2399 * looking like data was lost
2400 */
2401 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2402 }
2403
2404 pte = huge_ptep_get_and_clear(mm, address, ptep);
2405 tlb_remove_tlb_entry(tlb, ptep, address);
2384 continue;
2385 }
2386
2387 page = pte_page(pte);
2388 /*
2389 * If a reference page is supplied, it is because a specific
2390 * page is being unmapped, not a range. Ensure the page we
2391 * are about to unmap is the actual page of interest.

--- 7 unchanged lines hidden (view full) ---

2399 * future faults in this VMA will fail rather than
2400 * looking like data was lost
2401 */
2402 set_vma_resv_flags(vma, HPAGE_RESV_UNMAPPED);
2403 }
2404
2405 pte = huge_ptep_get_and_clear(mm, address, ptep);
2406 tlb_remove_tlb_entry(tlb, ptep, address);
2406 if (pte_dirty(pte))
2407 if (huge_pte_dirty(pte))
2407 set_page_dirty(page);
2408
2409 page_remove_rmap(page);
2410 force_flush = !__tlb_remove_page(tlb, page);
2411 if (force_flush)
2412 break;
2413 /* Bail out after unmapping reference page if supplied */
2414 if (ref_page)

--- 436 unchanged lines hidden (view full) ---

2851 /*
2852 * If we are going to COW the mapping later, we examine the pending
2853 * reservations for this page now. This will ensure that any
2854 * allocations necessary to record that reservation occur outside the
2855 * spinlock. For private mappings, we also lookup the pagecache
2856 * page now as it is used to determine if a reservation has been
2857 * consumed.
2858 */
2408 set_page_dirty(page);
2409
2410 page_remove_rmap(page);
2411 force_flush = !__tlb_remove_page(tlb, page);
2412 if (force_flush)
2413 break;
2414 /* Bail out after unmapping reference page if supplied */
2415 if (ref_page)

--- 436 unchanged lines hidden (view full) ---

2852 /*
2853 * If we are going to COW the mapping later, we examine the pending
2854 * reservations for this page now. This will ensure that any
2855 * allocations necessary to record that reservation occur outside the
2856 * spinlock. For private mappings, we also lookup the pagecache
2857 * page now as it is used to determine if a reservation has been
2858 * consumed.
2859 */
2859 if ((flags & FAULT_FLAG_WRITE) && !pte_write(entry)) {
2860 if ((flags & FAULT_FLAG_WRITE) && !huge_pte_write(entry)) {
2860 if (vma_needs_reservation(h, vma, address) < 0) {
2861 ret = VM_FAULT_OOM;
2862 goto out_mutex;
2863 }
2864
2865 if (!(vma->vm_flags & VM_MAYSHARE))
2866 pagecache_page = hugetlbfs_pagecache_page(h,
2867 vma, address);

--- 13 unchanged lines hidden (view full) ---

2881
2882 spin_lock(&mm->page_table_lock);
2883 /* Check for a racing update before calling hugetlb_cow */
2884 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2885 goto out_page_table_lock;
2886
2887
2888 if (flags & FAULT_FLAG_WRITE) {
2861 if (vma_needs_reservation(h, vma, address) < 0) {
2862 ret = VM_FAULT_OOM;
2863 goto out_mutex;
2864 }
2865
2866 if (!(vma->vm_flags & VM_MAYSHARE))
2867 pagecache_page = hugetlbfs_pagecache_page(h,
2868 vma, address);

--- 13 unchanged lines hidden (view full) ---

2882
2883 spin_lock(&mm->page_table_lock);
2884 /* Check for a racing update before calling hugetlb_cow */
2885 if (unlikely(!pte_same(entry, huge_ptep_get(ptep))))
2886 goto out_page_table_lock;
2887
2888
2889 if (flags & FAULT_FLAG_WRITE) {
2889 if (!pte_write(entry)) {
2890 if (!huge_pte_write(entry)) {
2890 ret = hugetlb_cow(mm, vma, address, ptep, entry,
2891 pagecache_page);
2892 goto out_page_table_lock;
2893 }
2891 ret = hugetlb_cow(mm, vma, address, ptep, entry,
2892 pagecache_page);
2893 goto out_page_table_lock;
2894 }
2894 entry = pte_mkdirty(entry);
2895 entry = huge_pte_mkdirty(entry);
2895 }
2896 entry = pte_mkyoung(entry);
2897 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2898 flags & FAULT_FLAG_WRITE))
2899 update_mmu_cache(vma, address, ptep);
2900
2901out_page_table_lock:
2902 spin_unlock(&mm->page_table_lock);

--- 64 unchanged lines hidden (view full) ---

2967 * hwpoisoned hugepages (in which case we need to prevent the
2968 * caller from accessing to them.) In order to do this, we use
2969 * here is_swap_pte instead of is_hugetlb_entry_migration and
2970 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
2971 * both cases, and because we can't follow correct pages
2972 * directly from any kind of swap entries.
2973 */
2974 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
2896 }
2897 entry = pte_mkyoung(entry);
2898 if (huge_ptep_set_access_flags(vma, address, ptep, entry,
2899 flags & FAULT_FLAG_WRITE))
2900 update_mmu_cache(vma, address, ptep);
2901
2902out_page_table_lock:
2903 spin_unlock(&mm->page_table_lock);

--- 64 unchanged lines hidden (view full) ---

2968 * hwpoisoned hugepages (in which case we need to prevent the
2969 * caller from accessing to them.) In order to do this, we use
2970 * here is_swap_pte instead of is_hugetlb_entry_migration and
2971 * is_hugetlb_entry_hwpoisoned. This is because it simply covers
2972 * both cases, and because we can't follow correct pages
2973 * directly from any kind of swap entries.
2974 */
2975 if (absent || is_swap_pte(huge_ptep_get(pte)) ||
2975 ((flags & FOLL_WRITE) && !pte_write(huge_ptep_get(pte)))) {
2976 ((flags & FOLL_WRITE) &&
2977 !huge_pte_write(huge_ptep_get(pte)))) {
2976 int ret;
2977
2978 spin_unlock(&mm->page_table_lock);
2979 ret = hugetlb_fault(mm, vma, vaddr,
2980 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2981 spin_lock(&mm->page_table_lock);
2982 if (!(ret & VM_FAULT_ERROR))
2983 continue;

--- 53 unchanged lines hidden (view full) ---

3037 if (!ptep)
3038 continue;
3039 if (huge_pmd_unshare(mm, &address, ptep)) {
3040 pages++;
3041 continue;
3042 }
3043 if (!huge_pte_none(huge_ptep_get(ptep))) {
3044 pte = huge_ptep_get_and_clear(mm, address, ptep);
2978 int ret;
2979
2980 spin_unlock(&mm->page_table_lock);
2981 ret = hugetlb_fault(mm, vma, vaddr,
2982 (flags & FOLL_WRITE) ? FAULT_FLAG_WRITE : 0);
2983 spin_lock(&mm->page_table_lock);
2984 if (!(ret & VM_FAULT_ERROR))
2985 continue;

--- 53 unchanged lines hidden (view full) ---

3039 if (!ptep)
3040 continue;
3041 if (huge_pmd_unshare(mm, &address, ptep)) {
3042 pages++;
3043 continue;
3044 }
3045 if (!huge_pte_none(huge_ptep_get(ptep))) {
3046 pte = huge_ptep_get_and_clear(mm, address, ptep);
3045 pte = pte_mkhuge(pte_modify(pte, newprot));
3047 pte = pte_mkhuge(huge_pte_modify(pte, newprot));
3046 pte = arch_make_huge_pte(pte, vma, NULL, 0);
3047 set_huge_pte_at(mm, address, ptep, pte);
3048 pages++;
3049 }
3050 }
3051 spin_unlock(&mm->page_table_lock);
3052 /*
3053 * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare

--- 145 unchanged lines hidden --- 		3048 pte = arch_make_huge_pte(pte, vma, NULL, 0);
3049 set_huge_pte_at(mm, address, ptep, pte);
3050 pages++;
3051 }
3052 }
3053 spin_unlock(&mm->page_table_lock);
3054 /*
3055 * Must flush TLB before releasing i_mmap_mutex: x86's huge_pmd_unshare

--- 145 unchanged lines hidden ---