shmem.c (95cc09d66f523895635eca8aa1f9bc7419e60e32) shmem.c (cfda05267f7bd02b5ae5ac6a37fbbdf3b9c41b57)
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.

--- 58 unchanged lines hidden (view full) ---

67#include <linux/ctype.h>
68#include <linux/migrate.h>
69#include <linux/highmem.h>
70#include <linux/seq_file.h>
71#include <linux/magic.h>
72#include <linux/syscalls.h>
73#include <linux/fcntl.h>
74#include <uapi/linux/memfd.h>
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.

--- 58 unchanged lines hidden (view full) ---

67#include <linux/ctype.h>
68#include <linux/migrate.h>
69#include <linux/highmem.h>
70#include <linux/seq_file.h>
71#include <linux/magic.h>
72#include <linux/syscalls.h>
73#include <linux/fcntl.h>
74#include <uapi/linux/memfd.h>
75#include <linux/userfaultfd_k.h>
75#include <linux/rmap.h>
76
77#include <linux/uaccess.h>
78#include <asm/pgtable.h>
79
80#include "internal.h"
81
82#define BLOCKS_PER_PAGE (PAGE_SIZE/512)

--- 30 unchanged lines hidden (view full) ---

113}
114#endif
115
116static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
117static int shmem_replace_page(struct page **pagep, gfp_t gfp,
118 struct shmem_inode_info *info, pgoff_t index);
119static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
120 struct page **pagep, enum sgp_type sgp,
76#include <linux/rmap.h>
77
78#include <linux/uaccess.h>
79#include <asm/pgtable.h>
80
81#include "internal.h"
82
83#define BLOCKS_PER_PAGE (PAGE_SIZE/512)

--- 30 unchanged lines hidden (view full) ---

114}
115#endif
116
117static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
118static int shmem_replace_page(struct page **pagep, gfp_t gfp,
119 struct shmem_inode_info *info, pgoff_t index);
120static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
121 struct page **pagep, enum sgp_type sgp,
121 gfp_t gfp, struct mm_struct *fault_mm, int *fault_type);
122 gfp_t gfp, struct vm_area_struct *vma,
123 struct vm_fault *vmf, int *fault_type);
122
123int shmem_getpage(struct inode *inode, pgoff_t index,
124 struct page **pagep, enum sgp_type sgp)
125{
126 return shmem_getpage_gfp(inode, index, pagep, sgp,
124
125int shmem_getpage(struct inode *inode, pgoff_t index,
126 struct page **pagep, enum sgp_type sgp)
127{
128 return shmem_getpage_gfp(inode, index, pagep, sgp,
127 mapping_gfp_mask(inode->i_mapping), NULL, NULL);
129 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
128}
129
130static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
131{
132 return sb->s_fs_info;
133}
134
135/*

--- 1437 unchanged lines hidden (view full) ---

1573 * vm. If we swap it in we mark it dirty since we also free the swap
1574 * entry since a page cannot live in both the swap and page cache.
1575 *
1576 * fault_mm and fault_type are only supplied by shmem_fault:
1577 * otherwise they are NULL.
1578 */
1579static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1580 struct page **pagep, enum sgp_type sgp, gfp_t gfp,
130}
131
132static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
133{
134 return sb->s_fs_info;
135}
136
137/*

--- 1437 unchanged lines hidden (view full) ---

1575 * vm. If we swap it in we mark it dirty since we also free the swap
1576 * entry since a page cannot live in both the swap and page cache.
1577 *
1578 * fault_mm and fault_type are only supplied by shmem_fault:
1579 * otherwise they are NULL.
1580 */
1581static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1582 struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1581 struct mm_struct *fault_mm, int *fault_type)
1583 struct vm_area_struct *vma, struct vm_fault *vmf, int *fault_type)
1582{
1583 struct address_space *mapping = inode->i_mapping;
1584 struct shmem_inode_info *info = SHMEM_I(inode);
1585 struct shmem_sb_info *sbinfo;
1586 struct mm_struct *charge_mm;
1587 struct mem_cgroup *memcg;
1588 struct page *page;
1589 swp_entry_t swap;

--- 37 unchanged lines hidden (view full) ---

1627 return 0;
1628 }
1629
1630 /*
1631 * Fast cache lookup did not find it:
1632 * bring it back from swap or allocate.
1633 */
1634 sbinfo = SHMEM_SB(inode->i_sb);
1584{
1585 struct address_space *mapping = inode->i_mapping;
1586 struct shmem_inode_info *info = SHMEM_I(inode);
1587 struct shmem_sb_info *sbinfo;
1588 struct mm_struct *charge_mm;
1589 struct mem_cgroup *memcg;
1590 struct page *page;
1591 swp_entry_t swap;

--- 37 unchanged lines hidden (view full) ---

1629 return 0;
1630 }
1631
1632 /*
1633 * Fast cache lookup did not find it:
1634 * bring it back from swap or allocate.
1635 */
1636 sbinfo = SHMEM_SB(inode->i_sb);
1635 charge_mm = fault_mm ? : current->mm;
1637 charge_mm = vma ? vma->vm_mm : current->mm;
1636
1637 if (swap.val) {
1638 /* Look it up and read it in.. */
1639 page = lookup_swap_cache(swap);
1640 if (!page) {
1641 /* Or update major stats only when swapin succeeds?? */
1642 if (fault_type) {
1643 *fault_type |= VM_FAULT_MAJOR;
1644 count_vm_event(PGMAJFAULT);
1638
1639 if (swap.val) {
1640 /* Look it up and read it in.. */
1641 page = lookup_swap_cache(swap);
1642 if (!page) {
1643 /* Or update major stats only when swapin succeeds?? */
1644 if (fault_type) {
1645 *fault_type |= VM_FAULT_MAJOR;
1646 count_vm_event(PGMAJFAULT);
1645 mem_cgroup_count_vm_event(fault_mm, PGMAJFAULT);
1647 mem_cgroup_count_vm_event(charge_mm,
1648 PGMAJFAULT);
1646 }
1647 /* Here we actually start the io */
1648 page = shmem_swapin(swap, gfp, info, index);
1649 if (!page) {
1650 error = -ENOMEM;
1651 goto failed;
1652 }
1653 }

--- 52 unchanged lines hidden (view full) ---

1706 if (sgp == SGP_WRITE)
1707 mark_page_accessed(page);
1708
1709 delete_from_swap_cache(page);
1710 set_page_dirty(page);
1711 swap_free(swap);
1712
1713 } else {
1649 }
1650 /* Here we actually start the io */
1651 page = shmem_swapin(swap, gfp, info, index);
1652 if (!page) {
1653 error = -ENOMEM;
1654 goto failed;
1655 }
1656 }

--- 52 unchanged lines hidden (view full) ---

1709 if (sgp == SGP_WRITE)
1710 mark_page_accessed(page);
1711
1712 delete_from_swap_cache(page);
1713 set_page_dirty(page);
1714 swap_free(swap);
1715
1716 } else {
1717 if (vma && userfaultfd_missing(vma)) {
1718 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1719 return 0;
1720 }
1721
1714 /* shmem_symlink() */
1715 if (mapping->a_ops != &shmem_aops)
1716 goto alloc_nohuge;
1717 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1718 goto alloc_nohuge;
1719 if (shmem_huge == SHMEM_HUGE_FORCE)
1720 goto alloc_huge;
1721 switch (sbinfo->huge) {

--- 246 unchanged lines hidden (view full) ---

1968
1969 sgp = SGP_CACHE;
1970 if (vma->vm_flags & VM_HUGEPAGE)
1971 sgp = SGP_HUGE;
1972 else if (vma->vm_flags & VM_NOHUGEPAGE)
1973 sgp = SGP_NOHUGE;
1974
1975 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
1722 /* shmem_symlink() */
1723 if (mapping->a_ops != &shmem_aops)
1724 goto alloc_nohuge;
1725 if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1726 goto alloc_nohuge;
1727 if (shmem_huge == SHMEM_HUGE_FORCE)
1728 goto alloc_huge;
1729 switch (sbinfo->huge) {

--- 246 unchanged lines hidden (view full) ---

1976
1977 sgp = SGP_CACHE;
1978 if (vma->vm_flags & VM_HUGEPAGE)
1979 sgp = SGP_HUGE;
1980 else if (vma->vm_flags & VM_NOHUGEPAGE)
1981 sgp = SGP_NOHUGE;
1982
1983 error = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
1976 gfp, vma->vm_mm, &ret);
1984 gfp, vma, vmf, &ret);
1977 if (error)
1978 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1979 return ret;
1980}
1981
1982unsigned long shmem_get_unmapped_area(struct file *file,
1983 unsigned long uaddr, unsigned long len,
1984 unsigned long pgoff, unsigned long flags)

--- 2264 unchanged lines hidden (view full) ---

4249{
4250#ifdef CONFIG_SHMEM
4251 struct inode *inode = mapping->host;
4252 struct page *page;
4253 int error;
4254
4255 BUG_ON(mapping->a_ops != &shmem_aops);
4256 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
1985 if (error)
1986 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1987 return ret;
1988}
1989
1990unsigned long shmem_get_unmapped_area(struct file *file,
1991 unsigned long uaddr, unsigned long len,
1992 unsigned long pgoff, unsigned long flags)

--- 2264 unchanged lines hidden (view full) ---

4257{
4258#ifdef CONFIG_SHMEM
4259 struct inode *inode = mapping->host;
4260 struct page *page;
4261 int error;
4262
4263 BUG_ON(mapping->a_ops != &shmem_aops);
4264 error = shmem_getpage_gfp(inode, index, &page, SGP_CACHE,
4257 gfp, NULL, NULL);
4265 gfp, NULL, NULL, NULL);
4258 if (error)
4259 page = ERR_PTR(error);
4260 else
4261 unlock_page(page);
4262 return page;
4263#else
4264 /*
4265 * The tiny !SHMEM case uses ramfs without swap
4266 */
4267 return read_cache_page_gfp(mapping, index, gfp);
4268#endif
4269}
4270EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);
4266 if (error)
4267 page = ERR_PTR(error);
4268 else
4269 unlock_page(page);
4270 return page;
4271#else
4272 /*
4273 * The tiny !SHMEM case uses ramfs without swap
4274 */
4275 return read_cache_page_gfp(mapping, index, gfp);
4276#endif
4277}
4278EXPORT_SYMBOL_GPL(shmem_read_mapping_page_gfp);