shmem.c (f3f0e1d2150b2b99da2cbdfaad000089efe9bf30) shmem.c (e496cf3d782135c1cca0d154d4b924517ff58de0)
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.

--- 349 unchanged lines hidden (view full) ---

358 * disables huge on shm_mnt and all mounts, for emergency use;
359 * SHMEM_HUGE_FORCE:
360 * enables huge on shm_mnt and all mounts, w/o needing option, for testing;
361 *
362 */
363#define SHMEM_HUGE_DENY (-1)
364#define SHMEM_HUGE_FORCE (-2)
365
1/*
2 * Resizable virtual memory filesystem for Linux.
3 *
4 * Copyright (C) 2000 Linus Torvalds.
5 * 2000 Transmeta Corp.
6 * 2000-2001 Christoph Rohland
7 * 2000-2001 SAP AG
8 * 2002 Red Hat Inc.

--- 349 unchanged lines hidden (view full) ---

358 * disables huge on shm_mnt and all mounts, for emergency use;
359 * SHMEM_HUGE_FORCE:
360 * enables huge on shm_mnt and all mounts, w/o needing option, for testing;
361 *
362 */
363#define SHMEM_HUGE_DENY (-1)
364#define SHMEM_HUGE_FORCE (-2)
365
366#ifdef CONFIG_TRANSPARENT_HUGEPAGE
366#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
367/* ifdef here to avoid bloating shmem.o when not necessary */
368
369int shmem_huge __read_mostly;
370
371static int shmem_parse_huge(const char *str)
372{
373 if (!strcmp(str, "never"))
374 return SHMEM_HUGE_NEVER;

--- 26 unchanged lines hidden (view full) ---

401 case SHMEM_HUGE_FORCE:
402 return "force";
403 default:
404 VM_BUG_ON(1);
405 return "bad_val";
406 }
407}
408
367/* ifdef here to avoid bloating shmem.o when not necessary */
368
369int shmem_huge __read_mostly;
370
371static int shmem_parse_huge(const char *str)
372{
373 if (!strcmp(str, "never"))
374 return SHMEM_HUGE_NEVER;

--- 26 unchanged lines hidden (view full) ---

401 case SHMEM_HUGE_FORCE:
402 return "force";
403 default:
404 VM_BUG_ON(1);
405 return "bad_val";
406 }
407}
408
409#else /* !CONFIG_TRANSPARENT_HUGEPAGE */
409#else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
410
411#define shmem_huge SHMEM_HUGE_DENY
412
410
411#define shmem_huge SHMEM_HUGE_DENY
412
413#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
413#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
414
415/*
416 * Like add_to_page_cache_locked, but error if expected item has gone.
417 */
418static int shmem_add_to_page_cache(struct page *page,
419 struct address_space *mapping,
420 pgoff_t index, void *expected)
421{

--- 802 unchanged lines hidden (view full) ---

1224{
1225 struct vm_area_struct pvma;
1226 struct inode *inode = &info->vfs_inode;
1227 struct address_space *mapping = inode->i_mapping;
1228 pgoff_t idx, hindex = round_down(index, HPAGE_PMD_NR);
1229 void __rcu **results;
1230 struct page *page;
1231
414
415/*
416 * Like add_to_page_cache_locked, but error if expected item has gone.
417 */
418static int shmem_add_to_page_cache(struct page *page,
419 struct address_space *mapping,
420 pgoff_t index, void *expected)
421{

--- 802 unchanged lines hidden (view full) ---

1224{
1225 struct vm_area_struct pvma;
1226 struct inode *inode = &info->vfs_inode;
1227 struct address_space *mapping = inode->i_mapping;
1228 pgoff_t idx, hindex = round_down(index, HPAGE_PMD_NR);
1229 void __rcu **results;
1230 struct page *page;
1231
1232 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1232 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1233 return NULL;
1234
1235 rcu_read_lock();
1236 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx,
1237 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1238 rcu_read_unlock();
1239 return NULL;
1240 }

--- 24 unchanged lines hidden (view full) ---

1265static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1266 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo,
1267 pgoff_t index, bool huge)
1268{
1269 struct page *page;
1270 int nr;
1271 int err = -ENOSPC;
1272
1233 return NULL;
1234
1235 rcu_read_lock();
1236 if (radix_tree_gang_lookup_slot(&mapping->page_tree, &results, &idx,
1237 hindex, 1) && idx < hindex + HPAGE_PMD_NR) {
1238 rcu_read_unlock();
1239 return NULL;
1240 }

--- 24 unchanged lines hidden (view full) ---

1265static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1266 struct shmem_inode_info *info, struct shmem_sb_info *sbinfo,
1267 pgoff_t index, bool huge)
1268{
1269 struct page *page;
1270 int nr;
1271 int err = -ENOSPC;
1272
1273 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1273 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1274 huge = false;
1275 nr = huge ? HPAGE_PMD_NR : 1;
1276
1277 if (shmem_acct_block(info->flags, nr))
1278 goto failed;
1279 if (sbinfo->max_blocks) {
1280 if (percpu_counter_compare(&sbinfo->used_blocks,
1281 sbinfo->max_blocks - nr) > 0)

--- 486 unchanged lines hidden (view full) ---

1768 unsigned long inflated_offset;
1769
1770 if (len > TASK_SIZE)
1771 return -ENOMEM;
1772
1773 get_area = current->mm->get_unmapped_area;
1774 addr = get_area(file, uaddr, len, pgoff, flags);
1775
1274 huge = false;
1275 nr = huge ? HPAGE_PMD_NR : 1;
1276
1277 if (shmem_acct_block(info->flags, nr))
1278 goto failed;
1279 if (sbinfo->max_blocks) {
1280 if (percpu_counter_compare(&sbinfo->used_blocks,
1281 sbinfo->max_blocks - nr) > 0)

--- 486 unchanged lines hidden (view full) ---

1768 unsigned long inflated_offset;
1769
1770 if (len > TASK_SIZE)
1771 return -ENOMEM;
1772
1773 get_area = current->mm->get_unmapped_area;
1774 addr = get_area(file, uaddr, len, pgoff, flags);
1775
1776 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE))
1776 if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1777 return addr;
1778 if (IS_ERR_VALUE(addr))
1779 return addr;
1780 if (addr & ~PAGE_MASK)
1781 return addr;
1782 if (addr > TASK_SIZE - len)
1783 return addr;
1784

--- 100 unchanged lines hidden (view full) ---

1885 spin_unlock_irq(&info->lock);
1886 return retval;
1887}
1888
1889static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1890{
1891 file_accessed(file);
1892 vma->vm_ops = &shmem_vm_ops;
1777 return addr;
1778 if (IS_ERR_VALUE(addr))
1779 return addr;
1780 if (addr & ~PAGE_MASK)
1781 return addr;
1782 if (addr > TASK_SIZE - len)
1783 return addr;
1784

--- 100 unchanged lines hidden (view full) ---

1885 spin_unlock_irq(&info->lock);
1886 return retval;
1887}
1888
1889static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1890{
1891 file_accessed(file);
1892 vma->vm_ops = &shmem_vm_ops;
1893 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
1893 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
1894 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
1895 (vma->vm_end & HPAGE_PMD_MASK)) {
1896 khugepaged_enter(vma, vma->vm_flags);
1897 }
1898 return 0;
1899}
1900
1901static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,

--- 1380 unchanged lines hidden (view full) ---

3282 if (remount)
3283 continue;
3284 gid = simple_strtoul(value, &rest, 0);
3285 if (*rest)
3286 goto bad_val;
3287 sbinfo->gid = make_kgid(current_user_ns(), gid);
3288 if (!gid_valid(sbinfo->gid))
3289 goto bad_val;
1894 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
1895 (vma->vm_end & HPAGE_PMD_MASK)) {
1896 khugepaged_enter(vma, vma->vm_flags);
1897 }
1898 return 0;
1899}
1900
1901static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,

--- 1380 unchanged lines hidden (view full) ---

3282 if (remount)
3283 continue;
3284 gid = simple_strtoul(value, &rest, 0);
3285 if (*rest)
3286 goto bad_val;
3287 sbinfo->gid = make_kgid(current_user_ns(), gid);
3288 if (!gid_valid(sbinfo->gid))
3289 goto bad_val;
3290#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3290#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3291 } else if (!strcmp(this_char, "huge")) {
3292 int huge;
3293 huge = shmem_parse_huge(value);
3294 if (huge < 0)
3295 goto bad_val;
3296 if (!has_transparent_hugepage() &&
3297 huge != SHMEM_HUGE_NEVER)
3298 goto bad_val;

--- 80 unchanged lines hidden (view full) ---

3379 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
3380 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3381 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3382 seq_printf(seq, ",uid=%u",
3383 from_kuid_munged(&init_user_ns, sbinfo->uid));
3384 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3385 seq_printf(seq, ",gid=%u",
3386 from_kgid_munged(&init_user_ns, sbinfo->gid));
3291 } else if (!strcmp(this_char, "huge")) {
3292 int huge;
3293 huge = shmem_parse_huge(value);
3294 if (huge < 0)
3295 goto bad_val;
3296 if (!has_transparent_hugepage() &&
3297 huge != SHMEM_HUGE_NEVER)
3298 goto bad_val;

--- 80 unchanged lines hidden (view full) ---

3379 if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
3380 seq_printf(seq, ",mode=%03ho", sbinfo->mode);
3381 if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
3382 seq_printf(seq, ",uid=%u",
3383 from_kuid_munged(&init_user_ns, sbinfo->uid));
3384 if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
3385 seq_printf(seq, ",gid=%u",
3386 from_kgid_munged(&init_user_ns, sbinfo->gid));
3387#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3387#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3388 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3389 if (sbinfo->huge)
3390 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3391#endif
3392 shmem_show_mpol(seq, sbinfo->mpol);
3393 return 0;
3394}
3395

--- 329 unchanged lines hidden (view full) ---

3725
3726 shm_mnt = kern_mount(&shmem_fs_type);
3727 if (IS_ERR(shm_mnt)) {
3728 error = PTR_ERR(shm_mnt);
3729 pr_err("Could not kern_mount tmpfs\n");
3730 goto out1;
3731 }
3732
3388 /* Rightly or wrongly, show huge mount option unmasked by shmem_huge */
3389 if (sbinfo->huge)
3390 seq_printf(seq, ",huge=%s", shmem_format_huge(sbinfo->huge));
3391#endif
3392 shmem_show_mpol(seq, sbinfo->mpol);
3393 return 0;
3394}
3395

--- 329 unchanged lines hidden (view full) ---

3725
3726 shm_mnt = kern_mount(&shmem_fs_type);
3727 if (IS_ERR(shm_mnt)) {
3728 error = PTR_ERR(shm_mnt);
3729 pr_err("Could not kern_mount tmpfs\n");
3730 goto out1;
3731 }
3732
3733#ifdef CONFIG_TRANSPARENT_HUGEPAGE
3733#ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
3734 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY)
3735 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3736 else
3737 shmem_huge = 0; /* just in case it was patched */
3738#endif
3739 return 0;
3740
3741out1:
3742 unregister_filesystem(&shmem_fs_type);
3743out2:
3744 shmem_destroy_inodecache();
3745out3:
3746 shm_mnt = ERR_PTR(error);
3747 return error;
3748}
3749
3734 if (has_transparent_hugepage() && shmem_huge < SHMEM_HUGE_DENY)
3735 SHMEM_SB(shm_mnt->mnt_sb)->huge = shmem_huge;
3736 else
3737 shmem_huge = 0; /* just in case it was patched */
3738#endif
3739 return 0;
3740
3741out1:
3742 unregister_filesystem(&shmem_fs_type);
3743out2:
3744 shmem_destroy_inodecache();
3745out3:
3746 shm_mnt = ERR_PTR(error);
3747 return error;
3748}
3749
3750#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)
3750#if defined(CONFIG_TRANSPARENT_HUGE_PAGECACHE) && defined(CONFIG_SYSFS)
3751static ssize_t shmem_enabled_show(struct kobject *kobj,
3752 struct kobj_attribute *attr, char *buf)
3753{
3754 int values[] = {
3755 SHMEM_HUGE_ALWAYS,
3756 SHMEM_HUGE_WITHIN_SIZE,
3757 SHMEM_HUGE_ADVISE,
3758 SHMEM_HUGE_NEVER,

--- 66 unchanged lines hidden (view full) ---

3825 case SHMEM_HUGE_ADVISE:
3826 /* TODO: implement fadvise() hints */
3827 return (vma->vm_flags & VM_HUGEPAGE);
3828 default:
3829 VM_BUG_ON(1);
3830 return false;
3831 }
3832}
3751static ssize_t shmem_enabled_show(struct kobject *kobj,
3752 struct kobj_attribute *attr, char *buf)
3753{
3754 int values[] = {
3755 SHMEM_HUGE_ALWAYS,
3756 SHMEM_HUGE_WITHIN_SIZE,
3757 SHMEM_HUGE_ADVISE,
3758 SHMEM_HUGE_NEVER,

--- 66 unchanged lines hidden (view full) ---

3825 case SHMEM_HUGE_ADVISE:
3826 /* TODO: implement fadvise() hints */
3827 return (vma->vm_flags & VM_HUGEPAGE);
3828 default:
3829 VM_BUG_ON(1);
3830 return false;
3831 }
3832}
3833#endif /* CONFIG_TRANSPARENT_HUGEPAGE && CONFIG_SYSFS */
3833#endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE && CONFIG_SYSFS */
3834
3835#else /* !CONFIG_SHMEM */
3836
3837/*
3838 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3839 *
3840 * This is intended for small system where the benefits of the full
3841 * shmem code (swap-backed and resource-limited) are outweighed by

--- 163 unchanged lines hidden (view full) ---

4005 if (IS_ERR(file))
4006 return PTR_ERR(file);
4007
4008 if (vma->vm_file)
4009 fput(vma->vm_file);
4010 vma->vm_file = file;
4011 vma->vm_ops = &shmem_vm_ops;
4012
3834
3835#else /* !CONFIG_SHMEM */
3836
3837/*
3838 * tiny-shmem: simple shmemfs and tmpfs using ramfs code
3839 *
3840 * This is intended for small system where the benefits of the full
3841 * shmem code (swap-backed and resource-limited) are outweighed by

--- 163 unchanged lines hidden (view full) ---

4005 if (IS_ERR(file))
4006 return PTR_ERR(file);
4007
4008 if (vma->vm_file)
4009 fput(vma->vm_file);
4010 vma->vm_file = file;
4011 vma->vm_ops = &shmem_vm_ops;
4012
4013 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGEPAGE) &&
4013 if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
4014 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4015 (vma->vm_end & HPAGE_PMD_MASK)) {
4016 khugepaged_enter(vma, vma->vm_flags);
4017 }
4018
4019 return 0;
4020}
4021

--- 39 unchanged lines hidden --- 		4014 ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
4015 (vma->vm_end & HPAGE_PMD_MASK)) {
4016 khugepaged_enter(vma, vma->vm_flags);
4017 }
4018
4019 return 0;
4020}
4021

--- 39 unchanged lines hidden ---