11da177e4SLinus Torvalds /*
21da177e4SLinus Torvalds * hugetlbpage-backed filesystem. Based on ramfs.
31da177e4SLinus Torvalds *
46d49e352SNadia Yvette Chambers * Nadia Yvette Chambers, 2002
51da177e4SLinus Torvalds *
61da177e4SLinus Torvalds * Copyright (C) 2002 Linus Torvalds.
73e89e1c5SPaul Gortmaker * License: GPL
81da177e4SLinus Torvalds */
91da177e4SLinus Torvalds
109b857d26SAndrew Morton #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
119b857d26SAndrew Morton
121da177e4SLinus Torvalds #include <linux/thread_info.h>
131da177e4SLinus Torvalds #include <asm/current.h>
1470c3547eSMike Kravetz #include <linux/falloc.h>
151da177e4SLinus Torvalds #include <linux/fs.h>
161da177e4SLinus Torvalds #include <linux/mount.h>
171da177e4SLinus Torvalds #include <linux/file.h>
18e73a75faSRandy Dunlap #include <linux/kernel.h>
191da177e4SLinus Torvalds #include <linux/writeback.h>
201da177e4SLinus Torvalds #include <linux/pagemap.h>
211da177e4SLinus Torvalds #include <linux/highmem.h>
221da177e4SLinus Torvalds #include <linux/init.h>
231da177e4SLinus Torvalds #include <linux/string.h>
2416f7e0feSRandy Dunlap #include <linux/capability.h>
25e73a75faSRandy Dunlap #include <linux/ctype.h>
261da177e4SLinus Torvalds #include <linux/backing-dev.h>
271da177e4SLinus Torvalds #include <linux/hugetlb.h>
281da177e4SLinus Torvalds #include <linux/pagevec.h>
2932021982SDavid Howells #include <linux/fs_parser.h>
30036e0856SBenjamin Herrenschmidt #include <linux/mman.h>
311da177e4SLinus Torvalds #include <linux/slab.h>
321da177e4SLinus Torvalds #include <linux/dnotify.h>
331da177e4SLinus Torvalds #include <linux/statfs.h>
341da177e4SLinus Torvalds #include <linux/security.h>
351fd7317dSNick Black #include <linux/magic.h>
36290408d4SNaoya Horiguchi #include <linux/migrate.h>
3734d0640eSAl Viro #include <linux/uio.h>
381da177e4SLinus Torvalds
397c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
4088590253SShijie Hu #include <linux/sched/mm.h>
411da177e4SLinus Torvalds
42f5e54d6eSChristoph Hellwig static const struct address_space_operations hugetlbfs_aops;
434b6f5d20SArjan van de Ven const struct file_operations hugetlbfs_file_operations;
4492e1d5beSArjan van de Ven static const struct inode_operations hugetlbfs_dir_inode_operations;
4592e1d5beSArjan van de Ven static const struct inode_operations hugetlbfs_inode_operations;
461da177e4SLinus Torvalds
4732021982SDavid Howells enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
4832021982SDavid Howells
4932021982SDavid Howells struct hugetlbfs_fs_context {
504a25220dSDavid Howells struct hstate *hstate;
5132021982SDavid Howells unsigned long long max_size_opt;
5232021982SDavid Howells unsigned long long min_size_opt;
534a25220dSDavid Howells long max_hpages;
544a25220dSDavid Howells long nr_inodes;
554a25220dSDavid Howells long min_hpages;
5632021982SDavid Howells enum hugetlbfs_size_type max_val_type;
5732021982SDavid Howells enum hugetlbfs_size_type min_val_type;
58a0eb3a05SEric W. Biederman kuid_t uid;
59a0eb3a05SEric W. Biederman kgid_t gid;
60a1d776eeSDavid Gibson umode_t mode;
61a1d776eeSDavid Gibson };
62a1d776eeSDavid Gibson
631da177e4SLinus Torvalds int sysctl_hugetlb_shm_group;
641da177e4SLinus Torvalds
6532021982SDavid Howells enum hugetlb_param {
6632021982SDavid Howells Opt_gid,
6732021982SDavid Howells Opt_min_size,
6832021982SDavid Howells Opt_mode,
6932021982SDavid Howells Opt_nr_inodes,
7032021982SDavid Howells Opt_pagesize,
7132021982SDavid Howells Opt_size,
7232021982SDavid Howells Opt_uid,
73e73a75faSRandy Dunlap };
74e73a75faSRandy Dunlap
75d7167b14SAl Viro static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
7632021982SDavid Howells fsparam_u32 ("gid", Opt_gid),
7732021982SDavid Howells fsparam_string("min_size", Opt_min_size),
78e0f7e2b2SMike Kravetz fsparam_u32oct("mode", Opt_mode),
7932021982SDavid Howells fsparam_string("nr_inodes", Opt_nr_inodes),
8032021982SDavid Howells fsparam_string("pagesize", Opt_pagesize),
8132021982SDavid Howells fsparam_string("size", Opt_size),
8232021982SDavid Howells fsparam_u32 ("uid", Opt_uid),
8332021982SDavid Howells {}
8432021982SDavid Howells };
8532021982SDavid Howells
8670c3547eSMike Kravetz #ifdef CONFIG_NUMA
hugetlb_set_vma_policy(struct vm_area_struct * vma,struct inode * inode,pgoff_t index)8770c3547eSMike Kravetz static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
8870c3547eSMike Kravetz struct inode *inode, pgoff_t index)
8970c3547eSMike Kravetz {
9070c3547eSMike Kravetz vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,
9170c3547eSMike Kravetz index);
9270c3547eSMike Kravetz }
9370c3547eSMike Kravetz
hugetlb_drop_vma_policy(struct vm_area_struct * vma)9470c3547eSMike Kravetz static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
9570c3547eSMike Kravetz {
9670c3547eSMike Kravetz mpol_cond_put(vma->vm_policy);
9770c3547eSMike Kravetz }
9870c3547eSMike Kravetz #else
hugetlb_set_vma_policy(struct vm_area_struct * vma,struct inode * inode,pgoff_t index)9970c3547eSMike Kravetz static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
10070c3547eSMike Kravetz struct inode *inode, pgoff_t index)
10170c3547eSMike Kravetz {
10270c3547eSMike Kravetz }
10370c3547eSMike Kravetz
hugetlb_drop_vma_policy(struct vm_area_struct * vma)10470c3547eSMike Kravetz static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
10570c3547eSMike Kravetz {
10670c3547eSMike Kravetz }
10770c3547eSMike Kravetz #endif
10870c3547eSMike Kravetz
10963489f8eSMike Kravetz /*
11063489f8eSMike Kravetz * Mask used when checking the page offset value passed in via system
11163489f8eSMike Kravetz * calls. This value will be converted to a loff_t which is signed.
11263489f8eSMike Kravetz * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
11363489f8eSMike Kravetz * value. The extra bit (- 1 in the shift value) is to take the sign
11463489f8eSMike Kravetz * bit into account.
11563489f8eSMike Kravetz */
11663489f8eSMike Kravetz #define PGOFF_LOFFT_MAX \
11763489f8eSMike Kravetz (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1)))
11863489f8eSMike Kravetz
hugetlbfs_file_mmap(struct file * file,struct vm_area_struct * vma)1191da177e4SLinus Torvalds static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
1201da177e4SLinus Torvalds {
121496ad9aaSAl Viro struct inode *inode = file_inode(file);
12222247efdSPeter Xu struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
1231da177e4SLinus Torvalds loff_t len, vma_len;
1241da177e4SLinus Torvalds int ret;
125a5516438SAndi Kleen struct hstate *h = hstate_file(file);
126*4d850ed7SPrakash Sangappa vm_flags_t vm_flags;
1271da177e4SLinus Torvalds
12868589bc3SHugh Dickins /*
129dec4ad86SDavid Gibson * vma address alignment (but not the pgoff alignment) has
130dec4ad86SDavid Gibson * already been checked by prepare_hugepage_range. If you add
131dec4ad86SDavid Gibson * any error returns here, do so after setting VM_HUGETLB, so
132dec4ad86SDavid Gibson * is_vm_hugetlb_page tests below unmap_region go the right
13345e55300SPeter Collingbourne * way when do_mmap unwinds (may be important on powerpc
134dec4ad86SDavid Gibson * and ia64).
13568589bc3SHugh Dickins */
1361c71222eSSuren Baghdasaryan vm_flags_set(vma, VM_HUGETLB | VM_DONTEXPAND);
13768589bc3SHugh Dickins vma->vm_ops = &hugetlb_vm_ops;
1381da177e4SLinus Torvalds
13922247efdSPeter Xu ret = seal_check_future_write(info->seals, vma);
14022247efdSPeter Xu if (ret)
14122247efdSPeter Xu return ret;
14222247efdSPeter Xu
143045c7a3fSMike Kravetz /*
14463489f8eSMike Kravetz * page based offset in vm_pgoff could be sufficiently large to
1455df63c2aSMike Kravetz * overflow a loff_t when converted to byte offset. This can
1465df63c2aSMike Kravetz * only happen on architectures where sizeof(loff_t) ==
1475df63c2aSMike Kravetz * sizeof(unsigned long). So, only check in those instances.
148045c7a3fSMike Kravetz */
1495df63c2aSMike Kravetz if (sizeof(unsigned long) == sizeof(loff_t)) {
15063489f8eSMike Kravetz if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
151045c7a3fSMike Kravetz return -EINVAL;
1525df63c2aSMike Kravetz }
153045c7a3fSMike Kravetz
15463489f8eSMike Kravetz /* must be huge page aligned */
1552b37c35eSBecky Bruce if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
156dec4ad86SDavid Gibson return -EINVAL;
157dec4ad86SDavid Gibson
1581da177e4SLinus Torvalds vma_len = (loff_t)(vma->vm_end - vma->vm_start);
159045c7a3fSMike Kravetz len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
160045c7a3fSMike Kravetz /* check for overflow */
161045c7a3fSMike Kravetz if (len < vma_len)
162045c7a3fSMike Kravetz return -EINVAL;
1631da177e4SLinus Torvalds
1645955102cSAl Viro inode_lock(inode);
1651da177e4SLinus Torvalds file_accessed(file);
1661da177e4SLinus Torvalds
1671da177e4SLinus Torvalds ret = -ENOMEM;
168*4d850ed7SPrakash Sangappa
169*4d850ed7SPrakash Sangappa vm_flags = vma->vm_flags;
170*4d850ed7SPrakash Sangappa /*
171*4d850ed7SPrakash Sangappa * for SHM_HUGETLB, the pages are reserved in the shmget() call so skip
172*4d850ed7SPrakash Sangappa * reserving here. Note: only for SHM hugetlbfs file, the inode
173*4d850ed7SPrakash Sangappa * flag S_PRIVATE is set.
174*4d850ed7SPrakash Sangappa */
175*4d850ed7SPrakash Sangappa if (inode->i_flags & S_PRIVATE)
176*4d850ed7SPrakash Sangappa vm_flags |= VM_NORESERVE;
177*4d850ed7SPrakash Sangappa
17833b8f84aSMike Kravetz if (!hugetlb_reserve_pages(inode,
179a5516438SAndi Kleen vma->vm_pgoff >> huge_page_order(h),
1805a6fe125SMel Gorman len >> huge_page_shift(h), vma,
181*4d850ed7SPrakash Sangappa vm_flags))
182b45b5bd6SDavid Gibson goto out;
183b45b5bd6SDavid Gibson
1844c887265SAdam Litke ret = 0;
185b6174df5SZhang, Yanmin if (vma->vm_flags & VM_WRITE && inode->i_size < len)
186045c7a3fSMike Kravetz i_size_write(inode, len);
1871da177e4SLinus Torvalds out:
1885955102cSAl Viro inode_unlock(inode);
1891da177e4SLinus Torvalds
1901da177e4SLinus Torvalds return ret;
1911da177e4SLinus Torvalds }
1921da177e4SLinus Torvalds
1931da177e4SLinus Torvalds /*
1943e4e28c5SMichel Lespinasse * Called under mmap_write_lock(mm).
1951da177e4SLinus Torvalds */
1961da177e4SLinus Torvalds
1971da177e4SLinus Torvalds static unsigned long
hugetlb_get_unmapped_area_bottomup(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)19888590253SShijie Hu hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr,
19988590253SShijie Hu unsigned long len, unsigned long pgoff, unsigned long flags)
20088590253SShijie Hu {
20188590253SShijie Hu struct hstate *h = hstate_file(file);
20288590253SShijie Hu struct vm_unmapped_area_info info;
20388590253SShijie Hu
20488590253SShijie Hu info.flags = 0;
20588590253SShijie Hu info.length = len;
20688590253SShijie Hu info.low_limit = current->mm->mmap_base;
2072cb4de08SChristophe Leroy info.high_limit = arch_get_mmap_end(addr, len, flags);
20888590253SShijie Hu info.align_mask = PAGE_MASK & ~huge_page_mask(h);
20988590253SShijie Hu info.align_offset = 0;
21088590253SShijie Hu return vm_unmapped_area(&info);
21188590253SShijie Hu }
21288590253SShijie Hu
21388590253SShijie Hu static unsigned long
hugetlb_get_unmapped_area_topdown(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)21488590253SShijie Hu hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr,
21588590253SShijie Hu unsigned long len, unsigned long pgoff, unsigned long flags)
21688590253SShijie Hu {
21788590253SShijie Hu struct hstate *h = hstate_file(file);
21888590253SShijie Hu struct vm_unmapped_area_info info;
21988590253SShijie Hu
22088590253SShijie Hu info.flags = VM_UNMAPPED_AREA_TOPDOWN;
22188590253SShijie Hu info.length = len;
2226b008640SLinus Torvalds info.low_limit = PAGE_SIZE;
2235f24d5a5SChristophe Leroy info.high_limit = arch_get_mmap_base(addr, current->mm->mmap_base);
22488590253SShijie Hu info.align_mask = PAGE_MASK & ~huge_page_mask(h);
22588590253SShijie Hu info.align_offset = 0;
22688590253SShijie Hu addr = vm_unmapped_area(&info);
22788590253SShijie Hu
22888590253SShijie Hu /*
22988590253SShijie Hu * A failed mmap() very likely causes application failure,
23088590253SShijie Hu * so fall back to the bottom-up function here. This scenario
23188590253SShijie Hu * can happen with large stack limits and large mmap()
23288590253SShijie Hu * allocations.
23388590253SShijie Hu */
23488590253SShijie Hu if (unlikely(offset_in_page(addr))) {
23588590253SShijie Hu VM_BUG_ON(addr != -ENOMEM);
23688590253SShijie Hu info.flags = 0;
23788590253SShijie Hu info.low_limit = current->mm->mmap_base;
2382cb4de08SChristophe Leroy info.high_limit = arch_get_mmap_end(addr, len, flags);
23988590253SShijie Hu addr = vm_unmapped_area(&info);
24088590253SShijie Hu }
24188590253SShijie Hu
24288590253SShijie Hu return addr;
24388590253SShijie Hu }
24488590253SShijie Hu
2454b439e25SChristophe Leroy unsigned long
generic_hugetlb_get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)2464b439e25SChristophe Leroy generic_hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
2474b439e25SChristophe Leroy unsigned long len, unsigned long pgoff,
2484b439e25SChristophe Leroy unsigned long flags)
2491da177e4SLinus Torvalds {
2501da177e4SLinus Torvalds struct mm_struct *mm = current->mm;
2511da177e4SLinus Torvalds struct vm_area_struct *vma;
252a5516438SAndi Kleen struct hstate *h = hstate_file(file);
2532cb4de08SChristophe Leroy const unsigned long mmap_end = arch_get_mmap_end(addr, len, flags);
2541da177e4SLinus Torvalds
255a5516438SAndi Kleen if (len & ~huge_page_mask(h))
2561da177e4SLinus Torvalds return -EINVAL;
2571da177e4SLinus Torvalds if (len > TASK_SIZE)
2581da177e4SLinus Torvalds return -ENOMEM;
2591da177e4SLinus Torvalds
260036e0856SBenjamin Herrenschmidt if (flags & MAP_FIXED) {
261a5516438SAndi Kleen if (prepare_hugepage_range(file, addr, len))
262036e0856SBenjamin Herrenschmidt return -EINVAL;
263036e0856SBenjamin Herrenschmidt return addr;
264036e0856SBenjamin Herrenschmidt }
265036e0856SBenjamin Herrenschmidt
2661da177e4SLinus Torvalds if (addr) {
267a5516438SAndi Kleen addr = ALIGN(addr, huge_page_size(h));
2681da177e4SLinus Torvalds vma = find_vma(mm, addr);
2695f24d5a5SChristophe Leroy if (mmap_end - len >= addr &&
2701be7107fSHugh Dickins (!vma || addr + len <= vm_start_gap(vma)))
2711da177e4SLinus Torvalds return addr;
2721da177e4SLinus Torvalds }
2731da177e4SLinus Torvalds
27488590253SShijie Hu /*
27588590253SShijie Hu * Use mm->get_unmapped_area value as a hint to use topdown routine.
27688590253SShijie Hu * If architectures have special needs, they should define their own
27788590253SShijie Hu * version of hugetlb_get_unmapped_area.
27888590253SShijie Hu */
27988590253SShijie Hu if (mm->get_unmapped_area == arch_get_unmapped_area_topdown)
28088590253SShijie Hu return hugetlb_get_unmapped_area_topdown(file, addr, len,
28188590253SShijie Hu pgoff, flags);
28288590253SShijie Hu return hugetlb_get_unmapped_area_bottomup(file, addr, len,
28388590253SShijie Hu pgoff, flags);
2841da177e4SLinus Torvalds }
2854b439e25SChristophe Leroy
2864b439e25SChristophe Leroy #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
2874b439e25SChristophe Leroy static unsigned long
hugetlb_get_unmapped_area(struct file * file,unsigned long addr,unsigned long len,unsigned long pgoff,unsigned long flags)2884b439e25SChristophe Leroy hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
2894b439e25SChristophe Leroy unsigned long len, unsigned long pgoff,
2904b439e25SChristophe Leroy unsigned long flags)
2914b439e25SChristophe Leroy {
2924b439e25SChristophe Leroy return generic_hugetlb_get_unmapped_area(file, addr, len, pgoff, flags);
2934b439e25SChristophe Leroy }
2941da177e4SLinus Torvalds #endif
2951da177e4SLinus Torvalds
296e63e1e5aSBadari Pulavarty /*
29738c1ddbdSJiaqi Yan * Someone wants to read @bytes from a HWPOISON hugetlb @page from @offset.
29838c1ddbdSJiaqi Yan * Returns the maximum number of bytes one can read without touching the 1st raw
29938c1ddbdSJiaqi Yan * HWPOISON subpage.
30038c1ddbdSJiaqi Yan *
30138c1ddbdSJiaqi Yan * The implementation borrows the iteration logic from copy_page_to_iter*.
30238c1ddbdSJiaqi Yan */
adjust_range_hwpoison(struct page * page,size_t offset,size_t bytes)30338c1ddbdSJiaqi Yan static size_t adjust_range_hwpoison(struct page *page, size_t offset, size_t bytes)
30438c1ddbdSJiaqi Yan {
30538c1ddbdSJiaqi Yan size_t n = 0;
30638c1ddbdSJiaqi Yan size_t res = 0;
30738c1ddbdSJiaqi Yan
30838c1ddbdSJiaqi Yan /* First subpage to start the loop. */
309cc64f5eaSZi Yan page = nth_page(page, offset / PAGE_SIZE);
31038c1ddbdSJiaqi Yan offset %= PAGE_SIZE;
31138c1ddbdSJiaqi Yan while (1) {
31238c1ddbdSJiaqi Yan if (is_raw_hwpoison_page_in_hugepage(page))
31338c1ddbdSJiaqi Yan break;
31438c1ddbdSJiaqi Yan
31538c1ddbdSJiaqi Yan /* Safe to read n bytes without touching HWPOISON subpage. */
31638c1ddbdSJiaqi Yan n = min(bytes, (size_t)PAGE_SIZE - offset);
31738c1ddbdSJiaqi Yan res += n;
31838c1ddbdSJiaqi Yan bytes -= n;
31938c1ddbdSJiaqi Yan if (!bytes || !n)
32038c1ddbdSJiaqi Yan break;
32138c1ddbdSJiaqi Yan offset += n;
32238c1ddbdSJiaqi Yan if (offset == PAGE_SIZE) {
323cc64f5eaSZi Yan page = nth_page(page, 1);
32438c1ddbdSJiaqi Yan offset = 0;
32538c1ddbdSJiaqi Yan }
32638c1ddbdSJiaqi Yan }
32738c1ddbdSJiaqi Yan
32838c1ddbdSJiaqi Yan return res;
32938c1ddbdSJiaqi Yan }
33038c1ddbdSJiaqi Yan
33138c1ddbdSJiaqi Yan /*
332e63e1e5aSBadari Pulavarty * Support for read() - Find the page attached to f_mapping and copy out the
333445c8098SMiaohe Lin * data. This provides functionality similar to filemap_read().
334e63e1e5aSBadari Pulavarty */
hugetlbfs_read_iter(struct kiocb * iocb,struct iov_iter * to)33534d0640eSAl Viro static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
336e63e1e5aSBadari Pulavarty {
33734d0640eSAl Viro struct file *file = iocb->ki_filp;
33834d0640eSAl Viro struct hstate *h = hstate_file(file);
33934d0640eSAl Viro struct address_space *mapping = file->f_mapping;
340e63e1e5aSBadari Pulavarty struct inode *inode = mapping->host;
34134d0640eSAl Viro unsigned long index = iocb->ki_pos >> huge_page_shift(h);
34234d0640eSAl Viro unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
343e63e1e5aSBadari Pulavarty unsigned long end_index;
344e63e1e5aSBadari Pulavarty loff_t isize;
345e63e1e5aSBadari Pulavarty ssize_t retval = 0;
346e63e1e5aSBadari Pulavarty
34734d0640eSAl Viro while (iov_iter_count(to)) {
348e63e1e5aSBadari Pulavarty struct page *page;
34938c1ddbdSJiaqi Yan size_t nr, copied, want;
350e63e1e5aSBadari Pulavarty
351e63e1e5aSBadari Pulavarty /* nr is the maximum number of bytes to copy from this page */
352a5516438SAndi Kleen nr = huge_page_size(h);
353a05b0855SAneesh Kumar K.V isize = i_size_read(inode);
354a05b0855SAneesh Kumar K.V if (!isize)
35534d0640eSAl Viro break;
356a05b0855SAneesh Kumar K.V end_index = (isize - 1) >> huge_page_shift(h);
357e63e1e5aSBadari Pulavarty if (index > end_index)
35834d0640eSAl Viro break;
35934d0640eSAl Viro if (index == end_index) {
360a5516438SAndi Kleen nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
361a05b0855SAneesh Kumar K.V if (nr <= offset)
36234d0640eSAl Viro break;
363e63e1e5aSBadari Pulavarty }
364e63e1e5aSBadari Pulavarty nr = nr - offset;
365e63e1e5aSBadari Pulavarty
366e63e1e5aSBadari Pulavarty /* Find the page */
367a05b0855SAneesh Kumar K.V page = find_lock_page(mapping, index);
368e63e1e5aSBadari Pulavarty if (unlikely(page == NULL)) {
369e63e1e5aSBadari Pulavarty /*
370e63e1e5aSBadari Pulavarty * We have a HOLE, zero out the user-buffer for the
371e63e1e5aSBadari Pulavarty * length of the hole or request.
372e63e1e5aSBadari Pulavarty */
37334d0640eSAl Viro copied = iov_iter_zero(nr, to);
374e63e1e5aSBadari Pulavarty } else {
375a05b0855SAneesh Kumar K.V unlock_page(page);
376a05b0855SAneesh Kumar K.V
37738c1ddbdSJiaqi Yan if (!PageHWPoison(page))
37838c1ddbdSJiaqi Yan want = nr;
37938c1ddbdSJiaqi Yan else {
38038c1ddbdSJiaqi Yan /*
38138c1ddbdSJiaqi Yan * Adjust how many bytes safe to read without
38238c1ddbdSJiaqi Yan * touching the 1st raw HWPOISON subpage after
38338c1ddbdSJiaqi Yan * offset.
38438c1ddbdSJiaqi Yan */
38538c1ddbdSJiaqi Yan want = adjust_range_hwpoison(page, offset, nr);
38638c1ddbdSJiaqi Yan if (want == 0) {
3878625147cSJames Houghton put_page(page);
3888625147cSJames Houghton retval = -EIO;
3898625147cSJames Houghton break;
3908625147cSJames Houghton }
39138c1ddbdSJiaqi Yan }
3928625147cSJames Houghton
393e63e1e5aSBadari Pulavarty /*
394e63e1e5aSBadari Pulavarty * We have the page, copy it to user space buffer.
395e63e1e5aSBadari Pulavarty */
39638c1ddbdSJiaqi Yan copied = copy_page_to_iter(page, offset, want, to);
39709cbfeafSKirill A. Shutemov put_page(page);
398e63e1e5aSBadari Pulavarty }
39934d0640eSAl Viro offset += copied;
40034d0640eSAl Viro retval += copied;
40134d0640eSAl Viro if (copied != nr && iov_iter_count(to)) {
40234d0640eSAl Viro if (!retval)
40334d0640eSAl Viro retval = -EFAULT;
404e63e1e5aSBadari Pulavarty break;
405e63e1e5aSBadari Pulavarty }
40634d0640eSAl Viro index += offset >> huge_page_shift(h);
40734d0640eSAl Viro offset &= ~huge_page_mask(h);
40834d0640eSAl Viro }
40934d0640eSAl Viro iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
410e63e1e5aSBadari Pulavarty return retval;
411e63e1e5aSBadari Pulavarty }
412e63e1e5aSBadari Pulavarty
hugetlbfs_write_begin(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,struct page ** pagep,void ** fsdata)413800d15a5SNick Piggin static int hugetlbfs_write_begin(struct file *file,
414800d15a5SNick Piggin struct address_space *mapping,
4159d6b0cd7SMatthew Wilcox (Oracle) loff_t pos, unsigned len,
416800d15a5SNick Piggin struct page **pagep, void **fsdata)
4171da177e4SLinus Torvalds {
4181da177e4SLinus Torvalds return -EINVAL;
4191da177e4SLinus Torvalds }
4201da177e4SLinus Torvalds
hugetlbfs_write_end(struct file * file,struct address_space * mapping,loff_t pos,unsigned len,unsigned copied,struct page * page,void * fsdata)421800d15a5SNick Piggin static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
422800d15a5SNick Piggin loff_t pos, unsigned len, unsigned copied,
423800d15a5SNick Piggin struct page *page, void *fsdata)
4241da177e4SLinus Torvalds {
425800d15a5SNick Piggin BUG();
4261da177e4SLinus Torvalds return -EINVAL;
4271da177e4SLinus Torvalds }
4281da177e4SLinus Torvalds
hugetlb_delete_from_page_cache(struct folio * folio)429ece62684SSidhartha Kumar static void hugetlb_delete_from_page_cache(struct folio *folio)
4301da177e4SLinus Torvalds {
431ece62684SSidhartha Kumar folio_clear_dirty(folio);
432ece62684SSidhartha Kumar folio_clear_uptodate(folio);
433ece62684SSidhartha Kumar filemap_remove_folio(folio);
4341da177e4SLinus Torvalds }
4351da177e4SLinus Torvalds
436378397ccSMike Kravetz /*
437378397ccSMike Kravetz * Called with i_mmap_rwsem held for inode based vma maps. This makes
438378397ccSMike Kravetz * sure vma (and vm_mm) will not go away. We also hold the hugetlb fault
439378397ccSMike Kravetz * mutex for the page in the mapping. So, we can not race with page being
440378397ccSMike Kravetz * faulted into the vma.
441378397ccSMike Kravetz */
hugetlb_vma_maps_page(struct vm_area_struct * vma,unsigned long addr,struct page * page)442378397ccSMike Kravetz static bool hugetlb_vma_maps_page(struct vm_area_struct *vma,
443378397ccSMike Kravetz unsigned long addr, struct page *page)
444378397ccSMike Kravetz {
445378397ccSMike Kravetz pte_t *ptep, pte;
446378397ccSMike Kravetz
4479c67a207SPeter Xu ptep = hugetlb_walk(vma, addr, huge_page_size(hstate_vma(vma)));
448378397ccSMike Kravetz if (!ptep)
449378397ccSMike Kravetz return false;
450378397ccSMike Kravetz
451378397ccSMike Kravetz pte = huge_ptep_get(ptep);
452378397ccSMike Kravetz if (huge_pte_none(pte) || !pte_present(pte))
453378397ccSMike Kravetz return false;
454378397ccSMike Kravetz
455378397ccSMike Kravetz if (pte_page(pte) == page)
456378397ccSMike Kravetz return true;
457378397ccSMike Kravetz
458378397ccSMike Kravetz return false;
459378397ccSMike Kravetz }
460378397ccSMike Kravetz
461378397ccSMike Kravetz /*
462378397ccSMike Kravetz * Can vma_offset_start/vma_offset_end overflow on 32-bit arches?
463378397ccSMike Kravetz * No, because the interval tree returns us only those vmas
464378397ccSMike Kravetz * which overlap the truncated area starting at pgoff,
465378397ccSMike Kravetz * and no vma on a 32-bit arch can span beyond the 4GB.
466378397ccSMike Kravetz */
vma_offset_start(struct vm_area_struct * vma,pgoff_t start)467378397ccSMike Kravetz static unsigned long vma_offset_start(struct vm_area_struct *vma, pgoff_t start)
468378397ccSMike Kravetz {
469243b1f2dSPeter Xu unsigned long offset = 0;
470243b1f2dSPeter Xu
471378397ccSMike Kravetz if (vma->vm_pgoff < start)
472243b1f2dSPeter Xu offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
473243b1f2dSPeter Xu
474243b1f2dSPeter Xu return vma->vm_start + offset;
475378397ccSMike Kravetz }
476378397ccSMike Kravetz
vma_offset_end(struct vm_area_struct * vma,pgoff_t end)477378397ccSMike Kravetz static unsigned long vma_offset_end(struct vm_area_struct *vma, pgoff_t end)
478378397ccSMike Kravetz {
479378397ccSMike Kravetz unsigned long t_end;
480378397ccSMike Kravetz
481378397ccSMike Kravetz if (!end)
482378397ccSMike Kravetz return vma->vm_end;
483378397ccSMike Kravetz
484378397ccSMike Kravetz t_end = ((end - vma->vm_pgoff) << PAGE_SHIFT) + vma->vm_start;
485378397ccSMike Kravetz if (t_end > vma->vm_end)
486378397ccSMike Kravetz t_end = vma->vm_end;
487378397ccSMike Kravetz return t_end;
488378397ccSMike Kravetz }
489378397ccSMike Kravetz
490378397ccSMike Kravetz /*
491378397ccSMike Kravetz * Called with hugetlb fault mutex held. Therefore, no more mappings to
492378397ccSMike Kravetz * this folio can be created while executing the routine.
493378397ccSMike Kravetz */
hugetlb_unmap_file_folio(struct hstate * h,struct address_space * mapping,struct folio * folio,pgoff_t index)494378397ccSMike Kravetz static void hugetlb_unmap_file_folio(struct hstate *h,
495378397ccSMike Kravetz struct address_space *mapping,
496378397ccSMike Kravetz struct folio *folio, pgoff_t index)
497378397ccSMike Kravetz {
498378397ccSMike Kravetz struct rb_root_cached *root = &mapping->i_mmap;
49940549ba8SMike Kravetz struct hugetlb_vma_lock *vma_lock;
500378397ccSMike Kravetz struct page *page = &folio->page;
501378397ccSMike Kravetz struct vm_area_struct *vma;
502378397ccSMike Kravetz unsigned long v_start;
503378397ccSMike Kravetz unsigned long v_end;
504378397ccSMike Kravetz pgoff_t start, end;
505378397ccSMike Kravetz
506378397ccSMike Kravetz start = index * pages_per_huge_page(h);
507378397ccSMike Kravetz end = (index + 1) * pages_per_huge_page(h);
508378397ccSMike Kravetz
509378397ccSMike Kravetz i_mmap_lock_write(mapping);
51040549ba8SMike Kravetz retry:
51140549ba8SMike Kravetz vma_lock = NULL;
512378397ccSMike Kravetz vma_interval_tree_foreach(vma, root, start, end - 1) {
513378397ccSMike Kravetz v_start = vma_offset_start(vma, start);
514378397ccSMike Kravetz v_end = vma_offset_end(vma, end);
515378397ccSMike Kravetz
516243b1f2dSPeter Xu if (!hugetlb_vma_maps_page(vma, v_start, page))
517378397ccSMike Kravetz continue;
518378397ccSMike Kravetz
51940549ba8SMike Kravetz if (!hugetlb_vma_trylock_write(vma)) {
52040549ba8SMike Kravetz vma_lock = vma->vm_private_data;
52140549ba8SMike Kravetz /*
52240549ba8SMike Kravetz * If we can not get vma lock, we need to drop
52340549ba8SMike Kravetz * immap_sema and take locks in order. First,
52440549ba8SMike Kravetz * take a ref on the vma_lock structure so that
52540549ba8SMike Kravetz * we can be guaranteed it will not go away when
52640549ba8SMike Kravetz * dropping immap_sema.
52740549ba8SMike Kravetz */
52840549ba8SMike Kravetz kref_get(&vma_lock->refs);
52940549ba8SMike Kravetz break;
53040549ba8SMike Kravetz }
53140549ba8SMike Kravetz
532243b1f2dSPeter Xu unmap_hugepage_range(vma, v_start, v_end, NULL,
533243b1f2dSPeter Xu ZAP_FLAG_DROP_MARKER);
53440549ba8SMike Kravetz hugetlb_vma_unlock_write(vma);
535378397ccSMike Kravetz }
536378397ccSMike Kravetz
537378397ccSMike Kravetz i_mmap_unlock_write(mapping);
53840549ba8SMike Kravetz
53940549ba8SMike Kravetz if (vma_lock) {
54040549ba8SMike Kravetz /*
54140549ba8SMike Kravetz * Wait on vma_lock. We know it is still valid as we have
54240549ba8SMike Kravetz * a reference. We must 'open code' vma locking as we do
54340549ba8SMike Kravetz * not know if vma_lock is still attached to vma.
54440549ba8SMike Kravetz */
54540549ba8SMike Kravetz down_write(&vma_lock->rw_sema);
54640549ba8SMike Kravetz i_mmap_lock_write(mapping);
54740549ba8SMike Kravetz
54840549ba8SMike Kravetz vma = vma_lock->vma;
54940549ba8SMike Kravetz if (!vma) {
55040549ba8SMike Kravetz /*
55140549ba8SMike Kravetz * If lock is no longer attached to vma, then just
55240549ba8SMike Kravetz * unlock, drop our reference and retry looking for
55340549ba8SMike Kravetz * other vmas.
55440549ba8SMike Kravetz */
55540549ba8SMike Kravetz up_write(&vma_lock->rw_sema);
55640549ba8SMike Kravetz kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
55740549ba8SMike Kravetz goto retry;
55840549ba8SMike Kravetz }
55940549ba8SMike Kravetz
56040549ba8SMike Kravetz /*
56140549ba8SMike Kravetz * vma_lock is still attached to vma. Check to see if vma
56240549ba8SMike Kravetz * still maps page and if so, unmap.
56340549ba8SMike Kravetz */
56440549ba8SMike Kravetz v_start = vma_offset_start(vma, start);
56540549ba8SMike Kravetz v_end = vma_offset_end(vma, end);
566243b1f2dSPeter Xu if (hugetlb_vma_maps_page(vma, v_start, page))
567243b1f2dSPeter Xu unmap_hugepage_range(vma, v_start, v_end, NULL,
56840549ba8SMike Kravetz ZAP_FLAG_DROP_MARKER);
56940549ba8SMike Kravetz
57040549ba8SMike Kravetz kref_put(&vma_lock->refs, hugetlb_vma_lock_release);
57140549ba8SMike Kravetz hugetlb_vma_unlock_write(vma);
57240549ba8SMike Kravetz
57340549ba8SMike Kravetz goto retry;
57440549ba8SMike Kravetz }
575378397ccSMike Kravetz }
576378397ccSMike Kravetz
5774aae8d1cSMike Kravetz static void
hugetlb_vmdelete_list(struct rb_root_cached * root,pgoff_t start,pgoff_t end,zap_flags_t zap_flags)57805e90bd0SPeter Xu hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end,
57905e90bd0SPeter Xu zap_flags_t zap_flags)
5804aae8d1cSMike Kravetz {
5814aae8d1cSMike Kravetz struct vm_area_struct *vma;
5824aae8d1cSMike Kravetz
5834aae8d1cSMike Kravetz /*
584d6aba4c8SSean Christopherson * end == 0 indicates that the entire range after start should be
585d6aba4c8SSean Christopherson * unmapped. Note, end is exclusive, whereas the interval tree takes
586d6aba4c8SSean Christopherson * an inclusive "last".
5874aae8d1cSMike Kravetz */
588d6aba4c8SSean Christopherson vma_interval_tree_foreach(vma, root, start, end ? end - 1 : ULONG_MAX) {
589378397ccSMike Kravetz unsigned long v_start;
5904aae8d1cSMike Kravetz unsigned long v_end;
5914aae8d1cSMike Kravetz
59240549ba8SMike Kravetz if (!hugetlb_vma_trylock_write(vma))
59340549ba8SMike Kravetz continue;
59440549ba8SMike Kravetz
595378397ccSMike Kravetz v_start = vma_offset_start(vma, start);
596378397ccSMike Kravetz v_end = vma_offset_end(vma, end);
5974aae8d1cSMike Kravetz
598243b1f2dSPeter Xu unmap_hugepage_range(vma, v_start, v_end, NULL, zap_flags);
59940549ba8SMike Kravetz
60040549ba8SMike Kravetz /*
60140549ba8SMike Kravetz * Note that vma lock only exists for shared/non-private
60240549ba8SMike Kravetz * vmas. Therefore, lock is not held when calling
60340549ba8SMike Kravetz * unmap_hugepage_range for private vmas.
60440549ba8SMike Kravetz */
60540549ba8SMike Kravetz hugetlb_vma_unlock_write(vma);
6064aae8d1cSMike Kravetz }
6074aae8d1cSMike Kravetz }
608b5cec28dSMike Kravetz
609b5cec28dSMike Kravetz /*
610c8627228SMike Kravetz * Called with hugetlb fault mutex held.
611c8627228SMike Kravetz * Returns true if page was actually removed, false otherwise.
612c8627228SMike Kravetz */
remove_inode_single_folio(struct hstate * h,struct inode * inode,struct address_space * mapping,struct folio * folio,pgoff_t index,bool truncate_op)613c8627228SMike Kravetz static bool remove_inode_single_folio(struct hstate *h, struct inode *inode,
614c8627228SMike Kravetz struct address_space *mapping,
615c8627228SMike Kravetz struct folio *folio, pgoff_t index,
616c8627228SMike Kravetz bool truncate_op)
617c8627228SMike Kravetz {
618c8627228SMike Kravetz bool ret = false;
619c8627228SMike Kravetz
620c8627228SMike Kravetz /*
621c8627228SMike Kravetz * If folio is mapped, it was faulted in after being
622c8627228SMike Kravetz * unmapped in caller. Unmap (again) while holding
623c8627228SMike Kravetz * the fault mutex. The mutex will prevent faults
624c8627228SMike Kravetz * until we finish removing the folio.
625c8627228SMike Kravetz */
626378397ccSMike Kravetz if (unlikely(folio_mapped(folio)))
627378397ccSMike Kravetz hugetlb_unmap_file_folio(h, mapping, folio, index);
628c8627228SMike Kravetz
629c8627228SMike Kravetz folio_lock(folio);
630c8627228SMike Kravetz /*
631c8627228SMike Kravetz * We must remove the folio from page cache before removing
632c8627228SMike Kravetz * the region/ reserve map (hugetlb_unreserve_pages). In
633c8627228SMike Kravetz * rare out of memory conditions, removal of the region/reserve
634c8627228SMike Kravetz * map could fail. Correspondingly, the subpool and global
635c8627228SMike Kravetz * reserve usage count can need to be adjusted.
636c8627228SMike Kravetz */
637ece62684SSidhartha Kumar VM_BUG_ON_FOLIO(folio_test_hugetlb_restore_reserve(folio), folio);
638ece62684SSidhartha Kumar hugetlb_delete_from_page_cache(folio);
639c8627228SMike Kravetz ret = true;
640c8627228SMike Kravetz if (!truncate_op) {
641c8627228SMike Kravetz if (unlikely(hugetlb_unreserve_pages(inode, index,
642c8627228SMike Kravetz index + 1, 1)))
643c8627228SMike Kravetz hugetlb_fix_reserve_counts(inode);
644c8627228SMike Kravetz }
645c8627228SMike Kravetz
646c8627228SMike Kravetz folio_unlock(folio);
647c8627228SMike Kravetz return ret;
648c8627228SMike Kravetz }
649c8627228SMike Kravetz
650c8627228SMike Kravetz /*
651b5cec28dSMike Kravetz * remove_inode_hugepages handles two distinct cases: truncation and hole
652b5cec28dSMike Kravetz * punch. There are subtle differences in operation for each case.
6534aae8d1cSMike Kravetz *
654b5cec28dSMike Kravetz * truncation is indicated by end of range being LLONG_MAX
655b5cec28dSMike Kravetz * In this case, we first scan the range and release found pages.
6561935ebd3SMiaohe Lin * After releasing pages, hugetlb_unreserve_pages cleans up region/reserve
657c8627228SMike Kravetz * maps and global counts. Page faults can race with truncation.
658c8627228SMike Kravetz * During faults, hugetlb_no_page() checks i_size before page allocation,
659c8627228SMike Kravetz * and again after obtaining page table lock. It will 'back out'
660c8627228SMike Kravetz * allocations in the truncated range.
661b5cec28dSMike Kravetz * hole punch is indicated if end is not LLONG_MAX
662b5cec28dSMike Kravetz * In the hole punch case we scan the range and release found pages.
6631935ebd3SMiaohe Lin * Only when releasing a page is the associated region/reserve map
6641935ebd3SMiaohe Lin * deleted. The region/reserve map for ranges without associated
665e7c58097SMike Kravetz * pages are not modified. Page faults can race with hole punch.
666e7c58097SMike Kravetz * This is indicated if we find a mapped page.
667b5cec28dSMike Kravetz * Note: If the passed end of range value is beyond the end of file, but
668b5cec28dSMike Kravetz * not LLONG_MAX this routine still performs a hole punch operation.
669b5cec28dSMike Kravetz */
remove_inode_hugepages(struct inode * inode,loff_t lstart,loff_t lend)670b5cec28dSMike Kravetz static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
671b5cec28dSMike Kravetz loff_t lend)
6721da177e4SLinus Torvalds {
673a5516438SAndi Kleen struct hstate *h = hstate_inode(inode);
674b45b5bd6SDavid Gibson struct address_space *mapping = &inode->i_data;
675a5516438SAndi Kleen const pgoff_t start = lstart >> huge_page_shift(h);
676b5cec28dSMike Kravetz const pgoff_t end = lend >> huge_page_shift(h);
6771508062eSMatthew Wilcox (Oracle) struct folio_batch fbatch;
678d72dc8a2SJan Kara pgoff_t next, index;
679a43a8c39SChen, Kenneth W int i, freed = 0;
680b5cec28dSMike Kravetz bool truncate_op = (lend == LLONG_MAX);
6811da177e4SLinus Torvalds
6821508062eSMatthew Wilcox (Oracle) folio_batch_init(&fbatch);
6831da177e4SLinus Torvalds next = start;
6841508062eSMatthew Wilcox (Oracle) while (filemap_get_folios(mapping, &next, end - 1, &fbatch)) {
6851508062eSMatthew Wilcox (Oracle) for (i = 0; i < folio_batch_count(&fbatch); ++i) {
6861508062eSMatthew Wilcox (Oracle) struct folio *folio = fbatch.folios[i];
687d4241a04SMiaohe Lin u32 hash = 0;
688b5cec28dSMike Kravetz
6891508062eSMatthew Wilcox (Oracle) index = folio->index;
690d4241a04SMiaohe Lin hash = hugetlb_fault_mutex_hash(mapping, index);
691e7c58097SMike Kravetz mutex_lock(&hugetlb_fault_mutex_table[hash]);
692e7c58097SMike Kravetz
693b5cec28dSMike Kravetz /*
694c8627228SMike Kravetz * Remove folio that was part of folio_batch.
695b5cec28dSMike Kravetz */
696c8627228SMike Kravetz if (remove_inode_single_folio(h, inode, mapping, folio,
697c8627228SMike Kravetz index, truncate_op))
698b5cec28dSMike Kravetz freed++;
699b5cec28dSMike Kravetz
700e7c58097SMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]);
7011da177e4SLinus Torvalds }
7021508062eSMatthew Wilcox (Oracle) folio_batch_release(&fbatch);
7031817889eSMike Kravetz cond_resched();
7041da177e4SLinus Torvalds }
705b5cec28dSMike Kravetz
706b5cec28dSMike Kravetz if (truncate_op)
707b5cec28dSMike Kravetz (void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
7081da177e4SLinus Torvalds }
7091da177e4SLinus Torvalds
hugetlbfs_evict_inode(struct inode * inode)7102bbbda30SAl Viro static void hugetlbfs_evict_inode(struct inode *inode)
7111da177e4SLinus Torvalds {
7129119a41eSJoonsoo Kim struct resv_map *resv_map;
7139119a41eSJoonsoo Kim
714b5cec28dSMike Kravetz remove_inode_hugepages(inode, 0, LLONG_MAX);
715f27a5136SMike Kravetz
716f27a5136SMike Kravetz /*
717f27a5136SMike Kravetz * Get the resv_map from the address space embedded in the inode.
718f27a5136SMike Kravetz * This is the address space which points to any resv_map allocated
719f27a5136SMike Kravetz * at inode creation time. If this is a device special inode,
720f27a5136SMike Kravetz * i_mapping may not point to the original address space.
721f27a5136SMike Kravetz */
722f27a5136SMike Kravetz resv_map = (struct resv_map *)(&inode->i_data)->private_data;
723f27a5136SMike Kravetz /* Only regular and link inodes have associated reserve maps */
7249119a41eSJoonsoo Kim if (resv_map)
7259119a41eSJoonsoo Kim resv_map_release(&resv_map->refs);
726dbd5768fSJan Kara clear_inode(inode);
727149f4211SChristoph Hellwig }
728149f4211SChristoph Hellwig
hugetlb_vmtruncate(struct inode * inode,loff_t offset)729e5d319deSMiaohe Lin static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
7301da177e4SLinus Torvalds {
731856fc295SHugh Dickins pgoff_t pgoff;
7321da177e4SLinus Torvalds struct address_space *mapping = inode->i_mapping;
733a5516438SAndi Kleen struct hstate *h = hstate_inode(inode);
7341da177e4SLinus Torvalds
735a5516438SAndi Kleen BUG_ON(offset & ~huge_page_mask(h));
736856fc295SHugh Dickins pgoff = offset >> PAGE_SHIFT;
7371da177e4SLinus Torvalds
73887bf91d3SMike Kravetz i_size_write(inode, offset);
739188a3972SMike Kravetz i_mmap_lock_write(mapping);
740f808c13fSDavidlohr Bueso if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
74105e90bd0SPeter Xu hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0,
74205e90bd0SPeter Xu ZAP_FLAG_DROP_MARKER);
743c86aa7bbSMike Kravetz i_mmap_unlock_write(mapping);
744e7c58097SMike Kravetz remove_inode_hugepages(inode, offset, LLONG_MAX);
7451da177e4SLinus Torvalds }
7461da177e4SLinus Torvalds
hugetlbfs_zero_partial_page(struct hstate * h,struct address_space * mapping,loff_t start,loff_t end)74768d32527SMike Kravetz static void hugetlbfs_zero_partial_page(struct hstate *h,
74868d32527SMike Kravetz struct address_space *mapping,
74968d32527SMike Kravetz loff_t start,
75068d32527SMike Kravetz loff_t end)
75168d32527SMike Kravetz {
75268d32527SMike Kravetz pgoff_t idx = start >> huge_page_shift(h);
75368d32527SMike Kravetz struct folio *folio;
75468d32527SMike Kravetz
75568d32527SMike Kravetz folio = filemap_lock_folio(mapping, idx);
75666dabbb6SChristoph Hellwig if (IS_ERR(folio))
75768d32527SMike Kravetz return;
75868d32527SMike Kravetz
75968d32527SMike Kravetz start = start & ~huge_page_mask(h);
76068d32527SMike Kravetz end = end & ~huge_page_mask(h);
76168d32527SMike Kravetz if (!end)
76268d32527SMike Kravetz end = huge_page_size(h);
76368d32527SMike Kravetz
76468d32527SMike Kravetz folio_zero_segment(folio, (size_t)start, (size_t)end);
76568d32527SMike Kravetz
76668d32527SMike Kravetz folio_unlock(folio);
76768d32527SMike Kravetz folio_put(folio);
76868d32527SMike Kravetz }
76968d32527SMike Kravetz
hugetlbfs_punch_hole(struct inode * inode,loff_t offset,loff_t len)77070c3547eSMike Kravetz static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
77170c3547eSMike Kravetz {
77268d32527SMike Kravetz struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
77368d32527SMike Kravetz struct address_space *mapping = inode->i_mapping;
77470c3547eSMike Kravetz struct hstate *h = hstate_inode(inode);
77570c3547eSMike Kravetz loff_t hpage_size = huge_page_size(h);
77670c3547eSMike Kravetz loff_t hole_start, hole_end;
77770c3547eSMike Kravetz
77870c3547eSMike Kravetz /*
77968d32527SMike Kravetz * hole_start and hole_end indicate the full pages within the hole.
78070c3547eSMike Kravetz */
78170c3547eSMike Kravetz hole_start = round_up(offset, hpage_size);
78270c3547eSMike Kravetz hole_end = round_down(offset + len, hpage_size);
78370c3547eSMike Kravetz
7845955102cSAl Viro inode_lock(inode);
785ff62a342SMarc-André Lureau
786398c0da7SMiaohe Lin /* protected by i_rwsem */
787ab3948f5SJoel Fernandes (Google) if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
788ff62a342SMarc-André Lureau inode_unlock(inode);
789ff62a342SMarc-André Lureau return -EPERM;
790ff62a342SMarc-André Lureau }
791ff62a342SMarc-André Lureau
79270c3547eSMike Kravetz i_mmap_lock_write(mapping);
79368d32527SMike Kravetz
79468d32527SMike Kravetz /* If range starts before first full page, zero partial page. */
79568d32527SMike Kravetz if (offset < hole_start)
79668d32527SMike Kravetz hugetlbfs_zero_partial_page(h, mapping,
79768d32527SMike Kravetz offset, min(offset + len, hole_start));
79868d32527SMike Kravetz
79968d32527SMike Kravetz /* Unmap users of full pages in the hole. */
80068d32527SMike Kravetz if (hole_end > hole_start) {
801f808c13fSDavidlohr Bueso if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
80270c3547eSMike Kravetz hugetlb_vmdelete_list(&mapping->i_mmap,
80370c3547eSMike Kravetz hole_start >> PAGE_SHIFT,
80405e90bd0SPeter Xu hole_end >> PAGE_SHIFT, 0);
80570c3547eSMike Kravetz }
80670c3547eSMike Kravetz
80768d32527SMike Kravetz /* If range extends beyond last full page, zero partial page. */
80868d32527SMike Kravetz if ((offset + len) > hole_end && (offset + len) > hole_start)
80968d32527SMike Kravetz hugetlbfs_zero_partial_page(h, mapping,
81068d32527SMike Kravetz hole_end, offset + len);
81168d32527SMike Kravetz
81268d32527SMike Kravetz i_mmap_unlock_write(mapping);
81368d32527SMike Kravetz
81468d32527SMike Kravetz /* Remove full pages from the file. */
81568d32527SMike Kravetz if (hole_end > hole_start)
81668d32527SMike Kravetz remove_inode_hugepages(inode, hole_start, hole_end);
81768d32527SMike Kravetz
81868d32527SMike Kravetz inode_unlock(inode);
81968d32527SMike Kravetz
82070c3547eSMike Kravetz return 0;
82170c3547eSMike Kravetz }
82270c3547eSMike Kravetz
hugetlbfs_fallocate(struct file * file,int mode,loff_t offset,loff_t len)82370c3547eSMike Kravetz static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
82470c3547eSMike Kravetz loff_t len)
82570c3547eSMike Kravetz {
82670c3547eSMike Kravetz struct inode *inode = file_inode(file);
827ff62a342SMarc-André Lureau struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
82870c3547eSMike Kravetz struct address_space *mapping = inode->i_mapping;
82970c3547eSMike Kravetz struct hstate *h = hstate_inode(inode);
83070c3547eSMike Kravetz struct vm_area_struct pseudo_vma;
83170c3547eSMike Kravetz struct mm_struct *mm = current->mm;
83270c3547eSMike Kravetz loff_t hpage_size = huge_page_size(h);
83370c3547eSMike Kravetz unsigned long hpage_shift = huge_page_shift(h);
83470c3547eSMike Kravetz pgoff_t start, index, end;
83570c3547eSMike Kravetz int error;
83670c3547eSMike Kravetz u32 hash;
83770c3547eSMike Kravetz
83870c3547eSMike Kravetz if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
83970c3547eSMike Kravetz return -EOPNOTSUPP;
84070c3547eSMike Kravetz
84170c3547eSMike Kravetz if (mode & FALLOC_FL_PUNCH_HOLE)
84270c3547eSMike Kravetz return hugetlbfs_punch_hole(inode, offset, len);
84370c3547eSMike Kravetz
84470c3547eSMike Kravetz /*
84570c3547eSMike Kravetz * Default preallocate case.
84670c3547eSMike Kravetz * For this range, start is rounded down and end is rounded up
84770c3547eSMike Kravetz * as well as being converted to page offsets.
84870c3547eSMike Kravetz */
84970c3547eSMike Kravetz start = offset >> hpage_shift;
85070c3547eSMike Kravetz end = (offset + len + hpage_size - 1) >> hpage_shift;
85170c3547eSMike Kravetz
8525955102cSAl Viro inode_lock(inode);
85370c3547eSMike Kravetz
85470c3547eSMike Kravetz /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
85570c3547eSMike Kravetz error = inode_newsize_ok(inode, offset + len);
85670c3547eSMike Kravetz if (error)
85770c3547eSMike Kravetz goto out;
85870c3547eSMike Kravetz
859ff62a342SMarc-André Lureau if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
860ff62a342SMarc-André Lureau error = -EPERM;
861ff62a342SMarc-André Lureau goto out;
862ff62a342SMarc-André Lureau }
863ff62a342SMarc-André Lureau
86470c3547eSMike Kravetz /*
86570c3547eSMike Kravetz * Initialize a pseudo vma as this is required by the huge page
86670c3547eSMike Kravetz * allocation routines. If NUMA is configured, use page index
86770c3547eSMike Kravetz * as input to create an allocation policy.
86870c3547eSMike Kravetz */
8692c4541e2SKirill A. Shutemov vma_init(&pseudo_vma, mm);
8701c71222eSSuren Baghdasaryan vm_flags_init(&pseudo_vma, VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
87170c3547eSMike Kravetz pseudo_vma.vm_file = file;
87270c3547eSMike Kravetz
87370c3547eSMike Kravetz for (index = start; index < end; index++) {
87470c3547eSMike Kravetz /*
87570c3547eSMike Kravetz * This is supposed to be the vaddr where the page is being
87670c3547eSMike Kravetz * faulted in, but we have no vaddr here.
87770c3547eSMike Kravetz */
878d0ce0e47SSidhartha Kumar struct folio *folio;
87970c3547eSMike Kravetz unsigned long addr;
88070c3547eSMike Kravetz
88170c3547eSMike Kravetz cond_resched();
88270c3547eSMike Kravetz
88370c3547eSMike Kravetz /*
88470c3547eSMike Kravetz * fallocate(2) manpage permits EINTR; we may have been
88570c3547eSMike Kravetz * interrupted because we are using up too much memory.
88670c3547eSMike Kravetz */
88770c3547eSMike Kravetz if (signal_pending(current)) {
88870c3547eSMike Kravetz error = -EINTR;
88970c3547eSMike Kravetz break;
89070c3547eSMike Kravetz }
89170c3547eSMike Kravetz
89270c3547eSMike Kravetz /* addr is the offset within the file (zero based) */
89370c3547eSMike Kravetz addr = index * hpage_size;
89470c3547eSMike Kravetz
895188a3972SMike Kravetz /* mutex taken here, fault path and hole punch */
896188b04a7SWei Yang hash = hugetlb_fault_mutex_hash(mapping, index);
89770c3547eSMike Kravetz mutex_lock(&hugetlb_fault_mutex_table[hash]);
89870c3547eSMike Kravetz
89970c3547eSMike Kravetz /* See if already present in mapping to avoid alloc/free */
900fd4aed8dSMike Kravetz folio = filemap_get_folio(mapping, index);
901fd4aed8dSMike Kravetz if (!IS_ERR(folio)) {
902fd4aed8dSMike Kravetz folio_put(folio);
90370c3547eSMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]);
90470c3547eSMike Kravetz continue;
90570c3547eSMike Kravetz }
90670c3547eSMike Kravetz
90788ce3fefSMiaohe Lin /*
908d0ce0e47SSidhartha Kumar * Allocate folio without setting the avoid_reserve argument.
90988ce3fefSMiaohe Lin * There certainly are no reserves associated with the
91088ce3fefSMiaohe Lin * pseudo_vma. However, there could be shared mappings with
91188ce3fefSMiaohe Lin * reserves for the file at the inode level. If we fallocate
912d0ce0e47SSidhartha Kumar * folios in these areas, we need to consume the reserves
91388ce3fefSMiaohe Lin * to keep reservation accounting consistent.
91488ce3fefSMiaohe Lin */
915adef0803SAckerley Tng hugetlb_set_vma_policy(&pseudo_vma, inode, index);
916d0ce0e47SSidhartha Kumar folio = alloc_hugetlb_folio(&pseudo_vma, addr, 0);
91770c3547eSMike Kravetz hugetlb_drop_vma_policy(&pseudo_vma);
918d0ce0e47SSidhartha Kumar if (IS_ERR(folio)) {
91970c3547eSMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]);
920d0ce0e47SSidhartha Kumar error = PTR_ERR(folio);
92170c3547eSMike Kravetz goto out;
92270c3547eSMike Kravetz }
923d0ce0e47SSidhartha Kumar clear_huge_page(&folio->page, addr, pages_per_huge_page(h));
924d0ce0e47SSidhartha Kumar __folio_mark_uptodate(folio);
9259b91c0e2SSidhartha Kumar error = hugetlb_add_to_page_cache(folio, mapping, index);
92670c3547eSMike Kravetz if (unlikely(error)) {
927d2d7bb44SSidhartha Kumar restore_reserve_on_error(h, &pseudo_vma, addr, folio);
928d0ce0e47SSidhartha Kumar folio_put(folio);
92970c3547eSMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]);
93070c3547eSMike Kravetz goto out;
93170c3547eSMike Kravetz }
93270c3547eSMike Kravetz
93370c3547eSMike Kravetz mutex_unlock(&hugetlb_fault_mutex_table[hash]);
93470c3547eSMike Kravetz
935d0ce0e47SSidhartha Kumar folio_set_hugetlb_migratable(folio);
93670c3547eSMike Kravetz /*
937d0ce0e47SSidhartha Kumar * folio_unlock because locked by hugetlb_add_to_page_cache()
938d0ce0e47SSidhartha Kumar * folio_put() due to reference from alloc_hugetlb_folio()
93970c3547eSMike Kravetz */
940d0ce0e47SSidhartha Kumar folio_unlock(folio);
941d0ce0e47SSidhartha Kumar folio_put(folio);
94270c3547eSMike Kravetz }
94370c3547eSMike Kravetz
94470c3547eSMike Kravetz if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
94570c3547eSMike Kravetz i_size_write(inode, offset + len);
946a72a7deaSJeff Layton inode_set_ctime_current(inode);
94770c3547eSMike Kravetz out:
9485955102cSAl Viro inode_unlock(inode);
94970c3547eSMike Kravetz return error;
95070c3547eSMike Kravetz }
95170c3547eSMike Kravetz
hugetlbfs_setattr(struct mnt_idmap * idmap,struct dentry * dentry,struct iattr * attr)952c1632a0fSChristian Brauner static int hugetlbfs_setattr(struct mnt_idmap *idmap,
953549c7297SChristian Brauner struct dentry *dentry, struct iattr *attr)
9541da177e4SLinus Torvalds {
9552b0143b5SDavid Howells struct inode *inode = d_inode(dentry);
956a5516438SAndi Kleen struct hstate *h = hstate_inode(inode);
9571da177e4SLinus Torvalds int error;
9581da177e4SLinus Torvalds unsigned int ia_valid = attr->ia_valid;
959ff62a342SMarc-André Lureau struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
9601da177e4SLinus Torvalds
961c1632a0fSChristian Brauner error = setattr_prepare(&nop_mnt_idmap, dentry, attr);
9621da177e4SLinus Torvalds if (error)
9631025774cSChristoph Hellwig return error;
9641da177e4SLinus Torvalds
9651da177e4SLinus Torvalds if (ia_valid & ATTR_SIZE) {
966ff62a342SMarc-André Lureau loff_t oldsize = inode->i_size;
967ff62a342SMarc-André Lureau loff_t newsize = attr->ia_size;
968ff62a342SMarc-André Lureau
969ff62a342SMarc-André Lureau if (newsize & ~huge_page_mask(h))
9701025774cSChristoph Hellwig return -EINVAL;
971398c0da7SMiaohe Lin /* protected by i_rwsem */
972ff62a342SMarc-André Lureau if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
973ff62a342SMarc-André Lureau (newsize > oldsize && (info->seals & F_SEAL_GROW)))
974ff62a342SMarc-André Lureau return -EPERM;
975e5d319deSMiaohe Lin hugetlb_vmtruncate(inode, newsize);
9761da177e4SLinus Torvalds }
9771da177e4SLinus Torvalds
978c1632a0fSChristian Brauner setattr_copy(&nop_mnt_idmap, inode, attr);
9791025774cSChristoph Hellwig mark_inode_dirty(inode);
9801025774cSChristoph Hellwig return 0;
9811025774cSChristoph Hellwig }
9821025774cSChristoph Hellwig
hugetlbfs_get_root(struct super_block * sb,struct hugetlbfs_fs_context * ctx)9837d54fa64SAl Viro static struct inode *hugetlbfs_get_root(struct super_block *sb,
98432021982SDavid Howells struct hugetlbfs_fs_context *ctx)
9851da177e4SLinus Torvalds {
9861da177e4SLinus Torvalds struct inode *inode;
9871da177e4SLinus Torvalds
9881da177e4SLinus Torvalds inode = new_inode(sb);
9891da177e4SLinus Torvalds if (inode) {
99085fe4025SChristoph Hellwig inode->i_ino = get_next_ino();
99132021982SDavid Howells inode->i_mode = S_IFDIR | ctx->mode;
99232021982SDavid Howells inode->i_uid = ctx->uid;
99332021982SDavid Howells inode->i_gid = ctx->gid;
994a72a7deaSJeff Layton inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
9957d54fa64SAl Viro inode->i_op = &hugetlbfs_dir_inode_operations;
9967d54fa64SAl Viro inode->i_fop = &simple_dir_operations;
9977d54fa64SAl Viro /* directory inodes start off with i_nlink == 2 (for "." entry) */
9987d54fa64SAl Viro inc_nlink(inode);
99965ed7601SAneesh Kumar K.V lockdep_annotate_inode_mutex_key(inode);
10007d54fa64SAl Viro }
10017d54fa64SAl Viro return inode;
10027d54fa64SAl Viro }
10037d54fa64SAl Viro
1004b610ded7SMichal Hocko /*
1005c8c06efaSDavidlohr Bueso * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
1006b610ded7SMichal Hocko * be taken from reclaim -- unlike regular filesystems. This needs an
100788f306b6SKirill A. Shutemov * annotation because huge_pmd_share() does an allocation under hugetlb's
1008c8c06efaSDavidlohr Bueso * i_mmap_rwsem.
1009b610ded7SMichal Hocko */
1010c8c06efaSDavidlohr Bueso static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
1011b610ded7SMichal Hocko
hugetlbfs_get_inode(struct super_block * sb,struct inode * dir,umode_t mode,dev_t dev)10127d54fa64SAl Viro static struct inode *hugetlbfs_get_inode(struct super_block *sb,
10137d54fa64SAl Viro struct inode *dir,
101418df2252SAl Viro umode_t mode, dev_t dev)
10157d54fa64SAl Viro {
10167d54fa64SAl Viro struct inode *inode;
101758b6e5e8SMike Kravetz struct resv_map *resv_map = NULL;
10189119a41eSJoonsoo Kim
101958b6e5e8SMike Kravetz /*
102058b6e5e8SMike Kravetz * Reserve maps are only needed for inodes that can have associated
102158b6e5e8SMike Kravetz * page allocations.
102258b6e5e8SMike Kravetz */
102358b6e5e8SMike Kravetz if (S_ISREG(mode) || S_ISLNK(mode)) {
10249119a41eSJoonsoo Kim resv_map = resv_map_alloc();
10259119a41eSJoonsoo Kim if (!resv_map)
10269119a41eSJoonsoo Kim return NULL;
102758b6e5e8SMike Kravetz }
10287d54fa64SAl Viro
10297d54fa64SAl Viro inode = new_inode(sb);
10307d54fa64SAl Viro if (inode) {
1031ff62a342SMarc-André Lureau struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
1032ff62a342SMarc-André Lureau
10337d54fa64SAl Viro inode->i_ino = get_next_ino();
1034f2d40141SChristian Brauner inode_init_owner(&nop_mnt_idmap, inode, dir, mode);
1035c8c06efaSDavidlohr Bueso lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
1036c8c06efaSDavidlohr Bueso &hugetlbfs_i_mmap_rwsem_key);
10371da177e4SLinus Torvalds inode->i_mapping->a_ops = &hugetlbfs_aops;
1038a72a7deaSJeff Layton inode->i_atime = inode->i_mtime = inode_set_ctime_current(inode);
10399119a41eSJoonsoo Kim inode->i_mapping->private_data = resv_map;
1040ff62a342SMarc-André Lureau info->seals = F_SEAL_SEAL;
10411da177e4SLinus Torvalds switch (mode & S_IFMT) {
10421da177e4SLinus Torvalds default:
10431da177e4SLinus Torvalds init_special_inode(inode, mode, dev);
10441da177e4SLinus Torvalds break;
10451da177e4SLinus Torvalds case S_IFREG:
10461da177e4SLinus Torvalds inode->i_op = &hugetlbfs_inode_operations;
10471da177e4SLinus Torvalds inode->i_fop = &hugetlbfs_file_operations;
10481da177e4SLinus Torvalds break;
10491da177e4SLinus Torvalds case S_IFDIR:
10501da177e4SLinus Torvalds inode->i_op = &hugetlbfs_dir_inode_operations;
10511da177e4SLinus Torvalds inode->i_fop = &simple_dir_operations;
10521da177e4SLinus Torvalds
10531da177e4SLinus Torvalds /* directory inodes start off with i_nlink == 2 (for "." entry) */
1054d8c76e6fSDave Hansen inc_nlink(inode);
10551da177e4SLinus Torvalds break;
10561da177e4SLinus Torvalds case S_IFLNK:
10571da177e4SLinus Torvalds inode->i_op = &page_symlink_inode_operations;
105821fc61c7SAl Viro inode_nohighmem(inode);
10591da177e4SLinus Torvalds break;
10601da177e4SLinus Torvalds }
1061e096d0c7SJosh Boyer lockdep_annotate_inode_mutex_key(inode);
106258b6e5e8SMike Kravetz } else {
106358b6e5e8SMike Kravetz if (resv_map)
10649119a41eSJoonsoo Kim kref_put(&resv_map->refs, resv_map_release);
106558b6e5e8SMike Kravetz }
10669119a41eSJoonsoo Kim
10671da177e4SLinus Torvalds return inode;
10681da177e4SLinus Torvalds }
10691da177e4SLinus Torvalds
10701da177e4SLinus Torvalds /*
10711da177e4SLinus Torvalds * File creation. Allocate an inode, and we're done..
10721da177e4SLinus Torvalds */
hugetlbfs_mknod(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,umode_t mode,dev_t dev)10735ebb29beSChristian Brauner static int hugetlbfs_mknod(struct mnt_idmap *idmap, struct inode *dir,
10741ab5b82fSPiotr Sarna struct dentry *dentry, umode_t mode, dev_t dev)
10751ab5b82fSPiotr Sarna {
107619ee5345SAl Viro struct inode *inode;
107719ee5345SAl Viro
107819ee5345SAl Viro inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
107919ee5345SAl Viro if (!inode)
108019ee5345SAl Viro return -ENOSPC;
1081a72a7deaSJeff Layton dir->i_mtime = inode_set_ctime_current(dir);
108219ee5345SAl Viro d_instantiate(dentry, inode);
108319ee5345SAl Viro dget(dentry);/* Extra count - pin the dentry in core */
108419ee5345SAl Viro return 0;
10851ab5b82fSPiotr Sarna }
10861ab5b82fSPiotr Sarna
hugetlbfs_mkdir(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,umode_t mode)1087c54bd91eSChristian Brauner static int hugetlbfs_mkdir(struct mnt_idmap *idmap, struct inode *dir,
1088549c7297SChristian Brauner struct dentry *dentry, umode_t mode)
10891da177e4SLinus Torvalds {
10905ebb29beSChristian Brauner int retval = hugetlbfs_mknod(&nop_mnt_idmap, dir, dentry,
1091549c7297SChristian Brauner mode | S_IFDIR, 0);
10921da177e4SLinus Torvalds if (!retval)
1093d8c76e6fSDave Hansen inc_nlink(dir);
10941da177e4SLinus Torvalds return retval;
10951da177e4SLinus Torvalds }
10961da177e4SLinus Torvalds
hugetlbfs_create(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,umode_t mode,bool excl)10976c960e68SChristian Brauner static int hugetlbfs_create(struct mnt_idmap *idmap,
1098549c7297SChristian Brauner struct inode *dir, struct dentry *dentry,
1099549c7297SChristian Brauner umode_t mode, bool excl)
11001da177e4SLinus Torvalds {
11015ebb29beSChristian Brauner return hugetlbfs_mknod(&nop_mnt_idmap, dir, dentry, mode | S_IFREG, 0);
11021da177e4SLinus Torvalds }
11031da177e4SLinus Torvalds
hugetlbfs_tmpfile(struct mnt_idmap * idmap,struct inode * dir,struct file * file,umode_t mode)1104011e2b71SChristian Brauner static int hugetlbfs_tmpfile(struct mnt_idmap *idmap,
1105863f144fSMiklos Szeredi struct inode *dir, struct file *file,
1106549c7297SChristian Brauner umode_t mode)
11071ab5b82fSPiotr Sarna {
110819ee5345SAl Viro struct inode *inode;
110919ee5345SAl Viro
111019ee5345SAl Viro inode = hugetlbfs_get_inode(dir->i_sb, dir, mode | S_IFREG, 0);
111119ee5345SAl Viro if (!inode)
111219ee5345SAl Viro return -ENOSPC;
1113a72a7deaSJeff Layton dir->i_mtime = inode_set_ctime_current(dir);
1114863f144fSMiklos Szeredi d_tmpfile(file, inode);
1115863f144fSMiklos Szeredi return finish_open_simple(file, 0);
11161ab5b82fSPiotr Sarna }
11171ab5b82fSPiotr Sarna
hugetlbfs_symlink(struct mnt_idmap * idmap,struct inode * dir,struct dentry * dentry,const char * symname)11187a77db95SChristian Brauner static int hugetlbfs_symlink(struct mnt_idmap *idmap,
1119549c7297SChristian Brauner struct inode *dir, struct dentry *dentry,
1120549c7297SChristian Brauner const char *symname)
11211da177e4SLinus Torvalds {
11221da177e4SLinus Torvalds struct inode *inode;
11231da177e4SLinus Torvalds int error = -ENOSPC;
11241da177e4SLinus Torvalds
11257d54fa64SAl Viro inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
11261da177e4SLinus Torvalds if (inode) {
11271da177e4SLinus Torvalds int l = strlen(symname)+1;
11281da177e4SLinus Torvalds error = page_symlink(inode, symname, l);
11291da177e4SLinus Torvalds if (!error) {
11301da177e4SLinus Torvalds d_instantiate(dentry, inode);
11311da177e4SLinus Torvalds dget(dentry);
11321da177e4SLinus Torvalds } else
11331da177e4SLinus Torvalds iput(inode);
11341da177e4SLinus Torvalds }
1135a72a7deaSJeff Layton dir->i_mtime = inode_set_ctime_current(dir);
11361da177e4SLinus Torvalds
11371da177e4SLinus Torvalds return error;
11381da177e4SLinus Torvalds }
11391da177e4SLinus Torvalds
1140b890ec2aSMatthew Wilcox (Oracle) #ifdef CONFIG_MIGRATION
hugetlbfs_migrate_folio(struct address_space * mapping,struct folio * dst,struct folio * src,enum migrate_mode mode)1141b890ec2aSMatthew Wilcox (Oracle) static int hugetlbfs_migrate_folio(struct address_space *mapping,
1142b890ec2aSMatthew Wilcox (Oracle) struct folio *dst, struct folio *src,
1143a6bc32b8SMel Gorman enum migrate_mode mode)
1144290408d4SNaoya Horiguchi {
1145290408d4SNaoya Horiguchi int rc;
1146290408d4SNaoya Horiguchi
1147b890ec2aSMatthew Wilcox (Oracle) rc = migrate_huge_page_move_mapping(mapping, dst, src);
114878bd5209SRafael Aquini if (rc != MIGRATEPAGE_SUCCESS)
1149290408d4SNaoya Horiguchi return rc;
1150cb6acd01SMike Kravetz
1151149562f7SSidhartha Kumar if (hugetlb_folio_subpool(src)) {
1152149562f7SSidhartha Kumar hugetlb_set_folio_subpool(dst,
1153149562f7SSidhartha Kumar hugetlb_folio_subpool(src));
1154149562f7SSidhartha Kumar hugetlb_set_folio_subpool(src, NULL);
1155cb6acd01SMike Kravetz }
1156cb6acd01SMike Kravetz
11572916ecc0SJérôme Glisse if (mode != MIGRATE_SYNC_NO_COPY)
1158b890ec2aSMatthew Wilcox (Oracle) folio_migrate_copy(dst, src);
11592916ecc0SJérôme Glisse else
1160b890ec2aSMatthew Wilcox (Oracle) folio_migrate_flags(dst, src);
1161290408d4SNaoya Horiguchi
116278bd5209SRafael Aquini return MIGRATEPAGE_SUCCESS;
1163290408d4SNaoya Horiguchi }
1164b890ec2aSMatthew Wilcox (Oracle) #else
1165b890ec2aSMatthew Wilcox (Oracle) #define hugetlbfs_migrate_folio NULL
1166b890ec2aSMatthew Wilcox (Oracle) #endif
1167290408d4SNaoya Horiguchi
hugetlbfs_error_remove_page(struct address_space * mapping,struct page * page)116878bb9203SNaoya Horiguchi static int hugetlbfs_error_remove_page(struct address_space *mapping,
116978bb9203SNaoya Horiguchi struct page *page)
117078bb9203SNaoya Horiguchi {
117178bb9203SNaoya Horiguchi return 0;
117278bb9203SNaoya Horiguchi }
117378bb9203SNaoya Horiguchi
11744a25220dSDavid Howells /*
11754a25220dSDavid Howells * Display the mount options in /proc/mounts.
11764a25220dSDavid Howells */
hugetlbfs_show_options(struct seq_file * m,struct dentry * root)11774a25220dSDavid Howells static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
11784a25220dSDavid Howells {
11794a25220dSDavid Howells struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
11804a25220dSDavid Howells struct hugepage_subpool *spool = sbinfo->spool;
11814a25220dSDavid Howells unsigned long hpage_size = huge_page_size(sbinfo->hstate);
11824a25220dSDavid Howells unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
11834a25220dSDavid Howells char mod;
11844a25220dSDavid Howells
11854a25220dSDavid Howells if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
11864a25220dSDavid Howells seq_printf(m, ",uid=%u",
11874a25220dSDavid Howells from_kuid_munged(&init_user_ns, sbinfo->uid));
11884a25220dSDavid Howells if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
11894a25220dSDavid Howells seq_printf(m, ",gid=%u",
11904a25220dSDavid Howells from_kgid_munged(&init_user_ns, sbinfo->gid));
11914a25220dSDavid Howells if (sbinfo->mode != 0755)
11924a25220dSDavid Howells seq_printf(m, ",mode=%o", sbinfo->mode);
11934a25220dSDavid Howells if (sbinfo->max_inodes != -1)
11944a25220dSDavid Howells seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
11954a25220dSDavid Howells
11964a25220dSDavid Howells hpage_size /= 1024;
11974a25220dSDavid Howells mod = 'K';
11984a25220dSDavid Howells if (hpage_size >= 1024) {
11994a25220dSDavid Howells hpage_size /= 1024;
12004a25220dSDavid Howells mod = 'M';
12014a25220dSDavid Howells }
12024a25220dSDavid Howells seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
12034a25220dSDavid Howells if (spool) {
12044a25220dSDavid Howells if (spool->max_hpages != -1)
12054a25220dSDavid Howells seq_printf(m, ",size=%llu",
12064a25220dSDavid Howells (unsigned long long)spool->max_hpages << hpage_shift);
12074a25220dSDavid Howells if (spool->min_hpages != -1)
12084a25220dSDavid Howells seq_printf(m, ",min_size=%llu",
12094a25220dSDavid Howells (unsigned long long)spool->min_hpages << hpage_shift);
12104a25220dSDavid Howells }
12114a25220dSDavid Howells return 0;
12124a25220dSDavid Howells }
12134a25220dSDavid Howells
hugetlbfs_statfs(struct dentry * dentry,struct kstatfs * buf)1214726c3342SDavid Howells static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
12151da177e4SLinus Torvalds {
1216726c3342SDavid Howells struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
12172b0143b5SDavid Howells struct hstate *h = hstate_inode(d_inode(dentry));
12181da177e4SLinus Torvalds
12191da177e4SLinus Torvalds buf->f_type = HUGETLBFS_MAGIC;
1220a5516438SAndi Kleen buf->f_bsize = huge_page_size(h);
12211da177e4SLinus Torvalds if (sbinfo) {
12221da177e4SLinus Torvalds spin_lock(&sbinfo->stat_lock);
122311680763SMiaohe Lin /* If no limits set, just report 0 or -1 for max/free/used
122474a8a65cSDavid Gibson * blocks, like simple_statfs() */
122590481622SDavid Gibson if (sbinfo->spool) {
122690481622SDavid Gibson long free_pages;
122790481622SDavid Gibson
12284b25f030SMina Almasry spin_lock_irq(&sbinfo->spool->lock);
122990481622SDavid Gibson buf->f_blocks = sbinfo->spool->max_hpages;
123090481622SDavid Gibson free_pages = sbinfo->spool->max_hpages
123190481622SDavid Gibson - sbinfo->spool->used_hpages;
123290481622SDavid Gibson buf->f_bavail = buf->f_bfree = free_pages;
12334b25f030SMina Almasry spin_unlock_irq(&sbinfo->spool->lock);
12341da177e4SLinus Torvalds buf->f_files = sbinfo->max_inodes;
12351da177e4SLinus Torvalds buf->f_ffree = sbinfo->free_inodes;
123674a8a65cSDavid Gibson }
12371da177e4SLinus Torvalds spin_unlock(&sbinfo->stat_lock);
12381da177e4SLinus Torvalds }
12391da177e4SLinus Torvalds buf->f_namelen = NAME_MAX;
12401da177e4SLinus Torvalds return 0;
12411da177e4SLinus Torvalds }
12421da177e4SLinus Torvalds
hugetlbfs_put_super(struct super_block * sb)12431da177e4SLinus Torvalds static void hugetlbfs_put_super(struct super_block *sb)
12441da177e4SLinus Torvalds {
12451da177e4SLinus Torvalds struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
12461da177e4SLinus Torvalds
12471da177e4SLinus Torvalds if (sbi) {
12481da177e4SLinus Torvalds sb->s_fs_info = NULL;
124990481622SDavid Gibson
125090481622SDavid Gibson if (sbi->spool)
125190481622SDavid Gibson hugepage_put_subpool(sbi->spool);
125290481622SDavid Gibson
12531da177e4SLinus Torvalds kfree(sbi);
12541da177e4SLinus Torvalds }
12551da177e4SLinus Torvalds }
12561da177e4SLinus Torvalds
hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info * sbinfo)125796527980SChristoph Hellwig static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
125896527980SChristoph Hellwig {
125996527980SChristoph Hellwig if (sbinfo->free_inodes >= 0) {
126096527980SChristoph Hellwig spin_lock(&sbinfo->stat_lock);
126196527980SChristoph Hellwig if (unlikely(!sbinfo->free_inodes)) {
126296527980SChristoph Hellwig spin_unlock(&sbinfo->stat_lock);
126396527980SChristoph Hellwig return 0;
126496527980SChristoph Hellwig }
126596527980SChristoph Hellwig sbinfo->free_inodes--;
126696527980SChristoph Hellwig spin_unlock(&sbinfo->stat_lock);
126796527980SChristoph Hellwig }
126896527980SChristoph Hellwig
126996527980SChristoph Hellwig return 1;
127096527980SChristoph Hellwig }
127196527980SChristoph Hellwig
hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info * sbinfo)127296527980SChristoph Hellwig static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
127396527980SChristoph Hellwig {
127496527980SChristoph Hellwig if (sbinfo->free_inodes >= 0) {
127596527980SChristoph Hellwig spin_lock(&sbinfo->stat_lock);
127696527980SChristoph Hellwig sbinfo->free_inodes++;
127796527980SChristoph Hellwig spin_unlock(&sbinfo->stat_lock);
127896527980SChristoph Hellwig }
127996527980SChristoph Hellwig }
128096527980SChristoph Hellwig
128196527980SChristoph Hellwig
1282e18b890bSChristoph Lameter static struct kmem_cache *hugetlbfs_inode_cachep;
12831da177e4SLinus Torvalds
hugetlbfs_alloc_inode(struct super_block * sb)12841da177e4SLinus Torvalds static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
12851da177e4SLinus Torvalds {
128696527980SChristoph Hellwig struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
12871da177e4SLinus Torvalds struct hugetlbfs_inode_info *p;
12881da177e4SLinus Torvalds
128996527980SChristoph Hellwig if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
12901da177e4SLinus Torvalds return NULL;
1291fd60b288SMuchun Song p = alloc_inode_sb(sb, hugetlbfs_inode_cachep, GFP_KERNEL);
129296527980SChristoph Hellwig if (unlikely(!p)) {
129396527980SChristoph Hellwig hugetlbfs_inc_free_inodes(sbinfo);
129496527980SChristoph Hellwig return NULL;
12951da177e4SLinus Torvalds }
12964742a35dSMike Kravetz
12974742a35dSMike Kravetz /*
12984742a35dSMike Kravetz * Any time after allocation, hugetlbfs_destroy_inode can be called
12994742a35dSMike Kravetz * for the inode. mpol_free_shared_policy is unconditionally called
13004742a35dSMike Kravetz * as part of hugetlbfs_destroy_inode. So, initialize policy here
13014742a35dSMike Kravetz * in case of a quick call to destroy.
13024742a35dSMike Kravetz *
13034742a35dSMike Kravetz * Note that the policy is initialized even if we are creating a
13044742a35dSMike Kravetz * private inode. This simplifies hugetlbfs_destroy_inode.
13054742a35dSMike Kravetz */
13064742a35dSMike Kravetz mpol_shared_policy_init(&p->policy, NULL);
13074742a35dSMike Kravetz
130896527980SChristoph Hellwig return &p->vfs_inode;
13091da177e4SLinus Torvalds }
13101da177e4SLinus Torvalds
hugetlbfs_free_inode(struct inode * inode)1311b62de322SAl Viro static void hugetlbfs_free_inode(struct inode *inode)
1312fa0d7e3dSNick Piggin {
1313fa0d7e3dSNick Piggin kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
1314fa0d7e3dSNick Piggin }
1315fa0d7e3dSNick Piggin
hugetlbfs_destroy_inode(struct inode * inode)13161da177e4SLinus Torvalds static void hugetlbfs_destroy_inode(struct inode *inode)
13171da177e4SLinus Torvalds {
131896527980SChristoph Hellwig hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
13191da177e4SLinus Torvalds mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
13201da177e4SLinus Torvalds }
13211da177e4SLinus Torvalds
1322f5e54d6eSChristoph Hellwig static const struct address_space_operations hugetlbfs_aops = {
1323800d15a5SNick Piggin .write_begin = hugetlbfs_write_begin,
1324800d15a5SNick Piggin .write_end = hugetlbfs_write_end,
132546de8b97SMatthew Wilcox (Oracle) .dirty_folio = noop_dirty_folio,
1326b890ec2aSMatthew Wilcox (Oracle) .migrate_folio = hugetlbfs_migrate_folio,
132778bb9203SNaoya Horiguchi .error_remove_page = hugetlbfs_error_remove_page,
13281da177e4SLinus Torvalds };
13291da177e4SLinus Torvalds
133096527980SChristoph Hellwig
init_once(void * foo)133151cc5068SAlexey Dobriyan static void init_once(void *foo)
133296527980SChristoph Hellwig {
1333dbaf7dc9SLi zeming struct hugetlbfs_inode_info *ei = foo;
133496527980SChristoph Hellwig
133596527980SChristoph Hellwig inode_init_once(&ei->vfs_inode);
133696527980SChristoph Hellwig }
133796527980SChristoph Hellwig
13384b6f5d20SArjan van de Ven const struct file_operations hugetlbfs_file_operations = {
133934d0640eSAl Viro .read_iter = hugetlbfs_read_iter,
13401da177e4SLinus Torvalds .mmap = hugetlbfs_file_mmap,
13411b061d92SChristoph Hellwig .fsync = noop_fsync,
13421da177e4SLinus Torvalds .get_unmapped_area = hugetlb_get_unmapped_area,
13436038f373SArnd Bergmann .llseek = default_llseek,
134470c3547eSMike Kravetz .fallocate = hugetlbfs_fallocate,
13451da177e4SLinus Torvalds };
13461da177e4SLinus Torvalds
134792e1d5beSArjan van de Ven static const struct inode_operations hugetlbfs_dir_inode_operations = {
13481da177e4SLinus Torvalds .create = hugetlbfs_create,
13491da177e4SLinus Torvalds .lookup = simple_lookup,
13501da177e4SLinus Torvalds .link = simple_link,
13511da177e4SLinus Torvalds .unlink = simple_unlink,
13521da177e4SLinus Torvalds .symlink = hugetlbfs_symlink,
13531da177e4SLinus Torvalds .mkdir = hugetlbfs_mkdir,
13541da177e4SLinus Torvalds .rmdir = simple_rmdir,
13551da177e4SLinus Torvalds .mknod = hugetlbfs_mknod,
13561da177e4SLinus Torvalds .rename = simple_rename,
13571da177e4SLinus Torvalds .setattr = hugetlbfs_setattr,
13581ab5b82fSPiotr Sarna .tmpfile = hugetlbfs_tmpfile,
13591da177e4SLinus Torvalds };
13601da177e4SLinus Torvalds
136192e1d5beSArjan van de Ven static const struct inode_operations hugetlbfs_inode_operations = {
13621da177e4SLinus Torvalds .setattr = hugetlbfs_setattr,
13631da177e4SLinus Torvalds };
13641da177e4SLinus Torvalds
1365ee9b6d61SJosef 'Jeff' Sipek static const struct super_operations hugetlbfs_ops = {
13661da177e4SLinus Torvalds .alloc_inode = hugetlbfs_alloc_inode,
1367b62de322SAl Viro .free_inode = hugetlbfs_free_inode,
13681da177e4SLinus Torvalds .destroy_inode = hugetlbfs_destroy_inode,
13692bbbda30SAl Viro .evict_inode = hugetlbfs_evict_inode,
13701da177e4SLinus Torvalds .statfs = hugetlbfs_statfs,
13711da177e4SLinus Torvalds .put_super = hugetlbfs_put_super,
13724a25220dSDavid Howells .show_options = hugetlbfs_show_options,
13731da177e4SLinus Torvalds };
13741da177e4SLinus Torvalds
13757ca02d0aSMike Kravetz /*
13767ca02d0aSMike Kravetz * Convert size option passed from command line to number of huge pages
13777ca02d0aSMike Kravetz * in the pool specified by hstate. Size option could be in bytes
13787ca02d0aSMike Kravetz * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
13797ca02d0aSMike Kravetz */
13804a25220dSDavid Howells static long
hugetlbfs_size_to_hpages(struct hstate * h,unsigned long long size_opt,enum hugetlbfs_size_type val_type)13817ca02d0aSMike Kravetz hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
13824a25220dSDavid Howells enum hugetlbfs_size_type val_type)
13837ca02d0aSMike Kravetz {
13847ca02d0aSMike Kravetz if (val_type == NO_SIZE)
13857ca02d0aSMike Kravetz return -1;
13867ca02d0aSMike Kravetz
13877ca02d0aSMike Kravetz if (val_type == SIZE_PERCENT) {
13887ca02d0aSMike Kravetz size_opt <<= huge_page_shift(h);
13897ca02d0aSMike Kravetz size_opt *= h->max_huge_pages;
13907ca02d0aSMike Kravetz do_div(size_opt, 100);
13917ca02d0aSMike Kravetz }
13927ca02d0aSMike Kravetz
13937ca02d0aSMike Kravetz size_opt >>= huge_page_shift(h);
13947ca02d0aSMike Kravetz return size_opt;
13957ca02d0aSMike Kravetz }
13967ca02d0aSMike Kravetz
139732021982SDavid Howells /*
139832021982SDavid Howells * Parse one mount parameter.
139932021982SDavid Howells */
hugetlbfs_parse_param(struct fs_context * fc,struct fs_parameter * param)140032021982SDavid Howells static int hugetlbfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
14011da177e4SLinus Torvalds {
140232021982SDavid Howells struct hugetlbfs_fs_context *ctx = fc->fs_private;
140332021982SDavid Howells struct fs_parse_result result;
140413c5a9fbSOscar Salvador struct hstate *h;
140532021982SDavid Howells char *rest;
140632021982SDavid Howells unsigned long ps;
140732021982SDavid Howells int opt;
14081da177e4SLinus Torvalds
1409d7167b14SAl Viro opt = fs_parse(fc, hugetlb_fs_parameters, param, &result);
141032021982SDavid Howells if (opt < 0)
141132021982SDavid Howells return opt;
141232021982SDavid Howells
141332021982SDavid Howells switch (opt) {
141432021982SDavid Howells case Opt_uid:
141532021982SDavid Howells ctx->uid = make_kuid(current_user_ns(), result.uint_32);
141632021982SDavid Howells if (!uid_valid(ctx->uid))
141732021982SDavid Howells goto bad_val;
14181da177e4SLinus Torvalds return 0;
14191da177e4SLinus Torvalds
1420e73a75faSRandy Dunlap case Opt_gid:
142132021982SDavid Howells ctx->gid = make_kgid(current_user_ns(), result.uint_32);
142232021982SDavid Howells if (!gid_valid(ctx->gid))
1423e73a75faSRandy Dunlap goto bad_val;
142432021982SDavid Howells return 0;
1425e73a75faSRandy Dunlap
1426e73a75faSRandy Dunlap case Opt_mode:
142732021982SDavid Howells ctx->mode = result.uint_32 & 01777U;
142832021982SDavid Howells return 0;
1429e73a75faSRandy Dunlap
143032021982SDavid Howells case Opt_size:
1431e73a75faSRandy Dunlap /* memparse() will accept a K/M/G without a digit */
143226215b7eSHawkins Jiawei if (!param->string || !isdigit(param->string[0]))
1433e73a75faSRandy Dunlap goto bad_val;
143432021982SDavid Howells ctx->max_size_opt = memparse(param->string, &rest);
143532021982SDavid Howells ctx->max_val_type = SIZE_STD;
1436a137e1ccSAndi Kleen if (*rest == '%')
143732021982SDavid Howells ctx->max_val_type = SIZE_PERCENT;
143832021982SDavid Howells return 0;
14391da177e4SLinus Torvalds
1440e73a75faSRandy Dunlap case Opt_nr_inodes:
1441e73a75faSRandy Dunlap /* memparse() will accept a K/M/G without a digit */
144226215b7eSHawkins Jiawei if (!param->string || !isdigit(param->string[0]))
1443e73a75faSRandy Dunlap goto bad_val;
144432021982SDavid Howells ctx->nr_inodes = memparse(param->string, &rest);
144532021982SDavid Howells return 0;
1446e73a75faSRandy Dunlap
144732021982SDavid Howells case Opt_pagesize:
144832021982SDavid Howells ps = memparse(param->string, &rest);
144913c5a9fbSOscar Salvador h = size_to_hstate(ps);
145013c5a9fbSOscar Salvador if (!h) {
1451d0036517SMiaohe Lin pr_err("Unsupported page size %lu MB\n", ps / SZ_1M);
1452a137e1ccSAndi Kleen return -EINVAL;
1453a137e1ccSAndi Kleen }
145413c5a9fbSOscar Salvador ctx->hstate = h;
145532021982SDavid Howells return 0;
1456a137e1ccSAndi Kleen
145732021982SDavid Howells case Opt_min_size:
14587ca02d0aSMike Kravetz /* memparse() will accept a K/M/G without a digit */
145926215b7eSHawkins Jiawei if (!param->string || !isdigit(param->string[0]))
14607ca02d0aSMike Kravetz goto bad_val;
146132021982SDavid Howells ctx->min_size_opt = memparse(param->string, &rest);
146232021982SDavid Howells ctx->min_val_type = SIZE_STD;
14637ca02d0aSMike Kravetz if (*rest == '%')
146432021982SDavid Howells ctx->min_val_type = SIZE_PERCENT;
146532021982SDavid Howells return 0;
14667ca02d0aSMike Kravetz
1467e73a75faSRandy Dunlap default:
1468b4c07bceSLee Schermerhorn return -EINVAL;
1469e73a75faSRandy Dunlap }
147032021982SDavid Howells
147132021982SDavid Howells bad_val:
1472b5db30cfSAl Viro return invalfc(fc, "Bad value '%s' for mount option '%s'\n",
147332021982SDavid Howells param->string, param->key);
14741da177e4SLinus Torvalds }
1475a137e1ccSAndi Kleen
14767ca02d0aSMike Kravetz /*
147732021982SDavid Howells * Validate the parsed options.
147832021982SDavid Howells */
hugetlbfs_validate(struct fs_context * fc)147932021982SDavid Howells static int hugetlbfs_validate(struct fs_context *fc)
148032021982SDavid Howells {
148132021982SDavid Howells struct hugetlbfs_fs_context *ctx = fc->fs_private;
148232021982SDavid Howells
148332021982SDavid Howells /*
14847ca02d0aSMike Kravetz * Use huge page pool size (in hstate) to convert the size
14857ca02d0aSMike Kravetz * options to number of huge pages. If NO_SIZE, -1 is returned.
14867ca02d0aSMike Kravetz */
148732021982SDavid Howells ctx->max_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
148832021982SDavid Howells ctx->max_size_opt,
148932021982SDavid Howells ctx->max_val_type);
149032021982SDavid Howells ctx->min_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
149132021982SDavid Howells ctx->min_size_opt,
149232021982SDavid Howells ctx->min_val_type);
14937ca02d0aSMike Kravetz
14947ca02d0aSMike Kravetz /*
14957ca02d0aSMike Kravetz * If max_size was specified, then min_size must be smaller
14967ca02d0aSMike Kravetz */
149732021982SDavid Howells if (ctx->max_val_type > NO_SIZE &&
149832021982SDavid Howells ctx->min_hpages > ctx->max_hpages) {
149932021982SDavid Howells pr_err("Minimum size can not be greater than maximum size\n");
15007ca02d0aSMike Kravetz return -EINVAL;
1501a137e1ccSAndi Kleen }
1502a137e1ccSAndi Kleen
15031da177e4SLinus Torvalds return 0;
15041da177e4SLinus Torvalds }
15051da177e4SLinus Torvalds
15061da177e4SLinus Torvalds static int
hugetlbfs_fill_super(struct super_block * sb,struct fs_context * fc)150732021982SDavid Howells hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc)
15081da177e4SLinus Torvalds {
150932021982SDavid Howells struct hugetlbfs_fs_context *ctx = fc->fs_private;
15101da177e4SLinus Torvalds struct hugetlbfs_sb_info *sbinfo;
15111da177e4SLinus Torvalds
15121da177e4SLinus Torvalds sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
15131da177e4SLinus Torvalds if (!sbinfo)
15141da177e4SLinus Torvalds return -ENOMEM;
15151da177e4SLinus Torvalds sb->s_fs_info = sbinfo;
15161da177e4SLinus Torvalds spin_lock_init(&sbinfo->stat_lock);
151732021982SDavid Howells sbinfo->hstate = ctx->hstate;
151832021982SDavid Howells sbinfo->max_inodes = ctx->nr_inodes;
151932021982SDavid Howells sbinfo->free_inodes = ctx->nr_inodes;
152090481622SDavid Gibson sbinfo->spool = NULL;
152132021982SDavid Howells sbinfo->uid = ctx->uid;
152232021982SDavid Howells sbinfo->gid = ctx->gid;
152332021982SDavid Howells sbinfo->mode = ctx->mode;
15244a25220dSDavid Howells
15257ca02d0aSMike Kravetz /*
15267ca02d0aSMike Kravetz * Allocate and initialize subpool if maximum or minimum size is
15271935ebd3SMiaohe Lin * specified. Any needed reservations (for minimum size) are taken
1528445c8098SMiaohe Lin * when the subpool is created.
15297ca02d0aSMike Kravetz */
153032021982SDavid Howells if (ctx->max_hpages != -1 || ctx->min_hpages != -1) {
153132021982SDavid Howells sbinfo->spool = hugepage_new_subpool(ctx->hstate,
153232021982SDavid Howells ctx->max_hpages,
153332021982SDavid Howells ctx->min_hpages);
153490481622SDavid Gibson if (!sbinfo->spool)
153590481622SDavid Gibson goto out_free;
153690481622SDavid Gibson }
15371da177e4SLinus Torvalds sb->s_maxbytes = MAX_LFS_FILESIZE;
153832021982SDavid Howells sb->s_blocksize = huge_page_size(ctx->hstate);
153932021982SDavid Howells sb->s_blocksize_bits = huge_page_shift(ctx->hstate);
15401da177e4SLinus Torvalds sb->s_magic = HUGETLBFS_MAGIC;
15411da177e4SLinus Torvalds sb->s_op = &hugetlbfs_ops;
15421da177e4SLinus Torvalds sb->s_time_gran = 1;
154315568299SMike Kravetz
154415568299SMike Kravetz /*
154515568299SMike Kravetz * Due to the special and limited functionality of hugetlbfs, it does
154615568299SMike Kravetz * not work well as a stacking filesystem.
154715568299SMike Kravetz */
154815568299SMike Kravetz sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH;
154932021982SDavid Howells sb->s_root = d_make_root(hugetlbfs_get_root(sb, ctx));
155048fde701SAl Viro if (!sb->s_root)
15511da177e4SLinus Torvalds goto out_free;
15521da177e4SLinus Torvalds return 0;
15531da177e4SLinus Torvalds out_free:
155490481622SDavid Gibson kfree(sbinfo->spool);
15551da177e4SLinus Torvalds kfree(sbinfo);
15561da177e4SLinus Torvalds return -ENOMEM;
15571da177e4SLinus Torvalds }
15581da177e4SLinus Torvalds
hugetlbfs_get_tree(struct fs_context * fc)155932021982SDavid Howells static int hugetlbfs_get_tree(struct fs_context *fc)
15601da177e4SLinus Torvalds {
156132021982SDavid Howells int err = hugetlbfs_validate(fc);
156232021982SDavid Howells if (err)
156332021982SDavid Howells return err;
15642ac295d4SAl Viro return get_tree_nodev(fc, hugetlbfs_fill_super);
156532021982SDavid Howells }
156632021982SDavid Howells
hugetlbfs_fs_context_free(struct fs_context * fc)156732021982SDavid Howells static void hugetlbfs_fs_context_free(struct fs_context *fc)
156832021982SDavid Howells {
156932021982SDavid Howells kfree(fc->fs_private);
157032021982SDavid Howells }
157132021982SDavid Howells
157232021982SDavid Howells static const struct fs_context_operations hugetlbfs_fs_context_ops = {
157332021982SDavid Howells .free = hugetlbfs_fs_context_free,
157432021982SDavid Howells .parse_param = hugetlbfs_parse_param,
157532021982SDavid Howells .get_tree = hugetlbfs_get_tree,
157632021982SDavid Howells };
157732021982SDavid Howells
hugetlbfs_init_fs_context(struct fs_context * fc)157832021982SDavid Howells static int hugetlbfs_init_fs_context(struct fs_context *fc)
157932021982SDavid Howells {
158032021982SDavid Howells struct hugetlbfs_fs_context *ctx;
158132021982SDavid Howells
158232021982SDavid Howells ctx = kzalloc(sizeof(struct hugetlbfs_fs_context), GFP_KERNEL);
158332021982SDavid Howells if (!ctx)
158432021982SDavid Howells return -ENOMEM;
158532021982SDavid Howells
158632021982SDavid Howells ctx->max_hpages = -1; /* No limit on size by default */
158732021982SDavid Howells ctx->nr_inodes = -1; /* No limit on number of inodes by default */
158832021982SDavid Howells ctx->uid = current_fsuid();
158932021982SDavid Howells ctx->gid = current_fsgid();
159032021982SDavid Howells ctx->mode = 0755;
159132021982SDavid Howells ctx->hstate = &default_hstate;
159232021982SDavid Howells ctx->min_hpages = -1; /* No default minimum size */
159332021982SDavid Howells ctx->max_val_type = NO_SIZE;
159432021982SDavid Howells ctx->min_val_type = NO_SIZE;
159532021982SDavid Howells fc->fs_private = ctx;
159632021982SDavid Howells fc->ops = &hugetlbfs_fs_context_ops;
159732021982SDavid Howells return 0;
15981da177e4SLinus Torvalds }
15991da177e4SLinus Torvalds
16001da177e4SLinus Torvalds static struct file_system_type hugetlbfs_fs_type = {
16011da177e4SLinus Torvalds .name = "hugetlbfs",
160232021982SDavid Howells .init_fs_context = hugetlbfs_init_fs_context,
1603d7167b14SAl Viro .parameters = hugetlb_fs_parameters,
16041da177e4SLinus Torvalds .kill_sb = kill_litter_super,
16051da177e4SLinus Torvalds };
16061da177e4SLinus Torvalds
160742d7395fSAndi Kleen static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
16081da177e4SLinus Torvalds
can_do_hugetlb_shm(void)1609ef1ff6b8SFrom: Mel Gorman static int can_do_hugetlb_shm(void)
16101da177e4SLinus Torvalds {
1611a0eb3a05SEric W. Biederman kgid_t shm_group;
1612a0eb3a05SEric W. Biederman shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
1613a0eb3a05SEric W. Biederman return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
16141da177e4SLinus Torvalds }
16151da177e4SLinus Torvalds
get_hstate_idx(int page_size_log)161642d7395fSAndi Kleen static int get_hstate_idx(int page_size_log)
161742d7395fSAndi Kleen {
1618af73e4d9SNaoya Horiguchi struct hstate *h = hstate_sizelog(page_size_log);
161942d7395fSAndi Kleen
162042d7395fSAndi Kleen if (!h)
162142d7395fSAndi Kleen return -1;
162204adbc3fSMiaohe Lin return hstate_index(h);
162342d7395fSAndi Kleen }
162442d7395fSAndi Kleen
1625af73e4d9SNaoya Horiguchi /*
1626af73e4d9SNaoya Horiguchi * Note that size should be aligned to proper hugepage size in caller side,
1627af73e4d9SNaoya Horiguchi * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
1628af73e4d9SNaoya Horiguchi */
hugetlb_file_setup(const char * name,size_t size,vm_flags_t acctflag,int creat_flags,int page_size_log)1629af73e4d9SNaoya Horiguchi struct file *hugetlb_file_setup(const char *name, size_t size,
163083c1fd76Szhangyiru vm_flags_t acctflag, int creat_flags,
163183c1fd76Szhangyiru int page_size_log)
16321da177e4SLinus Torvalds {
16331da177e4SLinus Torvalds struct inode *inode;
1634e68375c8SAl Viro struct vfsmount *mnt;
163542d7395fSAndi Kleen int hstate_idx;
1636e68375c8SAl Viro struct file *file;
163742d7395fSAndi Kleen
163842d7395fSAndi Kleen hstate_idx = get_hstate_idx(page_size_log);
163942d7395fSAndi Kleen if (hstate_idx < 0)
164042d7395fSAndi Kleen return ERR_PTR(-ENODEV);
16411da177e4SLinus Torvalds
1642e68375c8SAl Viro mnt = hugetlbfs_vfsmount[hstate_idx];
1643e68375c8SAl Viro if (!mnt)
16445bc98594SAkinobu Mita return ERR_PTR(-ENOENT);
16455bc98594SAkinobu Mita
1646ef1ff6b8SFrom: Mel Gorman if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
164783c1fd76Szhangyiru struct ucounts *ucounts = current_ucounts();
164883c1fd76Szhangyiru
164983c1fd76Szhangyiru if (user_shm_lock(size, ucounts)) {
165083c1fd76Szhangyiru pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is obsolete\n",
165121a3c273SDavid Rientjes current->comm, current->pid);
165283c1fd76Szhangyiru user_shm_unlock(size, ucounts);
16532584e517SRavikiran G Thirumalai }
165483c1fd76Szhangyiru return ERR_PTR(-EPERM);
1655353d5c30SHugh Dickins }
16561da177e4SLinus Torvalds
165739b65252SAnatol Pomozov file = ERR_PTR(-ENOSPC);
1658e68375c8SAl Viro inode = hugetlbfs_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0);
16591da177e4SLinus Torvalds if (!inode)
1660e68375c8SAl Viro goto out;
1661e1832f29SStephen Smalley if (creat_flags == HUGETLB_SHMFS_INODE)
1662e1832f29SStephen Smalley inode->i_flags |= S_PRIVATE;
16631da177e4SLinus Torvalds
16641da177e4SLinus Torvalds inode->i_size = size;
16656d6b77f1SMiklos Szeredi clear_nlink(inode);
1666ce8d2cdfSDave Hansen
166733b8f84aSMike Kravetz if (!hugetlb_reserve_pages(inode, 0,
1668e68375c8SAl Viro size >> huge_page_shift(hstate_inode(inode)), NULL,
1669e68375c8SAl Viro acctflag))
1670e68375c8SAl Viro file = ERR_PTR(-ENOMEM);
1671e68375c8SAl Viro else
1672e68375c8SAl Viro file = alloc_file_pseudo(inode, mnt, name, O_RDWR,
1673ce8d2cdfSDave Hansen &hugetlbfs_file_operations);
1674e68375c8SAl Viro if (!IS_ERR(file))
16751da177e4SLinus Torvalds return file;
16761da177e4SLinus Torvalds
1677b45b5bd6SDavid Gibson iput(inode);
1678e68375c8SAl Viro out:
167939b65252SAnatol Pomozov return file;
16801da177e4SLinus Torvalds }
16811da177e4SLinus Torvalds
mount_one_hugetlbfs(struct hstate * h)168232021982SDavid Howells static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h)
168332021982SDavid Howells {
168432021982SDavid Howells struct fs_context *fc;
168532021982SDavid Howells struct vfsmount *mnt;
168632021982SDavid Howells
168732021982SDavid Howells fc = fs_context_for_mount(&hugetlbfs_fs_type, SB_KERNMOUNT);
168832021982SDavid Howells if (IS_ERR(fc)) {
168932021982SDavid Howells mnt = ERR_CAST(fc);
169032021982SDavid Howells } else {
169132021982SDavid Howells struct hugetlbfs_fs_context *ctx = fc->fs_private;
169232021982SDavid Howells ctx->hstate = h;
169332021982SDavid Howells mnt = fc_mount(fc);
169432021982SDavid Howells put_fs_context(fc);
169532021982SDavid Howells }
169632021982SDavid Howells if (IS_ERR(mnt))
1697a25fddceSMiaohe Lin pr_err("Cannot mount internal hugetlbfs for page size %luK",
1698d0036517SMiaohe Lin huge_page_size(h) / SZ_1K);
169932021982SDavid Howells return mnt;
170032021982SDavid Howells }
170132021982SDavid Howells
init_hugetlbfs_fs(void)17021da177e4SLinus Torvalds static int __init init_hugetlbfs_fs(void)
17031da177e4SLinus Torvalds {
170432021982SDavid Howells struct vfsmount *mnt;
170542d7395fSAndi Kleen struct hstate *h;
17061da177e4SLinus Torvalds int error;
170742d7395fSAndi Kleen int i;
17081da177e4SLinus Torvalds
1709457c1b27SNishanth Aravamudan if (!hugepages_supported()) {
17109b857d26SAndrew Morton pr_info("disabling because there are no supported hugepage sizes\n");
1711457c1b27SNishanth Aravamudan return -ENOTSUPP;
1712457c1b27SNishanth Aravamudan }
1713457c1b27SNishanth Aravamudan
1714d1d5e05fSHillf Danton error = -ENOMEM;
17151da177e4SLinus Torvalds hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
17161da177e4SLinus Torvalds sizeof(struct hugetlbfs_inode_info),
17175d097056SVladimir Davydov 0, SLAB_ACCOUNT, init_once);
17181da177e4SLinus Torvalds if (hugetlbfs_inode_cachep == NULL)
17198fc312b3SMike Kravetz goto out;
17201da177e4SLinus Torvalds
17211da177e4SLinus Torvalds error = register_filesystem(&hugetlbfs_fs_type);
17221da177e4SLinus Torvalds if (error)
17238fc312b3SMike Kravetz goto out_free;
17241da177e4SLinus Torvalds
17258fc312b3SMike Kravetz /* default hstate mount is required */
17263b2275a8SMiaohe Lin mnt = mount_one_hugetlbfs(&default_hstate);
17278fc312b3SMike Kravetz if (IS_ERR(mnt)) {
17288fc312b3SMike Kravetz error = PTR_ERR(mnt);
17298fc312b3SMike Kravetz goto out_unreg;
17308fc312b3SMike Kravetz }
17318fc312b3SMike Kravetz hugetlbfs_vfsmount[default_hstate_idx] = mnt;
17328fc312b3SMike Kravetz
17338fc312b3SMike Kravetz /* other hstates are optional */
173442d7395fSAndi Kleen i = 0;
173542d7395fSAndi Kleen for_each_hstate(h) {
173615f0ec94SJan Stancek if (i == default_hstate_idx) {
173715f0ec94SJan Stancek i++;
17388fc312b3SMike Kravetz continue;
173915f0ec94SJan Stancek }
17408fc312b3SMike Kravetz
174132021982SDavid Howells mnt = mount_one_hugetlbfs(h);
17428fc312b3SMike Kravetz if (IS_ERR(mnt))
17438fc312b3SMike Kravetz hugetlbfs_vfsmount[i] = NULL;
17448fc312b3SMike Kravetz else
174532021982SDavid Howells hugetlbfs_vfsmount[i] = mnt;
174642d7395fSAndi Kleen i++;
174742d7395fSAndi Kleen }
174832021982SDavid Howells
174942d7395fSAndi Kleen return 0;
17501da177e4SLinus Torvalds
17518fc312b3SMike Kravetz out_unreg:
17528fc312b3SMike Kravetz (void)unregister_filesystem(&hugetlbfs_fs_type);
17538fc312b3SMike Kravetz out_free:
17541da177e4SLinus Torvalds kmem_cache_destroy(hugetlbfs_inode_cachep);
17558fc312b3SMike Kravetz out:
17561da177e4SLinus Torvalds return error;
17571da177e4SLinus Torvalds }
17583e89e1c5SPaul Gortmaker fs_initcall(init_hugetlbfs_fs)
1759