xref: /openbmc/linux/mm/shmem.c (revision f42b3800)
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *		 2000 Transmeta Corp.
6  *		 2000-2001 Christoph Rohland
7  *		 2000-2001 SAP AG
8  *		 2002 Red Hat Inc.
9  * Copyright (C) 2002-2005 Hugh Dickins.
10  * Copyright (C) 2002-2005 VERITAS Software Corporation.
11  * Copyright (C) 2004 Andi Kleen, SuSE Labs
12  *
13  * Extended attribute support for tmpfs:
14  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
16  *
17  * This file is released under the GPL.
18  */
19 
20 /*
21  * This virtual memory filesystem is heavily based on the ramfs. It
22  * extends ramfs by the ability to use swap and honor resource limits
23  * which makes it a completely usable filesystem.
24  */
25 
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/fs.h>
29 #include <linux/xattr.h>
30 #include <linux/exportfs.h>
31 #include <linux/generic_acl.h>
32 #include <linux/mm.h>
33 #include <linux/mman.h>
34 #include <linux/file.h>
35 #include <linux/swap.h>
36 #include <linux/pagemap.h>
37 #include <linux/string.h>
38 #include <linux/slab.h>
39 #include <linux/backing-dev.h>
40 #include <linux/shmem_fs.h>
41 #include <linux/mount.h>
42 #include <linux/writeback.h>
43 #include <linux/vfs.h>
44 #include <linux/blkdev.h>
45 #include <linux/security.h>
46 #include <linux/swapops.h>
47 #include <linux/mempolicy.h>
48 #include <linux/namei.h>
49 #include <linux/ctype.h>
50 #include <linux/migrate.h>
51 #include <linux/highmem.h>
52 #include <linux/seq_file.h>
53 
54 #include <asm/uaccess.h>
55 #include <asm/div64.h>
56 #include <asm/pgtable.h>
57 
58 /* This magic number is used in glibc for posix shared memory */
59 #define TMPFS_MAGIC	0x01021994
60 
61 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
62 #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
63 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
64 
65 #define SHMEM_MAX_INDEX  (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
66 #define SHMEM_MAX_BYTES  ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
67 
68 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
69 
70 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
71 #define SHMEM_PAGEIN	 VM_READ
72 #define SHMEM_TRUNCATE	 VM_WRITE
73 
74 /* Definition to limit shmem_truncate's steps between cond_rescheds */
75 #define LATENCY_LIMIT	 64
76 
77 /* Pretend that each entry is of this size in directory's i_size */
78 #define BOGO_DIRENT_SIZE 20
79 
80 /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
81 enum sgp_type {
82 	SGP_READ,	/* don't exceed i_size, don't allocate page */
83 	SGP_CACHE,	/* don't exceed i_size, may allocate page */
84 	SGP_DIRTY,	/* like SGP_CACHE, but set new page dirty */
85 	SGP_WRITE,	/* may exceed i_size, may allocate page */
86 };
87 
88 #ifdef CONFIG_TMPFS
89 static unsigned long shmem_default_max_blocks(void)
90 {
91 	return totalram_pages / 2;
92 }
93 
94 static unsigned long shmem_default_max_inodes(void)
95 {
96 	return min(totalram_pages - totalhigh_pages, totalram_pages / 2);
97 }
98 #endif
99 
100 static int shmem_getpage(struct inode *inode, unsigned long idx,
101 			 struct page **pagep, enum sgp_type sgp, int *type);
102 
103 static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
104 {
105 	/*
106 	 * The above definition of ENTRIES_PER_PAGE, and the use of
107 	 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
108 	 * might be reconsidered if it ever diverges from PAGE_SIZE.
109 	 *
110 	 * Mobility flags are masked out as swap vectors cannot move
111 	 */
112 	return alloc_pages((gfp_mask & ~GFP_MOVABLE_MASK) | __GFP_ZERO,
113 				PAGE_CACHE_SHIFT-PAGE_SHIFT);
114 }
115 
116 static inline void shmem_dir_free(struct page *page)
117 {
118 	__free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
119 }
120 
121 static struct page **shmem_dir_map(struct page *page)
122 {
123 	return (struct page **)kmap_atomic(page, KM_USER0);
124 }
125 
126 static inline void shmem_dir_unmap(struct page **dir)
127 {
128 	kunmap_atomic(dir, KM_USER0);
129 }
130 
131 static swp_entry_t *shmem_swp_map(struct page *page)
132 {
133 	return (swp_entry_t *)kmap_atomic(page, KM_USER1);
134 }
135 
136 static inline void shmem_swp_balance_unmap(void)
137 {
138 	/*
139 	 * When passing a pointer to an i_direct entry, to code which
140 	 * also handles indirect entries and so will shmem_swp_unmap,
141 	 * we must arrange for the preempt count to remain in balance.
142 	 * What kmap_atomic of a lowmem page does depends on config
143 	 * and architecture, so pretend to kmap_atomic some lowmem page.
144 	 */
145 	(void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
146 }
147 
148 static inline void shmem_swp_unmap(swp_entry_t *entry)
149 {
150 	kunmap_atomic(entry, KM_USER1);
151 }
152 
153 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
154 {
155 	return sb->s_fs_info;
156 }
157 
158 /*
159  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
160  * for shared memory and for shared anonymous (/dev/zero) mappings
161  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
162  * consistent with the pre-accounting of private mappings ...
163  */
164 static inline int shmem_acct_size(unsigned long flags, loff_t size)
165 {
166 	return (flags & VM_ACCOUNT)?
167 		security_vm_enough_memory(VM_ACCT(size)): 0;
168 }
169 
170 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
171 {
172 	if (flags & VM_ACCOUNT)
173 		vm_unacct_memory(VM_ACCT(size));
174 }
175 
176 /*
177  * ... whereas tmpfs objects are accounted incrementally as
178  * pages are allocated, in order to allow huge sparse files.
179  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
180  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
181  */
182 static inline int shmem_acct_block(unsigned long flags)
183 {
184 	return (flags & VM_ACCOUNT)?
185 		0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE));
186 }
187 
188 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
189 {
190 	if (!(flags & VM_ACCOUNT))
191 		vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
192 }
193 
194 static const struct super_operations shmem_ops;
195 static const struct address_space_operations shmem_aops;
196 static const struct file_operations shmem_file_operations;
197 static const struct inode_operations shmem_inode_operations;
198 static const struct inode_operations shmem_dir_inode_operations;
199 static const struct inode_operations shmem_special_inode_operations;
200 static struct vm_operations_struct shmem_vm_ops;
201 
202 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
203 	.ra_pages	= 0,	/* No readahead */
204 	.capabilities	= BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
205 	.unplug_io_fn	= default_unplug_io_fn,
206 };
207 
208 static LIST_HEAD(shmem_swaplist);
209 static DEFINE_MUTEX(shmem_swaplist_mutex);
210 
211 static void shmem_free_blocks(struct inode *inode, long pages)
212 {
213 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
214 	if (sbinfo->max_blocks) {
215 		spin_lock(&sbinfo->stat_lock);
216 		sbinfo->free_blocks += pages;
217 		inode->i_blocks -= pages*BLOCKS_PER_PAGE;
218 		spin_unlock(&sbinfo->stat_lock);
219 	}
220 }
221 
222 static int shmem_reserve_inode(struct super_block *sb)
223 {
224 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
225 	if (sbinfo->max_inodes) {
226 		spin_lock(&sbinfo->stat_lock);
227 		if (!sbinfo->free_inodes) {
228 			spin_unlock(&sbinfo->stat_lock);
229 			return -ENOSPC;
230 		}
231 		sbinfo->free_inodes--;
232 		spin_unlock(&sbinfo->stat_lock);
233 	}
234 	return 0;
235 }
236 
237 static void shmem_free_inode(struct super_block *sb)
238 {
239 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
240 	if (sbinfo->max_inodes) {
241 		spin_lock(&sbinfo->stat_lock);
242 		sbinfo->free_inodes++;
243 		spin_unlock(&sbinfo->stat_lock);
244 	}
245 }
246 
247 /**
248  * shmem_recalc_inode - recalculate the size of an inode
249  * @inode: inode to recalc
250  *
251  * We have to calculate the free blocks since the mm can drop
252  * undirtied hole pages behind our back.
253  *
254  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
255  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
256  *
257  * It has to be called with the spinlock held.
258  */
259 static void shmem_recalc_inode(struct inode *inode)
260 {
261 	struct shmem_inode_info *info = SHMEM_I(inode);
262 	long freed;
263 
264 	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
265 	if (freed > 0) {
266 		info->alloced -= freed;
267 		shmem_unacct_blocks(info->flags, freed);
268 		shmem_free_blocks(inode, freed);
269 	}
270 }
271 
272 /**
273  * shmem_swp_entry - find the swap vector position in the info structure
274  * @info:  info structure for the inode
275  * @index: index of the page to find
276  * @page:  optional page to add to the structure. Has to be preset to
277  *         all zeros
278  *
279  * If there is no space allocated yet it will return NULL when
280  * page is NULL, else it will use the page for the needed block,
281  * setting it to NULL on return to indicate that it has been used.
282  *
283  * The swap vector is organized the following way:
284  *
285  * There are SHMEM_NR_DIRECT entries directly stored in the
286  * shmem_inode_info structure. So small files do not need an addional
287  * allocation.
288  *
289  * For pages with index > SHMEM_NR_DIRECT there is the pointer
290  * i_indirect which points to a page which holds in the first half
291  * doubly indirect blocks, in the second half triple indirect blocks:
292  *
293  * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
294  * following layout (for SHMEM_NR_DIRECT == 16):
295  *
296  * i_indirect -> dir --> 16-19
297  * 	      |	     +-> 20-23
298  * 	      |
299  * 	      +-->dir2 --> 24-27
300  * 	      |	       +-> 28-31
301  * 	      |	       +-> 32-35
302  * 	      |	       +-> 36-39
303  * 	      |
304  * 	      +-->dir3 --> 40-43
305  * 	       	       +-> 44-47
306  * 	      	       +-> 48-51
307  * 	      	       +-> 52-55
308  */
309 static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
310 {
311 	unsigned long offset;
312 	struct page **dir;
313 	struct page *subdir;
314 
315 	if (index < SHMEM_NR_DIRECT) {
316 		shmem_swp_balance_unmap();
317 		return info->i_direct+index;
318 	}
319 	if (!info->i_indirect) {
320 		if (page) {
321 			info->i_indirect = *page;
322 			*page = NULL;
323 		}
324 		return NULL;			/* need another page */
325 	}
326 
327 	index -= SHMEM_NR_DIRECT;
328 	offset = index % ENTRIES_PER_PAGE;
329 	index /= ENTRIES_PER_PAGE;
330 	dir = shmem_dir_map(info->i_indirect);
331 
332 	if (index >= ENTRIES_PER_PAGE/2) {
333 		index -= ENTRIES_PER_PAGE/2;
334 		dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
335 		index %= ENTRIES_PER_PAGE;
336 		subdir = *dir;
337 		if (!subdir) {
338 			if (page) {
339 				*dir = *page;
340 				*page = NULL;
341 			}
342 			shmem_dir_unmap(dir);
343 			return NULL;		/* need another page */
344 		}
345 		shmem_dir_unmap(dir);
346 		dir = shmem_dir_map(subdir);
347 	}
348 
349 	dir += index;
350 	subdir = *dir;
351 	if (!subdir) {
352 		if (!page || !(subdir = *page)) {
353 			shmem_dir_unmap(dir);
354 			return NULL;		/* need a page */
355 		}
356 		*dir = subdir;
357 		*page = NULL;
358 	}
359 	shmem_dir_unmap(dir);
360 	return shmem_swp_map(subdir) + offset;
361 }
362 
363 static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
364 {
365 	long incdec = value? 1: -1;
366 
367 	entry->val = value;
368 	info->swapped += incdec;
369 	if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
370 		struct page *page = kmap_atomic_to_page(entry);
371 		set_page_private(page, page_private(page) + incdec);
372 	}
373 }
374 
375 /**
376  * shmem_swp_alloc - get the position of the swap entry for the page.
377  * @info:	info structure for the inode
378  * @index:	index of the page to find
379  * @sgp:	check and recheck i_size? skip allocation?
380  *
381  * If the entry does not exist, allocate it.
382  */
383 static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
384 {
385 	struct inode *inode = &info->vfs_inode;
386 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
387 	struct page *page = NULL;
388 	swp_entry_t *entry;
389 
390 	if (sgp != SGP_WRITE &&
391 	    ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
392 		return ERR_PTR(-EINVAL);
393 
394 	while (!(entry = shmem_swp_entry(info, index, &page))) {
395 		if (sgp == SGP_READ)
396 			return shmem_swp_map(ZERO_PAGE(0));
397 		/*
398 		 * Test free_blocks against 1 not 0, since we have 1 data
399 		 * page (and perhaps indirect index pages) yet to allocate:
400 		 * a waste to allocate index if we cannot allocate data.
401 		 */
402 		if (sbinfo->max_blocks) {
403 			spin_lock(&sbinfo->stat_lock);
404 			if (sbinfo->free_blocks <= 1) {
405 				spin_unlock(&sbinfo->stat_lock);
406 				return ERR_PTR(-ENOSPC);
407 			}
408 			sbinfo->free_blocks--;
409 			inode->i_blocks += BLOCKS_PER_PAGE;
410 			spin_unlock(&sbinfo->stat_lock);
411 		}
412 
413 		spin_unlock(&info->lock);
414 		page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping));
415 		if (page)
416 			set_page_private(page, 0);
417 		spin_lock(&info->lock);
418 
419 		if (!page) {
420 			shmem_free_blocks(inode, 1);
421 			return ERR_PTR(-ENOMEM);
422 		}
423 		if (sgp != SGP_WRITE &&
424 		    ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
425 			entry = ERR_PTR(-EINVAL);
426 			break;
427 		}
428 		if (info->next_index <= index)
429 			info->next_index = index + 1;
430 	}
431 	if (page) {
432 		/* another task gave its page, or truncated the file */
433 		shmem_free_blocks(inode, 1);
434 		shmem_dir_free(page);
435 	}
436 	if (info->next_index <= index && !IS_ERR(entry))
437 		info->next_index = index + 1;
438 	return entry;
439 }
440 
441 /**
442  * shmem_free_swp - free some swap entries in a directory
443  * @dir:        pointer to the directory
444  * @edir:       pointer after last entry of the directory
445  * @punch_lock: pointer to spinlock when needed for the holepunch case
446  */
447 static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
448 						spinlock_t *punch_lock)
449 {
450 	spinlock_t *punch_unlock = NULL;
451 	swp_entry_t *ptr;
452 	int freed = 0;
453 
454 	for (ptr = dir; ptr < edir; ptr++) {
455 		if (ptr->val) {
456 			if (unlikely(punch_lock)) {
457 				punch_unlock = punch_lock;
458 				punch_lock = NULL;
459 				spin_lock(punch_unlock);
460 				if (!ptr->val)
461 					continue;
462 			}
463 			free_swap_and_cache(*ptr);
464 			*ptr = (swp_entry_t){0};
465 			freed++;
466 		}
467 	}
468 	if (punch_unlock)
469 		spin_unlock(punch_unlock);
470 	return freed;
471 }
472 
473 static int shmem_map_and_free_swp(struct page *subdir, int offset,
474 		int limit, struct page ***dir, spinlock_t *punch_lock)
475 {
476 	swp_entry_t *ptr;
477 	int freed = 0;
478 
479 	ptr = shmem_swp_map(subdir);
480 	for (; offset < limit; offset += LATENCY_LIMIT) {
481 		int size = limit - offset;
482 		if (size > LATENCY_LIMIT)
483 			size = LATENCY_LIMIT;
484 		freed += shmem_free_swp(ptr+offset, ptr+offset+size,
485 							punch_lock);
486 		if (need_resched()) {
487 			shmem_swp_unmap(ptr);
488 			if (*dir) {
489 				shmem_dir_unmap(*dir);
490 				*dir = NULL;
491 			}
492 			cond_resched();
493 			ptr = shmem_swp_map(subdir);
494 		}
495 	}
496 	shmem_swp_unmap(ptr);
497 	return freed;
498 }
499 
500 static void shmem_free_pages(struct list_head *next)
501 {
502 	struct page *page;
503 	int freed = 0;
504 
505 	do {
506 		page = container_of(next, struct page, lru);
507 		next = next->next;
508 		shmem_dir_free(page);
509 		freed++;
510 		if (freed >= LATENCY_LIMIT) {
511 			cond_resched();
512 			freed = 0;
513 		}
514 	} while (next);
515 }
516 
517 static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
518 {
519 	struct shmem_inode_info *info = SHMEM_I(inode);
520 	unsigned long idx;
521 	unsigned long size;
522 	unsigned long limit;
523 	unsigned long stage;
524 	unsigned long diroff;
525 	struct page **dir;
526 	struct page *topdir;
527 	struct page *middir;
528 	struct page *subdir;
529 	swp_entry_t *ptr;
530 	LIST_HEAD(pages_to_free);
531 	long nr_pages_to_free = 0;
532 	long nr_swaps_freed = 0;
533 	int offset;
534 	int freed;
535 	int punch_hole;
536 	spinlock_t *needs_lock;
537 	spinlock_t *punch_lock;
538 	unsigned long upper_limit;
539 
540 	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
541 	idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
542 	if (idx >= info->next_index)
543 		return;
544 
545 	spin_lock(&info->lock);
546 	info->flags |= SHMEM_TRUNCATE;
547 	if (likely(end == (loff_t) -1)) {
548 		limit = info->next_index;
549 		upper_limit = SHMEM_MAX_INDEX;
550 		info->next_index = idx;
551 		needs_lock = NULL;
552 		punch_hole = 0;
553 	} else {
554 		if (end + 1 >= inode->i_size) {	/* we may free a little more */
555 			limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
556 							PAGE_CACHE_SHIFT;
557 			upper_limit = SHMEM_MAX_INDEX;
558 		} else {
559 			limit = (end + 1) >> PAGE_CACHE_SHIFT;
560 			upper_limit = limit;
561 		}
562 		needs_lock = &info->lock;
563 		punch_hole = 1;
564 	}
565 
566 	topdir = info->i_indirect;
567 	if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
568 		info->i_indirect = NULL;
569 		nr_pages_to_free++;
570 		list_add(&topdir->lru, &pages_to_free);
571 	}
572 	spin_unlock(&info->lock);
573 
574 	if (info->swapped && idx < SHMEM_NR_DIRECT) {
575 		ptr = info->i_direct;
576 		size = limit;
577 		if (size > SHMEM_NR_DIRECT)
578 			size = SHMEM_NR_DIRECT;
579 		nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
580 	}
581 
582 	/*
583 	 * If there are no indirect blocks or we are punching a hole
584 	 * below indirect blocks, nothing to be done.
585 	 */
586 	if (!topdir || limit <= SHMEM_NR_DIRECT)
587 		goto done2;
588 
589 	/*
590 	 * The truncation case has already dropped info->lock, and we're safe
591 	 * because i_size and next_index have already been lowered, preventing
592 	 * access beyond.  But in the punch_hole case, we still need to take
593 	 * the lock when updating the swap directory, because there might be
594 	 * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
595 	 * shmem_writepage.  However, whenever we find we can remove a whole
596 	 * directory page (not at the misaligned start or end of the range),
597 	 * we first NULLify its pointer in the level above, and then have no
598 	 * need to take the lock when updating its contents: needs_lock and
599 	 * punch_lock (either pointing to info->lock or NULL) manage this.
600 	 */
601 
602 	upper_limit -= SHMEM_NR_DIRECT;
603 	limit -= SHMEM_NR_DIRECT;
604 	idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
605 	offset = idx % ENTRIES_PER_PAGE;
606 	idx -= offset;
607 
608 	dir = shmem_dir_map(topdir);
609 	stage = ENTRIES_PER_PAGEPAGE/2;
610 	if (idx < ENTRIES_PER_PAGEPAGE/2) {
611 		middir = topdir;
612 		diroff = idx/ENTRIES_PER_PAGE;
613 	} else {
614 		dir += ENTRIES_PER_PAGE/2;
615 		dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
616 		while (stage <= idx)
617 			stage += ENTRIES_PER_PAGEPAGE;
618 		middir = *dir;
619 		if (*dir) {
620 			diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
621 				ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
622 			if (!diroff && !offset && upper_limit >= stage) {
623 				if (needs_lock) {
624 					spin_lock(needs_lock);
625 					*dir = NULL;
626 					spin_unlock(needs_lock);
627 					needs_lock = NULL;
628 				} else
629 					*dir = NULL;
630 				nr_pages_to_free++;
631 				list_add(&middir->lru, &pages_to_free);
632 			}
633 			shmem_dir_unmap(dir);
634 			dir = shmem_dir_map(middir);
635 		} else {
636 			diroff = 0;
637 			offset = 0;
638 			idx = stage;
639 		}
640 	}
641 
642 	for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
643 		if (unlikely(idx == stage)) {
644 			shmem_dir_unmap(dir);
645 			dir = shmem_dir_map(topdir) +
646 			    ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
647 			while (!*dir) {
648 				dir++;
649 				idx += ENTRIES_PER_PAGEPAGE;
650 				if (idx >= limit)
651 					goto done1;
652 			}
653 			stage = idx + ENTRIES_PER_PAGEPAGE;
654 			middir = *dir;
655 			if (punch_hole)
656 				needs_lock = &info->lock;
657 			if (upper_limit >= stage) {
658 				if (needs_lock) {
659 					spin_lock(needs_lock);
660 					*dir = NULL;
661 					spin_unlock(needs_lock);
662 					needs_lock = NULL;
663 				} else
664 					*dir = NULL;
665 				nr_pages_to_free++;
666 				list_add(&middir->lru, &pages_to_free);
667 			}
668 			shmem_dir_unmap(dir);
669 			cond_resched();
670 			dir = shmem_dir_map(middir);
671 			diroff = 0;
672 		}
673 		punch_lock = needs_lock;
674 		subdir = dir[diroff];
675 		if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
676 			if (needs_lock) {
677 				spin_lock(needs_lock);
678 				dir[diroff] = NULL;
679 				spin_unlock(needs_lock);
680 				punch_lock = NULL;
681 			} else
682 				dir[diroff] = NULL;
683 			nr_pages_to_free++;
684 			list_add(&subdir->lru, &pages_to_free);
685 		}
686 		if (subdir && page_private(subdir) /* has swap entries */) {
687 			size = limit - idx;
688 			if (size > ENTRIES_PER_PAGE)
689 				size = ENTRIES_PER_PAGE;
690 			freed = shmem_map_and_free_swp(subdir,
691 					offset, size, &dir, punch_lock);
692 			if (!dir)
693 				dir = shmem_dir_map(middir);
694 			nr_swaps_freed += freed;
695 			if (offset || punch_lock) {
696 				spin_lock(&info->lock);
697 				set_page_private(subdir,
698 					page_private(subdir) - freed);
699 				spin_unlock(&info->lock);
700 			} else
701 				BUG_ON(page_private(subdir) != freed);
702 		}
703 		offset = 0;
704 	}
705 done1:
706 	shmem_dir_unmap(dir);
707 done2:
708 	if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
709 		/*
710 		 * Call truncate_inode_pages again: racing shmem_unuse_inode
711 		 * may have swizzled a page in from swap since vmtruncate or
712 		 * generic_delete_inode did it, before we lowered next_index.
713 		 * Also, though shmem_getpage checks i_size before adding to
714 		 * cache, no recheck after: so fix the narrow window there too.
715 		 *
716 		 * Recalling truncate_inode_pages_range and unmap_mapping_range
717 		 * every time for punch_hole (which never got a chance to clear
718 		 * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
719 		 * yet hardly ever necessary: try to optimize them out later.
720 		 */
721 		truncate_inode_pages_range(inode->i_mapping, start, end);
722 		if (punch_hole)
723 			unmap_mapping_range(inode->i_mapping, start,
724 							end - start, 1);
725 	}
726 
727 	spin_lock(&info->lock);
728 	info->flags &= ~SHMEM_TRUNCATE;
729 	info->swapped -= nr_swaps_freed;
730 	if (nr_pages_to_free)
731 		shmem_free_blocks(inode, nr_pages_to_free);
732 	shmem_recalc_inode(inode);
733 	spin_unlock(&info->lock);
734 
735 	/*
736 	 * Empty swap vector directory pages to be freed?
737 	 */
738 	if (!list_empty(&pages_to_free)) {
739 		pages_to_free.prev->next = NULL;
740 		shmem_free_pages(pages_to_free.next);
741 	}
742 }
743 
744 static void shmem_truncate(struct inode *inode)
745 {
746 	shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
747 }
748 
749 static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
750 {
751 	struct inode *inode = dentry->d_inode;
752 	struct page *page = NULL;
753 	int error;
754 
755 	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
756 		if (attr->ia_size < inode->i_size) {
757 			/*
758 			 * If truncating down to a partial page, then
759 			 * if that page is already allocated, hold it
760 			 * in memory until the truncation is over, so
761 			 * truncate_partial_page cannnot miss it were
762 			 * it assigned to swap.
763 			 */
764 			if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
765 				(void) shmem_getpage(inode,
766 					attr->ia_size>>PAGE_CACHE_SHIFT,
767 						&page, SGP_READ, NULL);
768 				if (page)
769 					unlock_page(page);
770 			}
771 			/*
772 			 * Reset SHMEM_PAGEIN flag so that shmem_truncate can
773 			 * detect if any pages might have been added to cache
774 			 * after truncate_inode_pages.  But we needn't bother
775 			 * if it's being fully truncated to zero-length: the
776 			 * nrpages check is efficient enough in that case.
777 			 */
778 			if (attr->ia_size) {
779 				struct shmem_inode_info *info = SHMEM_I(inode);
780 				spin_lock(&info->lock);
781 				info->flags &= ~SHMEM_PAGEIN;
782 				spin_unlock(&info->lock);
783 			}
784 		}
785 	}
786 
787 	error = inode_change_ok(inode, attr);
788 	if (!error)
789 		error = inode_setattr(inode, attr);
790 #ifdef CONFIG_TMPFS_POSIX_ACL
791 	if (!error && (attr->ia_valid & ATTR_MODE))
792 		error = generic_acl_chmod(inode, &shmem_acl_ops);
793 #endif
794 	if (page)
795 		page_cache_release(page);
796 	return error;
797 }
798 
799 static void shmem_delete_inode(struct inode *inode)
800 {
801 	struct shmem_inode_info *info = SHMEM_I(inode);
802 
803 	if (inode->i_op->truncate == shmem_truncate) {
804 		truncate_inode_pages(inode->i_mapping, 0);
805 		shmem_unacct_size(info->flags, inode->i_size);
806 		inode->i_size = 0;
807 		shmem_truncate(inode);
808 		if (!list_empty(&info->swaplist)) {
809 			mutex_lock(&shmem_swaplist_mutex);
810 			list_del_init(&info->swaplist);
811 			mutex_unlock(&shmem_swaplist_mutex);
812 		}
813 	}
814 	BUG_ON(inode->i_blocks);
815 	shmem_free_inode(inode->i_sb);
816 	clear_inode(inode);
817 }
818 
819 static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
820 {
821 	swp_entry_t *ptr;
822 
823 	for (ptr = dir; ptr < edir; ptr++) {
824 		if (ptr->val == entry.val)
825 			return ptr - dir;
826 	}
827 	return -1;
828 }
829 
830 static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
831 {
832 	struct inode *inode;
833 	unsigned long idx;
834 	unsigned long size;
835 	unsigned long limit;
836 	unsigned long stage;
837 	struct page **dir;
838 	struct page *subdir;
839 	swp_entry_t *ptr;
840 	int offset;
841 	int error;
842 
843 	idx = 0;
844 	ptr = info->i_direct;
845 	spin_lock(&info->lock);
846 	if (!info->swapped) {
847 		list_del_init(&info->swaplist);
848 		goto lost2;
849 	}
850 	limit = info->next_index;
851 	size = limit;
852 	if (size > SHMEM_NR_DIRECT)
853 		size = SHMEM_NR_DIRECT;
854 	offset = shmem_find_swp(entry, ptr, ptr+size);
855 	if (offset >= 0)
856 		goto found;
857 	if (!info->i_indirect)
858 		goto lost2;
859 
860 	dir = shmem_dir_map(info->i_indirect);
861 	stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
862 
863 	for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
864 		if (unlikely(idx == stage)) {
865 			shmem_dir_unmap(dir-1);
866 			if (cond_resched_lock(&info->lock)) {
867 				/* check it has not been truncated */
868 				if (limit > info->next_index) {
869 					limit = info->next_index;
870 					if (idx >= limit)
871 						goto lost2;
872 				}
873 			}
874 			dir = shmem_dir_map(info->i_indirect) +
875 			    ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
876 			while (!*dir) {
877 				dir++;
878 				idx += ENTRIES_PER_PAGEPAGE;
879 				if (idx >= limit)
880 					goto lost1;
881 			}
882 			stage = idx + ENTRIES_PER_PAGEPAGE;
883 			subdir = *dir;
884 			shmem_dir_unmap(dir);
885 			dir = shmem_dir_map(subdir);
886 		}
887 		subdir = *dir;
888 		if (subdir && page_private(subdir)) {
889 			ptr = shmem_swp_map(subdir);
890 			size = limit - idx;
891 			if (size > ENTRIES_PER_PAGE)
892 				size = ENTRIES_PER_PAGE;
893 			offset = shmem_find_swp(entry, ptr, ptr+size);
894 			shmem_swp_unmap(ptr);
895 			if (offset >= 0) {
896 				shmem_dir_unmap(dir);
897 				goto found;
898 			}
899 		}
900 	}
901 lost1:
902 	shmem_dir_unmap(dir-1);
903 lost2:
904 	spin_unlock(&info->lock);
905 	return 0;
906 found:
907 	idx += offset;
908 	inode = igrab(&info->vfs_inode);
909 	spin_unlock(&info->lock);
910 
911 	/*
912 	 * Move _head_ to start search for next from here.
913 	 * But be careful: shmem_delete_inode checks list_empty without taking
914 	 * mutex, and there's an instant in list_move_tail when info->swaplist
915 	 * would appear empty, if it were the only one on shmem_swaplist.  We
916 	 * could avoid doing it if inode NULL; or use this minor optimization.
917 	 */
918 	if (shmem_swaplist.next != &info->swaplist)
919 		list_move_tail(&shmem_swaplist, &info->swaplist);
920 	mutex_unlock(&shmem_swaplist_mutex);
921 
922 	error = 1;
923 	if (!inode)
924 		goto out;
925 	/* Precharge page while we can wait, compensate afterwards */
926 	error = mem_cgroup_cache_charge(page, current->mm, GFP_KERNEL);
927 	if (error)
928 		goto out;
929 	error = radix_tree_preload(GFP_KERNEL);
930 	if (error)
931 		goto uncharge;
932 	error = 1;
933 
934 	spin_lock(&info->lock);
935 	ptr = shmem_swp_entry(info, idx, NULL);
936 	if (ptr && ptr->val == entry.val)
937 		error = add_to_page_cache(page, inode->i_mapping,
938 						idx, GFP_NOWAIT);
939 	if (error == -EEXIST) {
940 		struct page *filepage = find_get_page(inode->i_mapping, idx);
941 		error = 1;
942 		if (filepage) {
943 			/*
944 			 * There might be a more uptodate page coming down
945 			 * from a stacked writepage: forget our swappage if so.
946 			 */
947 			if (PageUptodate(filepage))
948 				error = 0;
949 			page_cache_release(filepage);
950 		}
951 	}
952 	if (!error) {
953 		delete_from_swap_cache(page);
954 		set_page_dirty(page);
955 		info->flags |= SHMEM_PAGEIN;
956 		shmem_swp_set(info, ptr, 0);
957 		swap_free(entry);
958 		error = 1;	/* not an error, but entry was found */
959 	}
960 	if (ptr)
961 		shmem_swp_unmap(ptr);
962 	spin_unlock(&info->lock);
963 	radix_tree_preload_end();
964 uncharge:
965 	mem_cgroup_uncharge_page(page);
966 out:
967 	unlock_page(page);
968 	page_cache_release(page);
969 	iput(inode);		/* allows for NULL */
970 	return error;
971 }
972 
973 /*
974  * shmem_unuse() search for an eventually swapped out shmem page.
975  */
976 int shmem_unuse(swp_entry_t entry, struct page *page)
977 {
978 	struct list_head *p, *next;
979 	struct shmem_inode_info *info;
980 	int found = 0;
981 
982 	mutex_lock(&shmem_swaplist_mutex);
983 	list_for_each_safe(p, next, &shmem_swaplist) {
984 		info = list_entry(p, struct shmem_inode_info, swaplist);
985 		found = shmem_unuse_inode(info, entry, page);
986 		cond_resched();
987 		if (found)
988 			goto out;
989 	}
990 	mutex_unlock(&shmem_swaplist_mutex);
991 out:	return found;	/* 0 or 1 or -ENOMEM */
992 }
993 
994 /*
995  * Move the page from the page cache to the swap cache.
996  */
997 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
998 {
999 	struct shmem_inode_info *info;
1000 	swp_entry_t *entry, swap;
1001 	struct address_space *mapping;
1002 	unsigned long index;
1003 	struct inode *inode;
1004 
1005 	BUG_ON(!PageLocked(page));
1006 	mapping = page->mapping;
1007 	index = page->index;
1008 	inode = mapping->host;
1009 	info = SHMEM_I(inode);
1010 	if (info->flags & VM_LOCKED)
1011 		goto redirty;
1012 	if (!total_swap_pages)
1013 		goto redirty;
1014 
1015 	/*
1016 	 * shmem_backing_dev_info's capabilities prevent regular writeback or
1017 	 * sync from ever calling shmem_writepage; but a stacking filesystem
1018 	 * may use the ->writepage of its underlying filesystem, in which case
1019 	 * tmpfs should write out to swap only in response to memory pressure,
1020 	 * and not for pdflush or sync.  However, in those cases, we do still
1021 	 * want to check if there's a redundant swappage to be discarded.
1022 	 */
1023 	if (wbc->for_reclaim)
1024 		swap = get_swap_page();
1025 	else
1026 		swap.val = 0;
1027 
1028 	spin_lock(&info->lock);
1029 	if (index >= info->next_index) {
1030 		BUG_ON(!(info->flags & SHMEM_TRUNCATE));
1031 		goto unlock;
1032 	}
1033 	entry = shmem_swp_entry(info, index, NULL);
1034 	if (entry->val) {
1035 		/*
1036 		 * The more uptodate page coming down from a stacked
1037 		 * writepage should replace our old swappage.
1038 		 */
1039 		free_swap_and_cache(*entry);
1040 		shmem_swp_set(info, entry, 0);
1041 	}
1042 	shmem_recalc_inode(inode);
1043 
1044 	if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1045 		remove_from_page_cache(page);
1046 		shmem_swp_set(info, entry, swap.val);
1047 		shmem_swp_unmap(entry);
1048 		if (list_empty(&info->swaplist))
1049 			inode = igrab(inode);
1050 		else
1051 			inode = NULL;
1052 		spin_unlock(&info->lock);
1053 		swap_duplicate(swap);
1054 		BUG_ON(page_mapped(page));
1055 		page_cache_release(page);	/* pagecache ref */
1056 		set_page_dirty(page);
1057 		unlock_page(page);
1058 		if (inode) {
1059 			mutex_lock(&shmem_swaplist_mutex);
1060 			/* move instead of add in case we're racing */
1061 			list_move_tail(&info->swaplist, &shmem_swaplist);
1062 			mutex_unlock(&shmem_swaplist_mutex);
1063 			iput(inode);
1064 		}
1065 		return 0;
1066 	}
1067 
1068 	shmem_swp_unmap(entry);
1069 unlock:
1070 	spin_unlock(&info->lock);
1071 	swap_free(swap);
1072 redirty:
1073 	set_page_dirty(page);
1074 	if (wbc->for_reclaim)
1075 		return AOP_WRITEPAGE_ACTIVATE;	/* Return with page locked */
1076 	unlock_page(page);
1077 	return 0;
1078 }
1079 
1080 #ifdef CONFIG_NUMA
1081 #ifdef CONFIG_TMPFS
1082 static int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
1083 {
1084 	char *nodelist = strchr(value, ':');
1085 	int err = 1;
1086 
1087 	if (nodelist) {
1088 		/* NUL-terminate policy string */
1089 		*nodelist++ = '\0';
1090 		if (nodelist_parse(nodelist, *policy_nodes))
1091 			goto out;
1092 		if (!nodes_subset(*policy_nodes, node_states[N_HIGH_MEMORY]))
1093 			goto out;
1094 	}
1095 	if (!strcmp(value, "default")) {
1096 		*policy = MPOL_DEFAULT;
1097 		/* Don't allow a nodelist */
1098 		if (!nodelist)
1099 			err = 0;
1100 	} else if (!strcmp(value, "prefer")) {
1101 		*policy = MPOL_PREFERRED;
1102 		/* Insist on a nodelist of one node only */
1103 		if (nodelist) {
1104 			char *rest = nodelist;
1105 			while (isdigit(*rest))
1106 				rest++;
1107 			if (!*rest)
1108 				err = 0;
1109 		}
1110 	} else if (!strcmp(value, "bind")) {
1111 		*policy = MPOL_BIND;
1112 		/* Insist on a nodelist */
1113 		if (nodelist)
1114 			err = 0;
1115 	} else if (!strcmp(value, "interleave")) {
1116 		*policy = MPOL_INTERLEAVE;
1117 		/*
1118 		 * Default to online nodes with memory if no nodelist
1119 		 */
1120 		if (!nodelist)
1121 			*policy_nodes = node_states[N_HIGH_MEMORY];
1122 		err = 0;
1123 	}
1124 out:
1125 	/* Restore string for error message */
1126 	if (nodelist)
1127 		*--nodelist = ':';
1128 	return err;
1129 }
1130 
1131 static void shmem_show_mpol(struct seq_file *seq, int policy,
1132 			    const nodemask_t policy_nodes)
1133 {
1134 	char *policy_string;
1135 
1136 	switch (policy) {
1137 	case MPOL_PREFERRED:
1138 		policy_string = "prefer";
1139 		break;
1140 	case MPOL_BIND:
1141 		policy_string = "bind";
1142 		break;
1143 	case MPOL_INTERLEAVE:
1144 		policy_string = "interleave";
1145 		break;
1146 	default:
1147 		/* MPOL_DEFAULT */
1148 		return;
1149 	}
1150 
1151 	seq_printf(seq, ",mpol=%s", policy_string);
1152 
1153 	if (policy != MPOL_INTERLEAVE ||
1154 	    !nodes_equal(policy_nodes, node_states[N_HIGH_MEMORY])) {
1155 		char buffer[64];
1156 		int len;
1157 
1158 		len = nodelist_scnprintf(buffer, sizeof(buffer), policy_nodes);
1159 		if (len < sizeof(buffer))
1160 			seq_printf(seq, ":%s", buffer);
1161 		else
1162 			seq_printf(seq, ":?");
1163 	}
1164 }
1165 #endif /* CONFIG_TMPFS */
1166 
1167 static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1168 			struct shmem_inode_info *info, unsigned long idx)
1169 {
1170 	struct vm_area_struct pvma;
1171 	struct page *page;
1172 
1173 	/* Create a pseudo vma that just contains the policy */
1174 	pvma.vm_start = 0;
1175 	pvma.vm_pgoff = idx;
1176 	pvma.vm_ops = NULL;
1177 	pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1178 	page = swapin_readahead(entry, gfp, &pvma, 0);
1179 	mpol_free(pvma.vm_policy);
1180 	return page;
1181 }
1182 
1183 static struct page *shmem_alloc_page(gfp_t gfp,
1184 			struct shmem_inode_info *info, unsigned long idx)
1185 {
1186 	struct vm_area_struct pvma;
1187 	struct page *page;
1188 
1189 	/* Create a pseudo vma that just contains the policy */
1190 	pvma.vm_start = 0;
1191 	pvma.vm_pgoff = idx;
1192 	pvma.vm_ops = NULL;
1193 	pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1194 	page = alloc_page_vma(gfp, &pvma, 0);
1195 	mpol_free(pvma.vm_policy);
1196 	return page;
1197 }
1198 #else /* !CONFIG_NUMA */
1199 #ifdef CONFIG_TMPFS
1200 static inline int shmem_parse_mpol(char *value, int *policy,
1201 						nodemask_t *policy_nodes)
1202 {
1203 	return 1;
1204 }
1205 
1206 static inline void shmem_show_mpol(struct seq_file *seq, int policy,
1207 			    const nodemask_t policy_nodes)
1208 {
1209 }
1210 #endif /* CONFIG_TMPFS */
1211 
1212 static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
1213 			struct shmem_inode_info *info, unsigned long idx)
1214 {
1215 	return swapin_readahead(entry, gfp, NULL, 0);
1216 }
1217 
1218 static inline struct page *shmem_alloc_page(gfp_t gfp,
1219 			struct shmem_inode_info *info, unsigned long idx)
1220 {
1221 	return alloc_page(gfp);
1222 }
1223 #endif /* CONFIG_NUMA */
1224 
1225 /*
1226  * shmem_getpage - either get the page from swap or allocate a new one
1227  *
1228  * If we allocate a new one we do not mark it dirty. That's up to the
1229  * vm. If we swap it in we mark it dirty since we also free the swap
1230  * entry since a page cannot live in both the swap and page cache
1231  */
1232 static int shmem_getpage(struct inode *inode, unsigned long idx,
1233 			struct page **pagep, enum sgp_type sgp, int *type)
1234 {
1235 	struct address_space *mapping = inode->i_mapping;
1236 	struct shmem_inode_info *info = SHMEM_I(inode);
1237 	struct shmem_sb_info *sbinfo;
1238 	struct page *filepage = *pagep;
1239 	struct page *swappage;
1240 	swp_entry_t *entry;
1241 	swp_entry_t swap;
1242 	gfp_t gfp;
1243 	int error;
1244 
1245 	if (idx >= SHMEM_MAX_INDEX)
1246 		return -EFBIG;
1247 
1248 	if (type)
1249 		*type = 0;
1250 
1251 	/*
1252 	 * Normally, filepage is NULL on entry, and either found
1253 	 * uptodate immediately, or allocated and zeroed, or read
1254 	 * in under swappage, which is then assigned to filepage.
1255 	 * But shmem_readpage (required for splice) passes in a locked
1256 	 * filepage, which may be found not uptodate by other callers
1257 	 * too, and may need to be copied from the swappage read in.
1258 	 */
1259 repeat:
1260 	if (!filepage)
1261 		filepage = find_lock_page(mapping, idx);
1262 	if (filepage && PageUptodate(filepage))
1263 		goto done;
1264 	error = 0;
1265 	gfp = mapping_gfp_mask(mapping);
1266 	if (!filepage) {
1267 		/*
1268 		 * Try to preload while we can wait, to not make a habit of
1269 		 * draining atomic reserves; but don't latch on to this cpu.
1270 		 */
1271 		error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
1272 		if (error)
1273 			goto failed;
1274 		radix_tree_preload_end();
1275 	}
1276 
1277 	spin_lock(&info->lock);
1278 	shmem_recalc_inode(inode);
1279 	entry = shmem_swp_alloc(info, idx, sgp);
1280 	if (IS_ERR(entry)) {
1281 		spin_unlock(&info->lock);
1282 		error = PTR_ERR(entry);
1283 		goto failed;
1284 	}
1285 	swap = *entry;
1286 
1287 	if (swap.val) {
1288 		/* Look it up and read it in.. */
1289 		swappage = lookup_swap_cache(swap);
1290 		if (!swappage) {
1291 			shmem_swp_unmap(entry);
1292 			/* here we actually do the io */
1293 			if (type && !(*type & VM_FAULT_MAJOR)) {
1294 				__count_vm_event(PGMAJFAULT);
1295 				*type |= VM_FAULT_MAJOR;
1296 			}
1297 			spin_unlock(&info->lock);
1298 			swappage = shmem_swapin(swap, gfp, info, idx);
1299 			if (!swappage) {
1300 				spin_lock(&info->lock);
1301 				entry = shmem_swp_alloc(info, idx, sgp);
1302 				if (IS_ERR(entry))
1303 					error = PTR_ERR(entry);
1304 				else {
1305 					if (entry->val == swap.val)
1306 						error = -ENOMEM;
1307 					shmem_swp_unmap(entry);
1308 				}
1309 				spin_unlock(&info->lock);
1310 				if (error)
1311 					goto failed;
1312 				goto repeat;
1313 			}
1314 			wait_on_page_locked(swappage);
1315 			page_cache_release(swappage);
1316 			goto repeat;
1317 		}
1318 
1319 		/* We have to do this with page locked to prevent races */
1320 		if (TestSetPageLocked(swappage)) {
1321 			shmem_swp_unmap(entry);
1322 			spin_unlock(&info->lock);
1323 			wait_on_page_locked(swappage);
1324 			page_cache_release(swappage);
1325 			goto repeat;
1326 		}
1327 		if (PageWriteback(swappage)) {
1328 			shmem_swp_unmap(entry);
1329 			spin_unlock(&info->lock);
1330 			wait_on_page_writeback(swappage);
1331 			unlock_page(swappage);
1332 			page_cache_release(swappage);
1333 			goto repeat;
1334 		}
1335 		if (!PageUptodate(swappage)) {
1336 			shmem_swp_unmap(entry);
1337 			spin_unlock(&info->lock);
1338 			unlock_page(swappage);
1339 			page_cache_release(swappage);
1340 			error = -EIO;
1341 			goto failed;
1342 		}
1343 
1344 		if (filepage) {
1345 			shmem_swp_set(info, entry, 0);
1346 			shmem_swp_unmap(entry);
1347 			delete_from_swap_cache(swappage);
1348 			spin_unlock(&info->lock);
1349 			copy_highpage(filepage, swappage);
1350 			unlock_page(swappage);
1351 			page_cache_release(swappage);
1352 			flush_dcache_page(filepage);
1353 			SetPageUptodate(filepage);
1354 			set_page_dirty(filepage);
1355 			swap_free(swap);
1356 		} else if (!(error = add_to_page_cache(
1357 				swappage, mapping, idx, GFP_NOWAIT))) {
1358 			info->flags |= SHMEM_PAGEIN;
1359 			shmem_swp_set(info, entry, 0);
1360 			shmem_swp_unmap(entry);
1361 			delete_from_swap_cache(swappage);
1362 			spin_unlock(&info->lock);
1363 			filepage = swappage;
1364 			set_page_dirty(filepage);
1365 			swap_free(swap);
1366 		} else {
1367 			shmem_swp_unmap(entry);
1368 			spin_unlock(&info->lock);
1369 			unlock_page(swappage);
1370 			if (error == -ENOMEM) {
1371 				/* allow reclaim from this memory cgroup */
1372 				error = mem_cgroup_cache_charge(swappage,
1373 					current->mm, gfp & ~__GFP_HIGHMEM);
1374 				if (error) {
1375 					page_cache_release(swappage);
1376 					goto failed;
1377 				}
1378 				mem_cgroup_uncharge_page(swappage);
1379 			}
1380 			page_cache_release(swappage);
1381 			goto repeat;
1382 		}
1383 	} else if (sgp == SGP_READ && !filepage) {
1384 		shmem_swp_unmap(entry);
1385 		filepage = find_get_page(mapping, idx);
1386 		if (filepage &&
1387 		    (!PageUptodate(filepage) || TestSetPageLocked(filepage))) {
1388 			spin_unlock(&info->lock);
1389 			wait_on_page_locked(filepage);
1390 			page_cache_release(filepage);
1391 			filepage = NULL;
1392 			goto repeat;
1393 		}
1394 		spin_unlock(&info->lock);
1395 	} else {
1396 		shmem_swp_unmap(entry);
1397 		sbinfo = SHMEM_SB(inode->i_sb);
1398 		if (sbinfo->max_blocks) {
1399 			spin_lock(&sbinfo->stat_lock);
1400 			if (sbinfo->free_blocks == 0 ||
1401 			    shmem_acct_block(info->flags)) {
1402 				spin_unlock(&sbinfo->stat_lock);
1403 				spin_unlock(&info->lock);
1404 				error = -ENOSPC;
1405 				goto failed;
1406 			}
1407 			sbinfo->free_blocks--;
1408 			inode->i_blocks += BLOCKS_PER_PAGE;
1409 			spin_unlock(&sbinfo->stat_lock);
1410 		} else if (shmem_acct_block(info->flags)) {
1411 			spin_unlock(&info->lock);
1412 			error = -ENOSPC;
1413 			goto failed;
1414 		}
1415 
1416 		if (!filepage) {
1417 			spin_unlock(&info->lock);
1418 			filepage = shmem_alloc_page(gfp, info, idx);
1419 			if (!filepage) {
1420 				shmem_unacct_blocks(info->flags, 1);
1421 				shmem_free_blocks(inode, 1);
1422 				error = -ENOMEM;
1423 				goto failed;
1424 			}
1425 
1426 			/* Precharge page while we can wait, compensate after */
1427 			error = mem_cgroup_cache_charge(filepage, current->mm,
1428 							gfp & ~__GFP_HIGHMEM);
1429 			if (error) {
1430 				page_cache_release(filepage);
1431 				shmem_unacct_blocks(info->flags, 1);
1432 				shmem_free_blocks(inode, 1);
1433 				filepage = NULL;
1434 				goto failed;
1435 			}
1436 
1437 			spin_lock(&info->lock);
1438 			entry = shmem_swp_alloc(info, idx, sgp);
1439 			if (IS_ERR(entry))
1440 				error = PTR_ERR(entry);
1441 			else {
1442 				swap = *entry;
1443 				shmem_swp_unmap(entry);
1444 			}
1445 			if (error || swap.val || 0 != add_to_page_cache_lru(
1446 					filepage, mapping, idx, GFP_NOWAIT)) {
1447 				spin_unlock(&info->lock);
1448 				mem_cgroup_uncharge_page(filepage);
1449 				page_cache_release(filepage);
1450 				shmem_unacct_blocks(info->flags, 1);
1451 				shmem_free_blocks(inode, 1);
1452 				filepage = NULL;
1453 				if (error)
1454 					goto failed;
1455 				goto repeat;
1456 			}
1457 			mem_cgroup_uncharge_page(filepage);
1458 			info->flags |= SHMEM_PAGEIN;
1459 		}
1460 
1461 		info->alloced++;
1462 		spin_unlock(&info->lock);
1463 		clear_highpage(filepage);
1464 		flush_dcache_page(filepage);
1465 		SetPageUptodate(filepage);
1466 		if (sgp == SGP_DIRTY)
1467 			set_page_dirty(filepage);
1468 	}
1469 done:
1470 	*pagep = filepage;
1471 	return 0;
1472 
1473 failed:
1474 	if (*pagep != filepage) {
1475 		unlock_page(filepage);
1476 		page_cache_release(filepage);
1477 	}
1478 	return error;
1479 }
1480 
1481 static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1482 {
1483 	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1484 	int error;
1485 	int ret;
1486 
1487 	if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1488 		return VM_FAULT_SIGBUS;
1489 
1490 	error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
1491 	if (error)
1492 		return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
1493 
1494 	mark_page_accessed(vmf->page);
1495 	return ret | VM_FAULT_LOCKED;
1496 }
1497 
1498 #ifdef CONFIG_NUMA
1499 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1500 {
1501 	struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1502 	return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1503 }
1504 
1505 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
1506 					  unsigned long addr)
1507 {
1508 	struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1509 	unsigned long idx;
1510 
1511 	idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1512 	return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1513 }
1514 #endif
1515 
1516 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1517 {
1518 	struct inode *inode = file->f_path.dentry->d_inode;
1519 	struct shmem_inode_info *info = SHMEM_I(inode);
1520 	int retval = -ENOMEM;
1521 
1522 	spin_lock(&info->lock);
1523 	if (lock && !(info->flags & VM_LOCKED)) {
1524 		if (!user_shm_lock(inode->i_size, user))
1525 			goto out_nomem;
1526 		info->flags |= VM_LOCKED;
1527 	}
1528 	if (!lock && (info->flags & VM_LOCKED) && user) {
1529 		user_shm_unlock(inode->i_size, user);
1530 		info->flags &= ~VM_LOCKED;
1531 	}
1532 	retval = 0;
1533 out_nomem:
1534 	spin_unlock(&info->lock);
1535 	return retval;
1536 }
1537 
1538 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1539 {
1540 	file_accessed(file);
1541 	vma->vm_ops = &shmem_vm_ops;
1542 	vma->vm_flags |= VM_CAN_NONLINEAR;
1543 	return 0;
1544 }
1545 
1546 static struct inode *
1547 shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
1548 {
1549 	struct inode *inode;
1550 	struct shmem_inode_info *info;
1551 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1552 
1553 	if (shmem_reserve_inode(sb))
1554 		return NULL;
1555 
1556 	inode = new_inode(sb);
1557 	if (inode) {
1558 		inode->i_mode = mode;
1559 		inode->i_uid = current->fsuid;
1560 		inode->i_gid = current->fsgid;
1561 		inode->i_blocks = 0;
1562 		inode->i_mapping->a_ops = &shmem_aops;
1563 		inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1564 		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1565 		inode->i_generation = get_seconds();
1566 		info = SHMEM_I(inode);
1567 		memset(info, 0, (char *)inode - (char *)info);
1568 		spin_lock_init(&info->lock);
1569 		INIT_LIST_HEAD(&info->swaplist);
1570 
1571 		switch (mode & S_IFMT) {
1572 		default:
1573 			inode->i_op = &shmem_special_inode_operations;
1574 			init_special_inode(inode, mode, dev);
1575 			break;
1576 		case S_IFREG:
1577 			inode->i_op = &shmem_inode_operations;
1578 			inode->i_fop = &shmem_file_operations;
1579 			mpol_shared_policy_init(&info->policy, sbinfo->policy,
1580 							&sbinfo->policy_nodes);
1581 			break;
1582 		case S_IFDIR:
1583 			inc_nlink(inode);
1584 			/* Some things misbehave if size == 0 on a directory */
1585 			inode->i_size = 2 * BOGO_DIRENT_SIZE;
1586 			inode->i_op = &shmem_dir_inode_operations;
1587 			inode->i_fop = &simple_dir_operations;
1588 			break;
1589 		case S_IFLNK:
1590 			/*
1591 			 * Must not load anything in the rbtree,
1592 			 * mpol_free_shared_policy will not be called.
1593 			 */
1594 			mpol_shared_policy_init(&info->policy, MPOL_DEFAULT,
1595 						NULL);
1596 			break;
1597 		}
1598 	} else
1599 		shmem_free_inode(sb);
1600 	return inode;
1601 }
1602 
1603 #ifdef CONFIG_TMPFS
1604 static const struct inode_operations shmem_symlink_inode_operations;
1605 static const struct inode_operations shmem_symlink_inline_operations;
1606 
1607 /*
1608  * Normally tmpfs avoids the use of shmem_readpage and shmem_write_begin;
1609  * but providing them allows a tmpfs file to be used for splice, sendfile, and
1610  * below the loop driver, in the generic fashion that many filesystems support.
1611  */
1612 static int shmem_readpage(struct file *file, struct page *page)
1613 {
1614 	struct inode *inode = page->mapping->host;
1615 	int error = shmem_getpage(inode, page->index, &page, SGP_CACHE, NULL);
1616 	unlock_page(page);
1617 	return error;
1618 }
1619 
1620 static int
1621 shmem_write_begin(struct file *file, struct address_space *mapping,
1622 			loff_t pos, unsigned len, unsigned flags,
1623 			struct page **pagep, void **fsdata)
1624 {
1625 	struct inode *inode = mapping->host;
1626 	pgoff_t index = pos >> PAGE_CACHE_SHIFT;
1627 	*pagep = NULL;
1628 	return shmem_getpage(inode, index, pagep, SGP_WRITE, NULL);
1629 }
1630 
1631 static int
1632 shmem_write_end(struct file *file, struct address_space *mapping,
1633 			loff_t pos, unsigned len, unsigned copied,
1634 			struct page *page, void *fsdata)
1635 {
1636 	struct inode *inode = mapping->host;
1637 
1638 	if (pos + copied > inode->i_size)
1639 		i_size_write(inode, pos + copied);
1640 
1641 	unlock_page(page);
1642 	set_page_dirty(page);
1643 	page_cache_release(page);
1644 
1645 	return copied;
1646 }
1647 
1648 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1649 {
1650 	struct inode *inode = filp->f_path.dentry->d_inode;
1651 	struct address_space *mapping = inode->i_mapping;
1652 	unsigned long index, offset;
1653 	enum sgp_type sgp = SGP_READ;
1654 
1655 	/*
1656 	 * Might this read be for a stacking filesystem?  Then when reading
1657 	 * holes of a sparse file, we actually need to allocate those pages,
1658 	 * and even mark them dirty, so it cannot exceed the max_blocks limit.
1659 	 */
1660 	if (segment_eq(get_fs(), KERNEL_DS))
1661 		sgp = SGP_DIRTY;
1662 
1663 	index = *ppos >> PAGE_CACHE_SHIFT;
1664 	offset = *ppos & ~PAGE_CACHE_MASK;
1665 
1666 	for (;;) {
1667 		struct page *page = NULL;
1668 		unsigned long end_index, nr, ret;
1669 		loff_t i_size = i_size_read(inode);
1670 
1671 		end_index = i_size >> PAGE_CACHE_SHIFT;
1672 		if (index > end_index)
1673 			break;
1674 		if (index == end_index) {
1675 			nr = i_size & ~PAGE_CACHE_MASK;
1676 			if (nr <= offset)
1677 				break;
1678 		}
1679 
1680 		desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
1681 		if (desc->error) {
1682 			if (desc->error == -EINVAL)
1683 				desc->error = 0;
1684 			break;
1685 		}
1686 		if (page)
1687 			unlock_page(page);
1688 
1689 		/*
1690 		 * We must evaluate after, since reads (unlike writes)
1691 		 * are called without i_mutex protection against truncate
1692 		 */
1693 		nr = PAGE_CACHE_SIZE;
1694 		i_size = i_size_read(inode);
1695 		end_index = i_size >> PAGE_CACHE_SHIFT;
1696 		if (index == end_index) {
1697 			nr = i_size & ~PAGE_CACHE_MASK;
1698 			if (nr <= offset) {
1699 				if (page)
1700 					page_cache_release(page);
1701 				break;
1702 			}
1703 		}
1704 		nr -= offset;
1705 
1706 		if (page) {
1707 			/*
1708 			 * If users can be writing to this page using arbitrary
1709 			 * virtual addresses, take care about potential aliasing
1710 			 * before reading the page on the kernel side.
1711 			 */
1712 			if (mapping_writably_mapped(mapping))
1713 				flush_dcache_page(page);
1714 			/*
1715 			 * Mark the page accessed if we read the beginning.
1716 			 */
1717 			if (!offset)
1718 				mark_page_accessed(page);
1719 		} else {
1720 			page = ZERO_PAGE(0);
1721 			page_cache_get(page);
1722 		}
1723 
1724 		/*
1725 		 * Ok, we have the page, and it's up-to-date, so
1726 		 * now we can copy it to user space...
1727 		 *
1728 		 * The actor routine returns how many bytes were actually used..
1729 		 * NOTE! This may not be the same as how much of a user buffer
1730 		 * we filled up (we may be padding etc), so we can only update
1731 		 * "pos" here (the actor routine has to update the user buffer
1732 		 * pointers and the remaining count).
1733 		 */
1734 		ret = actor(desc, page, offset, nr);
1735 		offset += ret;
1736 		index += offset >> PAGE_CACHE_SHIFT;
1737 		offset &= ~PAGE_CACHE_MASK;
1738 
1739 		page_cache_release(page);
1740 		if (ret != nr || !desc->count)
1741 			break;
1742 
1743 		cond_resched();
1744 	}
1745 
1746 	*ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1747 	file_accessed(filp);
1748 }
1749 
1750 static ssize_t shmem_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
1751 {
1752 	read_descriptor_t desc;
1753 
1754 	if ((ssize_t) count < 0)
1755 		return -EINVAL;
1756 	if (!access_ok(VERIFY_WRITE, buf, count))
1757 		return -EFAULT;
1758 	if (!count)
1759 		return 0;
1760 
1761 	desc.written = 0;
1762 	desc.count = count;
1763 	desc.arg.buf = buf;
1764 	desc.error = 0;
1765 
1766 	do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1767 	if (desc.written)
1768 		return desc.written;
1769 	return desc.error;
1770 }
1771 
1772 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1773 {
1774 	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1775 
1776 	buf->f_type = TMPFS_MAGIC;
1777 	buf->f_bsize = PAGE_CACHE_SIZE;
1778 	buf->f_namelen = NAME_MAX;
1779 	spin_lock(&sbinfo->stat_lock);
1780 	if (sbinfo->max_blocks) {
1781 		buf->f_blocks = sbinfo->max_blocks;
1782 		buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
1783 	}
1784 	if (sbinfo->max_inodes) {
1785 		buf->f_files = sbinfo->max_inodes;
1786 		buf->f_ffree = sbinfo->free_inodes;
1787 	}
1788 	/* else leave those fields 0 like simple_statfs */
1789 	spin_unlock(&sbinfo->stat_lock);
1790 	return 0;
1791 }
1792 
1793 /*
1794  * File creation. Allocate an inode, and we're done..
1795  */
1796 static int
1797 shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1798 {
1799 	struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
1800 	int error = -ENOSPC;
1801 
1802 	if (inode) {
1803 		error = security_inode_init_security(inode, dir, NULL, NULL,
1804 						     NULL);
1805 		if (error) {
1806 			if (error != -EOPNOTSUPP) {
1807 				iput(inode);
1808 				return error;
1809 			}
1810 		}
1811 		error = shmem_acl_init(inode, dir);
1812 		if (error) {
1813 			iput(inode);
1814 			return error;
1815 		}
1816 		if (dir->i_mode & S_ISGID) {
1817 			inode->i_gid = dir->i_gid;
1818 			if (S_ISDIR(mode))
1819 				inode->i_mode |= S_ISGID;
1820 		}
1821 		dir->i_size += BOGO_DIRENT_SIZE;
1822 		dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1823 		d_instantiate(dentry, inode);
1824 		dget(dentry); /* Extra count - pin the dentry in core */
1825 	}
1826 	return error;
1827 }
1828 
1829 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1830 {
1831 	int error;
1832 
1833 	if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1834 		return error;
1835 	inc_nlink(dir);
1836 	return 0;
1837 }
1838 
1839 static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1840 		struct nameidata *nd)
1841 {
1842 	return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1843 }
1844 
1845 /*
1846  * Link a file..
1847  */
1848 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1849 {
1850 	struct inode *inode = old_dentry->d_inode;
1851 	int ret;
1852 
1853 	/*
1854 	 * No ordinary (disk based) filesystem counts links as inodes;
1855 	 * but each new link needs a new dentry, pinning lowmem, and
1856 	 * tmpfs dentries cannot be pruned until they are unlinked.
1857 	 */
1858 	ret = shmem_reserve_inode(inode->i_sb);
1859 	if (ret)
1860 		goto out;
1861 
1862 	dir->i_size += BOGO_DIRENT_SIZE;
1863 	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1864 	inc_nlink(inode);
1865 	atomic_inc(&inode->i_count);	/* New dentry reference */
1866 	dget(dentry);		/* Extra pinning count for the created dentry */
1867 	d_instantiate(dentry, inode);
1868 out:
1869 	return ret;
1870 }
1871 
1872 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1873 {
1874 	struct inode *inode = dentry->d_inode;
1875 
1876 	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
1877 		shmem_free_inode(inode->i_sb);
1878 
1879 	dir->i_size -= BOGO_DIRENT_SIZE;
1880 	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1881 	drop_nlink(inode);
1882 	dput(dentry);	/* Undo the count from "create" - this does all the work */
1883 	return 0;
1884 }
1885 
1886 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1887 {
1888 	if (!simple_empty(dentry))
1889 		return -ENOTEMPTY;
1890 
1891 	drop_nlink(dentry->d_inode);
1892 	drop_nlink(dir);
1893 	return shmem_unlink(dir, dentry);
1894 }
1895 
1896 /*
1897  * The VFS layer already does all the dentry stuff for rename,
1898  * we just have to decrement the usage count for the target if
1899  * it exists so that the VFS layer correctly free's it when it
1900  * gets overwritten.
1901  */
1902 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1903 {
1904 	struct inode *inode = old_dentry->d_inode;
1905 	int they_are_dirs = S_ISDIR(inode->i_mode);
1906 
1907 	if (!simple_empty(new_dentry))
1908 		return -ENOTEMPTY;
1909 
1910 	if (new_dentry->d_inode) {
1911 		(void) shmem_unlink(new_dir, new_dentry);
1912 		if (they_are_dirs)
1913 			drop_nlink(old_dir);
1914 	} else if (they_are_dirs) {
1915 		drop_nlink(old_dir);
1916 		inc_nlink(new_dir);
1917 	}
1918 
1919 	old_dir->i_size -= BOGO_DIRENT_SIZE;
1920 	new_dir->i_size += BOGO_DIRENT_SIZE;
1921 	old_dir->i_ctime = old_dir->i_mtime =
1922 	new_dir->i_ctime = new_dir->i_mtime =
1923 	inode->i_ctime = CURRENT_TIME;
1924 	return 0;
1925 }
1926 
1927 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1928 {
1929 	int error;
1930 	int len;
1931 	struct inode *inode;
1932 	struct page *page = NULL;
1933 	char *kaddr;
1934 	struct shmem_inode_info *info;
1935 
1936 	len = strlen(symname) + 1;
1937 	if (len > PAGE_CACHE_SIZE)
1938 		return -ENAMETOOLONG;
1939 
1940 	inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
1941 	if (!inode)
1942 		return -ENOSPC;
1943 
1944 	error = security_inode_init_security(inode, dir, NULL, NULL,
1945 					     NULL);
1946 	if (error) {
1947 		if (error != -EOPNOTSUPP) {
1948 			iput(inode);
1949 			return error;
1950 		}
1951 		error = 0;
1952 	}
1953 
1954 	info = SHMEM_I(inode);
1955 	inode->i_size = len-1;
1956 	if (len <= (char *)inode - (char *)info) {
1957 		/* do it inline */
1958 		memcpy(info, symname, len);
1959 		inode->i_op = &shmem_symlink_inline_operations;
1960 	} else {
1961 		error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1962 		if (error) {
1963 			iput(inode);
1964 			return error;
1965 		}
1966 		unlock_page(page);
1967 		inode->i_op = &shmem_symlink_inode_operations;
1968 		kaddr = kmap_atomic(page, KM_USER0);
1969 		memcpy(kaddr, symname, len);
1970 		kunmap_atomic(kaddr, KM_USER0);
1971 		set_page_dirty(page);
1972 		page_cache_release(page);
1973 	}
1974 	if (dir->i_mode & S_ISGID)
1975 		inode->i_gid = dir->i_gid;
1976 	dir->i_size += BOGO_DIRENT_SIZE;
1977 	dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1978 	d_instantiate(dentry, inode);
1979 	dget(dentry);
1980 	return 0;
1981 }
1982 
1983 static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1984 {
1985 	nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
1986 	return NULL;
1987 }
1988 
1989 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1990 {
1991 	struct page *page = NULL;
1992 	int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1993 	nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
1994 	if (page)
1995 		unlock_page(page);
1996 	return page;
1997 }
1998 
1999 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
2000 {
2001 	if (!IS_ERR(nd_get_link(nd))) {
2002 		struct page *page = cookie;
2003 		kunmap(page);
2004 		mark_page_accessed(page);
2005 		page_cache_release(page);
2006 	}
2007 }
2008 
2009 static const struct inode_operations shmem_symlink_inline_operations = {
2010 	.readlink	= generic_readlink,
2011 	.follow_link	= shmem_follow_link_inline,
2012 };
2013 
2014 static const struct inode_operations shmem_symlink_inode_operations = {
2015 	.truncate	= shmem_truncate,
2016 	.readlink	= generic_readlink,
2017 	.follow_link	= shmem_follow_link,
2018 	.put_link	= shmem_put_link,
2019 };
2020 
2021 #ifdef CONFIG_TMPFS_POSIX_ACL
2022 /*
2023  * Superblocks without xattr inode operations will get security.* xattr
2024  * support from the VFS "for free". As soon as we have any other xattrs
2025  * like ACLs, we also need to implement the security.* handlers at
2026  * filesystem level, though.
2027  */
2028 
2029 static size_t shmem_xattr_security_list(struct inode *inode, char *list,
2030 					size_t list_len, const char *name,
2031 					size_t name_len)
2032 {
2033 	return security_inode_listsecurity(inode, list, list_len);
2034 }
2035 
2036 static int shmem_xattr_security_get(struct inode *inode, const char *name,
2037 				    void *buffer, size_t size)
2038 {
2039 	if (strcmp(name, "") == 0)
2040 		return -EINVAL;
2041 	return xattr_getsecurity(inode, name, buffer, size);
2042 }
2043 
2044 static int shmem_xattr_security_set(struct inode *inode, const char *name,
2045 				    const void *value, size_t size, int flags)
2046 {
2047 	if (strcmp(name, "") == 0)
2048 		return -EINVAL;
2049 	return security_inode_setsecurity(inode, name, value, size, flags);
2050 }
2051 
2052 static struct xattr_handler shmem_xattr_security_handler = {
2053 	.prefix = XATTR_SECURITY_PREFIX,
2054 	.list   = shmem_xattr_security_list,
2055 	.get    = shmem_xattr_security_get,
2056 	.set    = shmem_xattr_security_set,
2057 };
2058 
2059 static struct xattr_handler *shmem_xattr_handlers[] = {
2060 	&shmem_xattr_acl_access_handler,
2061 	&shmem_xattr_acl_default_handler,
2062 	&shmem_xattr_security_handler,
2063 	NULL
2064 };
2065 #endif
2066 
2067 static struct dentry *shmem_get_parent(struct dentry *child)
2068 {
2069 	return ERR_PTR(-ESTALE);
2070 }
2071 
2072 static int shmem_match(struct inode *ino, void *vfh)
2073 {
2074 	__u32 *fh = vfh;
2075 	__u64 inum = fh[2];
2076 	inum = (inum << 32) | fh[1];
2077 	return ino->i_ino == inum && fh[0] == ino->i_generation;
2078 }
2079 
2080 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
2081 		struct fid *fid, int fh_len, int fh_type)
2082 {
2083 	struct inode *inode;
2084 	struct dentry *dentry = NULL;
2085 	u64 inum = fid->raw[2];
2086 	inum = (inum << 32) | fid->raw[1];
2087 
2088 	if (fh_len < 3)
2089 		return NULL;
2090 
2091 	inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
2092 			shmem_match, fid->raw);
2093 	if (inode) {
2094 		dentry = d_find_alias(inode);
2095 		iput(inode);
2096 	}
2097 
2098 	return dentry;
2099 }
2100 
2101 static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2102 				int connectable)
2103 {
2104 	struct inode *inode = dentry->d_inode;
2105 
2106 	if (*len < 3)
2107 		return 255;
2108 
2109 	if (hlist_unhashed(&inode->i_hash)) {
2110 		/* Unfortunately insert_inode_hash is not idempotent,
2111 		 * so as we hash inodes here rather than at creation
2112 		 * time, we need a lock to ensure we only try
2113 		 * to do it once
2114 		 */
2115 		static DEFINE_SPINLOCK(lock);
2116 		spin_lock(&lock);
2117 		if (hlist_unhashed(&inode->i_hash))
2118 			__insert_inode_hash(inode,
2119 					    inode->i_ino + inode->i_generation);
2120 		spin_unlock(&lock);
2121 	}
2122 
2123 	fh[0] = inode->i_generation;
2124 	fh[1] = inode->i_ino;
2125 	fh[2] = ((__u64)inode->i_ino) >> 32;
2126 
2127 	*len = 3;
2128 	return 1;
2129 }
2130 
2131 static const struct export_operations shmem_export_ops = {
2132 	.get_parent     = shmem_get_parent,
2133 	.encode_fh      = shmem_encode_fh,
2134 	.fh_to_dentry	= shmem_fh_to_dentry,
2135 };
2136 
2137 static int shmem_parse_options(char *options, struct shmem_sb_info *sbinfo,
2138 			       bool remount)
2139 {
2140 	char *this_char, *value, *rest;
2141 
2142 	while (options != NULL) {
2143 		this_char = options;
2144 		for (;;) {
2145 			/*
2146 			 * NUL-terminate this option: unfortunately,
2147 			 * mount options form a comma-separated list,
2148 			 * but mpol's nodelist may also contain commas.
2149 			 */
2150 			options = strchr(options, ',');
2151 			if (options == NULL)
2152 				break;
2153 			options++;
2154 			if (!isdigit(*options)) {
2155 				options[-1] = '\0';
2156 				break;
2157 			}
2158 		}
2159 		if (!*this_char)
2160 			continue;
2161 		if ((value = strchr(this_char,'=')) != NULL) {
2162 			*value++ = 0;
2163 		} else {
2164 			printk(KERN_ERR
2165 			    "tmpfs: No value for mount option '%s'\n",
2166 			    this_char);
2167 			return 1;
2168 		}
2169 
2170 		if (!strcmp(this_char,"size")) {
2171 			unsigned long long size;
2172 			size = memparse(value,&rest);
2173 			if (*rest == '%') {
2174 				size <<= PAGE_SHIFT;
2175 				size *= totalram_pages;
2176 				do_div(size, 100);
2177 				rest++;
2178 			}
2179 			if (*rest)
2180 				goto bad_val;
2181 			sbinfo->max_blocks =
2182 				DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
2183 		} else if (!strcmp(this_char,"nr_blocks")) {
2184 			sbinfo->max_blocks = memparse(value, &rest);
2185 			if (*rest)
2186 				goto bad_val;
2187 		} else if (!strcmp(this_char,"nr_inodes")) {
2188 			sbinfo->max_inodes = memparse(value, &rest);
2189 			if (*rest)
2190 				goto bad_val;
2191 		} else if (!strcmp(this_char,"mode")) {
2192 			if (remount)
2193 				continue;
2194 			sbinfo->mode = simple_strtoul(value, &rest, 8) & 07777;
2195 			if (*rest)
2196 				goto bad_val;
2197 		} else if (!strcmp(this_char,"uid")) {
2198 			if (remount)
2199 				continue;
2200 			sbinfo->uid = simple_strtoul(value, &rest, 0);
2201 			if (*rest)
2202 				goto bad_val;
2203 		} else if (!strcmp(this_char,"gid")) {
2204 			if (remount)
2205 				continue;
2206 			sbinfo->gid = simple_strtoul(value, &rest, 0);
2207 			if (*rest)
2208 				goto bad_val;
2209 		} else if (!strcmp(this_char,"mpol")) {
2210 			if (shmem_parse_mpol(value, &sbinfo->policy,
2211 					     &sbinfo->policy_nodes))
2212 				goto bad_val;
2213 		} else {
2214 			printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2215 			       this_char);
2216 			return 1;
2217 		}
2218 	}
2219 	return 0;
2220 
2221 bad_val:
2222 	printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2223 	       value, this_char);
2224 	return 1;
2225 
2226 }
2227 
2228 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2229 {
2230 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2231 	struct shmem_sb_info config = *sbinfo;
2232 	unsigned long blocks;
2233 	unsigned long inodes;
2234 	int error = -EINVAL;
2235 
2236 	if (shmem_parse_options(data, &config, true))
2237 		return error;
2238 
2239 	spin_lock(&sbinfo->stat_lock);
2240 	blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2241 	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2242 	if (config.max_blocks < blocks)
2243 		goto out;
2244 	if (config.max_inodes < inodes)
2245 		goto out;
2246 	/*
2247 	 * Those tests also disallow limited->unlimited while any are in
2248 	 * use, so i_blocks will always be zero when max_blocks is zero;
2249 	 * but we must separately disallow unlimited->limited, because
2250 	 * in that case we have no record of how much is already in use.
2251 	 */
2252 	if (config.max_blocks && !sbinfo->max_blocks)
2253 		goto out;
2254 	if (config.max_inodes && !sbinfo->max_inodes)
2255 		goto out;
2256 
2257 	error = 0;
2258 	sbinfo->max_blocks  = config.max_blocks;
2259 	sbinfo->free_blocks = config.max_blocks - blocks;
2260 	sbinfo->max_inodes  = config.max_inodes;
2261 	sbinfo->free_inodes = config.max_inodes - inodes;
2262 	sbinfo->policy      = config.policy;
2263 	sbinfo->policy_nodes = config.policy_nodes;
2264 out:
2265 	spin_unlock(&sbinfo->stat_lock);
2266 	return error;
2267 }
2268 
2269 static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
2270 {
2271 	struct shmem_sb_info *sbinfo = SHMEM_SB(vfs->mnt_sb);
2272 
2273 	if (sbinfo->max_blocks != shmem_default_max_blocks())
2274 		seq_printf(seq, ",size=%luk",
2275 			sbinfo->max_blocks << (PAGE_CACHE_SHIFT - 10));
2276 	if (sbinfo->max_inodes != shmem_default_max_inodes())
2277 		seq_printf(seq, ",nr_inodes=%lu", sbinfo->max_inodes);
2278 	if (sbinfo->mode != (S_IRWXUGO | S_ISVTX))
2279 		seq_printf(seq, ",mode=%03o", sbinfo->mode);
2280 	if (sbinfo->uid != 0)
2281 		seq_printf(seq, ",uid=%u", sbinfo->uid);
2282 	if (sbinfo->gid != 0)
2283 		seq_printf(seq, ",gid=%u", sbinfo->gid);
2284 	shmem_show_mpol(seq, sbinfo->policy, sbinfo->policy_nodes);
2285 	return 0;
2286 }
2287 #endif /* CONFIG_TMPFS */
2288 
2289 static void shmem_put_super(struct super_block *sb)
2290 {
2291 	kfree(sb->s_fs_info);
2292 	sb->s_fs_info = NULL;
2293 }
2294 
2295 static int shmem_fill_super(struct super_block *sb,
2296 			    void *data, int silent)
2297 {
2298 	struct inode *inode;
2299 	struct dentry *root;
2300 	struct shmem_sb_info *sbinfo;
2301 	int err = -ENOMEM;
2302 
2303 	/* Round up to L1_CACHE_BYTES to resist false sharing */
2304 	sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
2305 				L1_CACHE_BYTES), GFP_KERNEL);
2306 	if (!sbinfo)
2307 		return -ENOMEM;
2308 
2309 	sbinfo->max_blocks = 0;
2310 	sbinfo->max_inodes = 0;
2311 	sbinfo->mode = S_IRWXUGO | S_ISVTX;
2312 	sbinfo->uid = current->fsuid;
2313 	sbinfo->gid = current->fsgid;
2314 	sbinfo->policy = MPOL_DEFAULT;
2315 	sbinfo->policy_nodes = node_states[N_HIGH_MEMORY];
2316 	sb->s_fs_info = sbinfo;
2317 
2318 #ifdef CONFIG_TMPFS
2319 	/*
2320 	 * Per default we only allow half of the physical ram per
2321 	 * tmpfs instance, limiting inodes to one per page of lowmem;
2322 	 * but the internal instance is left unlimited.
2323 	 */
2324 	if (!(sb->s_flags & MS_NOUSER)) {
2325 		sbinfo->max_blocks = shmem_default_max_blocks();
2326 		sbinfo->max_inodes = shmem_default_max_inodes();
2327 		if (shmem_parse_options(data, sbinfo, false)) {
2328 			err = -EINVAL;
2329 			goto failed;
2330 		}
2331 	}
2332 	sb->s_export_op = &shmem_export_ops;
2333 #else
2334 	sb->s_flags |= MS_NOUSER;
2335 #endif
2336 
2337 	spin_lock_init(&sbinfo->stat_lock);
2338 	sbinfo->free_blocks = sbinfo->max_blocks;
2339 	sbinfo->free_inodes = sbinfo->max_inodes;
2340 
2341 	sb->s_maxbytes = SHMEM_MAX_BYTES;
2342 	sb->s_blocksize = PAGE_CACHE_SIZE;
2343 	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2344 	sb->s_magic = TMPFS_MAGIC;
2345 	sb->s_op = &shmem_ops;
2346 	sb->s_time_gran = 1;
2347 #ifdef CONFIG_TMPFS_POSIX_ACL
2348 	sb->s_xattr = shmem_xattr_handlers;
2349 	sb->s_flags |= MS_POSIXACL;
2350 #endif
2351 
2352 	inode = shmem_get_inode(sb, S_IFDIR | sbinfo->mode, 0);
2353 	if (!inode)
2354 		goto failed;
2355 	inode->i_uid = sbinfo->uid;
2356 	inode->i_gid = sbinfo->gid;
2357 	root = d_alloc_root(inode);
2358 	if (!root)
2359 		goto failed_iput;
2360 	sb->s_root = root;
2361 	return 0;
2362 
2363 failed_iput:
2364 	iput(inode);
2365 failed:
2366 	shmem_put_super(sb);
2367 	return err;
2368 }
2369 
2370 static struct kmem_cache *shmem_inode_cachep;
2371 
2372 static struct inode *shmem_alloc_inode(struct super_block *sb)
2373 {
2374 	struct shmem_inode_info *p;
2375 	p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2376 	if (!p)
2377 		return NULL;
2378 	return &p->vfs_inode;
2379 }
2380 
2381 static void shmem_destroy_inode(struct inode *inode)
2382 {
2383 	if ((inode->i_mode & S_IFMT) == S_IFREG) {
2384 		/* only struct inode is valid if it's an inline symlink */
2385 		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2386 	}
2387 	shmem_acl_destroy_inode(inode);
2388 	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2389 }
2390 
2391 static void init_once(struct kmem_cache *cachep, void *foo)
2392 {
2393 	struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2394 
2395 	inode_init_once(&p->vfs_inode);
2396 #ifdef CONFIG_TMPFS_POSIX_ACL
2397 	p->i_acl = NULL;
2398 	p->i_default_acl = NULL;
2399 #endif
2400 }
2401 
2402 static int init_inodecache(void)
2403 {
2404 	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2405 				sizeof(struct shmem_inode_info),
2406 				0, SLAB_PANIC, init_once);
2407 	return 0;
2408 }
2409 
2410 static void destroy_inodecache(void)
2411 {
2412 	kmem_cache_destroy(shmem_inode_cachep);
2413 }
2414 
2415 static const struct address_space_operations shmem_aops = {
2416 	.writepage	= shmem_writepage,
2417 	.set_page_dirty	= __set_page_dirty_no_writeback,
2418 #ifdef CONFIG_TMPFS
2419 	.readpage	= shmem_readpage,
2420 	.write_begin	= shmem_write_begin,
2421 	.write_end	= shmem_write_end,
2422 #endif
2423 	.migratepage	= migrate_page,
2424 };
2425 
2426 static const struct file_operations shmem_file_operations = {
2427 	.mmap		= shmem_mmap,
2428 #ifdef CONFIG_TMPFS
2429 	.llseek		= generic_file_llseek,
2430 	.read		= shmem_file_read,
2431 	.write		= do_sync_write,
2432 	.aio_write	= generic_file_aio_write,
2433 	.fsync		= simple_sync_file,
2434 	.splice_read	= generic_file_splice_read,
2435 	.splice_write	= generic_file_splice_write,
2436 #endif
2437 };
2438 
2439 static const struct inode_operations shmem_inode_operations = {
2440 	.truncate	= shmem_truncate,
2441 	.setattr	= shmem_notify_change,
2442 	.truncate_range	= shmem_truncate_range,
2443 #ifdef CONFIG_TMPFS_POSIX_ACL
2444 	.setxattr	= generic_setxattr,
2445 	.getxattr	= generic_getxattr,
2446 	.listxattr	= generic_listxattr,
2447 	.removexattr	= generic_removexattr,
2448 	.permission	= shmem_permission,
2449 #endif
2450 
2451 };
2452 
2453 static const struct inode_operations shmem_dir_inode_operations = {
2454 #ifdef CONFIG_TMPFS
2455 	.create		= shmem_create,
2456 	.lookup		= simple_lookup,
2457 	.link		= shmem_link,
2458 	.unlink		= shmem_unlink,
2459 	.symlink	= shmem_symlink,
2460 	.mkdir		= shmem_mkdir,
2461 	.rmdir		= shmem_rmdir,
2462 	.mknod		= shmem_mknod,
2463 	.rename		= shmem_rename,
2464 #endif
2465 #ifdef CONFIG_TMPFS_POSIX_ACL
2466 	.setattr	= shmem_notify_change,
2467 	.setxattr	= generic_setxattr,
2468 	.getxattr	= generic_getxattr,
2469 	.listxattr	= generic_listxattr,
2470 	.removexattr	= generic_removexattr,
2471 	.permission	= shmem_permission,
2472 #endif
2473 };
2474 
2475 static const struct inode_operations shmem_special_inode_operations = {
2476 #ifdef CONFIG_TMPFS_POSIX_ACL
2477 	.setattr	= shmem_notify_change,
2478 	.setxattr	= generic_setxattr,
2479 	.getxattr	= generic_getxattr,
2480 	.listxattr	= generic_listxattr,
2481 	.removexattr	= generic_removexattr,
2482 	.permission	= shmem_permission,
2483 #endif
2484 };
2485 
2486 static const struct super_operations shmem_ops = {
2487 	.alloc_inode	= shmem_alloc_inode,
2488 	.destroy_inode	= shmem_destroy_inode,
2489 #ifdef CONFIG_TMPFS
2490 	.statfs		= shmem_statfs,
2491 	.remount_fs	= shmem_remount_fs,
2492 	.show_options	= shmem_show_options,
2493 #endif
2494 	.delete_inode	= shmem_delete_inode,
2495 	.drop_inode	= generic_delete_inode,
2496 	.put_super	= shmem_put_super,
2497 };
2498 
2499 static struct vm_operations_struct shmem_vm_ops = {
2500 	.fault		= shmem_fault,
2501 #ifdef CONFIG_NUMA
2502 	.set_policy     = shmem_set_policy,
2503 	.get_policy     = shmem_get_policy,
2504 #endif
2505 };
2506 
2507 
2508 static int shmem_get_sb(struct file_system_type *fs_type,
2509 	int flags, const char *dev_name, void *data, struct vfsmount *mnt)
2510 {
2511 	return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
2512 }
2513 
2514 static struct file_system_type tmpfs_fs_type = {
2515 	.owner		= THIS_MODULE,
2516 	.name		= "tmpfs",
2517 	.get_sb		= shmem_get_sb,
2518 	.kill_sb	= kill_litter_super,
2519 };
2520 static struct vfsmount *shm_mnt;
2521 
2522 static int __init init_tmpfs(void)
2523 {
2524 	int error;
2525 
2526 	error = bdi_init(&shmem_backing_dev_info);
2527 	if (error)
2528 		goto out4;
2529 
2530 	error = init_inodecache();
2531 	if (error)
2532 		goto out3;
2533 
2534 	error = register_filesystem(&tmpfs_fs_type);
2535 	if (error) {
2536 		printk(KERN_ERR "Could not register tmpfs\n");
2537 		goto out2;
2538 	}
2539 
2540 	shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2541 				tmpfs_fs_type.name, NULL);
2542 	if (IS_ERR(shm_mnt)) {
2543 		error = PTR_ERR(shm_mnt);
2544 		printk(KERN_ERR "Could not kern_mount tmpfs\n");
2545 		goto out1;
2546 	}
2547 	return 0;
2548 
2549 out1:
2550 	unregister_filesystem(&tmpfs_fs_type);
2551 out2:
2552 	destroy_inodecache();
2553 out3:
2554 	bdi_destroy(&shmem_backing_dev_info);
2555 out4:
2556 	shm_mnt = ERR_PTR(error);
2557 	return error;
2558 }
2559 module_init(init_tmpfs)
2560 
2561 /**
2562  * shmem_file_setup - get an unlinked file living in tmpfs
2563  * @name: name for dentry (to be seen in /proc/<pid>/maps
2564  * @size: size to be set for the file
2565  * @flags: vm_flags
2566  */
2567 struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
2568 {
2569 	int error;
2570 	struct file *file;
2571 	struct inode *inode;
2572 	struct dentry *dentry, *root;
2573 	struct qstr this;
2574 
2575 	if (IS_ERR(shm_mnt))
2576 		return (void *)shm_mnt;
2577 
2578 	if (size < 0 || size > SHMEM_MAX_BYTES)
2579 		return ERR_PTR(-EINVAL);
2580 
2581 	if (shmem_acct_size(flags, size))
2582 		return ERR_PTR(-ENOMEM);
2583 
2584 	error = -ENOMEM;
2585 	this.name = name;
2586 	this.len = strlen(name);
2587 	this.hash = 0; /* will go */
2588 	root = shm_mnt->mnt_root;
2589 	dentry = d_alloc(root, &this);
2590 	if (!dentry)
2591 		goto put_memory;
2592 
2593 	error = -ENFILE;
2594 	file = get_empty_filp();
2595 	if (!file)
2596 		goto put_dentry;
2597 
2598 	error = -ENOSPC;
2599 	inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
2600 	if (!inode)
2601 		goto close_file;
2602 
2603 	SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
2604 	d_instantiate(dentry, inode);
2605 	inode->i_size = size;
2606 	inode->i_nlink = 0;	/* It is unlinked */
2607 	init_file(file, shm_mnt, dentry, FMODE_WRITE | FMODE_READ,
2608 			&shmem_file_operations);
2609 	return file;
2610 
2611 close_file:
2612 	put_filp(file);
2613 put_dentry:
2614 	dput(dentry);
2615 put_memory:
2616 	shmem_unacct_size(flags, size);
2617 	return ERR_PTR(error);
2618 }
2619 
2620 /**
2621  * shmem_zero_setup - setup a shared anonymous mapping
2622  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2623  */
2624 int shmem_zero_setup(struct vm_area_struct *vma)
2625 {
2626 	struct file *file;
2627 	loff_t size = vma->vm_end - vma->vm_start;
2628 
2629 	file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2630 	if (IS_ERR(file))
2631 		return PTR_ERR(file);
2632 
2633 	if (vma->vm_file)
2634 		fput(vma->vm_file);
2635 	vma->vm_file = file;
2636 	vma->vm_ops = &shmem_vm_ops;
2637 	return 0;
2638 }
2639