xref: /openbmc/linux/mm/shmem.c (revision e868d61272caa648214046a096e5a6bfc068dc8c)
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *		 2000 Transmeta Corp.
6  *		 2000-2001 Christoph Rohland
7  *		 2000-2001 SAP AG
8  *		 2002 Red Hat Inc.
9  * Copyright (C) 2002-2005 Hugh Dickins.
10  * Copyright (C) 2002-2005 VERITAS Software Corporation.
11  * Copyright (C) 2004 Andi Kleen, SuSE Labs
12  *
13  * Extended attribute support for tmpfs:
14  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
15  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
16  *
17  * This file is released under the GPL.
18  */
19 
20 /*
21  * This virtual memory filesystem is heavily based on the ramfs. It
22  * extends ramfs by the ability to use swap and honor resource limits
23  * which makes it a completely usable filesystem.
24  */
25 
26 #include <linux/module.h>
27 #include <linux/init.h>
28 #include <linux/fs.h>
29 #include <linux/xattr.h>
30 #include <linux/generic_acl.h>
31 #include <linux/mm.h>
32 #include <linux/mman.h>
33 #include <linux/file.h>
34 #include <linux/swap.h>
35 #include <linux/pagemap.h>
36 #include <linux/string.h>
37 #include <linux/slab.h>
38 #include <linux/backing-dev.h>
39 #include <linux/shmem_fs.h>
40 #include <linux/mount.h>
41 #include <linux/writeback.h>
42 #include <linux/vfs.h>
43 #include <linux/blkdev.h>
44 #include <linux/security.h>
45 #include <linux/swapops.h>
46 #include <linux/mempolicy.h>
47 #include <linux/namei.h>
48 #include <linux/ctype.h>
49 #include <linux/migrate.h>
50 #include <linux/highmem.h>
51 #include <linux/backing-dev.h>
52 
53 #include <asm/uaccess.h>
54 #include <asm/div64.h>
55 #include <asm/pgtable.h>
56 
57 /* This magic number is used in glibc for posix shared memory */
58 #define TMPFS_MAGIC	0x01021994
59 
60 #define ENTRIES_PER_PAGE (PAGE_CACHE_SIZE/sizeof(unsigned long))
61 #define ENTRIES_PER_PAGEPAGE (ENTRIES_PER_PAGE*ENTRIES_PER_PAGE)
62 #define BLOCKS_PER_PAGE  (PAGE_CACHE_SIZE/512)
63 
64 #define SHMEM_MAX_INDEX  (SHMEM_NR_DIRECT + (ENTRIES_PER_PAGEPAGE/2) * (ENTRIES_PER_PAGE+1))
65 #define SHMEM_MAX_BYTES  ((unsigned long long)SHMEM_MAX_INDEX << PAGE_CACHE_SHIFT)
66 
67 #define VM_ACCT(size)    (PAGE_CACHE_ALIGN(size) >> PAGE_SHIFT)
68 
69 /* info->flags needs VM_flags to handle pagein/truncate races efficiently */
70 #define SHMEM_PAGEIN	 VM_READ
71 #define SHMEM_TRUNCATE	 VM_WRITE
72 
73 /* Definition to limit shmem_truncate's steps between cond_rescheds */
74 #define LATENCY_LIMIT	 64
75 
76 /* Pretend that each entry is of this size in directory's i_size */
77 #define BOGO_DIRENT_SIZE 20
78 
79 /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
80 enum sgp_type {
81 	SGP_QUICK,	/* don't try more than file page cache lookup */
82 	SGP_READ,	/* don't exceed i_size, don't allocate page */
83 	SGP_CACHE,	/* don't exceed i_size, may allocate page */
84 	SGP_WRITE,	/* may exceed i_size, may allocate page */
85 };
86 
87 static int shmem_getpage(struct inode *inode, unsigned long idx,
88 			 struct page **pagep, enum sgp_type sgp, int *type);
89 
90 static inline struct page *shmem_dir_alloc(gfp_t gfp_mask)
91 {
92 	/*
93 	 * The above definition of ENTRIES_PER_PAGE, and the use of
94 	 * BLOCKS_PER_PAGE on indirect pages, assume PAGE_CACHE_SIZE:
95 	 * might be reconsidered if it ever diverges from PAGE_SIZE.
96 	 */
97 	return alloc_pages(gfp_mask, PAGE_CACHE_SHIFT-PAGE_SHIFT);
98 }
99 
100 static inline void shmem_dir_free(struct page *page)
101 {
102 	__free_pages(page, PAGE_CACHE_SHIFT-PAGE_SHIFT);
103 }
104 
105 static struct page **shmem_dir_map(struct page *page)
106 {
107 	return (struct page **)kmap_atomic(page, KM_USER0);
108 }
109 
110 static inline void shmem_dir_unmap(struct page **dir)
111 {
112 	kunmap_atomic(dir, KM_USER0);
113 }
114 
115 static swp_entry_t *shmem_swp_map(struct page *page)
116 {
117 	return (swp_entry_t *)kmap_atomic(page, KM_USER1);
118 }
119 
120 static inline void shmem_swp_balance_unmap(void)
121 {
122 	/*
123 	 * When passing a pointer to an i_direct entry, to code which
124 	 * also handles indirect entries and so will shmem_swp_unmap,
125 	 * we must arrange for the preempt count to remain in balance.
126 	 * What kmap_atomic of a lowmem page does depends on config
127 	 * and architecture, so pretend to kmap_atomic some lowmem page.
128 	 */
129 	(void) kmap_atomic(ZERO_PAGE(0), KM_USER1);
130 }
131 
132 static inline void shmem_swp_unmap(swp_entry_t *entry)
133 {
134 	kunmap_atomic(entry, KM_USER1);
135 }
136 
137 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
138 {
139 	return sb->s_fs_info;
140 }
141 
142 /*
143  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
144  * for shared memory and for shared anonymous (/dev/zero) mappings
145  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
146  * consistent with the pre-accounting of private mappings ...
147  */
148 static inline int shmem_acct_size(unsigned long flags, loff_t size)
149 {
150 	return (flags & VM_ACCOUNT)?
151 		security_vm_enough_memory(VM_ACCT(size)): 0;
152 }
153 
154 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
155 {
156 	if (flags & VM_ACCOUNT)
157 		vm_unacct_memory(VM_ACCT(size));
158 }
159 
160 /*
161  * ... whereas tmpfs objects are accounted incrementally as
162  * pages are allocated, in order to allow huge sparse files.
163  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
164  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
165  */
166 static inline int shmem_acct_block(unsigned long flags)
167 {
168 	return (flags & VM_ACCOUNT)?
169 		0: security_vm_enough_memory(VM_ACCT(PAGE_CACHE_SIZE));
170 }
171 
172 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
173 {
174 	if (!(flags & VM_ACCOUNT))
175 		vm_unacct_memory(pages * VM_ACCT(PAGE_CACHE_SIZE));
176 }
177 
178 static const struct super_operations shmem_ops;
179 static const struct address_space_operations shmem_aops;
180 static const struct file_operations shmem_file_operations;
181 static const struct inode_operations shmem_inode_operations;
182 static const struct inode_operations shmem_dir_inode_operations;
183 static const struct inode_operations shmem_special_inode_operations;
184 static struct vm_operations_struct shmem_vm_ops;
185 
186 static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
187 	.ra_pages	= 0,	/* No readahead */
188 	.capabilities	= BDI_CAP_NO_ACCT_DIRTY | BDI_CAP_NO_WRITEBACK,
189 	.unplug_io_fn	= default_unplug_io_fn,
190 };
191 
192 static LIST_HEAD(shmem_swaplist);
193 static DEFINE_SPINLOCK(shmem_swaplist_lock);
194 
195 static void shmem_free_blocks(struct inode *inode, long pages)
196 {
197 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
198 	if (sbinfo->max_blocks) {
199 		spin_lock(&sbinfo->stat_lock);
200 		sbinfo->free_blocks += pages;
201 		inode->i_blocks -= pages*BLOCKS_PER_PAGE;
202 		spin_unlock(&sbinfo->stat_lock);
203 	}
204 }
205 
206 /*
207  * shmem_recalc_inode - recalculate the size of an inode
208  *
209  * @inode: inode to recalc
210  *
211  * We have to calculate the free blocks since the mm can drop
212  * undirtied hole pages behind our back.
213  *
214  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
215  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
216  *
217  * It has to be called with the spinlock held.
218  */
219 static void shmem_recalc_inode(struct inode *inode)
220 {
221 	struct shmem_inode_info *info = SHMEM_I(inode);
222 	long freed;
223 
224 	freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
225 	if (freed > 0) {
226 		info->alloced -= freed;
227 		shmem_unacct_blocks(info->flags, freed);
228 		shmem_free_blocks(inode, freed);
229 	}
230 }
231 
232 /*
233  * shmem_swp_entry - find the swap vector position in the info structure
234  *
235  * @info:  info structure for the inode
236  * @index: index of the page to find
237  * @page:  optional page to add to the structure. Has to be preset to
238  *         all zeros
239  *
240  * If there is no space allocated yet it will return NULL when
241  * page is NULL, else it will use the page for the needed block,
242  * setting it to NULL on return to indicate that it has been used.
243  *
244  * The swap vector is organized the following way:
245  *
246  * There are SHMEM_NR_DIRECT entries directly stored in the
247  * shmem_inode_info structure. So small files do not need an addional
248  * allocation.
249  *
250  * For pages with index > SHMEM_NR_DIRECT there is the pointer
251  * i_indirect which points to a page which holds in the first half
252  * doubly indirect blocks, in the second half triple indirect blocks:
253  *
254  * For an artificial ENTRIES_PER_PAGE = 4 this would lead to the
255  * following layout (for SHMEM_NR_DIRECT == 16):
256  *
257  * i_indirect -> dir --> 16-19
258  * 	      |	     +-> 20-23
259  * 	      |
260  * 	      +-->dir2 --> 24-27
261  * 	      |	       +-> 28-31
262  * 	      |	       +-> 32-35
263  * 	      |	       +-> 36-39
264  * 	      |
265  * 	      +-->dir3 --> 40-43
266  * 	       	       +-> 44-47
267  * 	      	       +-> 48-51
268  * 	      	       +-> 52-55
269  */
270 static swp_entry_t *shmem_swp_entry(struct shmem_inode_info *info, unsigned long index, struct page **page)
271 {
272 	unsigned long offset;
273 	struct page **dir;
274 	struct page *subdir;
275 
276 	if (index < SHMEM_NR_DIRECT) {
277 		shmem_swp_balance_unmap();
278 		return info->i_direct+index;
279 	}
280 	if (!info->i_indirect) {
281 		if (page) {
282 			info->i_indirect = *page;
283 			*page = NULL;
284 		}
285 		return NULL;			/* need another page */
286 	}
287 
288 	index -= SHMEM_NR_DIRECT;
289 	offset = index % ENTRIES_PER_PAGE;
290 	index /= ENTRIES_PER_PAGE;
291 	dir = shmem_dir_map(info->i_indirect);
292 
293 	if (index >= ENTRIES_PER_PAGE/2) {
294 		index -= ENTRIES_PER_PAGE/2;
295 		dir += ENTRIES_PER_PAGE/2 + index/ENTRIES_PER_PAGE;
296 		index %= ENTRIES_PER_PAGE;
297 		subdir = *dir;
298 		if (!subdir) {
299 			if (page) {
300 				*dir = *page;
301 				*page = NULL;
302 			}
303 			shmem_dir_unmap(dir);
304 			return NULL;		/* need another page */
305 		}
306 		shmem_dir_unmap(dir);
307 		dir = shmem_dir_map(subdir);
308 	}
309 
310 	dir += index;
311 	subdir = *dir;
312 	if (!subdir) {
313 		if (!page || !(subdir = *page)) {
314 			shmem_dir_unmap(dir);
315 			return NULL;		/* need a page */
316 		}
317 		*dir = subdir;
318 		*page = NULL;
319 	}
320 	shmem_dir_unmap(dir);
321 	return shmem_swp_map(subdir) + offset;
322 }
323 
324 static void shmem_swp_set(struct shmem_inode_info *info, swp_entry_t *entry, unsigned long value)
325 {
326 	long incdec = value? 1: -1;
327 
328 	entry->val = value;
329 	info->swapped += incdec;
330 	if ((unsigned long)(entry - info->i_direct) >= SHMEM_NR_DIRECT) {
331 		struct page *page = kmap_atomic_to_page(entry);
332 		set_page_private(page, page_private(page) + incdec);
333 	}
334 }
335 
336 /*
337  * shmem_swp_alloc - get the position of the swap entry for the page.
338  *                   If it does not exist allocate the entry.
339  *
340  * @info:	info structure for the inode
341  * @index:	index of the page to find
342  * @sgp:	check and recheck i_size? skip allocation?
343  */
344 static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long index, enum sgp_type sgp)
345 {
346 	struct inode *inode = &info->vfs_inode;
347 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
348 	struct page *page = NULL;
349 	swp_entry_t *entry;
350 
351 	if (sgp != SGP_WRITE &&
352 	    ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode))
353 		return ERR_PTR(-EINVAL);
354 
355 	while (!(entry = shmem_swp_entry(info, index, &page))) {
356 		if (sgp == SGP_READ)
357 			return shmem_swp_map(ZERO_PAGE(0));
358 		/*
359 		 * Test free_blocks against 1 not 0, since we have 1 data
360 		 * page (and perhaps indirect index pages) yet to allocate:
361 		 * a waste to allocate index if we cannot allocate data.
362 		 */
363 		if (sbinfo->max_blocks) {
364 			spin_lock(&sbinfo->stat_lock);
365 			if (sbinfo->free_blocks <= 1) {
366 				spin_unlock(&sbinfo->stat_lock);
367 				return ERR_PTR(-ENOSPC);
368 			}
369 			sbinfo->free_blocks--;
370 			inode->i_blocks += BLOCKS_PER_PAGE;
371 			spin_unlock(&sbinfo->stat_lock);
372 		}
373 
374 		spin_unlock(&info->lock);
375 		page = shmem_dir_alloc(mapping_gfp_mask(inode->i_mapping) | __GFP_ZERO);
376 		if (page)
377 			set_page_private(page, 0);
378 		spin_lock(&info->lock);
379 
380 		if (!page) {
381 			shmem_free_blocks(inode, 1);
382 			return ERR_PTR(-ENOMEM);
383 		}
384 		if (sgp != SGP_WRITE &&
385 		    ((loff_t) index << PAGE_CACHE_SHIFT) >= i_size_read(inode)) {
386 			entry = ERR_PTR(-EINVAL);
387 			break;
388 		}
389 		if (info->next_index <= index)
390 			info->next_index = index + 1;
391 	}
392 	if (page) {
393 		/* another task gave its page, or truncated the file */
394 		shmem_free_blocks(inode, 1);
395 		shmem_dir_free(page);
396 	}
397 	if (info->next_index <= index && !IS_ERR(entry))
398 		info->next_index = index + 1;
399 	return entry;
400 }
401 
402 /*
403  * shmem_free_swp - free some swap entries in a directory
404  *
405  * @dir:        pointer to the directory
406  * @edir:       pointer after last entry of the directory
407  * @punch_lock: pointer to spinlock when needed for the holepunch case
408  */
409 static int shmem_free_swp(swp_entry_t *dir, swp_entry_t *edir,
410 						spinlock_t *punch_lock)
411 {
412 	spinlock_t *punch_unlock = NULL;
413 	swp_entry_t *ptr;
414 	int freed = 0;
415 
416 	for (ptr = dir; ptr < edir; ptr++) {
417 		if (ptr->val) {
418 			if (unlikely(punch_lock)) {
419 				punch_unlock = punch_lock;
420 				punch_lock = NULL;
421 				spin_lock(punch_unlock);
422 				if (!ptr->val)
423 					continue;
424 			}
425 			free_swap_and_cache(*ptr);
426 			*ptr = (swp_entry_t){0};
427 			freed++;
428 		}
429 	}
430 	if (punch_unlock)
431 		spin_unlock(punch_unlock);
432 	return freed;
433 }
434 
435 static int shmem_map_and_free_swp(struct page *subdir, int offset,
436 		int limit, struct page ***dir, spinlock_t *punch_lock)
437 {
438 	swp_entry_t *ptr;
439 	int freed = 0;
440 
441 	ptr = shmem_swp_map(subdir);
442 	for (; offset < limit; offset += LATENCY_LIMIT) {
443 		int size = limit - offset;
444 		if (size > LATENCY_LIMIT)
445 			size = LATENCY_LIMIT;
446 		freed += shmem_free_swp(ptr+offset, ptr+offset+size,
447 							punch_lock);
448 		if (need_resched()) {
449 			shmem_swp_unmap(ptr);
450 			if (*dir) {
451 				shmem_dir_unmap(*dir);
452 				*dir = NULL;
453 			}
454 			cond_resched();
455 			ptr = shmem_swp_map(subdir);
456 		}
457 	}
458 	shmem_swp_unmap(ptr);
459 	return freed;
460 }
461 
462 static void shmem_free_pages(struct list_head *next)
463 {
464 	struct page *page;
465 	int freed = 0;
466 
467 	do {
468 		page = container_of(next, struct page, lru);
469 		next = next->next;
470 		shmem_dir_free(page);
471 		freed++;
472 		if (freed >= LATENCY_LIMIT) {
473 			cond_resched();
474 			freed = 0;
475 		}
476 	} while (next);
477 }
478 
479 static void shmem_truncate_range(struct inode *inode, loff_t start, loff_t end)
480 {
481 	struct shmem_inode_info *info = SHMEM_I(inode);
482 	unsigned long idx;
483 	unsigned long size;
484 	unsigned long limit;
485 	unsigned long stage;
486 	unsigned long diroff;
487 	struct page **dir;
488 	struct page *topdir;
489 	struct page *middir;
490 	struct page *subdir;
491 	swp_entry_t *ptr;
492 	LIST_HEAD(pages_to_free);
493 	long nr_pages_to_free = 0;
494 	long nr_swaps_freed = 0;
495 	int offset;
496 	int freed;
497 	int punch_hole;
498 	spinlock_t *needs_lock;
499 	spinlock_t *punch_lock;
500 	unsigned long upper_limit;
501 
502 	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
503 	idx = (start + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
504 	if (idx >= info->next_index)
505 		return;
506 
507 	spin_lock(&info->lock);
508 	info->flags |= SHMEM_TRUNCATE;
509 	if (likely(end == (loff_t) -1)) {
510 		limit = info->next_index;
511 		upper_limit = SHMEM_MAX_INDEX;
512 		info->next_index = idx;
513 		needs_lock = NULL;
514 		punch_hole = 0;
515 	} else {
516 		if (end + 1 >= inode->i_size) {	/* we may free a little more */
517 			limit = (inode->i_size + PAGE_CACHE_SIZE - 1) >>
518 							PAGE_CACHE_SHIFT;
519 			upper_limit = SHMEM_MAX_INDEX;
520 		} else {
521 			limit = (end + 1) >> PAGE_CACHE_SHIFT;
522 			upper_limit = limit;
523 		}
524 		needs_lock = &info->lock;
525 		punch_hole = 1;
526 	}
527 
528 	topdir = info->i_indirect;
529 	if (topdir && idx <= SHMEM_NR_DIRECT && !punch_hole) {
530 		info->i_indirect = NULL;
531 		nr_pages_to_free++;
532 		list_add(&topdir->lru, &pages_to_free);
533 	}
534 	spin_unlock(&info->lock);
535 
536 	if (info->swapped && idx < SHMEM_NR_DIRECT) {
537 		ptr = info->i_direct;
538 		size = limit;
539 		if (size > SHMEM_NR_DIRECT)
540 			size = SHMEM_NR_DIRECT;
541 		nr_swaps_freed = shmem_free_swp(ptr+idx, ptr+size, needs_lock);
542 	}
543 
544 	/*
545 	 * If there are no indirect blocks or we are punching a hole
546 	 * below indirect blocks, nothing to be done.
547 	 */
548 	if (!topdir || limit <= SHMEM_NR_DIRECT)
549 		goto done2;
550 
551 	/*
552 	 * The truncation case has already dropped info->lock, and we're safe
553 	 * because i_size and next_index have already been lowered, preventing
554 	 * access beyond.  But in the punch_hole case, we still need to take
555 	 * the lock when updating the swap directory, because there might be
556 	 * racing accesses by shmem_getpage(SGP_CACHE), shmem_unuse_inode or
557 	 * shmem_writepage.  However, whenever we find we can remove a whole
558 	 * directory page (not at the misaligned start or end of the range),
559 	 * we first NULLify its pointer in the level above, and then have no
560 	 * need to take the lock when updating its contents: needs_lock and
561 	 * punch_lock (either pointing to info->lock or NULL) manage this.
562 	 */
563 
564 	upper_limit -= SHMEM_NR_DIRECT;
565 	limit -= SHMEM_NR_DIRECT;
566 	idx = (idx > SHMEM_NR_DIRECT)? (idx - SHMEM_NR_DIRECT): 0;
567 	offset = idx % ENTRIES_PER_PAGE;
568 	idx -= offset;
569 
570 	dir = shmem_dir_map(topdir);
571 	stage = ENTRIES_PER_PAGEPAGE/2;
572 	if (idx < ENTRIES_PER_PAGEPAGE/2) {
573 		middir = topdir;
574 		diroff = idx/ENTRIES_PER_PAGE;
575 	} else {
576 		dir += ENTRIES_PER_PAGE/2;
577 		dir += (idx - ENTRIES_PER_PAGEPAGE/2)/ENTRIES_PER_PAGEPAGE;
578 		while (stage <= idx)
579 			stage += ENTRIES_PER_PAGEPAGE;
580 		middir = *dir;
581 		if (*dir) {
582 			diroff = ((idx - ENTRIES_PER_PAGEPAGE/2) %
583 				ENTRIES_PER_PAGEPAGE) / ENTRIES_PER_PAGE;
584 			if (!diroff && !offset && upper_limit >= stage) {
585 				if (needs_lock) {
586 					spin_lock(needs_lock);
587 					*dir = NULL;
588 					spin_unlock(needs_lock);
589 					needs_lock = NULL;
590 				} else
591 					*dir = NULL;
592 				nr_pages_to_free++;
593 				list_add(&middir->lru, &pages_to_free);
594 			}
595 			shmem_dir_unmap(dir);
596 			dir = shmem_dir_map(middir);
597 		} else {
598 			diroff = 0;
599 			offset = 0;
600 			idx = stage;
601 		}
602 	}
603 
604 	for (; idx < limit; idx += ENTRIES_PER_PAGE, diroff++) {
605 		if (unlikely(idx == stage)) {
606 			shmem_dir_unmap(dir);
607 			dir = shmem_dir_map(topdir) +
608 			    ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
609 			while (!*dir) {
610 				dir++;
611 				idx += ENTRIES_PER_PAGEPAGE;
612 				if (idx >= limit)
613 					goto done1;
614 			}
615 			stage = idx + ENTRIES_PER_PAGEPAGE;
616 			middir = *dir;
617 			if (punch_hole)
618 				needs_lock = &info->lock;
619 			if (upper_limit >= stage) {
620 				if (needs_lock) {
621 					spin_lock(needs_lock);
622 					*dir = NULL;
623 					spin_unlock(needs_lock);
624 					needs_lock = NULL;
625 				} else
626 					*dir = NULL;
627 				nr_pages_to_free++;
628 				list_add(&middir->lru, &pages_to_free);
629 			}
630 			shmem_dir_unmap(dir);
631 			cond_resched();
632 			dir = shmem_dir_map(middir);
633 			diroff = 0;
634 		}
635 		punch_lock = needs_lock;
636 		subdir = dir[diroff];
637 		if (subdir && !offset && upper_limit-idx >= ENTRIES_PER_PAGE) {
638 			if (needs_lock) {
639 				spin_lock(needs_lock);
640 				dir[diroff] = NULL;
641 				spin_unlock(needs_lock);
642 				punch_lock = NULL;
643 			} else
644 				dir[diroff] = NULL;
645 			nr_pages_to_free++;
646 			list_add(&subdir->lru, &pages_to_free);
647 		}
648 		if (subdir && page_private(subdir) /* has swap entries */) {
649 			size = limit - idx;
650 			if (size > ENTRIES_PER_PAGE)
651 				size = ENTRIES_PER_PAGE;
652 			freed = shmem_map_and_free_swp(subdir,
653 					offset, size, &dir, punch_lock);
654 			if (!dir)
655 				dir = shmem_dir_map(middir);
656 			nr_swaps_freed += freed;
657 			if (offset || punch_lock) {
658 				spin_lock(&info->lock);
659 				set_page_private(subdir,
660 					page_private(subdir) - freed);
661 				spin_unlock(&info->lock);
662 			} else
663 				BUG_ON(page_private(subdir) != freed);
664 		}
665 		offset = 0;
666 	}
667 done1:
668 	shmem_dir_unmap(dir);
669 done2:
670 	if (inode->i_mapping->nrpages && (info->flags & SHMEM_PAGEIN)) {
671 		/*
672 		 * Call truncate_inode_pages again: racing shmem_unuse_inode
673 		 * may have swizzled a page in from swap since vmtruncate or
674 		 * generic_delete_inode did it, before we lowered next_index.
675 		 * Also, though shmem_getpage checks i_size before adding to
676 		 * cache, no recheck after: so fix the narrow window there too.
677 		 *
678 		 * Recalling truncate_inode_pages_range and unmap_mapping_range
679 		 * every time for punch_hole (which never got a chance to clear
680 		 * SHMEM_PAGEIN at the start of vmtruncate_range) is expensive,
681 		 * yet hardly ever necessary: try to optimize them out later.
682 		 */
683 		truncate_inode_pages_range(inode->i_mapping, start, end);
684 		if (punch_hole)
685 			unmap_mapping_range(inode->i_mapping, start,
686 							end - start, 1);
687 	}
688 
689 	spin_lock(&info->lock);
690 	info->flags &= ~SHMEM_TRUNCATE;
691 	info->swapped -= nr_swaps_freed;
692 	if (nr_pages_to_free)
693 		shmem_free_blocks(inode, nr_pages_to_free);
694 	shmem_recalc_inode(inode);
695 	spin_unlock(&info->lock);
696 
697 	/*
698 	 * Empty swap vector directory pages to be freed?
699 	 */
700 	if (!list_empty(&pages_to_free)) {
701 		pages_to_free.prev->next = NULL;
702 		shmem_free_pages(pages_to_free.next);
703 	}
704 }
705 
706 static void shmem_truncate(struct inode *inode)
707 {
708 	shmem_truncate_range(inode, inode->i_size, (loff_t)-1);
709 }
710 
711 static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
712 {
713 	struct inode *inode = dentry->d_inode;
714 	struct page *page = NULL;
715 	int error;
716 
717 	if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
718 		if (attr->ia_size < inode->i_size) {
719 			/*
720 			 * If truncating down to a partial page, then
721 			 * if that page is already allocated, hold it
722 			 * in memory until the truncation is over, so
723 			 * truncate_partial_page cannnot miss it were
724 			 * it assigned to swap.
725 			 */
726 			if (attr->ia_size & (PAGE_CACHE_SIZE-1)) {
727 				(void) shmem_getpage(inode,
728 					attr->ia_size>>PAGE_CACHE_SHIFT,
729 						&page, SGP_READ, NULL);
730 			}
731 			/*
732 			 * Reset SHMEM_PAGEIN flag so that shmem_truncate can
733 			 * detect if any pages might have been added to cache
734 			 * after truncate_inode_pages.  But we needn't bother
735 			 * if it's being fully truncated to zero-length: the
736 			 * nrpages check is efficient enough in that case.
737 			 */
738 			if (attr->ia_size) {
739 				struct shmem_inode_info *info = SHMEM_I(inode);
740 				spin_lock(&info->lock);
741 				info->flags &= ~SHMEM_PAGEIN;
742 				spin_unlock(&info->lock);
743 			}
744 		}
745 	}
746 
747 	error = inode_change_ok(inode, attr);
748 	if (!error)
749 		error = inode_setattr(inode, attr);
750 #ifdef CONFIG_TMPFS_POSIX_ACL
751 	if (!error && (attr->ia_valid & ATTR_MODE))
752 		error = generic_acl_chmod(inode, &shmem_acl_ops);
753 #endif
754 	if (page)
755 		page_cache_release(page);
756 	return error;
757 }
758 
759 static void shmem_delete_inode(struct inode *inode)
760 {
761 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
762 	struct shmem_inode_info *info = SHMEM_I(inode);
763 
764 	if (inode->i_op->truncate == shmem_truncate) {
765 		truncate_inode_pages(inode->i_mapping, 0);
766 		shmem_unacct_size(info->flags, inode->i_size);
767 		inode->i_size = 0;
768 		shmem_truncate(inode);
769 		if (!list_empty(&info->swaplist)) {
770 			spin_lock(&shmem_swaplist_lock);
771 			list_del_init(&info->swaplist);
772 			spin_unlock(&shmem_swaplist_lock);
773 		}
774 	}
775 	BUG_ON(inode->i_blocks);
776 	if (sbinfo->max_inodes) {
777 		spin_lock(&sbinfo->stat_lock);
778 		sbinfo->free_inodes++;
779 		spin_unlock(&sbinfo->stat_lock);
780 	}
781 	clear_inode(inode);
782 }
783 
784 static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
785 {
786 	swp_entry_t *ptr;
787 
788 	for (ptr = dir; ptr < edir; ptr++) {
789 		if (ptr->val == entry.val)
790 			return ptr - dir;
791 	}
792 	return -1;
793 }
794 
795 static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, struct page *page)
796 {
797 	struct inode *inode;
798 	unsigned long idx;
799 	unsigned long size;
800 	unsigned long limit;
801 	unsigned long stage;
802 	struct page **dir;
803 	struct page *subdir;
804 	swp_entry_t *ptr;
805 	int offset;
806 
807 	idx = 0;
808 	ptr = info->i_direct;
809 	spin_lock(&info->lock);
810 	limit = info->next_index;
811 	size = limit;
812 	if (size > SHMEM_NR_DIRECT)
813 		size = SHMEM_NR_DIRECT;
814 	offset = shmem_find_swp(entry, ptr, ptr+size);
815 	if (offset >= 0) {
816 		shmem_swp_balance_unmap();
817 		goto found;
818 	}
819 	if (!info->i_indirect)
820 		goto lost2;
821 
822 	dir = shmem_dir_map(info->i_indirect);
823 	stage = SHMEM_NR_DIRECT + ENTRIES_PER_PAGEPAGE/2;
824 
825 	for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
826 		if (unlikely(idx == stage)) {
827 			shmem_dir_unmap(dir-1);
828 			dir = shmem_dir_map(info->i_indirect) +
829 			    ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
830 			while (!*dir) {
831 				dir++;
832 				idx += ENTRIES_PER_PAGEPAGE;
833 				if (idx >= limit)
834 					goto lost1;
835 			}
836 			stage = idx + ENTRIES_PER_PAGEPAGE;
837 			subdir = *dir;
838 			shmem_dir_unmap(dir);
839 			dir = shmem_dir_map(subdir);
840 		}
841 		subdir = *dir;
842 		if (subdir && page_private(subdir)) {
843 			ptr = shmem_swp_map(subdir);
844 			size = limit - idx;
845 			if (size > ENTRIES_PER_PAGE)
846 				size = ENTRIES_PER_PAGE;
847 			offset = shmem_find_swp(entry, ptr, ptr+size);
848 			if (offset >= 0) {
849 				shmem_dir_unmap(dir);
850 				goto found;
851 			}
852 			shmem_swp_unmap(ptr);
853 		}
854 	}
855 lost1:
856 	shmem_dir_unmap(dir-1);
857 lost2:
858 	spin_unlock(&info->lock);
859 	return 0;
860 found:
861 	idx += offset;
862 	inode = &info->vfs_inode;
863 	if (move_from_swap_cache(page, idx, inode->i_mapping) == 0) {
864 		info->flags |= SHMEM_PAGEIN;
865 		shmem_swp_set(info, ptr + offset, 0);
866 	}
867 	shmem_swp_unmap(ptr);
868 	spin_unlock(&info->lock);
869 	/*
870 	 * Decrement swap count even when the entry is left behind:
871 	 * try_to_unuse will skip over mms, then reincrement count.
872 	 */
873 	swap_free(entry);
874 	return 1;
875 }
876 
877 /*
878  * shmem_unuse() search for an eventually swapped out shmem page.
879  */
880 int shmem_unuse(swp_entry_t entry, struct page *page)
881 {
882 	struct list_head *p, *next;
883 	struct shmem_inode_info *info;
884 	int found = 0;
885 
886 	spin_lock(&shmem_swaplist_lock);
887 	list_for_each_safe(p, next, &shmem_swaplist) {
888 		info = list_entry(p, struct shmem_inode_info, swaplist);
889 		if (!info->swapped)
890 			list_del_init(&info->swaplist);
891 		else if (shmem_unuse_inode(info, entry, page)) {
892 			/* move head to start search for next from here */
893 			list_move_tail(&shmem_swaplist, &info->swaplist);
894 			found = 1;
895 			break;
896 		}
897 	}
898 	spin_unlock(&shmem_swaplist_lock);
899 	return found;
900 }
901 
902 /*
903  * Move the page from the page cache to the swap cache.
904  */
905 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
906 {
907 	struct shmem_inode_info *info;
908 	swp_entry_t *entry, swap;
909 	struct address_space *mapping;
910 	unsigned long index;
911 	struct inode *inode;
912 
913 	BUG_ON(!PageLocked(page));
914 	BUG_ON(page_mapped(page));
915 
916 	mapping = page->mapping;
917 	index = page->index;
918 	inode = mapping->host;
919 	info = SHMEM_I(inode);
920 	if (info->flags & VM_LOCKED)
921 		goto redirty;
922 	swap = get_swap_page();
923 	if (!swap.val)
924 		goto redirty;
925 
926 	spin_lock(&info->lock);
927 	shmem_recalc_inode(inode);
928 	if (index >= info->next_index) {
929 		BUG_ON(!(info->flags & SHMEM_TRUNCATE));
930 		goto unlock;
931 	}
932 	entry = shmem_swp_entry(info, index, NULL);
933 	BUG_ON(!entry);
934 	BUG_ON(entry->val);
935 
936 	if (move_to_swap_cache(page, swap) == 0) {
937 		shmem_swp_set(info, entry, swap.val);
938 		shmem_swp_unmap(entry);
939 		spin_unlock(&info->lock);
940 		if (list_empty(&info->swaplist)) {
941 			spin_lock(&shmem_swaplist_lock);
942 			/* move instead of add in case we're racing */
943 			list_move_tail(&info->swaplist, &shmem_swaplist);
944 			spin_unlock(&shmem_swaplist_lock);
945 		}
946 		unlock_page(page);
947 		return 0;
948 	}
949 
950 	shmem_swp_unmap(entry);
951 unlock:
952 	spin_unlock(&info->lock);
953 	swap_free(swap);
954 redirty:
955 	set_page_dirty(page);
956 	return AOP_WRITEPAGE_ACTIVATE;	/* Return with the page locked */
957 }
958 
959 #ifdef CONFIG_NUMA
960 static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
961 {
962 	char *nodelist = strchr(value, ':');
963 	int err = 1;
964 
965 	if (nodelist) {
966 		/* NUL-terminate policy string */
967 		*nodelist++ = '\0';
968 		if (nodelist_parse(nodelist, *policy_nodes))
969 			goto out;
970 	}
971 	if (!strcmp(value, "default")) {
972 		*policy = MPOL_DEFAULT;
973 		/* Don't allow a nodelist */
974 		if (!nodelist)
975 			err = 0;
976 	} else if (!strcmp(value, "prefer")) {
977 		*policy = MPOL_PREFERRED;
978 		/* Insist on a nodelist of one node only */
979 		if (nodelist) {
980 			char *rest = nodelist;
981 			while (isdigit(*rest))
982 				rest++;
983 			if (!*rest)
984 				err = 0;
985 		}
986 	} else if (!strcmp(value, "bind")) {
987 		*policy = MPOL_BIND;
988 		/* Insist on a nodelist */
989 		if (nodelist)
990 			err = 0;
991 	} else if (!strcmp(value, "interleave")) {
992 		*policy = MPOL_INTERLEAVE;
993 		/* Default to nodes online if no nodelist */
994 		if (!nodelist)
995 			*policy_nodes = node_online_map;
996 		err = 0;
997 	}
998 out:
999 	/* Restore string for error message */
1000 	if (nodelist)
1001 		*--nodelist = ':';
1002 	return err;
1003 }
1004 
1005 static struct page *shmem_swapin_async(struct shared_policy *p,
1006 				       swp_entry_t entry, unsigned long idx)
1007 {
1008 	struct page *page;
1009 	struct vm_area_struct pvma;
1010 
1011 	/* Create a pseudo vma that just contains the policy */
1012 	memset(&pvma, 0, sizeof(struct vm_area_struct));
1013 	pvma.vm_end = PAGE_SIZE;
1014 	pvma.vm_pgoff = idx;
1015 	pvma.vm_policy = mpol_shared_policy_lookup(p, idx);
1016 	page = read_swap_cache_async(entry, &pvma, 0);
1017 	mpol_free(pvma.vm_policy);
1018 	return page;
1019 }
1020 
1021 struct page *shmem_swapin(struct shmem_inode_info *info, swp_entry_t entry,
1022 			  unsigned long idx)
1023 {
1024 	struct shared_policy *p = &info->policy;
1025 	int i, num;
1026 	struct page *page;
1027 	unsigned long offset;
1028 
1029 	num = valid_swaphandles(entry, &offset);
1030 	for (i = 0; i < num; offset++, i++) {
1031 		page = shmem_swapin_async(p,
1032 				swp_entry(swp_type(entry), offset), idx);
1033 		if (!page)
1034 			break;
1035 		page_cache_release(page);
1036 	}
1037 	lru_add_drain();	/* Push any new pages onto the LRU now */
1038 	return shmem_swapin_async(p, entry, idx);
1039 }
1040 
1041 static struct page *
1042 shmem_alloc_page(gfp_t gfp, struct shmem_inode_info *info,
1043 		 unsigned long idx)
1044 {
1045 	struct vm_area_struct pvma;
1046 	struct page *page;
1047 
1048 	memset(&pvma, 0, sizeof(struct vm_area_struct));
1049 	pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
1050 	pvma.vm_pgoff = idx;
1051 	pvma.vm_end = PAGE_SIZE;
1052 	page = alloc_page_vma(gfp | __GFP_ZERO, &pvma, 0);
1053 	mpol_free(pvma.vm_policy);
1054 	return page;
1055 }
1056 #else
1057 static inline int shmem_parse_mpol(char *value, int *policy, nodemask_t *policy_nodes)
1058 {
1059 	return 1;
1060 }
1061 
1062 static inline struct page *
1063 shmem_swapin(struct shmem_inode_info *info,swp_entry_t entry,unsigned long idx)
1064 {
1065 	swapin_readahead(entry, 0, NULL);
1066 	return read_swap_cache_async(entry, NULL, 0);
1067 }
1068 
1069 static inline struct page *
1070 shmem_alloc_page(gfp_t gfp,struct shmem_inode_info *info, unsigned long idx)
1071 {
1072 	return alloc_page(gfp | __GFP_ZERO);
1073 }
1074 #endif
1075 
1076 /*
1077  * shmem_getpage - either get the page from swap or allocate a new one
1078  *
1079  * If we allocate a new one we do not mark it dirty. That's up to the
1080  * vm. If we swap it in we mark it dirty since we also free the swap
1081  * entry since a page cannot live in both the swap and page cache
1082  */
1083 static int shmem_getpage(struct inode *inode, unsigned long idx,
1084 			struct page **pagep, enum sgp_type sgp, int *type)
1085 {
1086 	struct address_space *mapping = inode->i_mapping;
1087 	struct shmem_inode_info *info = SHMEM_I(inode);
1088 	struct shmem_sb_info *sbinfo;
1089 	struct page *filepage = *pagep;
1090 	struct page *swappage;
1091 	swp_entry_t *entry;
1092 	swp_entry_t swap;
1093 	int error;
1094 
1095 	if (idx >= SHMEM_MAX_INDEX)
1096 		return -EFBIG;
1097 	/*
1098 	 * Normally, filepage is NULL on entry, and either found
1099 	 * uptodate immediately, or allocated and zeroed, or read
1100 	 * in under swappage, which is then assigned to filepage.
1101 	 * But shmem_prepare_write passes in a locked filepage,
1102 	 * which may be found not uptodate by other callers too,
1103 	 * and may need to be copied from the swappage read in.
1104 	 */
1105 repeat:
1106 	if (!filepage)
1107 		filepage = find_lock_page(mapping, idx);
1108 	if (filepage && PageUptodate(filepage))
1109 		goto done;
1110 	error = 0;
1111 	if (sgp == SGP_QUICK)
1112 		goto failed;
1113 
1114 	spin_lock(&info->lock);
1115 	shmem_recalc_inode(inode);
1116 	entry = shmem_swp_alloc(info, idx, sgp);
1117 	if (IS_ERR(entry)) {
1118 		spin_unlock(&info->lock);
1119 		error = PTR_ERR(entry);
1120 		goto failed;
1121 	}
1122 	swap = *entry;
1123 
1124 	if (swap.val) {
1125 		/* Look it up and read it in.. */
1126 		swappage = lookup_swap_cache(swap);
1127 		if (!swappage) {
1128 			shmem_swp_unmap(entry);
1129 			/* here we actually do the io */
1130 			if (type && *type == VM_FAULT_MINOR) {
1131 				__count_vm_event(PGMAJFAULT);
1132 				*type = VM_FAULT_MAJOR;
1133 			}
1134 			spin_unlock(&info->lock);
1135 			swappage = shmem_swapin(info, swap, idx);
1136 			if (!swappage) {
1137 				spin_lock(&info->lock);
1138 				entry = shmem_swp_alloc(info, idx, sgp);
1139 				if (IS_ERR(entry))
1140 					error = PTR_ERR(entry);
1141 				else {
1142 					if (entry->val == swap.val)
1143 						error = -ENOMEM;
1144 					shmem_swp_unmap(entry);
1145 				}
1146 				spin_unlock(&info->lock);
1147 				if (error)
1148 					goto failed;
1149 				goto repeat;
1150 			}
1151 			wait_on_page_locked(swappage);
1152 			page_cache_release(swappage);
1153 			goto repeat;
1154 		}
1155 
1156 		/* We have to do this with page locked to prevent races */
1157 		if (TestSetPageLocked(swappage)) {
1158 			shmem_swp_unmap(entry);
1159 			spin_unlock(&info->lock);
1160 			wait_on_page_locked(swappage);
1161 			page_cache_release(swappage);
1162 			goto repeat;
1163 		}
1164 		if (PageWriteback(swappage)) {
1165 			shmem_swp_unmap(entry);
1166 			spin_unlock(&info->lock);
1167 			wait_on_page_writeback(swappage);
1168 			unlock_page(swappage);
1169 			page_cache_release(swappage);
1170 			goto repeat;
1171 		}
1172 		if (!PageUptodate(swappage)) {
1173 			shmem_swp_unmap(entry);
1174 			spin_unlock(&info->lock);
1175 			unlock_page(swappage);
1176 			page_cache_release(swappage);
1177 			error = -EIO;
1178 			goto failed;
1179 		}
1180 
1181 		if (filepage) {
1182 			shmem_swp_set(info, entry, 0);
1183 			shmem_swp_unmap(entry);
1184 			delete_from_swap_cache(swappage);
1185 			spin_unlock(&info->lock);
1186 			copy_highpage(filepage, swappage);
1187 			unlock_page(swappage);
1188 			page_cache_release(swappage);
1189 			flush_dcache_page(filepage);
1190 			SetPageUptodate(filepage);
1191 			set_page_dirty(filepage);
1192 			swap_free(swap);
1193 		} else if (!(error = move_from_swap_cache(
1194 				swappage, idx, mapping))) {
1195 			info->flags |= SHMEM_PAGEIN;
1196 			shmem_swp_set(info, entry, 0);
1197 			shmem_swp_unmap(entry);
1198 			spin_unlock(&info->lock);
1199 			filepage = swappage;
1200 			swap_free(swap);
1201 		} else {
1202 			shmem_swp_unmap(entry);
1203 			spin_unlock(&info->lock);
1204 			unlock_page(swappage);
1205 			page_cache_release(swappage);
1206 			if (error == -ENOMEM) {
1207 				/* let kswapd refresh zone for GFP_ATOMICs */
1208 				congestion_wait(WRITE, HZ/50);
1209 			}
1210 			goto repeat;
1211 		}
1212 	} else if (sgp == SGP_READ && !filepage) {
1213 		shmem_swp_unmap(entry);
1214 		filepage = find_get_page(mapping, idx);
1215 		if (filepage &&
1216 		    (!PageUptodate(filepage) || TestSetPageLocked(filepage))) {
1217 			spin_unlock(&info->lock);
1218 			wait_on_page_locked(filepage);
1219 			page_cache_release(filepage);
1220 			filepage = NULL;
1221 			goto repeat;
1222 		}
1223 		spin_unlock(&info->lock);
1224 	} else {
1225 		shmem_swp_unmap(entry);
1226 		sbinfo = SHMEM_SB(inode->i_sb);
1227 		if (sbinfo->max_blocks) {
1228 			spin_lock(&sbinfo->stat_lock);
1229 			if (sbinfo->free_blocks == 0 ||
1230 			    shmem_acct_block(info->flags)) {
1231 				spin_unlock(&sbinfo->stat_lock);
1232 				spin_unlock(&info->lock);
1233 				error = -ENOSPC;
1234 				goto failed;
1235 			}
1236 			sbinfo->free_blocks--;
1237 			inode->i_blocks += BLOCKS_PER_PAGE;
1238 			spin_unlock(&sbinfo->stat_lock);
1239 		} else if (shmem_acct_block(info->flags)) {
1240 			spin_unlock(&info->lock);
1241 			error = -ENOSPC;
1242 			goto failed;
1243 		}
1244 
1245 		if (!filepage) {
1246 			spin_unlock(&info->lock);
1247 			filepage = shmem_alloc_page(mapping_gfp_mask(mapping),
1248 						    info,
1249 						    idx);
1250 			if (!filepage) {
1251 				shmem_unacct_blocks(info->flags, 1);
1252 				shmem_free_blocks(inode, 1);
1253 				error = -ENOMEM;
1254 				goto failed;
1255 			}
1256 
1257 			spin_lock(&info->lock);
1258 			entry = shmem_swp_alloc(info, idx, sgp);
1259 			if (IS_ERR(entry))
1260 				error = PTR_ERR(entry);
1261 			else {
1262 				swap = *entry;
1263 				shmem_swp_unmap(entry);
1264 			}
1265 			if (error || swap.val || 0 != add_to_page_cache_lru(
1266 					filepage, mapping, idx, GFP_ATOMIC)) {
1267 				spin_unlock(&info->lock);
1268 				page_cache_release(filepage);
1269 				shmem_unacct_blocks(info->flags, 1);
1270 				shmem_free_blocks(inode, 1);
1271 				filepage = NULL;
1272 				if (error)
1273 					goto failed;
1274 				goto repeat;
1275 			}
1276 			info->flags |= SHMEM_PAGEIN;
1277 		}
1278 
1279 		info->alloced++;
1280 		spin_unlock(&info->lock);
1281 		flush_dcache_page(filepage);
1282 		SetPageUptodate(filepage);
1283 	}
1284 done:
1285 	if (*pagep != filepage) {
1286 		unlock_page(filepage);
1287 		*pagep = filepage;
1288 	}
1289 	return 0;
1290 
1291 failed:
1292 	if (*pagep != filepage) {
1293 		unlock_page(filepage);
1294 		page_cache_release(filepage);
1295 	}
1296 	return error;
1297 }
1298 
1299 static struct page *shmem_nopage(struct vm_area_struct *vma,
1300 				 unsigned long address, int *type)
1301 {
1302 	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1303 	struct page *page = NULL;
1304 	unsigned long idx;
1305 	int error;
1306 
1307 	idx = (address - vma->vm_start) >> PAGE_SHIFT;
1308 	idx += vma->vm_pgoff;
1309 	idx >>= PAGE_CACHE_SHIFT - PAGE_SHIFT;
1310 	if (((loff_t) idx << PAGE_CACHE_SHIFT) >= i_size_read(inode))
1311 		return NOPAGE_SIGBUS;
1312 
1313 	error = shmem_getpage(inode, idx, &page, SGP_CACHE, type);
1314 	if (error)
1315 		return (error == -ENOMEM)? NOPAGE_OOM: NOPAGE_SIGBUS;
1316 
1317 	mark_page_accessed(page);
1318 	return page;
1319 }
1320 
1321 static int shmem_populate(struct vm_area_struct *vma,
1322 	unsigned long addr, unsigned long len,
1323 	pgprot_t prot, unsigned long pgoff, int nonblock)
1324 {
1325 	struct inode *inode = vma->vm_file->f_path.dentry->d_inode;
1326 	struct mm_struct *mm = vma->vm_mm;
1327 	enum sgp_type sgp = nonblock? SGP_QUICK: SGP_CACHE;
1328 	unsigned long size;
1329 
1330 	size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
1331 	if (pgoff >= size || pgoff + (len >> PAGE_SHIFT) > size)
1332 		return -EINVAL;
1333 
1334 	while ((long) len > 0) {
1335 		struct page *page = NULL;
1336 		int err;
1337 		/*
1338 		 * Will need changing if PAGE_CACHE_SIZE != PAGE_SIZE
1339 		 */
1340 		err = shmem_getpage(inode, pgoff, &page, sgp, NULL);
1341 		if (err)
1342 			return err;
1343 		/* Page may still be null, but only if nonblock was set. */
1344 		if (page) {
1345 			mark_page_accessed(page);
1346 			err = install_page(mm, vma, addr, page, prot);
1347 			if (err) {
1348 				page_cache_release(page);
1349 				return err;
1350 			}
1351 		} else if (vma->vm_flags & VM_NONLINEAR) {
1352 			/* No page was found just because we can't read it in
1353 			 * now (being here implies nonblock != 0), but the page
1354 			 * may exist, so set the PTE to fault it in later. */
1355     			err = install_file_pte(mm, vma, addr, pgoff, prot);
1356 			if (err)
1357 	    			return err;
1358 		}
1359 
1360 		len -= PAGE_SIZE;
1361 		addr += PAGE_SIZE;
1362 		pgoff++;
1363 	}
1364 	return 0;
1365 }
1366 
1367 #ifdef CONFIG_NUMA
1368 int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *new)
1369 {
1370 	struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1371 	return mpol_set_shared_policy(&SHMEM_I(i)->policy, vma, new);
1372 }
1373 
1374 struct mempolicy *
1375 shmem_get_policy(struct vm_area_struct *vma, unsigned long addr)
1376 {
1377 	struct inode *i = vma->vm_file->f_path.dentry->d_inode;
1378 	unsigned long idx;
1379 
1380 	idx = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
1381 	return mpol_shared_policy_lookup(&SHMEM_I(i)->policy, idx);
1382 }
1383 #endif
1384 
1385 int shmem_lock(struct file *file, int lock, struct user_struct *user)
1386 {
1387 	struct inode *inode = file->f_path.dentry->d_inode;
1388 	struct shmem_inode_info *info = SHMEM_I(inode);
1389 	int retval = -ENOMEM;
1390 
1391 	spin_lock(&info->lock);
1392 	if (lock && !(info->flags & VM_LOCKED)) {
1393 		if (!user_shm_lock(inode->i_size, user))
1394 			goto out_nomem;
1395 		info->flags |= VM_LOCKED;
1396 	}
1397 	if (!lock && (info->flags & VM_LOCKED) && user) {
1398 		user_shm_unlock(inode->i_size, user);
1399 		info->flags &= ~VM_LOCKED;
1400 	}
1401 	retval = 0;
1402 out_nomem:
1403 	spin_unlock(&info->lock);
1404 	return retval;
1405 }
1406 
1407 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
1408 {
1409 	file_accessed(file);
1410 	vma->vm_ops = &shmem_vm_ops;
1411 	return 0;
1412 }
1413 
1414 static struct inode *
1415 shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
1416 {
1417 	struct inode *inode;
1418 	struct shmem_inode_info *info;
1419 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
1420 
1421 	if (sbinfo->max_inodes) {
1422 		spin_lock(&sbinfo->stat_lock);
1423 		if (!sbinfo->free_inodes) {
1424 			spin_unlock(&sbinfo->stat_lock);
1425 			return NULL;
1426 		}
1427 		sbinfo->free_inodes--;
1428 		spin_unlock(&sbinfo->stat_lock);
1429 	}
1430 
1431 	inode = new_inode(sb);
1432 	if (inode) {
1433 		inode->i_mode = mode;
1434 		inode->i_uid = current->fsuid;
1435 		inode->i_gid = current->fsgid;
1436 		inode->i_blocks = 0;
1437 		inode->i_mapping->a_ops = &shmem_aops;
1438 		inode->i_mapping->backing_dev_info = &shmem_backing_dev_info;
1439 		inode->i_atime = inode->i_mtime = inode->i_ctime = CURRENT_TIME;
1440 		inode->i_generation = get_seconds();
1441 		info = SHMEM_I(inode);
1442 		memset(info, 0, (char *)inode - (char *)info);
1443 		spin_lock_init(&info->lock);
1444 		INIT_LIST_HEAD(&info->swaplist);
1445 
1446 		switch (mode & S_IFMT) {
1447 		default:
1448 			inode->i_op = &shmem_special_inode_operations;
1449 			init_special_inode(inode, mode, dev);
1450 			break;
1451 		case S_IFREG:
1452 			inode->i_op = &shmem_inode_operations;
1453 			inode->i_fop = &shmem_file_operations;
1454 			mpol_shared_policy_init(&info->policy, sbinfo->policy,
1455 							&sbinfo->policy_nodes);
1456 			break;
1457 		case S_IFDIR:
1458 			inc_nlink(inode);
1459 			/* Some things misbehave if size == 0 on a directory */
1460 			inode->i_size = 2 * BOGO_DIRENT_SIZE;
1461 			inode->i_op = &shmem_dir_inode_operations;
1462 			inode->i_fop = &simple_dir_operations;
1463 			break;
1464 		case S_IFLNK:
1465 			/*
1466 			 * Must not load anything in the rbtree,
1467 			 * mpol_free_shared_policy will not be called.
1468 			 */
1469 			mpol_shared_policy_init(&info->policy, MPOL_DEFAULT,
1470 						NULL);
1471 			break;
1472 		}
1473 	} else if (sbinfo->max_inodes) {
1474 		spin_lock(&sbinfo->stat_lock);
1475 		sbinfo->free_inodes++;
1476 		spin_unlock(&sbinfo->stat_lock);
1477 	}
1478 	return inode;
1479 }
1480 
1481 #ifdef CONFIG_TMPFS
1482 static const struct inode_operations shmem_symlink_inode_operations;
1483 static const struct inode_operations shmem_symlink_inline_operations;
1484 
1485 /*
1486  * Normally tmpfs makes no use of shmem_prepare_write, but it
1487  * lets a tmpfs file be used read-write below the loop driver.
1488  */
1489 static int
1490 shmem_prepare_write(struct file *file, struct page *page, unsigned offset, unsigned to)
1491 {
1492 	struct inode *inode = page->mapping->host;
1493 	return shmem_getpage(inode, page->index, &page, SGP_WRITE, NULL);
1494 }
1495 
1496 static ssize_t
1497 shmem_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
1498 {
1499 	struct inode	*inode = file->f_path.dentry->d_inode;
1500 	loff_t		pos;
1501 	unsigned long	written;
1502 	ssize_t		err;
1503 
1504 	if ((ssize_t) count < 0)
1505 		return -EINVAL;
1506 
1507 	if (!access_ok(VERIFY_READ, buf, count))
1508 		return -EFAULT;
1509 
1510 	mutex_lock(&inode->i_mutex);
1511 
1512 	pos = *ppos;
1513 	written = 0;
1514 
1515 	err = generic_write_checks(file, &pos, &count, 0);
1516 	if (err || !count)
1517 		goto out;
1518 
1519 	err = remove_suid(file->f_path.dentry);
1520 	if (err)
1521 		goto out;
1522 
1523 	inode->i_ctime = inode->i_mtime = CURRENT_TIME;
1524 
1525 	do {
1526 		struct page *page = NULL;
1527 		unsigned long bytes, index, offset;
1528 		char *kaddr;
1529 		int left;
1530 
1531 		offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
1532 		index = pos >> PAGE_CACHE_SHIFT;
1533 		bytes = PAGE_CACHE_SIZE - offset;
1534 		if (bytes > count)
1535 			bytes = count;
1536 
1537 		/*
1538 		 * We don't hold page lock across copy from user -
1539 		 * what would it guard against? - so no deadlock here.
1540 		 * But it still may be a good idea to prefault below.
1541 		 */
1542 
1543 		err = shmem_getpage(inode, index, &page, SGP_WRITE, NULL);
1544 		if (err)
1545 			break;
1546 
1547 		left = bytes;
1548 		if (PageHighMem(page)) {
1549 			volatile unsigned char dummy;
1550 			__get_user(dummy, buf);
1551 			__get_user(dummy, buf + bytes - 1);
1552 
1553 			kaddr = kmap_atomic(page, KM_USER0);
1554 			left = __copy_from_user_inatomic(kaddr + offset,
1555 							buf, bytes);
1556 			kunmap_atomic(kaddr, KM_USER0);
1557 		}
1558 		if (left) {
1559 			kaddr = kmap(page);
1560 			left = __copy_from_user(kaddr + offset, buf, bytes);
1561 			kunmap(page);
1562 		}
1563 
1564 		written += bytes;
1565 		count -= bytes;
1566 		pos += bytes;
1567 		buf += bytes;
1568 		if (pos > inode->i_size)
1569 			i_size_write(inode, pos);
1570 
1571 		flush_dcache_page(page);
1572 		set_page_dirty(page);
1573 		mark_page_accessed(page);
1574 		page_cache_release(page);
1575 
1576 		if (left) {
1577 			pos -= left;
1578 			written -= left;
1579 			err = -EFAULT;
1580 			break;
1581 		}
1582 
1583 		/*
1584 		 * Our dirty pages are not counted in nr_dirty,
1585 		 * and we do not attempt to balance dirty pages.
1586 		 */
1587 
1588 		cond_resched();
1589 	} while (count);
1590 
1591 	*ppos = pos;
1592 	if (written)
1593 		err = written;
1594 out:
1595 	mutex_unlock(&inode->i_mutex);
1596 	return err;
1597 }
1598 
1599 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
1600 {
1601 	struct inode *inode = filp->f_path.dentry->d_inode;
1602 	struct address_space *mapping = inode->i_mapping;
1603 	unsigned long index, offset;
1604 
1605 	index = *ppos >> PAGE_CACHE_SHIFT;
1606 	offset = *ppos & ~PAGE_CACHE_MASK;
1607 
1608 	for (;;) {
1609 		struct page *page = NULL;
1610 		unsigned long end_index, nr, ret;
1611 		loff_t i_size = i_size_read(inode);
1612 
1613 		end_index = i_size >> PAGE_CACHE_SHIFT;
1614 		if (index > end_index)
1615 			break;
1616 		if (index == end_index) {
1617 			nr = i_size & ~PAGE_CACHE_MASK;
1618 			if (nr <= offset)
1619 				break;
1620 		}
1621 
1622 		desc->error = shmem_getpage(inode, index, &page, SGP_READ, NULL);
1623 		if (desc->error) {
1624 			if (desc->error == -EINVAL)
1625 				desc->error = 0;
1626 			break;
1627 		}
1628 
1629 		/*
1630 		 * We must evaluate after, since reads (unlike writes)
1631 		 * are called without i_mutex protection against truncate
1632 		 */
1633 		nr = PAGE_CACHE_SIZE;
1634 		i_size = i_size_read(inode);
1635 		end_index = i_size >> PAGE_CACHE_SHIFT;
1636 		if (index == end_index) {
1637 			nr = i_size & ~PAGE_CACHE_MASK;
1638 			if (nr <= offset) {
1639 				if (page)
1640 					page_cache_release(page);
1641 				break;
1642 			}
1643 		}
1644 		nr -= offset;
1645 
1646 		if (page) {
1647 			/*
1648 			 * If users can be writing to this page using arbitrary
1649 			 * virtual addresses, take care about potential aliasing
1650 			 * before reading the page on the kernel side.
1651 			 */
1652 			if (mapping_writably_mapped(mapping))
1653 				flush_dcache_page(page);
1654 			/*
1655 			 * Mark the page accessed if we read the beginning.
1656 			 */
1657 			if (!offset)
1658 				mark_page_accessed(page);
1659 		} else {
1660 			page = ZERO_PAGE(0);
1661 			page_cache_get(page);
1662 		}
1663 
1664 		/*
1665 		 * Ok, we have the page, and it's up-to-date, so
1666 		 * now we can copy it to user space...
1667 		 *
1668 		 * The actor routine returns how many bytes were actually used..
1669 		 * NOTE! This may not be the same as how much of a user buffer
1670 		 * we filled up (we may be padding etc), so we can only update
1671 		 * "pos" here (the actor routine has to update the user buffer
1672 		 * pointers and the remaining count).
1673 		 */
1674 		ret = actor(desc, page, offset, nr);
1675 		offset += ret;
1676 		index += offset >> PAGE_CACHE_SHIFT;
1677 		offset &= ~PAGE_CACHE_MASK;
1678 
1679 		page_cache_release(page);
1680 		if (ret != nr || !desc->count)
1681 			break;
1682 
1683 		cond_resched();
1684 	}
1685 
1686 	*ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
1687 	file_accessed(filp);
1688 }
1689 
1690 static ssize_t shmem_file_read(struct file *filp, char __user *buf, size_t count, loff_t *ppos)
1691 {
1692 	read_descriptor_t desc;
1693 
1694 	if ((ssize_t) count < 0)
1695 		return -EINVAL;
1696 	if (!access_ok(VERIFY_WRITE, buf, count))
1697 		return -EFAULT;
1698 	if (!count)
1699 		return 0;
1700 
1701 	desc.written = 0;
1702 	desc.count = count;
1703 	desc.arg.buf = buf;
1704 	desc.error = 0;
1705 
1706 	do_shmem_file_read(filp, ppos, &desc, file_read_actor);
1707 	if (desc.written)
1708 		return desc.written;
1709 	return desc.error;
1710 }
1711 
1712 static ssize_t shmem_file_sendfile(struct file *in_file, loff_t *ppos,
1713 			 size_t count, read_actor_t actor, void *target)
1714 {
1715 	read_descriptor_t desc;
1716 
1717 	if (!count)
1718 		return 0;
1719 
1720 	desc.written = 0;
1721 	desc.count = count;
1722 	desc.arg.data = target;
1723 	desc.error = 0;
1724 
1725 	do_shmem_file_read(in_file, ppos, &desc, actor);
1726 	if (desc.written)
1727 		return desc.written;
1728 	return desc.error;
1729 }
1730 
1731 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
1732 {
1733 	struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
1734 
1735 	buf->f_type = TMPFS_MAGIC;
1736 	buf->f_bsize = PAGE_CACHE_SIZE;
1737 	buf->f_namelen = NAME_MAX;
1738 	spin_lock(&sbinfo->stat_lock);
1739 	if (sbinfo->max_blocks) {
1740 		buf->f_blocks = sbinfo->max_blocks;
1741 		buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
1742 	}
1743 	if (sbinfo->max_inodes) {
1744 		buf->f_files = sbinfo->max_inodes;
1745 		buf->f_ffree = sbinfo->free_inodes;
1746 	}
1747 	/* else leave those fields 0 like simple_statfs */
1748 	spin_unlock(&sbinfo->stat_lock);
1749 	return 0;
1750 }
1751 
1752 /*
1753  * File creation. Allocate an inode, and we're done..
1754  */
1755 static int
1756 shmem_mknod(struct inode *dir, struct dentry *dentry, int mode, dev_t dev)
1757 {
1758 	struct inode *inode = shmem_get_inode(dir->i_sb, mode, dev);
1759 	int error = -ENOSPC;
1760 
1761 	if (inode) {
1762 		error = security_inode_init_security(inode, dir, NULL, NULL,
1763 						     NULL);
1764 		if (error) {
1765 			if (error != -EOPNOTSUPP) {
1766 				iput(inode);
1767 				return error;
1768 			}
1769 		}
1770 		error = shmem_acl_init(inode, dir);
1771 		if (error) {
1772 			iput(inode);
1773 			return error;
1774 		}
1775 		if (dir->i_mode & S_ISGID) {
1776 			inode->i_gid = dir->i_gid;
1777 			if (S_ISDIR(mode))
1778 				inode->i_mode |= S_ISGID;
1779 		}
1780 		dir->i_size += BOGO_DIRENT_SIZE;
1781 		dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1782 		d_instantiate(dentry, inode);
1783 		dget(dentry); /* Extra count - pin the dentry in core */
1784 	}
1785 	return error;
1786 }
1787 
1788 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, int mode)
1789 {
1790 	int error;
1791 
1792 	if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
1793 		return error;
1794 	inc_nlink(dir);
1795 	return 0;
1796 }
1797 
1798 static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
1799 		struct nameidata *nd)
1800 {
1801 	return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
1802 }
1803 
1804 /*
1805  * Link a file..
1806  */
1807 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
1808 {
1809 	struct inode *inode = old_dentry->d_inode;
1810 	struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1811 
1812 	/*
1813 	 * No ordinary (disk based) filesystem counts links as inodes;
1814 	 * but each new link needs a new dentry, pinning lowmem, and
1815 	 * tmpfs dentries cannot be pruned until they are unlinked.
1816 	 */
1817 	if (sbinfo->max_inodes) {
1818 		spin_lock(&sbinfo->stat_lock);
1819 		if (!sbinfo->free_inodes) {
1820 			spin_unlock(&sbinfo->stat_lock);
1821 			return -ENOSPC;
1822 		}
1823 		sbinfo->free_inodes--;
1824 		spin_unlock(&sbinfo->stat_lock);
1825 	}
1826 
1827 	dir->i_size += BOGO_DIRENT_SIZE;
1828 	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1829 	inc_nlink(inode);
1830 	atomic_inc(&inode->i_count);	/* New dentry reference */
1831 	dget(dentry);		/* Extra pinning count for the created dentry */
1832 	d_instantiate(dentry, inode);
1833 	return 0;
1834 }
1835 
1836 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
1837 {
1838 	struct inode *inode = dentry->d_inode;
1839 
1840 	if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) {
1841 		struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1842 		if (sbinfo->max_inodes) {
1843 			spin_lock(&sbinfo->stat_lock);
1844 			sbinfo->free_inodes++;
1845 			spin_unlock(&sbinfo->stat_lock);
1846 		}
1847 	}
1848 
1849 	dir->i_size -= BOGO_DIRENT_SIZE;
1850 	inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1851 	drop_nlink(inode);
1852 	dput(dentry);	/* Undo the count from "create" - this does all the work */
1853 	return 0;
1854 }
1855 
1856 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
1857 {
1858 	if (!simple_empty(dentry))
1859 		return -ENOTEMPTY;
1860 
1861 	drop_nlink(dentry->d_inode);
1862 	drop_nlink(dir);
1863 	return shmem_unlink(dir, dentry);
1864 }
1865 
1866 /*
1867  * The VFS layer already does all the dentry stuff for rename,
1868  * we just have to decrement the usage count for the target if
1869  * it exists so that the VFS layer correctly free's it when it
1870  * gets overwritten.
1871  */
1872 static int shmem_rename(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
1873 {
1874 	struct inode *inode = old_dentry->d_inode;
1875 	int they_are_dirs = S_ISDIR(inode->i_mode);
1876 
1877 	if (!simple_empty(new_dentry))
1878 		return -ENOTEMPTY;
1879 
1880 	if (new_dentry->d_inode) {
1881 		(void) shmem_unlink(new_dir, new_dentry);
1882 		if (they_are_dirs)
1883 			drop_nlink(old_dir);
1884 	} else if (they_are_dirs) {
1885 		drop_nlink(old_dir);
1886 		inc_nlink(new_dir);
1887 	}
1888 
1889 	old_dir->i_size -= BOGO_DIRENT_SIZE;
1890 	new_dir->i_size += BOGO_DIRENT_SIZE;
1891 	old_dir->i_ctime = old_dir->i_mtime =
1892 	new_dir->i_ctime = new_dir->i_mtime =
1893 	inode->i_ctime = CURRENT_TIME;
1894 	return 0;
1895 }
1896 
1897 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
1898 {
1899 	int error;
1900 	int len;
1901 	struct inode *inode;
1902 	struct page *page = NULL;
1903 	char *kaddr;
1904 	struct shmem_inode_info *info;
1905 
1906 	len = strlen(symname) + 1;
1907 	if (len > PAGE_CACHE_SIZE)
1908 		return -ENAMETOOLONG;
1909 
1910 	inode = shmem_get_inode(dir->i_sb, S_IFLNK|S_IRWXUGO, 0);
1911 	if (!inode)
1912 		return -ENOSPC;
1913 
1914 	error = security_inode_init_security(inode, dir, NULL, NULL,
1915 					     NULL);
1916 	if (error) {
1917 		if (error != -EOPNOTSUPP) {
1918 			iput(inode);
1919 			return error;
1920 		}
1921 		error = 0;
1922 	}
1923 
1924 	info = SHMEM_I(inode);
1925 	inode->i_size = len-1;
1926 	if (len <= (char *)inode - (char *)info) {
1927 		/* do it inline */
1928 		memcpy(info, symname, len);
1929 		inode->i_op = &shmem_symlink_inline_operations;
1930 	} else {
1931 		error = shmem_getpage(inode, 0, &page, SGP_WRITE, NULL);
1932 		if (error) {
1933 			iput(inode);
1934 			return error;
1935 		}
1936 		inode->i_op = &shmem_symlink_inode_operations;
1937 		kaddr = kmap_atomic(page, KM_USER0);
1938 		memcpy(kaddr, symname, len);
1939 		kunmap_atomic(kaddr, KM_USER0);
1940 		set_page_dirty(page);
1941 		page_cache_release(page);
1942 	}
1943 	if (dir->i_mode & S_ISGID)
1944 		inode->i_gid = dir->i_gid;
1945 	dir->i_size += BOGO_DIRENT_SIZE;
1946 	dir->i_ctime = dir->i_mtime = CURRENT_TIME;
1947 	d_instantiate(dentry, inode);
1948 	dget(dentry);
1949 	return 0;
1950 }
1951 
1952 static void *shmem_follow_link_inline(struct dentry *dentry, struct nameidata *nd)
1953 {
1954 	nd_set_link(nd, (char *)SHMEM_I(dentry->d_inode));
1955 	return NULL;
1956 }
1957 
1958 static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
1959 {
1960 	struct page *page = NULL;
1961 	int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
1962 	nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
1963 	return page;
1964 }
1965 
1966 static void shmem_put_link(struct dentry *dentry, struct nameidata *nd, void *cookie)
1967 {
1968 	if (!IS_ERR(nd_get_link(nd))) {
1969 		struct page *page = cookie;
1970 		kunmap(page);
1971 		mark_page_accessed(page);
1972 		page_cache_release(page);
1973 	}
1974 }
1975 
1976 static const struct inode_operations shmem_symlink_inline_operations = {
1977 	.readlink	= generic_readlink,
1978 	.follow_link	= shmem_follow_link_inline,
1979 };
1980 
1981 static const struct inode_operations shmem_symlink_inode_operations = {
1982 	.truncate	= shmem_truncate,
1983 	.readlink	= generic_readlink,
1984 	.follow_link	= shmem_follow_link,
1985 	.put_link	= shmem_put_link,
1986 };
1987 
1988 #ifdef CONFIG_TMPFS_POSIX_ACL
1989 /**
1990  * Superblocks without xattr inode operations will get security.* xattr
1991  * support from the VFS "for free". As soon as we have any other xattrs
1992  * like ACLs, we also need to implement the security.* handlers at
1993  * filesystem level, though.
1994  */
1995 
1996 static size_t shmem_xattr_security_list(struct inode *inode, char *list,
1997 					size_t list_len, const char *name,
1998 					size_t name_len)
1999 {
2000 	return security_inode_listsecurity(inode, list, list_len);
2001 }
2002 
2003 static int shmem_xattr_security_get(struct inode *inode, const char *name,
2004 				    void *buffer, size_t size)
2005 {
2006 	if (strcmp(name, "") == 0)
2007 		return -EINVAL;
2008 	return security_inode_getsecurity(inode, name, buffer, size,
2009 					  -EOPNOTSUPP);
2010 }
2011 
2012 static int shmem_xattr_security_set(struct inode *inode, const char *name,
2013 				    const void *value, size_t size, int flags)
2014 {
2015 	if (strcmp(name, "") == 0)
2016 		return -EINVAL;
2017 	return security_inode_setsecurity(inode, name, value, size, flags);
2018 }
2019 
2020 static struct xattr_handler shmem_xattr_security_handler = {
2021 	.prefix = XATTR_SECURITY_PREFIX,
2022 	.list   = shmem_xattr_security_list,
2023 	.get    = shmem_xattr_security_get,
2024 	.set    = shmem_xattr_security_set,
2025 };
2026 
2027 static struct xattr_handler *shmem_xattr_handlers[] = {
2028 	&shmem_xattr_acl_access_handler,
2029 	&shmem_xattr_acl_default_handler,
2030 	&shmem_xattr_security_handler,
2031 	NULL
2032 };
2033 #endif
2034 
2035 static struct dentry *shmem_get_parent(struct dentry *child)
2036 {
2037 	return ERR_PTR(-ESTALE);
2038 }
2039 
2040 static int shmem_match(struct inode *ino, void *vfh)
2041 {
2042 	__u32 *fh = vfh;
2043 	__u64 inum = fh[2];
2044 	inum = (inum << 32) | fh[1];
2045 	return ino->i_ino == inum && fh[0] == ino->i_generation;
2046 }
2047 
2048 static struct dentry *shmem_get_dentry(struct super_block *sb, void *vfh)
2049 {
2050 	struct dentry *de = NULL;
2051 	struct inode *inode;
2052 	__u32 *fh = vfh;
2053 	__u64 inum = fh[2];
2054 	inum = (inum << 32) | fh[1];
2055 
2056 	inode = ilookup5(sb, (unsigned long)(inum+fh[0]), shmem_match, vfh);
2057 	if (inode) {
2058 		de = d_find_alias(inode);
2059 		iput(inode);
2060 	}
2061 
2062 	return de? de: ERR_PTR(-ESTALE);
2063 }
2064 
2065 static struct dentry *shmem_decode_fh(struct super_block *sb, __u32 *fh,
2066 		int len, int type,
2067 		int (*acceptable)(void *context, struct dentry *de),
2068 		void *context)
2069 {
2070 	if (len < 3)
2071 		return ERR_PTR(-ESTALE);
2072 
2073 	return sb->s_export_op->find_exported_dentry(sb, fh, NULL, acceptable,
2074 							context);
2075 }
2076 
2077 static int shmem_encode_fh(struct dentry *dentry, __u32 *fh, int *len,
2078 				int connectable)
2079 {
2080 	struct inode *inode = dentry->d_inode;
2081 
2082 	if (*len < 3)
2083 		return 255;
2084 
2085 	if (hlist_unhashed(&inode->i_hash)) {
2086 		/* Unfortunately insert_inode_hash is not idempotent,
2087 		 * so as we hash inodes here rather than at creation
2088 		 * time, we need a lock to ensure we only try
2089 		 * to do it once
2090 		 */
2091 		static DEFINE_SPINLOCK(lock);
2092 		spin_lock(&lock);
2093 		if (hlist_unhashed(&inode->i_hash))
2094 			__insert_inode_hash(inode,
2095 					    inode->i_ino + inode->i_generation);
2096 		spin_unlock(&lock);
2097 	}
2098 
2099 	fh[0] = inode->i_generation;
2100 	fh[1] = inode->i_ino;
2101 	fh[2] = ((__u64)inode->i_ino) >> 32;
2102 
2103 	*len = 3;
2104 	return 1;
2105 }
2106 
2107 static struct export_operations shmem_export_ops = {
2108 	.get_parent     = shmem_get_parent,
2109 	.get_dentry     = shmem_get_dentry,
2110 	.encode_fh      = shmem_encode_fh,
2111 	.decode_fh      = shmem_decode_fh,
2112 };
2113 
2114 static int shmem_parse_options(char *options, int *mode, uid_t *uid,
2115 	gid_t *gid, unsigned long *blocks, unsigned long *inodes,
2116 	int *policy, nodemask_t *policy_nodes)
2117 {
2118 	char *this_char, *value, *rest;
2119 
2120 	while (options != NULL) {
2121 		this_char = options;
2122 		for (;;) {
2123 			/*
2124 			 * NUL-terminate this option: unfortunately,
2125 			 * mount options form a comma-separated list,
2126 			 * but mpol's nodelist may also contain commas.
2127 			 */
2128 			options = strchr(options, ',');
2129 			if (options == NULL)
2130 				break;
2131 			options++;
2132 			if (!isdigit(*options)) {
2133 				options[-1] = '\0';
2134 				break;
2135 			}
2136 		}
2137 		if (!*this_char)
2138 			continue;
2139 		if ((value = strchr(this_char,'=')) != NULL) {
2140 			*value++ = 0;
2141 		} else {
2142 			printk(KERN_ERR
2143 			    "tmpfs: No value for mount option '%s'\n",
2144 			    this_char);
2145 			return 1;
2146 		}
2147 
2148 		if (!strcmp(this_char,"size")) {
2149 			unsigned long long size;
2150 			size = memparse(value,&rest);
2151 			if (*rest == '%') {
2152 				size <<= PAGE_SHIFT;
2153 				size *= totalram_pages;
2154 				do_div(size, 100);
2155 				rest++;
2156 			}
2157 			if (*rest)
2158 				goto bad_val;
2159 			*blocks = size >> PAGE_CACHE_SHIFT;
2160 		} else if (!strcmp(this_char,"nr_blocks")) {
2161 			*blocks = memparse(value,&rest);
2162 			if (*rest)
2163 				goto bad_val;
2164 		} else if (!strcmp(this_char,"nr_inodes")) {
2165 			*inodes = memparse(value,&rest);
2166 			if (*rest)
2167 				goto bad_val;
2168 		} else if (!strcmp(this_char,"mode")) {
2169 			if (!mode)
2170 				continue;
2171 			*mode = simple_strtoul(value,&rest,8);
2172 			if (*rest)
2173 				goto bad_val;
2174 		} else if (!strcmp(this_char,"uid")) {
2175 			if (!uid)
2176 				continue;
2177 			*uid = simple_strtoul(value,&rest,0);
2178 			if (*rest)
2179 				goto bad_val;
2180 		} else if (!strcmp(this_char,"gid")) {
2181 			if (!gid)
2182 				continue;
2183 			*gid = simple_strtoul(value,&rest,0);
2184 			if (*rest)
2185 				goto bad_val;
2186 		} else if (!strcmp(this_char,"mpol")) {
2187 			if (shmem_parse_mpol(value,policy,policy_nodes))
2188 				goto bad_val;
2189 		} else {
2190 			printk(KERN_ERR "tmpfs: Bad mount option %s\n",
2191 			       this_char);
2192 			return 1;
2193 		}
2194 	}
2195 	return 0;
2196 
2197 bad_val:
2198 	printk(KERN_ERR "tmpfs: Bad value '%s' for mount option '%s'\n",
2199 	       value, this_char);
2200 	return 1;
2201 
2202 }
2203 
2204 static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
2205 {
2206 	struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2207 	unsigned long max_blocks = sbinfo->max_blocks;
2208 	unsigned long max_inodes = sbinfo->max_inodes;
2209 	int policy = sbinfo->policy;
2210 	nodemask_t policy_nodes = sbinfo->policy_nodes;
2211 	unsigned long blocks;
2212 	unsigned long inodes;
2213 	int error = -EINVAL;
2214 
2215 	if (shmem_parse_options(data, NULL, NULL, NULL, &max_blocks,
2216 				&max_inodes, &policy, &policy_nodes))
2217 		return error;
2218 
2219 	spin_lock(&sbinfo->stat_lock);
2220 	blocks = sbinfo->max_blocks - sbinfo->free_blocks;
2221 	inodes = sbinfo->max_inodes - sbinfo->free_inodes;
2222 	if (max_blocks < blocks)
2223 		goto out;
2224 	if (max_inodes < inodes)
2225 		goto out;
2226 	/*
2227 	 * Those tests also disallow limited->unlimited while any are in
2228 	 * use, so i_blocks will always be zero when max_blocks is zero;
2229 	 * but we must separately disallow unlimited->limited, because
2230 	 * in that case we have no record of how much is already in use.
2231 	 */
2232 	if (max_blocks && !sbinfo->max_blocks)
2233 		goto out;
2234 	if (max_inodes && !sbinfo->max_inodes)
2235 		goto out;
2236 
2237 	error = 0;
2238 	sbinfo->max_blocks  = max_blocks;
2239 	sbinfo->free_blocks = max_blocks - blocks;
2240 	sbinfo->max_inodes  = max_inodes;
2241 	sbinfo->free_inodes = max_inodes - inodes;
2242 	sbinfo->policy = policy;
2243 	sbinfo->policy_nodes = policy_nodes;
2244 out:
2245 	spin_unlock(&sbinfo->stat_lock);
2246 	return error;
2247 }
2248 #endif
2249 
2250 static void shmem_put_super(struct super_block *sb)
2251 {
2252 	kfree(sb->s_fs_info);
2253 	sb->s_fs_info = NULL;
2254 }
2255 
2256 static int shmem_fill_super(struct super_block *sb,
2257 			    void *data, int silent)
2258 {
2259 	struct inode *inode;
2260 	struct dentry *root;
2261 	int mode   = S_IRWXUGO | S_ISVTX;
2262 	uid_t uid = current->fsuid;
2263 	gid_t gid = current->fsgid;
2264 	int err = -ENOMEM;
2265 	struct shmem_sb_info *sbinfo;
2266 	unsigned long blocks = 0;
2267 	unsigned long inodes = 0;
2268 	int policy = MPOL_DEFAULT;
2269 	nodemask_t policy_nodes = node_online_map;
2270 
2271 #ifdef CONFIG_TMPFS
2272 	/*
2273 	 * Per default we only allow half of the physical ram per
2274 	 * tmpfs instance, limiting inodes to one per page of lowmem;
2275 	 * but the internal instance is left unlimited.
2276 	 */
2277 	if (!(sb->s_flags & MS_NOUSER)) {
2278 		blocks = totalram_pages / 2;
2279 		inodes = totalram_pages - totalhigh_pages;
2280 		if (inodes > blocks)
2281 			inodes = blocks;
2282 		if (shmem_parse_options(data, &mode, &uid, &gid, &blocks,
2283 					&inodes, &policy, &policy_nodes))
2284 			return -EINVAL;
2285 	}
2286 	sb->s_export_op = &shmem_export_ops;
2287 #else
2288 	sb->s_flags |= MS_NOUSER;
2289 #endif
2290 
2291 	/* Round up to L1_CACHE_BYTES to resist false sharing */
2292 	sbinfo = kmalloc(max((int)sizeof(struct shmem_sb_info),
2293 				L1_CACHE_BYTES), GFP_KERNEL);
2294 	if (!sbinfo)
2295 		return -ENOMEM;
2296 
2297 	spin_lock_init(&sbinfo->stat_lock);
2298 	sbinfo->max_blocks = blocks;
2299 	sbinfo->free_blocks = blocks;
2300 	sbinfo->max_inodes = inodes;
2301 	sbinfo->free_inodes = inodes;
2302 	sbinfo->policy = policy;
2303 	sbinfo->policy_nodes = policy_nodes;
2304 
2305 	sb->s_fs_info = sbinfo;
2306 	sb->s_maxbytes = SHMEM_MAX_BYTES;
2307 	sb->s_blocksize = PAGE_CACHE_SIZE;
2308 	sb->s_blocksize_bits = PAGE_CACHE_SHIFT;
2309 	sb->s_magic = TMPFS_MAGIC;
2310 	sb->s_op = &shmem_ops;
2311 	sb->s_time_gran = 1;
2312 #ifdef CONFIG_TMPFS_POSIX_ACL
2313 	sb->s_xattr = shmem_xattr_handlers;
2314 	sb->s_flags |= MS_POSIXACL;
2315 #endif
2316 
2317 	inode = shmem_get_inode(sb, S_IFDIR | mode, 0);
2318 	if (!inode)
2319 		goto failed;
2320 	inode->i_uid = uid;
2321 	inode->i_gid = gid;
2322 	root = d_alloc_root(inode);
2323 	if (!root)
2324 		goto failed_iput;
2325 	sb->s_root = root;
2326 	return 0;
2327 
2328 failed_iput:
2329 	iput(inode);
2330 failed:
2331 	shmem_put_super(sb);
2332 	return err;
2333 }
2334 
2335 static struct kmem_cache *shmem_inode_cachep;
2336 
2337 static struct inode *shmem_alloc_inode(struct super_block *sb)
2338 {
2339 	struct shmem_inode_info *p;
2340 	p = (struct shmem_inode_info *)kmem_cache_alloc(shmem_inode_cachep, GFP_KERNEL);
2341 	if (!p)
2342 		return NULL;
2343 	return &p->vfs_inode;
2344 }
2345 
2346 static void shmem_destroy_inode(struct inode *inode)
2347 {
2348 	if ((inode->i_mode & S_IFMT) == S_IFREG) {
2349 		/* only struct inode is valid if it's an inline symlink */
2350 		mpol_free_shared_policy(&SHMEM_I(inode)->policy);
2351 	}
2352 	shmem_acl_destroy_inode(inode);
2353 	kmem_cache_free(shmem_inode_cachep, SHMEM_I(inode));
2354 }
2355 
2356 static void init_once(void *foo, struct kmem_cache *cachep,
2357 		      unsigned long flags)
2358 {
2359 	struct shmem_inode_info *p = (struct shmem_inode_info *) foo;
2360 
2361 	if (flags & SLAB_CTOR_CONSTRUCTOR) {
2362 		inode_init_once(&p->vfs_inode);
2363 #ifdef CONFIG_TMPFS_POSIX_ACL
2364 		p->i_acl = NULL;
2365 		p->i_default_acl = NULL;
2366 #endif
2367 	}
2368 }
2369 
2370 static int init_inodecache(void)
2371 {
2372 	shmem_inode_cachep = kmem_cache_create("shmem_inode_cache",
2373 				sizeof(struct shmem_inode_info),
2374 				0, 0, init_once, NULL);
2375 	if (shmem_inode_cachep == NULL)
2376 		return -ENOMEM;
2377 	return 0;
2378 }
2379 
2380 static void destroy_inodecache(void)
2381 {
2382 	kmem_cache_destroy(shmem_inode_cachep);
2383 }
2384 
2385 static const struct address_space_operations shmem_aops = {
2386 	.writepage	= shmem_writepage,
2387 	.set_page_dirty	= __set_page_dirty_no_writeback,
2388 #ifdef CONFIG_TMPFS
2389 	.prepare_write	= shmem_prepare_write,
2390 	.commit_write	= simple_commit_write,
2391 #endif
2392 	.migratepage	= migrate_page,
2393 };
2394 
2395 static const struct file_operations shmem_file_operations = {
2396 	.mmap		= shmem_mmap,
2397 #ifdef CONFIG_TMPFS
2398 	.llseek		= generic_file_llseek,
2399 	.read		= shmem_file_read,
2400 	.write		= shmem_file_write,
2401 	.fsync		= simple_sync_file,
2402 	.sendfile	= shmem_file_sendfile,
2403 #endif
2404 };
2405 
2406 static const struct inode_operations shmem_inode_operations = {
2407 	.truncate	= shmem_truncate,
2408 	.setattr	= shmem_notify_change,
2409 	.truncate_range	= shmem_truncate_range,
2410 #ifdef CONFIG_TMPFS_POSIX_ACL
2411 	.setxattr	= generic_setxattr,
2412 	.getxattr	= generic_getxattr,
2413 	.listxattr	= generic_listxattr,
2414 	.removexattr	= generic_removexattr,
2415 	.permission	= shmem_permission,
2416 #endif
2417 
2418 };
2419 
2420 static const struct inode_operations shmem_dir_inode_operations = {
2421 #ifdef CONFIG_TMPFS
2422 	.create		= shmem_create,
2423 	.lookup		= simple_lookup,
2424 	.link		= shmem_link,
2425 	.unlink		= shmem_unlink,
2426 	.symlink	= shmem_symlink,
2427 	.mkdir		= shmem_mkdir,
2428 	.rmdir		= shmem_rmdir,
2429 	.mknod		= shmem_mknod,
2430 	.rename		= shmem_rename,
2431 #endif
2432 #ifdef CONFIG_TMPFS_POSIX_ACL
2433 	.setattr	= shmem_notify_change,
2434 	.setxattr	= generic_setxattr,
2435 	.getxattr	= generic_getxattr,
2436 	.listxattr	= generic_listxattr,
2437 	.removexattr	= generic_removexattr,
2438 	.permission	= shmem_permission,
2439 #endif
2440 };
2441 
2442 static const struct inode_operations shmem_special_inode_operations = {
2443 #ifdef CONFIG_TMPFS_POSIX_ACL
2444 	.setattr	= shmem_notify_change,
2445 	.setxattr	= generic_setxattr,
2446 	.getxattr	= generic_getxattr,
2447 	.listxattr	= generic_listxattr,
2448 	.removexattr	= generic_removexattr,
2449 	.permission	= shmem_permission,
2450 #endif
2451 };
2452 
2453 static const struct super_operations shmem_ops = {
2454 	.alloc_inode	= shmem_alloc_inode,
2455 	.destroy_inode	= shmem_destroy_inode,
2456 #ifdef CONFIG_TMPFS
2457 	.statfs		= shmem_statfs,
2458 	.remount_fs	= shmem_remount_fs,
2459 #endif
2460 	.delete_inode	= shmem_delete_inode,
2461 	.drop_inode	= generic_delete_inode,
2462 	.put_super	= shmem_put_super,
2463 };
2464 
2465 static struct vm_operations_struct shmem_vm_ops = {
2466 	.nopage		= shmem_nopage,
2467 	.populate	= shmem_populate,
2468 #ifdef CONFIG_NUMA
2469 	.set_policy     = shmem_set_policy,
2470 	.get_policy     = shmem_get_policy,
2471 #endif
2472 };
2473 
2474 
2475 static int shmem_get_sb(struct file_system_type *fs_type,
2476 	int flags, const char *dev_name, void *data, struct vfsmount *mnt)
2477 {
2478 	return get_sb_nodev(fs_type, flags, data, shmem_fill_super, mnt);
2479 }
2480 
2481 static struct file_system_type tmpfs_fs_type = {
2482 	.owner		= THIS_MODULE,
2483 	.name		= "tmpfs",
2484 	.get_sb		= shmem_get_sb,
2485 	.kill_sb	= kill_litter_super,
2486 };
2487 static struct vfsmount *shm_mnt;
2488 
2489 static int __init init_tmpfs(void)
2490 {
2491 	int error;
2492 
2493 	error = init_inodecache();
2494 	if (error)
2495 		goto out3;
2496 
2497 	error = register_filesystem(&tmpfs_fs_type);
2498 	if (error) {
2499 		printk(KERN_ERR "Could not register tmpfs\n");
2500 		goto out2;
2501 	}
2502 
2503 	shm_mnt = vfs_kern_mount(&tmpfs_fs_type, MS_NOUSER,
2504 				tmpfs_fs_type.name, NULL);
2505 	if (IS_ERR(shm_mnt)) {
2506 		error = PTR_ERR(shm_mnt);
2507 		printk(KERN_ERR "Could not kern_mount tmpfs\n");
2508 		goto out1;
2509 	}
2510 	return 0;
2511 
2512 out1:
2513 	unregister_filesystem(&tmpfs_fs_type);
2514 out2:
2515 	destroy_inodecache();
2516 out3:
2517 	shm_mnt = ERR_PTR(error);
2518 	return error;
2519 }
2520 module_init(init_tmpfs)
2521 
2522 /*
2523  * shmem_file_setup - get an unlinked file living in tmpfs
2524  *
2525  * @name: name for dentry (to be seen in /proc/<pid>/maps
2526  * @size: size to be set for the file
2527  *
2528  */
2529 struct file *shmem_file_setup(char *name, loff_t size, unsigned long flags)
2530 {
2531 	int error;
2532 	struct file *file;
2533 	struct inode *inode;
2534 	struct dentry *dentry, *root;
2535 	struct qstr this;
2536 
2537 	if (IS_ERR(shm_mnt))
2538 		return (void *)shm_mnt;
2539 
2540 	if (size < 0 || size > SHMEM_MAX_BYTES)
2541 		return ERR_PTR(-EINVAL);
2542 
2543 	if (shmem_acct_size(flags, size))
2544 		return ERR_PTR(-ENOMEM);
2545 
2546 	error = -ENOMEM;
2547 	this.name = name;
2548 	this.len = strlen(name);
2549 	this.hash = 0; /* will go */
2550 	root = shm_mnt->mnt_root;
2551 	dentry = d_alloc(root, &this);
2552 	if (!dentry)
2553 		goto put_memory;
2554 
2555 	error = -ENFILE;
2556 	file = get_empty_filp();
2557 	if (!file)
2558 		goto put_dentry;
2559 
2560 	error = -ENOSPC;
2561 	inode = shmem_get_inode(root->d_sb, S_IFREG | S_IRWXUGO, 0);
2562 	if (!inode)
2563 		goto close_file;
2564 
2565 	SHMEM_I(inode)->flags = flags & VM_ACCOUNT;
2566 	d_instantiate(dentry, inode);
2567 	inode->i_size = size;
2568 	inode->i_nlink = 0;	/* It is unlinked */
2569 	file->f_path.mnt = mntget(shm_mnt);
2570 	file->f_path.dentry = dentry;
2571 	file->f_mapping = inode->i_mapping;
2572 	file->f_op = &shmem_file_operations;
2573 	file->f_mode = FMODE_WRITE | FMODE_READ;
2574 	return file;
2575 
2576 close_file:
2577 	put_filp(file);
2578 put_dentry:
2579 	dput(dentry);
2580 put_memory:
2581 	shmem_unacct_size(flags, size);
2582 	return ERR_PTR(error);
2583 }
2584 
2585 /*
2586  * shmem_zero_setup - setup a shared anonymous mapping
2587  *
2588  * @vma: the vma to be mmapped is prepared by do_mmap_pgoff
2589  */
2590 int shmem_zero_setup(struct vm_area_struct *vma)
2591 {
2592 	struct file *file;
2593 	loff_t size = vma->vm_end - vma->vm_start;
2594 
2595 	file = shmem_file_setup("dev/zero", size, vma->vm_flags);
2596 	if (IS_ERR(file))
2597 		return PTR_ERR(file);
2598 
2599 	if (vma->vm_file)
2600 		fput(vma->vm_file);
2601 	vma->vm_file = file;
2602 	vma->vm_ops = &shmem_vm_ops;
2603 	return 0;
2604 }
2605