xref: /openbmc/linux/fs/namei.c (revision adb57164)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *  linux/fs/namei.c
4  *
5  *  Copyright (C) 1991, 1992  Linus Torvalds
6  */
7 
8 /*
9  * Some corrections by tytso.
10  */
11 
12 /* [Feb 1997 T. Schoebel-Theuer] Complete rewrite of the pathname
13  * lookup logic.
14  */
15 /* [Feb-Apr 2000, AV] Rewrite to the new namespace architecture.
16  */
17 
18 #include <linux/init.h>
19 #include <linux/export.h>
20 #include <linux/kernel.h>
21 #include <linux/slab.h>
22 #include <linux/fs.h>
23 #include <linux/namei.h>
24 #include <linux/pagemap.h>
25 #include <linux/fsnotify.h>
26 #include <linux/personality.h>
27 #include <linux/security.h>
28 #include <linux/ima.h>
29 #include <linux/syscalls.h>
30 #include <linux/mount.h>
31 #include <linux/audit.h>
32 #include <linux/capability.h>
33 #include <linux/file.h>
34 #include <linux/fcntl.h>
35 #include <linux/device_cgroup.h>
36 #include <linux/fs_struct.h>
37 #include <linux/posix_acl.h>
38 #include <linux/hash.h>
39 #include <linux/bitops.h>
40 #include <linux/init_task.h>
41 #include <linux/uaccess.h>
42 
43 #include "internal.h"
44 #include "mount.h"
45 
46 /* [Feb-1997 T. Schoebel-Theuer]
47  * Fundamental changes in the pathname lookup mechanisms (namei)
48  * were necessary because of omirr.  The reason is that omirr needs
49  * to know the _real_ pathname, not the user-supplied one, in case
50  * of symlinks (and also when transname replacements occur).
51  *
52  * The new code replaces the old recursive symlink resolution with
53  * an iterative one (in case of non-nested symlink chains).  It does
54  * this with calls to <fs>_follow_link().
55  * As a side effect, dir_namei(), _namei() and follow_link() are now
56  * replaced with a single function lookup_dentry() that can handle all
57  * the special cases of the former code.
58  *
59  * With the new dcache, the pathname is stored at each inode, at least as
60  * long as the refcount of the inode is positive.  As a side effect, the
61  * size of the dcache depends on the inode cache and thus is dynamic.
62  *
63  * [29-Apr-1998 C. Scott Ananian] Updated above description of symlink
64  * resolution to correspond with current state of the code.
65  *
66  * Note that the symlink resolution is not *completely* iterative.
67  * There is still a significant amount of tail- and mid- recursion in
68  * the algorithm.  Also, note that <fs>_readlink() is not used in
69  * lookup_dentry(): lookup_dentry() on the result of <fs>_readlink()
70  * may return different results than <fs>_follow_link().  Many virtual
71  * filesystems (including /proc) exhibit this behavior.
72  */
73 
74 /* [24-Feb-97 T. Schoebel-Theuer] Side effects caused by new implementation:
75  * New symlink semantics: when open() is called with flags O_CREAT | O_EXCL
76  * and the name already exists in form of a symlink, try to create the new
77  * name indicated by the symlink. The old code always complained that the
78  * name already exists, due to not following the symlink even if its target
79  * is nonexistent.  The new semantics affects also mknod() and link() when
80  * the name is a symlink pointing to a non-existent name.
81  *
82  * I don't know which semantics is the right one, since I have no access
83  * to standards. But I found by trial that HP-UX 9.0 has the full "new"
84  * semantics implemented, while SunOS 4.1.1 and Solaris (SunOS 5.4) have the
85  * "old" one. Personally, I think the new semantics is much more logical.
86  * Note that "ln old new" where "new" is a symlink pointing to a non-existing
87  * file does succeed in both HP-UX and SunOs, but not in Solaris
88  * and in the old Linux semantics.
89  */
90 
91 /* [16-Dec-97 Kevin Buhr] For security reasons, we change some symlink
92  * semantics.  See the comments in "open_namei" and "do_link" below.
93  *
94  * [10-Sep-98 Alan Modra] Another symlink change.
95  */
96 
97 /* [Feb-Apr 2000 AV] Complete rewrite. Rules for symlinks:
98  *	inside the path - always follow.
99  *	in the last component in creation/removal/renaming - never follow.
100  *	if LOOKUP_FOLLOW passed - follow.
101  *	if the pathname has trailing slashes - follow.
102  *	otherwise - don't follow.
103  * (applied in that order).
104  *
105  * [Jun 2000 AV] Inconsistent behaviour of open() in case if flags==O_CREAT
106  * restored for 2.4. This is the last surviving part of old 4.2BSD bug.
107  * During the 2.4 we need to fix the userland stuff depending on it -
108  * hopefully we will be able to get rid of that wart in 2.5. So far only
109  * XEmacs seems to be relying on it...
110  */
111 /*
112  * [Sep 2001 AV] Single-semaphore locking scheme (kudos to David Holland)
113  * implemented.  Let's see if raised priority of ->s_vfs_rename_mutex gives
114  * any extra contention...
115  */
116 
117 /* In order to reduce some races, while at the same time doing additional
118  * checking and hopefully speeding things up, we copy filenames to the
119  * kernel data space before using them..
120  *
121  * POSIX.1 2.4: an empty pathname is invalid (ENOENT).
122  * PATH_MAX includes the nul terminator --RR.
123  */
124 
125 #define EMBEDDED_NAME_MAX	(PATH_MAX - offsetof(struct filename, iname))
126 
127 struct filename *
128 getname_flags(const char __user *filename, int flags, int *empty)
129 {
130 	struct filename *result;
131 	char *kname;
132 	int len;
133 
134 	result = audit_reusename(filename);
135 	if (result)
136 		return result;
137 
138 	result = __getname();
139 	if (unlikely(!result))
140 		return ERR_PTR(-ENOMEM);
141 
142 	/*
143 	 * First, try to embed the struct filename inside the names_cache
144 	 * allocation
145 	 */
146 	kname = (char *)result->iname;
147 	result->name = kname;
148 
149 	len = strncpy_from_user(kname, filename, EMBEDDED_NAME_MAX);
150 	if (unlikely(len < 0)) {
151 		__putname(result);
152 		return ERR_PTR(len);
153 	}
154 
155 	/*
156 	 * Uh-oh. We have a name that's approaching PATH_MAX. Allocate a
157 	 * separate struct filename so we can dedicate the entire
158 	 * names_cache allocation for the pathname, and re-do the copy from
159 	 * userland.
160 	 */
161 	if (unlikely(len == EMBEDDED_NAME_MAX)) {
162 		const size_t size = offsetof(struct filename, iname[1]);
163 		kname = (char *)result;
164 
165 		/*
166 		 * size is chosen that way we to guarantee that
167 		 * result->iname[0] is within the same object and that
168 		 * kname can't be equal to result->iname, no matter what.
169 		 */
170 		result = kzalloc(size, GFP_KERNEL);
171 		if (unlikely(!result)) {
172 			__putname(kname);
173 			return ERR_PTR(-ENOMEM);
174 		}
175 		result->name = kname;
176 		len = strncpy_from_user(kname, filename, PATH_MAX);
177 		if (unlikely(len < 0)) {
178 			__putname(kname);
179 			kfree(result);
180 			return ERR_PTR(len);
181 		}
182 		if (unlikely(len == PATH_MAX)) {
183 			__putname(kname);
184 			kfree(result);
185 			return ERR_PTR(-ENAMETOOLONG);
186 		}
187 	}
188 
189 	result->refcnt = 1;
190 	/* The empty path is special. */
191 	if (unlikely(!len)) {
192 		if (empty)
193 			*empty = 1;
194 		if (!(flags & LOOKUP_EMPTY)) {
195 			putname(result);
196 			return ERR_PTR(-ENOENT);
197 		}
198 	}
199 
200 	result->uptr = filename;
201 	result->aname = NULL;
202 	audit_getname(result);
203 	return result;
204 }
205 
206 struct filename *
207 getname(const char __user * filename)
208 {
209 	return getname_flags(filename, 0, NULL);
210 }
211 
212 struct filename *
213 getname_kernel(const char * filename)
214 {
215 	struct filename *result;
216 	int len = strlen(filename) + 1;
217 
218 	result = __getname();
219 	if (unlikely(!result))
220 		return ERR_PTR(-ENOMEM);
221 
222 	if (len <= EMBEDDED_NAME_MAX) {
223 		result->name = (char *)result->iname;
224 	} else if (len <= PATH_MAX) {
225 		const size_t size = offsetof(struct filename, iname[1]);
226 		struct filename *tmp;
227 
228 		tmp = kmalloc(size, GFP_KERNEL);
229 		if (unlikely(!tmp)) {
230 			__putname(result);
231 			return ERR_PTR(-ENOMEM);
232 		}
233 		tmp->name = (char *)result;
234 		result = tmp;
235 	} else {
236 		__putname(result);
237 		return ERR_PTR(-ENAMETOOLONG);
238 	}
239 	memcpy((char *)result->name, filename, len);
240 	result->uptr = NULL;
241 	result->aname = NULL;
242 	result->refcnt = 1;
243 	audit_getname(result);
244 
245 	return result;
246 }
247 
248 void putname(struct filename *name)
249 {
250 	BUG_ON(name->refcnt <= 0);
251 
252 	if (--name->refcnt > 0)
253 		return;
254 
255 	if (name->name != name->iname) {
256 		__putname(name->name);
257 		kfree(name);
258 	} else
259 		__putname(name);
260 }
261 
262 static int check_acl(struct inode *inode, int mask)
263 {
264 #ifdef CONFIG_FS_POSIX_ACL
265 	struct posix_acl *acl;
266 
267 	if (mask & MAY_NOT_BLOCK) {
268 		acl = get_cached_acl_rcu(inode, ACL_TYPE_ACCESS);
269 	        if (!acl)
270 	                return -EAGAIN;
271 		/* no ->get_acl() calls in RCU mode... */
272 		if (is_uncached_acl(acl))
273 			return -ECHILD;
274 	        return posix_acl_permission(inode, acl, mask & ~MAY_NOT_BLOCK);
275 	}
276 
277 	acl = get_acl(inode, ACL_TYPE_ACCESS);
278 	if (IS_ERR(acl))
279 		return PTR_ERR(acl);
280 	if (acl) {
281 	        int error = posix_acl_permission(inode, acl, mask);
282 	        posix_acl_release(acl);
283 	        return error;
284 	}
285 #endif
286 
287 	return -EAGAIN;
288 }
289 
290 /*
291  * This does the basic permission checking
292  */
293 static int acl_permission_check(struct inode *inode, int mask)
294 {
295 	unsigned int mode = inode->i_mode;
296 
297 	if (likely(uid_eq(current_fsuid(), inode->i_uid)))
298 		mode >>= 6;
299 	else {
300 		if (IS_POSIXACL(inode) && (mode & S_IRWXG)) {
301 			int error = check_acl(inode, mask);
302 			if (error != -EAGAIN)
303 				return error;
304 		}
305 
306 		if (in_group_p(inode->i_gid))
307 			mode >>= 3;
308 	}
309 
310 	/*
311 	 * If the DACs are ok we don't need any capability check.
312 	 */
313 	if ((mask & ~mode & (MAY_READ | MAY_WRITE | MAY_EXEC)) == 0)
314 		return 0;
315 	return -EACCES;
316 }
317 
318 /**
319  * generic_permission -  check for access rights on a Posix-like filesystem
320  * @inode:	inode to check access rights for
321  * @mask:	right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC, ...)
322  *
323  * Used to check for read/write/execute permissions on a file.
324  * We use "fsuid" for this, letting us set arbitrary permissions
325  * for filesystem access without changing the "normal" uids which
326  * are used for other things.
327  *
328  * generic_permission is rcu-walk aware. It returns -ECHILD in case an rcu-walk
329  * request cannot be satisfied (eg. requires blocking or too much complexity).
330  * It would then be called again in ref-walk mode.
331  */
332 int generic_permission(struct inode *inode, int mask)
333 {
334 	int ret;
335 
336 	/*
337 	 * Do the basic permission checks.
338 	 */
339 	ret = acl_permission_check(inode, mask);
340 	if (ret != -EACCES)
341 		return ret;
342 
343 	if (S_ISDIR(inode->i_mode)) {
344 		/* DACs are overridable for directories */
345 		if (!(mask & MAY_WRITE))
346 			if (capable_wrt_inode_uidgid(inode,
347 						     CAP_DAC_READ_SEARCH))
348 				return 0;
349 		if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
350 			return 0;
351 		return -EACCES;
352 	}
353 
354 	/*
355 	 * Searching includes executable on directories, else just read.
356 	 */
357 	mask &= MAY_READ | MAY_WRITE | MAY_EXEC;
358 	if (mask == MAY_READ)
359 		if (capable_wrt_inode_uidgid(inode, CAP_DAC_READ_SEARCH))
360 			return 0;
361 	/*
362 	 * Read/write DACs are always overridable.
363 	 * Executable DACs are overridable when there is
364 	 * at least one exec bit set.
365 	 */
366 	if (!(mask & MAY_EXEC) || (inode->i_mode & S_IXUGO))
367 		if (capable_wrt_inode_uidgid(inode, CAP_DAC_OVERRIDE))
368 			return 0;
369 
370 	return -EACCES;
371 }
372 EXPORT_SYMBOL(generic_permission);
373 
374 /*
375  * We _really_ want to just do "generic_permission()" without
376  * even looking at the inode->i_op values. So we keep a cache
377  * flag in inode->i_opflags, that says "this has not special
378  * permission function, use the fast case".
379  */
380 static inline int do_inode_permission(struct inode *inode, int mask)
381 {
382 	if (unlikely(!(inode->i_opflags & IOP_FASTPERM))) {
383 		if (likely(inode->i_op->permission))
384 			return inode->i_op->permission(inode, mask);
385 
386 		/* This gets set once for the inode lifetime */
387 		spin_lock(&inode->i_lock);
388 		inode->i_opflags |= IOP_FASTPERM;
389 		spin_unlock(&inode->i_lock);
390 	}
391 	return generic_permission(inode, mask);
392 }
393 
394 /**
395  * sb_permission - Check superblock-level permissions
396  * @sb: Superblock of inode to check permission on
397  * @inode: Inode to check permission on
398  * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
399  *
400  * Separate out file-system wide checks from inode-specific permission checks.
401  */
402 static int sb_permission(struct super_block *sb, struct inode *inode, int mask)
403 {
404 	if (unlikely(mask & MAY_WRITE)) {
405 		umode_t mode = inode->i_mode;
406 
407 		/* Nobody gets write access to a read-only fs. */
408 		if (sb_rdonly(sb) && (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode)))
409 			return -EROFS;
410 	}
411 	return 0;
412 }
413 
414 /**
415  * inode_permission - Check for access rights to a given inode
416  * @inode: Inode to check permission on
417  * @mask: Right to check for (%MAY_READ, %MAY_WRITE, %MAY_EXEC)
418  *
419  * Check for read/write/execute permissions on an inode.  We use fs[ug]id for
420  * this, letting us set arbitrary permissions for filesystem access without
421  * changing the "normal" UIDs which are used for other things.
422  *
423  * When checking for MAY_APPEND, MAY_WRITE must also be set in @mask.
424  */
425 int inode_permission(struct inode *inode, int mask)
426 {
427 	int retval;
428 
429 	retval = sb_permission(inode->i_sb, inode, mask);
430 	if (retval)
431 		return retval;
432 
433 	if (unlikely(mask & MAY_WRITE)) {
434 		/*
435 		 * Nobody gets write access to an immutable file.
436 		 */
437 		if (IS_IMMUTABLE(inode))
438 			return -EPERM;
439 
440 		/*
441 		 * Updating mtime will likely cause i_uid and i_gid to be
442 		 * written back improperly if their true value is unknown
443 		 * to the vfs.
444 		 */
445 		if (HAS_UNMAPPED_ID(inode))
446 			return -EACCES;
447 	}
448 
449 	retval = do_inode_permission(inode, mask);
450 	if (retval)
451 		return retval;
452 
453 	retval = devcgroup_inode_permission(inode, mask);
454 	if (retval)
455 		return retval;
456 
457 	return security_inode_permission(inode, mask);
458 }
459 EXPORT_SYMBOL(inode_permission);
460 
461 /**
462  * path_get - get a reference to a path
463  * @path: path to get the reference to
464  *
465  * Given a path increment the reference count to the dentry and the vfsmount.
466  */
467 void path_get(const struct path *path)
468 {
469 	mntget(path->mnt);
470 	dget(path->dentry);
471 }
472 EXPORT_SYMBOL(path_get);
473 
474 /**
475  * path_put - put a reference to a path
476  * @path: path to put the reference to
477  *
478  * Given a path decrement the reference count to the dentry and the vfsmount.
479  */
480 void path_put(const struct path *path)
481 {
482 	dput(path->dentry);
483 	mntput(path->mnt);
484 }
485 EXPORT_SYMBOL(path_put);
486 
487 #define EMBEDDED_LEVELS 2
488 struct nameidata {
489 	struct path	path;
490 	struct qstr	last;
491 	struct path	root;
492 	struct inode	*inode; /* path.dentry.d_inode */
493 	unsigned int	flags;
494 	unsigned	seq, m_seq, r_seq;
495 	int		last_type;
496 	unsigned	depth;
497 	int		total_link_count;
498 	struct saved {
499 		struct path link;
500 		struct delayed_call done;
501 		const char *name;
502 		unsigned seq;
503 	} *stack, internal[EMBEDDED_LEVELS];
504 	struct filename	*name;
505 	struct nameidata *saved;
506 	unsigned	root_seq;
507 	int		dfd;
508 	kuid_t		dir_uid;
509 	umode_t		dir_mode;
510 } __randomize_layout;
511 
512 static void set_nameidata(struct nameidata *p, int dfd, struct filename *name)
513 {
514 	struct nameidata *old = current->nameidata;
515 	p->stack = p->internal;
516 	p->dfd = dfd;
517 	p->name = name;
518 	p->total_link_count = old ? old->total_link_count : 0;
519 	p->saved = old;
520 	current->nameidata = p;
521 }
522 
523 static void restore_nameidata(void)
524 {
525 	struct nameidata *now = current->nameidata, *old = now->saved;
526 
527 	current->nameidata = old;
528 	if (old)
529 		old->total_link_count = now->total_link_count;
530 	if (now->stack != now->internal)
531 		kfree(now->stack);
532 }
533 
534 static bool nd_alloc_stack(struct nameidata *nd)
535 {
536 	struct saved *p;
537 
538 	p= kmalloc_array(MAXSYMLINKS, sizeof(struct saved),
539 			 nd->flags & LOOKUP_RCU ? GFP_ATOMIC : GFP_KERNEL);
540 	if (unlikely(!p))
541 		return false;
542 	memcpy(p, nd->internal, sizeof(nd->internal));
543 	nd->stack = p;
544 	return true;
545 }
546 
547 /**
548  * path_connected - Verify that a dentry is below mnt.mnt_root
549  *
550  * Rename can sometimes move a file or directory outside of a bind
551  * mount, path_connected allows those cases to be detected.
552  */
553 static bool path_connected(struct vfsmount *mnt, struct dentry *dentry)
554 {
555 	struct super_block *sb = mnt->mnt_sb;
556 
557 	/* Bind mounts and multi-root filesystems can have disconnected paths */
558 	if (!(sb->s_iflags & SB_I_MULTIROOT) && (mnt->mnt_root == sb->s_root))
559 		return true;
560 
561 	return is_subdir(dentry, mnt->mnt_root);
562 }
563 
564 static void drop_links(struct nameidata *nd)
565 {
566 	int i = nd->depth;
567 	while (i--) {
568 		struct saved *last = nd->stack + i;
569 		do_delayed_call(&last->done);
570 		clear_delayed_call(&last->done);
571 	}
572 }
573 
574 static void terminate_walk(struct nameidata *nd)
575 {
576 	drop_links(nd);
577 	if (!(nd->flags & LOOKUP_RCU)) {
578 		int i;
579 		path_put(&nd->path);
580 		for (i = 0; i < nd->depth; i++)
581 			path_put(&nd->stack[i].link);
582 		if (nd->flags & LOOKUP_ROOT_GRABBED) {
583 			path_put(&nd->root);
584 			nd->flags &= ~LOOKUP_ROOT_GRABBED;
585 		}
586 	} else {
587 		nd->flags &= ~LOOKUP_RCU;
588 		rcu_read_unlock();
589 	}
590 	nd->depth = 0;
591 }
592 
593 /* path_put is needed afterwards regardless of success or failure */
594 static bool __legitimize_path(struct path *path, unsigned seq, unsigned mseq)
595 {
596 	int res = __legitimize_mnt(path->mnt, mseq);
597 	if (unlikely(res)) {
598 		if (res > 0)
599 			path->mnt = NULL;
600 		path->dentry = NULL;
601 		return false;
602 	}
603 	if (unlikely(!lockref_get_not_dead(&path->dentry->d_lockref))) {
604 		path->dentry = NULL;
605 		return false;
606 	}
607 	return !read_seqcount_retry(&path->dentry->d_seq, seq);
608 }
609 
610 static inline bool legitimize_path(struct nameidata *nd,
611 			    struct path *path, unsigned seq)
612 {
613 	return __legitimize_path(path, seq, nd->m_seq);
614 }
615 
616 static bool legitimize_links(struct nameidata *nd)
617 {
618 	int i;
619 	for (i = 0; i < nd->depth; i++) {
620 		struct saved *last = nd->stack + i;
621 		if (unlikely(!legitimize_path(nd, &last->link, last->seq))) {
622 			drop_links(nd);
623 			nd->depth = i + 1;
624 			return false;
625 		}
626 	}
627 	return true;
628 }
629 
630 static bool legitimize_root(struct nameidata *nd)
631 {
632 	/*
633 	 * For scoped-lookups (where nd->root has been zeroed), we need to
634 	 * restart the whole lookup from scratch -- because set_root() is wrong
635 	 * for these lookups (nd->dfd is the root, not the filesystem root).
636 	 */
637 	if (!nd->root.mnt && (nd->flags & LOOKUP_IS_SCOPED))
638 		return false;
639 	/* Nothing to do if nd->root is zero or is managed by the VFS user. */
640 	if (!nd->root.mnt || (nd->flags & LOOKUP_ROOT))
641 		return true;
642 	nd->flags |= LOOKUP_ROOT_GRABBED;
643 	return legitimize_path(nd, &nd->root, nd->root_seq);
644 }
645 
646 /*
647  * Path walking has 2 modes, rcu-walk and ref-walk (see
648  * Documentation/filesystems/path-lookup.txt).  In situations when we can't
649  * continue in RCU mode, we attempt to drop out of rcu-walk mode and grab
650  * normal reference counts on dentries and vfsmounts to transition to ref-walk
651  * mode.  Refcounts are grabbed at the last known good point before rcu-walk
652  * got stuck, so ref-walk may continue from there. If this is not successful
653  * (eg. a seqcount has changed), then failure is returned and it's up to caller
654  * to restart the path walk from the beginning in ref-walk mode.
655  */
656 
657 /**
658  * unlazy_walk - try to switch to ref-walk mode.
659  * @nd: nameidata pathwalk data
660  * Returns: 0 on success, -ECHILD on failure
661  *
662  * unlazy_walk attempts to legitimize the current nd->path and nd->root
663  * for ref-walk mode.
664  * Must be called from rcu-walk context.
665  * Nothing should touch nameidata between unlazy_walk() failure and
666  * terminate_walk().
667  */
668 static int unlazy_walk(struct nameidata *nd)
669 {
670 	struct dentry *parent = nd->path.dentry;
671 
672 	BUG_ON(!(nd->flags & LOOKUP_RCU));
673 
674 	nd->flags &= ~LOOKUP_RCU;
675 	if (unlikely(!legitimize_links(nd)))
676 		goto out1;
677 	if (unlikely(!legitimize_path(nd, &nd->path, nd->seq)))
678 		goto out;
679 	if (unlikely(!legitimize_root(nd)))
680 		goto out;
681 	rcu_read_unlock();
682 	BUG_ON(nd->inode != parent->d_inode);
683 	return 0;
684 
685 out1:
686 	nd->path.mnt = NULL;
687 	nd->path.dentry = NULL;
688 out:
689 	rcu_read_unlock();
690 	return -ECHILD;
691 }
692 
693 /**
694  * unlazy_child - try to switch to ref-walk mode.
695  * @nd: nameidata pathwalk data
696  * @dentry: child of nd->path.dentry
697  * @seq: seq number to check dentry against
698  * Returns: 0 on success, -ECHILD on failure
699  *
700  * unlazy_child attempts to legitimize the current nd->path, nd->root and dentry
701  * for ref-walk mode.  @dentry must be a path found by a do_lookup call on
702  * @nd.  Must be called from rcu-walk context.
703  * Nothing should touch nameidata between unlazy_child() failure and
704  * terminate_walk().
705  */
706 static int unlazy_child(struct nameidata *nd, struct dentry *dentry, unsigned seq)
707 {
708 	BUG_ON(!(nd->flags & LOOKUP_RCU));
709 
710 	nd->flags &= ~LOOKUP_RCU;
711 	if (unlikely(!legitimize_links(nd)))
712 		goto out2;
713 	if (unlikely(!legitimize_mnt(nd->path.mnt, nd->m_seq)))
714 		goto out2;
715 	if (unlikely(!lockref_get_not_dead(&nd->path.dentry->d_lockref)))
716 		goto out1;
717 
718 	/*
719 	 * We need to move both the parent and the dentry from the RCU domain
720 	 * to be properly refcounted. And the sequence number in the dentry
721 	 * validates *both* dentry counters, since we checked the sequence
722 	 * number of the parent after we got the child sequence number. So we
723 	 * know the parent must still be valid if the child sequence number is
724 	 */
725 	if (unlikely(!lockref_get_not_dead(&dentry->d_lockref)))
726 		goto out;
727 	if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
728 		goto out_dput;
729 	/*
730 	 * Sequence counts matched. Now make sure that the root is
731 	 * still valid and get it if required.
732 	 */
733 	if (unlikely(!legitimize_root(nd)))
734 		goto out_dput;
735 	rcu_read_unlock();
736 	return 0;
737 
738 out2:
739 	nd->path.mnt = NULL;
740 out1:
741 	nd->path.dentry = NULL;
742 out:
743 	rcu_read_unlock();
744 	return -ECHILD;
745 out_dput:
746 	rcu_read_unlock();
747 	dput(dentry);
748 	return -ECHILD;
749 }
750 
751 static inline int d_revalidate(struct dentry *dentry, unsigned int flags)
752 {
753 	if (unlikely(dentry->d_flags & DCACHE_OP_REVALIDATE))
754 		return dentry->d_op->d_revalidate(dentry, flags);
755 	else
756 		return 1;
757 }
758 
759 /**
760  * complete_walk - successful completion of path walk
761  * @nd:  pointer nameidata
762  *
763  * If we had been in RCU mode, drop out of it and legitimize nd->path.
764  * Revalidate the final result, unless we'd already done that during
765  * the path walk or the filesystem doesn't ask for it.  Return 0 on
766  * success, -error on failure.  In case of failure caller does not
767  * need to drop nd->path.
768  */
769 static int complete_walk(struct nameidata *nd)
770 {
771 	struct dentry *dentry = nd->path.dentry;
772 	int status;
773 
774 	if (nd->flags & LOOKUP_RCU) {
775 		/*
776 		 * We don't want to zero nd->root for scoped-lookups or
777 		 * externally-managed nd->root.
778 		 */
779 		if (!(nd->flags & (LOOKUP_ROOT | LOOKUP_IS_SCOPED)))
780 			nd->root.mnt = NULL;
781 		if (unlikely(unlazy_walk(nd)))
782 			return -ECHILD;
783 	}
784 
785 	if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
786 		/*
787 		 * While the guarantee of LOOKUP_IS_SCOPED is (roughly) "don't
788 		 * ever step outside the root during lookup" and should already
789 		 * be guaranteed by the rest of namei, we want to avoid a namei
790 		 * BUG resulting in userspace being given a path that was not
791 		 * scoped within the root at some point during the lookup.
792 		 *
793 		 * So, do a final sanity-check to make sure that in the
794 		 * worst-case scenario (a complete bypass of LOOKUP_IS_SCOPED)
795 		 * we won't silently return an fd completely outside of the
796 		 * requested root to userspace.
797 		 *
798 		 * Userspace could move the path outside the root after this
799 		 * check, but as discussed elsewhere this is not a concern (the
800 		 * resolved file was inside the root at some point).
801 		 */
802 		if (!path_is_under(&nd->path, &nd->root))
803 			return -EXDEV;
804 	}
805 
806 	if (likely(!(nd->flags & LOOKUP_JUMPED)))
807 		return 0;
808 
809 	if (likely(!(dentry->d_flags & DCACHE_OP_WEAK_REVALIDATE)))
810 		return 0;
811 
812 	status = dentry->d_op->d_weak_revalidate(dentry, nd->flags);
813 	if (status > 0)
814 		return 0;
815 
816 	if (!status)
817 		status = -ESTALE;
818 
819 	return status;
820 }
821 
822 static int set_root(struct nameidata *nd)
823 {
824 	struct fs_struct *fs = current->fs;
825 
826 	/*
827 	 * Jumping to the real root in a scoped-lookup is a BUG in namei, but we
828 	 * still have to ensure it doesn't happen because it will cause a breakout
829 	 * from the dirfd.
830 	 */
831 	if (WARN_ON(nd->flags & LOOKUP_IS_SCOPED))
832 		return -ENOTRECOVERABLE;
833 
834 	if (nd->flags & LOOKUP_RCU) {
835 		unsigned seq;
836 
837 		do {
838 			seq = read_seqcount_begin(&fs->seq);
839 			nd->root = fs->root;
840 			nd->root_seq = __read_seqcount_begin(&nd->root.dentry->d_seq);
841 		} while (read_seqcount_retry(&fs->seq, seq));
842 	} else {
843 		get_fs_root(fs, &nd->root);
844 		nd->flags |= LOOKUP_ROOT_GRABBED;
845 	}
846 	return 0;
847 }
848 
849 static int nd_jump_root(struct nameidata *nd)
850 {
851 	if (unlikely(nd->flags & LOOKUP_BENEATH))
852 		return -EXDEV;
853 	if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
854 		/* Absolute path arguments to path_init() are allowed. */
855 		if (nd->path.mnt != NULL && nd->path.mnt != nd->root.mnt)
856 			return -EXDEV;
857 	}
858 	if (!nd->root.mnt) {
859 		int error = set_root(nd);
860 		if (error)
861 			return error;
862 	}
863 	if (nd->flags & LOOKUP_RCU) {
864 		struct dentry *d;
865 		nd->path = nd->root;
866 		d = nd->path.dentry;
867 		nd->inode = d->d_inode;
868 		nd->seq = nd->root_seq;
869 		if (unlikely(read_seqcount_retry(&d->d_seq, nd->seq)))
870 			return -ECHILD;
871 	} else {
872 		path_put(&nd->path);
873 		nd->path = nd->root;
874 		path_get(&nd->path);
875 		nd->inode = nd->path.dentry->d_inode;
876 	}
877 	nd->flags |= LOOKUP_JUMPED;
878 	return 0;
879 }
880 
881 /*
882  * Helper to directly jump to a known parsed path from ->get_link,
883  * caller must have taken a reference to path beforehand.
884  */
885 int nd_jump_link(struct path *path)
886 {
887 	int error = -ELOOP;
888 	struct nameidata *nd = current->nameidata;
889 
890 	if (unlikely(nd->flags & LOOKUP_NO_MAGICLINKS))
891 		goto err;
892 
893 	error = -EXDEV;
894 	if (unlikely(nd->flags & LOOKUP_NO_XDEV)) {
895 		if (nd->path.mnt != path->mnt)
896 			goto err;
897 	}
898 	/* Not currently safe for scoped-lookups. */
899 	if (unlikely(nd->flags & LOOKUP_IS_SCOPED))
900 		goto err;
901 
902 	path_put(&nd->path);
903 	nd->path = *path;
904 	nd->inode = nd->path.dentry->d_inode;
905 	nd->flags |= LOOKUP_JUMPED;
906 	return 0;
907 
908 err:
909 	path_put(path);
910 	return error;
911 }
912 
913 static inline void put_link(struct nameidata *nd)
914 {
915 	struct saved *last = nd->stack + --nd->depth;
916 	do_delayed_call(&last->done);
917 	if (!(nd->flags & LOOKUP_RCU))
918 		path_put(&last->link);
919 }
920 
921 int sysctl_protected_symlinks __read_mostly = 0;
922 int sysctl_protected_hardlinks __read_mostly = 0;
923 int sysctl_protected_fifos __read_mostly;
924 int sysctl_protected_regular __read_mostly;
925 
926 /**
927  * may_follow_link - Check symlink following for unsafe situations
928  * @nd: nameidata pathwalk data
929  *
930  * In the case of the sysctl_protected_symlinks sysctl being enabled,
931  * CAP_DAC_OVERRIDE needs to be specifically ignored if the symlink is
932  * in a sticky world-writable directory. This is to protect privileged
933  * processes from failing races against path names that may change out
934  * from under them by way of other users creating malicious symlinks.
935  * It will permit symlinks to be followed only when outside a sticky
936  * world-writable directory, or when the uid of the symlink and follower
937  * match, or when the directory owner matches the symlink's owner.
938  *
939  * Returns 0 if following the symlink is allowed, -ve on error.
940  */
941 static inline int may_follow_link(struct nameidata *nd, const struct inode *inode)
942 {
943 	if (!sysctl_protected_symlinks)
944 		return 0;
945 
946 	/* Allowed if owner and follower match. */
947 	if (uid_eq(current_cred()->fsuid, inode->i_uid))
948 		return 0;
949 
950 	/* Allowed if parent directory not sticky and world-writable. */
951 	if ((nd->dir_mode & (S_ISVTX|S_IWOTH)) != (S_ISVTX|S_IWOTH))
952 		return 0;
953 
954 	/* Allowed if parent directory and link owner match. */
955 	if (uid_valid(nd->dir_uid) && uid_eq(nd->dir_uid, inode->i_uid))
956 		return 0;
957 
958 	if (nd->flags & LOOKUP_RCU)
959 		return -ECHILD;
960 
961 	audit_inode(nd->name, nd->stack[0].link.dentry, 0);
962 	audit_log_path_denied(AUDIT_ANOM_LINK, "follow_link");
963 	return -EACCES;
964 }
965 
966 /**
967  * safe_hardlink_source - Check for safe hardlink conditions
968  * @inode: the source inode to hardlink from
969  *
970  * Return false if at least one of the following conditions:
971  *    - inode is not a regular file
972  *    - inode is setuid
973  *    - inode is setgid and group-exec
974  *    - access failure for read and write
975  *
976  * Otherwise returns true.
977  */
978 static bool safe_hardlink_source(struct inode *inode)
979 {
980 	umode_t mode = inode->i_mode;
981 
982 	/* Special files should not get pinned to the filesystem. */
983 	if (!S_ISREG(mode))
984 		return false;
985 
986 	/* Setuid files should not get pinned to the filesystem. */
987 	if (mode & S_ISUID)
988 		return false;
989 
990 	/* Executable setgid files should not get pinned to the filesystem. */
991 	if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP))
992 		return false;
993 
994 	/* Hardlinking to unreadable or unwritable sources is dangerous. */
995 	if (inode_permission(inode, MAY_READ | MAY_WRITE))
996 		return false;
997 
998 	return true;
999 }
1000 
1001 /**
1002  * may_linkat - Check permissions for creating a hardlink
1003  * @link: the source to hardlink from
1004  *
1005  * Block hardlink when all of:
1006  *  - sysctl_protected_hardlinks enabled
1007  *  - fsuid does not match inode
1008  *  - hardlink source is unsafe (see safe_hardlink_source() above)
1009  *  - not CAP_FOWNER in a namespace with the inode owner uid mapped
1010  *
1011  * Returns 0 if successful, -ve on error.
1012  */
1013 static int may_linkat(struct path *link)
1014 {
1015 	struct inode *inode = link->dentry->d_inode;
1016 
1017 	/* Inode writeback is not safe when the uid or gid are invalid. */
1018 	if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid))
1019 		return -EOVERFLOW;
1020 
1021 	if (!sysctl_protected_hardlinks)
1022 		return 0;
1023 
1024 	/* Source inode owner (or CAP_FOWNER) can hardlink all they like,
1025 	 * otherwise, it must be a safe source.
1026 	 */
1027 	if (safe_hardlink_source(inode) || inode_owner_or_capable(inode))
1028 		return 0;
1029 
1030 	audit_log_path_denied(AUDIT_ANOM_LINK, "linkat");
1031 	return -EPERM;
1032 }
1033 
1034 /**
1035  * may_create_in_sticky - Check whether an O_CREAT open in a sticky directory
1036  *			  should be allowed, or not, on files that already
1037  *			  exist.
1038  * @dir_mode: mode bits of directory
1039  * @dir_uid: owner of directory
1040  * @inode: the inode of the file to open
1041  *
1042  * Block an O_CREAT open of a FIFO (or a regular file) when:
1043  *   - sysctl_protected_fifos (or sysctl_protected_regular) is enabled
1044  *   - the file already exists
1045  *   - we are in a sticky directory
1046  *   - we don't own the file
1047  *   - the owner of the directory doesn't own the file
1048  *   - the directory is world writable
1049  * If the sysctl_protected_fifos (or sysctl_protected_regular) is set to 2
1050  * the directory doesn't have to be world writable: being group writable will
1051  * be enough.
1052  *
1053  * Returns 0 if the open is allowed, -ve on error.
1054  */
1055 static int may_create_in_sticky(umode_t dir_mode, kuid_t dir_uid,
1056 				struct inode * const inode)
1057 {
1058 	if ((!sysctl_protected_fifos && S_ISFIFO(inode->i_mode)) ||
1059 	    (!sysctl_protected_regular && S_ISREG(inode->i_mode)) ||
1060 	    likely(!(dir_mode & S_ISVTX)) ||
1061 	    uid_eq(inode->i_uid, dir_uid) ||
1062 	    uid_eq(current_fsuid(), inode->i_uid))
1063 		return 0;
1064 
1065 	if (likely(dir_mode & 0002) ||
1066 	    (dir_mode & 0020 &&
1067 	     ((sysctl_protected_fifos >= 2 && S_ISFIFO(inode->i_mode)) ||
1068 	      (sysctl_protected_regular >= 2 && S_ISREG(inode->i_mode))))) {
1069 		const char *operation = S_ISFIFO(inode->i_mode) ?
1070 					"sticky_create_fifo" :
1071 					"sticky_create_regular";
1072 		audit_log_path_denied(AUDIT_ANOM_CREAT, operation);
1073 		return -EACCES;
1074 	}
1075 	return 0;
1076 }
1077 
1078 /*
1079  * follow_up - Find the mountpoint of path's vfsmount
1080  *
1081  * Given a path, find the mountpoint of its source file system.
1082  * Replace @path with the path of the mountpoint in the parent mount.
1083  * Up is towards /.
1084  *
1085  * Return 1 if we went up a level and 0 if we were already at the
1086  * root.
1087  */
1088 int follow_up(struct path *path)
1089 {
1090 	struct mount *mnt = real_mount(path->mnt);
1091 	struct mount *parent;
1092 	struct dentry *mountpoint;
1093 
1094 	read_seqlock_excl(&mount_lock);
1095 	parent = mnt->mnt_parent;
1096 	if (parent == mnt) {
1097 		read_sequnlock_excl(&mount_lock);
1098 		return 0;
1099 	}
1100 	mntget(&parent->mnt);
1101 	mountpoint = dget(mnt->mnt_mountpoint);
1102 	read_sequnlock_excl(&mount_lock);
1103 	dput(path->dentry);
1104 	path->dentry = mountpoint;
1105 	mntput(path->mnt);
1106 	path->mnt = &parent->mnt;
1107 	return 1;
1108 }
1109 EXPORT_SYMBOL(follow_up);
1110 
1111 static bool choose_mountpoint_rcu(struct mount *m, const struct path *root,
1112 				  struct path *path, unsigned *seqp)
1113 {
1114 	while (mnt_has_parent(m)) {
1115 		struct dentry *mountpoint = m->mnt_mountpoint;
1116 
1117 		m = m->mnt_parent;
1118 		if (unlikely(root->dentry == mountpoint &&
1119 			     root->mnt == &m->mnt))
1120 			break;
1121 		if (mountpoint != m->mnt.mnt_root) {
1122 			path->mnt = &m->mnt;
1123 			path->dentry = mountpoint;
1124 			*seqp = read_seqcount_begin(&mountpoint->d_seq);
1125 			return true;
1126 		}
1127 	}
1128 	return false;
1129 }
1130 
1131 static bool choose_mountpoint(struct mount *m, const struct path *root,
1132 			      struct path *path)
1133 {
1134 	bool found;
1135 
1136 	rcu_read_lock();
1137 	while (1) {
1138 		unsigned seq, mseq = read_seqbegin(&mount_lock);
1139 
1140 		found = choose_mountpoint_rcu(m, root, path, &seq);
1141 		if (unlikely(!found)) {
1142 			if (!read_seqretry(&mount_lock, mseq))
1143 				break;
1144 		} else {
1145 			if (likely(__legitimize_path(path, seq, mseq)))
1146 				break;
1147 			rcu_read_unlock();
1148 			path_put(path);
1149 			rcu_read_lock();
1150 		}
1151 	}
1152 	rcu_read_unlock();
1153 	return found;
1154 }
1155 
1156 /*
1157  * Perform an automount
1158  * - return -EISDIR to tell follow_managed() to stop and return the path we
1159  *   were called with.
1160  */
1161 static int follow_automount(struct path *path, int *count, unsigned lookup_flags)
1162 {
1163 	struct dentry *dentry = path->dentry;
1164 
1165 	/* We don't want to mount if someone's just doing a stat -
1166 	 * unless they're stat'ing a directory and appended a '/' to
1167 	 * the name.
1168 	 *
1169 	 * We do, however, want to mount if someone wants to open or
1170 	 * create a file of any type under the mountpoint, wants to
1171 	 * traverse through the mountpoint or wants to open the
1172 	 * mounted directory.  Also, autofs may mark negative dentries
1173 	 * as being automount points.  These will need the attentions
1174 	 * of the daemon to instantiate them before they can be used.
1175 	 */
1176 	if (!(lookup_flags & (LOOKUP_PARENT | LOOKUP_DIRECTORY |
1177 			   LOOKUP_OPEN | LOOKUP_CREATE | LOOKUP_AUTOMOUNT)) &&
1178 	    dentry->d_inode)
1179 		return -EISDIR;
1180 
1181 	if (count && (*count)++ >= MAXSYMLINKS)
1182 		return -ELOOP;
1183 
1184 	return finish_automount(dentry->d_op->d_automount(path), path);
1185 }
1186 
1187 /*
1188  * mount traversal - out-of-line part.  One note on ->d_flags accesses -
1189  * dentries are pinned but not locked here, so negative dentry can go
1190  * positive right under us.  Use of smp_load_acquire() provides a barrier
1191  * sufficient for ->d_inode and ->d_flags consistency.
1192  */
1193 static int __traverse_mounts(struct path *path, unsigned flags, bool *jumped,
1194 			     int *count, unsigned lookup_flags)
1195 {
1196 	struct vfsmount *mnt = path->mnt;
1197 	bool need_mntput = false;
1198 	int ret = 0;
1199 
1200 	while (flags & DCACHE_MANAGED_DENTRY) {
1201 		/* Allow the filesystem to manage the transit without i_mutex
1202 		 * being held. */
1203 		if (flags & DCACHE_MANAGE_TRANSIT) {
1204 			ret = path->dentry->d_op->d_manage(path, false);
1205 			flags = smp_load_acquire(&path->dentry->d_flags);
1206 			if (ret < 0)
1207 				break;
1208 		}
1209 
1210 		if (flags & DCACHE_MOUNTED) {	// something's mounted on it..
1211 			struct vfsmount *mounted = lookup_mnt(path);
1212 			if (mounted) {		// ... in our namespace
1213 				dput(path->dentry);
1214 				if (need_mntput)
1215 					mntput(path->mnt);
1216 				path->mnt = mounted;
1217 				path->dentry = dget(mounted->mnt_root);
1218 				// here we know it's positive
1219 				flags = path->dentry->d_flags;
1220 				need_mntput = true;
1221 				continue;
1222 			}
1223 		}
1224 
1225 		if (!(flags & DCACHE_NEED_AUTOMOUNT))
1226 			break;
1227 
1228 		// uncovered automount point
1229 		ret = follow_automount(path, count, lookup_flags);
1230 		flags = smp_load_acquire(&path->dentry->d_flags);
1231 		if (ret < 0)
1232 			break;
1233 	}
1234 
1235 	if (ret == -EISDIR)
1236 		ret = 0;
1237 	// possible if you race with several mount --move
1238 	if (need_mntput && path->mnt == mnt)
1239 		mntput(path->mnt);
1240 	if (!ret && unlikely(d_flags_negative(flags)))
1241 		ret = -ENOENT;
1242 	*jumped = need_mntput;
1243 	return ret;
1244 }
1245 
1246 static inline int traverse_mounts(struct path *path, bool *jumped,
1247 				  int *count, unsigned lookup_flags)
1248 {
1249 	unsigned flags = smp_load_acquire(&path->dentry->d_flags);
1250 
1251 	/* fastpath */
1252 	if (likely(!(flags & DCACHE_MANAGED_DENTRY))) {
1253 		*jumped = false;
1254 		if (unlikely(d_flags_negative(flags)))
1255 			return -ENOENT;
1256 		return 0;
1257 	}
1258 	return __traverse_mounts(path, flags, jumped, count, lookup_flags);
1259 }
1260 
1261 int follow_down_one(struct path *path)
1262 {
1263 	struct vfsmount *mounted;
1264 
1265 	mounted = lookup_mnt(path);
1266 	if (mounted) {
1267 		dput(path->dentry);
1268 		mntput(path->mnt);
1269 		path->mnt = mounted;
1270 		path->dentry = dget(mounted->mnt_root);
1271 		return 1;
1272 	}
1273 	return 0;
1274 }
1275 EXPORT_SYMBOL(follow_down_one);
1276 
1277 /*
1278  * Follow down to the covering mount currently visible to userspace.  At each
1279  * point, the filesystem owning that dentry may be queried as to whether the
1280  * caller is permitted to proceed or not.
1281  */
1282 int follow_down(struct path *path)
1283 {
1284 	struct vfsmount *mnt = path->mnt;
1285 	bool jumped;
1286 	int ret = traverse_mounts(path, &jumped, NULL, 0);
1287 
1288 	if (path->mnt != mnt)
1289 		mntput(mnt);
1290 	return ret;
1291 }
1292 EXPORT_SYMBOL(follow_down);
1293 
1294 /*
1295  * Try to skip to top of mountpoint pile in rcuwalk mode.  Fail if
1296  * we meet a managed dentry that would need blocking.
1297  */
1298 static bool __follow_mount_rcu(struct nameidata *nd, struct path *path,
1299 			       struct inode **inode, unsigned *seqp)
1300 {
1301 	struct dentry *dentry = path->dentry;
1302 	unsigned int flags = dentry->d_flags;
1303 
1304 	if (likely(!(flags & DCACHE_MANAGED_DENTRY)))
1305 		return true;
1306 
1307 	if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1308 		return false;
1309 
1310 	for (;;) {
1311 		/*
1312 		 * Don't forget we might have a non-mountpoint managed dentry
1313 		 * that wants to block transit.
1314 		 */
1315 		if (unlikely(flags & DCACHE_MANAGE_TRANSIT)) {
1316 			int res = dentry->d_op->d_manage(path, true);
1317 			if (res)
1318 				return res == -EISDIR;
1319 			flags = dentry->d_flags;
1320 		}
1321 
1322 		if (flags & DCACHE_MOUNTED) {
1323 			struct mount *mounted = __lookup_mnt(path->mnt, dentry);
1324 			if (mounted) {
1325 				path->mnt = &mounted->mnt;
1326 				dentry = path->dentry = mounted->mnt.mnt_root;
1327 				nd->flags |= LOOKUP_JUMPED;
1328 				*seqp = read_seqcount_begin(&dentry->d_seq);
1329 				*inode = dentry->d_inode;
1330 				/*
1331 				 * We don't need to re-check ->d_seq after this
1332 				 * ->d_inode read - there will be an RCU delay
1333 				 * between mount hash removal and ->mnt_root
1334 				 * becoming unpinned.
1335 				 */
1336 				flags = dentry->d_flags;
1337 				continue;
1338 			}
1339 			if (read_seqretry(&mount_lock, nd->m_seq))
1340 				return false;
1341 		}
1342 		return !(flags & DCACHE_NEED_AUTOMOUNT);
1343 	}
1344 }
1345 
1346 static inline int handle_mounts(struct nameidata *nd, struct dentry *dentry,
1347 			  struct path *path, struct inode **inode,
1348 			  unsigned int *seqp)
1349 {
1350 	bool jumped;
1351 	int ret;
1352 
1353 	path->mnt = nd->path.mnt;
1354 	path->dentry = dentry;
1355 	if (nd->flags & LOOKUP_RCU) {
1356 		unsigned int seq = *seqp;
1357 		if (unlikely(!*inode))
1358 			return -ENOENT;
1359 		if (likely(__follow_mount_rcu(nd, path, inode, seqp)))
1360 			return 0;
1361 		if (unlazy_child(nd, dentry, seq))
1362 			return -ECHILD;
1363 		// *path might've been clobbered by __follow_mount_rcu()
1364 		path->mnt = nd->path.mnt;
1365 		path->dentry = dentry;
1366 	}
1367 	ret = traverse_mounts(path, &jumped, &nd->total_link_count, nd->flags);
1368 	if (jumped) {
1369 		if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1370 			ret = -EXDEV;
1371 		else
1372 			nd->flags |= LOOKUP_JUMPED;
1373 	}
1374 	if (unlikely(ret)) {
1375 		dput(path->dentry);
1376 		if (path->mnt != nd->path.mnt)
1377 			mntput(path->mnt);
1378 	} else {
1379 		*inode = d_backing_inode(path->dentry);
1380 		*seqp = 0; /* out of RCU mode, so the value doesn't matter */
1381 	}
1382 	return ret;
1383 }
1384 
1385 /*
1386  * This looks up the name in dcache and possibly revalidates the found dentry.
1387  * NULL is returned if the dentry does not exist in the cache.
1388  */
1389 static struct dentry *lookup_dcache(const struct qstr *name,
1390 				    struct dentry *dir,
1391 				    unsigned int flags)
1392 {
1393 	struct dentry *dentry = d_lookup(dir, name);
1394 	if (dentry) {
1395 		int error = d_revalidate(dentry, flags);
1396 		if (unlikely(error <= 0)) {
1397 			if (!error)
1398 				d_invalidate(dentry);
1399 			dput(dentry);
1400 			return ERR_PTR(error);
1401 		}
1402 	}
1403 	return dentry;
1404 }
1405 
1406 /*
1407  * Parent directory has inode locked exclusive.  This is one
1408  * and only case when ->lookup() gets called on non in-lookup
1409  * dentries - as the matter of fact, this only gets called
1410  * when directory is guaranteed to have no in-lookup children
1411  * at all.
1412  */
1413 static struct dentry *__lookup_hash(const struct qstr *name,
1414 		struct dentry *base, unsigned int flags)
1415 {
1416 	struct dentry *dentry = lookup_dcache(name, base, flags);
1417 	struct dentry *old;
1418 	struct inode *dir = base->d_inode;
1419 
1420 	if (dentry)
1421 		return dentry;
1422 
1423 	/* Don't create child dentry for a dead directory. */
1424 	if (unlikely(IS_DEADDIR(dir)))
1425 		return ERR_PTR(-ENOENT);
1426 
1427 	dentry = d_alloc(base, name);
1428 	if (unlikely(!dentry))
1429 		return ERR_PTR(-ENOMEM);
1430 
1431 	old = dir->i_op->lookup(dir, dentry, flags);
1432 	if (unlikely(old)) {
1433 		dput(dentry);
1434 		dentry = old;
1435 	}
1436 	return dentry;
1437 }
1438 
1439 static struct dentry *lookup_fast(struct nameidata *nd,
1440 				  struct inode **inode,
1441 			          unsigned *seqp)
1442 {
1443 	struct dentry *dentry, *parent = nd->path.dentry;
1444 	int status = 1;
1445 
1446 	/*
1447 	 * Rename seqlock is not required here because in the off chance
1448 	 * of a false negative due to a concurrent rename, the caller is
1449 	 * going to fall back to non-racy lookup.
1450 	 */
1451 	if (nd->flags & LOOKUP_RCU) {
1452 		unsigned seq;
1453 		dentry = __d_lookup_rcu(parent, &nd->last, &seq);
1454 		if (unlikely(!dentry)) {
1455 			if (unlazy_walk(nd))
1456 				return ERR_PTR(-ECHILD);
1457 			return NULL;
1458 		}
1459 
1460 		/*
1461 		 * This sequence count validates that the inode matches
1462 		 * the dentry name information from lookup.
1463 		 */
1464 		*inode = d_backing_inode(dentry);
1465 		if (unlikely(read_seqcount_retry(&dentry->d_seq, seq)))
1466 			return ERR_PTR(-ECHILD);
1467 
1468 		/*
1469 		 * This sequence count validates that the parent had no
1470 		 * changes while we did the lookup of the dentry above.
1471 		 *
1472 		 * The memory barrier in read_seqcount_begin of child is
1473 		 *  enough, we can use __read_seqcount_retry here.
1474 		 */
1475 		if (unlikely(__read_seqcount_retry(&parent->d_seq, nd->seq)))
1476 			return ERR_PTR(-ECHILD);
1477 
1478 		*seqp = seq;
1479 		status = d_revalidate(dentry, nd->flags);
1480 		if (likely(status > 0))
1481 			return dentry;
1482 		if (unlazy_child(nd, dentry, seq))
1483 			return ERR_PTR(-ECHILD);
1484 		if (unlikely(status == -ECHILD))
1485 			/* we'd been told to redo it in non-rcu mode */
1486 			status = d_revalidate(dentry, nd->flags);
1487 	} else {
1488 		dentry = __d_lookup(parent, &nd->last);
1489 		if (unlikely(!dentry))
1490 			return NULL;
1491 		status = d_revalidate(dentry, nd->flags);
1492 	}
1493 	if (unlikely(status <= 0)) {
1494 		if (!status)
1495 			d_invalidate(dentry);
1496 		dput(dentry);
1497 		return ERR_PTR(status);
1498 	}
1499 	return dentry;
1500 }
1501 
1502 /* Fast lookup failed, do it the slow way */
1503 static struct dentry *__lookup_slow(const struct qstr *name,
1504 				    struct dentry *dir,
1505 				    unsigned int flags)
1506 {
1507 	struct dentry *dentry, *old;
1508 	struct inode *inode = dir->d_inode;
1509 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
1510 
1511 	/* Don't go there if it's already dead */
1512 	if (unlikely(IS_DEADDIR(inode)))
1513 		return ERR_PTR(-ENOENT);
1514 again:
1515 	dentry = d_alloc_parallel(dir, name, &wq);
1516 	if (IS_ERR(dentry))
1517 		return dentry;
1518 	if (unlikely(!d_in_lookup(dentry))) {
1519 		int error = d_revalidate(dentry, flags);
1520 		if (unlikely(error <= 0)) {
1521 			if (!error) {
1522 				d_invalidate(dentry);
1523 				dput(dentry);
1524 				goto again;
1525 			}
1526 			dput(dentry);
1527 			dentry = ERR_PTR(error);
1528 		}
1529 	} else {
1530 		old = inode->i_op->lookup(inode, dentry, flags);
1531 		d_lookup_done(dentry);
1532 		if (unlikely(old)) {
1533 			dput(dentry);
1534 			dentry = old;
1535 		}
1536 	}
1537 	return dentry;
1538 }
1539 
1540 static struct dentry *lookup_slow(const struct qstr *name,
1541 				  struct dentry *dir,
1542 				  unsigned int flags)
1543 {
1544 	struct inode *inode = dir->d_inode;
1545 	struct dentry *res;
1546 	inode_lock_shared(inode);
1547 	res = __lookup_slow(name, dir, flags);
1548 	inode_unlock_shared(inode);
1549 	return res;
1550 }
1551 
1552 static inline int may_lookup(struct nameidata *nd)
1553 {
1554 	if (nd->flags & LOOKUP_RCU) {
1555 		int err = inode_permission(nd->inode, MAY_EXEC|MAY_NOT_BLOCK);
1556 		if (err != -ECHILD)
1557 			return err;
1558 		if (unlazy_walk(nd))
1559 			return -ECHILD;
1560 	}
1561 	return inode_permission(nd->inode, MAY_EXEC);
1562 }
1563 
1564 static int reserve_stack(struct nameidata *nd, struct path *link, unsigned seq)
1565 {
1566 	if (unlikely(nd->total_link_count++ >= MAXSYMLINKS))
1567 		return -ELOOP;
1568 
1569 	if (likely(nd->depth != EMBEDDED_LEVELS))
1570 		return 0;
1571 	if (likely(nd->stack != nd->internal))
1572 		return 0;
1573 	if (likely(nd_alloc_stack(nd)))
1574 		return 0;
1575 
1576 	if (nd->flags & LOOKUP_RCU) {
1577 		// we need to grab link before we do unlazy.  And we can't skip
1578 		// unlazy even if we fail to grab the link - cleanup needs it
1579 		bool grabbed_link = legitimize_path(nd, link, seq);
1580 
1581 		if (unlazy_walk(nd) != 0 || !grabbed_link)
1582 			return -ECHILD;
1583 
1584 		if (nd_alloc_stack(nd))
1585 			return 0;
1586 	}
1587 	return -ENOMEM;
1588 }
1589 
1590 enum {WALK_TRAILING = 1, WALK_MORE = 2, WALK_NOFOLLOW = 4};
1591 
1592 static const char *pick_link(struct nameidata *nd, struct path *link,
1593 		     struct inode *inode, unsigned seq, int flags)
1594 {
1595 	struct saved *last;
1596 	const char *res;
1597 	int error = reserve_stack(nd, link, seq);
1598 
1599 	if (unlikely(error)) {
1600 		if (!(nd->flags & LOOKUP_RCU))
1601 			path_put(link);
1602 		return ERR_PTR(error);
1603 	}
1604 	last = nd->stack + nd->depth++;
1605 	last->link = *link;
1606 	clear_delayed_call(&last->done);
1607 	last->seq = seq;
1608 
1609 	if (flags & WALK_TRAILING) {
1610 		error = may_follow_link(nd, inode);
1611 		if (unlikely(error))
1612 			return ERR_PTR(error);
1613 	}
1614 
1615 	if (unlikely(nd->flags & LOOKUP_NO_SYMLINKS))
1616 		return ERR_PTR(-ELOOP);
1617 
1618 	if (!(nd->flags & LOOKUP_RCU)) {
1619 		touch_atime(&last->link);
1620 		cond_resched();
1621 	} else if (atime_needs_update(&last->link, inode)) {
1622 		if (unlikely(unlazy_walk(nd)))
1623 			return ERR_PTR(-ECHILD);
1624 		touch_atime(&last->link);
1625 	}
1626 
1627 	error = security_inode_follow_link(link->dentry, inode,
1628 					   nd->flags & LOOKUP_RCU);
1629 	if (unlikely(error))
1630 		return ERR_PTR(error);
1631 
1632 	res = READ_ONCE(inode->i_link);
1633 	if (!res) {
1634 		const char * (*get)(struct dentry *, struct inode *,
1635 				struct delayed_call *);
1636 		get = inode->i_op->get_link;
1637 		if (nd->flags & LOOKUP_RCU) {
1638 			res = get(NULL, inode, &last->done);
1639 			if (res == ERR_PTR(-ECHILD)) {
1640 				if (unlikely(unlazy_walk(nd)))
1641 					return ERR_PTR(-ECHILD);
1642 				res = get(link->dentry, inode, &last->done);
1643 			}
1644 		} else {
1645 			res = get(link->dentry, inode, &last->done);
1646 		}
1647 		if (!res)
1648 			goto all_done;
1649 		if (IS_ERR(res))
1650 			return res;
1651 	}
1652 	if (*res == '/') {
1653 		error = nd_jump_root(nd);
1654 		if (unlikely(error))
1655 			return ERR_PTR(error);
1656 		while (unlikely(*++res == '/'))
1657 			;
1658 	}
1659 	if (*res)
1660 		return res;
1661 all_done: // pure jump
1662 	put_link(nd);
1663 	return NULL;
1664 }
1665 
1666 /*
1667  * Do we need to follow links? We _really_ want to be able
1668  * to do this check without having to look at inode->i_op,
1669  * so we keep a cache of "no, this doesn't need follow_link"
1670  * for the common case.
1671  */
1672 static const char *step_into(struct nameidata *nd, int flags,
1673 		     struct dentry *dentry, struct inode *inode, unsigned seq)
1674 {
1675 	struct path path;
1676 	int err = handle_mounts(nd, dentry, &path, &inode, &seq);
1677 
1678 	if (err < 0)
1679 		return ERR_PTR(err);
1680 	if (likely(!d_is_symlink(path.dentry)) ||
1681 	   ((flags & WALK_TRAILING) && !(nd->flags & LOOKUP_FOLLOW)) ||
1682 	   (flags & WALK_NOFOLLOW)) {
1683 		/* not a symlink or should not follow */
1684 		if (!(nd->flags & LOOKUP_RCU)) {
1685 			dput(nd->path.dentry);
1686 			if (nd->path.mnt != path.mnt)
1687 				mntput(nd->path.mnt);
1688 		}
1689 		nd->path = path;
1690 		nd->inode = inode;
1691 		nd->seq = seq;
1692 		return NULL;
1693 	}
1694 	if (nd->flags & LOOKUP_RCU) {
1695 		/* make sure that d_is_symlink above matches inode */
1696 		if (read_seqcount_retry(&path.dentry->d_seq, seq))
1697 			return ERR_PTR(-ECHILD);
1698 	} else {
1699 		if (path.mnt == nd->path.mnt)
1700 			mntget(path.mnt);
1701 	}
1702 	return pick_link(nd, &path, inode, seq, flags);
1703 }
1704 
1705 static struct dentry *follow_dotdot_rcu(struct nameidata *nd,
1706 					struct inode **inodep,
1707 					unsigned *seqp)
1708 {
1709 	struct dentry *parent, *old;
1710 
1711 	if (path_equal(&nd->path, &nd->root))
1712 		goto in_root;
1713 	if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) {
1714 		struct path path;
1715 		unsigned seq;
1716 		if (!choose_mountpoint_rcu(real_mount(nd->path.mnt),
1717 					   &nd->root, &path, &seq))
1718 			goto in_root;
1719 		if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1720 			return ERR_PTR(-ECHILD);
1721 		nd->path = path;
1722 		nd->inode = path.dentry->d_inode;
1723 		nd->seq = seq;
1724 		if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
1725 			return ERR_PTR(-ECHILD);
1726 		/* we know that mountpoint was pinned */
1727 	}
1728 	old = nd->path.dentry;
1729 	parent = old->d_parent;
1730 	*inodep = parent->d_inode;
1731 	*seqp = read_seqcount_begin(&parent->d_seq);
1732 	if (unlikely(read_seqcount_retry(&old->d_seq, nd->seq)))
1733 		return ERR_PTR(-ECHILD);
1734 	if (unlikely(!path_connected(nd->path.mnt, parent)))
1735 		return ERR_PTR(-ECHILD);
1736 	return parent;
1737 in_root:
1738 	if (unlikely(read_seqretry(&mount_lock, nd->m_seq)))
1739 		return ERR_PTR(-ECHILD);
1740 	if (unlikely(nd->flags & LOOKUP_BENEATH))
1741 		return ERR_PTR(-ECHILD);
1742 	return NULL;
1743 }
1744 
1745 static struct dentry *follow_dotdot(struct nameidata *nd,
1746 				 struct inode **inodep,
1747 				 unsigned *seqp)
1748 {
1749 	struct dentry *parent;
1750 
1751 	if (path_equal(&nd->path, &nd->root))
1752 		goto in_root;
1753 	if (unlikely(nd->path.dentry == nd->path.mnt->mnt_root)) {
1754 		struct path path;
1755 
1756 		if (!choose_mountpoint(real_mount(nd->path.mnt),
1757 				       &nd->root, &path))
1758 			goto in_root;
1759 		path_put(&nd->path);
1760 		nd->path = path;
1761 		nd->inode = path.dentry->d_inode;
1762 		if (unlikely(nd->flags & LOOKUP_NO_XDEV))
1763 			return ERR_PTR(-EXDEV);
1764 	}
1765 	/* rare case of legitimate dget_parent()... */
1766 	parent = dget_parent(nd->path.dentry);
1767 	if (unlikely(!path_connected(nd->path.mnt, parent))) {
1768 		dput(parent);
1769 		return ERR_PTR(-ENOENT);
1770 	}
1771 	*seqp = 0;
1772 	*inodep = parent->d_inode;
1773 	return parent;
1774 
1775 in_root:
1776 	if (unlikely(nd->flags & LOOKUP_BENEATH))
1777 		return ERR_PTR(-EXDEV);
1778 	dget(nd->path.dentry);
1779 	return NULL;
1780 }
1781 
1782 static const char *handle_dots(struct nameidata *nd, int type)
1783 {
1784 	if (type == LAST_DOTDOT) {
1785 		const char *error = NULL;
1786 		struct dentry *parent;
1787 		struct inode *inode;
1788 		unsigned seq;
1789 
1790 		if (!nd->root.mnt) {
1791 			error = ERR_PTR(set_root(nd));
1792 			if (error)
1793 				return error;
1794 		}
1795 		if (nd->flags & LOOKUP_RCU)
1796 			parent = follow_dotdot_rcu(nd, &inode, &seq);
1797 		else
1798 			parent = follow_dotdot(nd, &inode, &seq);
1799 		if (IS_ERR(parent))
1800 			return ERR_CAST(parent);
1801 		if (unlikely(!parent))
1802 			error = step_into(nd, WALK_NOFOLLOW,
1803 					 nd->path.dentry, nd->inode, nd->seq);
1804 		else
1805 			error = step_into(nd, WALK_NOFOLLOW,
1806 					 parent, inode, seq);
1807 		if (unlikely(error))
1808 			return error;
1809 
1810 		if (unlikely(nd->flags & LOOKUP_IS_SCOPED)) {
1811 			/*
1812 			 * If there was a racing rename or mount along our
1813 			 * path, then we can't be sure that ".." hasn't jumped
1814 			 * above nd->root (and so userspace should retry or use
1815 			 * some fallback).
1816 			 */
1817 			smp_rmb();
1818 			if (unlikely(__read_seqcount_retry(&mount_lock.seqcount, nd->m_seq)))
1819 				return ERR_PTR(-EAGAIN);
1820 			if (unlikely(__read_seqcount_retry(&rename_lock.seqcount, nd->r_seq)))
1821 				return ERR_PTR(-EAGAIN);
1822 		}
1823 	}
1824 	return NULL;
1825 }
1826 
1827 static const char *walk_component(struct nameidata *nd, int flags)
1828 {
1829 	struct dentry *dentry;
1830 	struct inode *inode;
1831 	unsigned seq;
1832 	/*
1833 	 * "." and ".." are special - ".." especially so because it has
1834 	 * to be able to know about the current root directory and
1835 	 * parent relationships.
1836 	 */
1837 	if (unlikely(nd->last_type != LAST_NORM)) {
1838 		if (!(flags & WALK_MORE) && nd->depth)
1839 			put_link(nd);
1840 		return handle_dots(nd, nd->last_type);
1841 	}
1842 	dentry = lookup_fast(nd, &inode, &seq);
1843 	if (IS_ERR(dentry))
1844 		return ERR_CAST(dentry);
1845 	if (unlikely(!dentry)) {
1846 		dentry = lookup_slow(&nd->last, nd->path.dentry, nd->flags);
1847 		if (IS_ERR(dentry))
1848 			return ERR_CAST(dentry);
1849 	}
1850 	if (!(flags & WALK_MORE) && nd->depth)
1851 		put_link(nd);
1852 	return step_into(nd, flags, dentry, inode, seq);
1853 }
1854 
1855 /*
1856  * We can do the critical dentry name comparison and hashing
1857  * operations one word at a time, but we are limited to:
1858  *
1859  * - Architectures with fast unaligned word accesses. We could
1860  *   do a "get_unaligned()" if this helps and is sufficiently
1861  *   fast.
1862  *
1863  * - non-CONFIG_DEBUG_PAGEALLOC configurations (so that we
1864  *   do not trap on the (extremely unlikely) case of a page
1865  *   crossing operation.
1866  *
1867  * - Furthermore, we need an efficient 64-bit compile for the
1868  *   64-bit case in order to generate the "number of bytes in
1869  *   the final mask". Again, that could be replaced with a
1870  *   efficient population count instruction or similar.
1871  */
1872 #ifdef CONFIG_DCACHE_WORD_ACCESS
1873 
1874 #include <asm/word-at-a-time.h>
1875 
1876 #ifdef HASH_MIX
1877 
1878 /* Architecture provides HASH_MIX and fold_hash() in <asm/hash.h> */
1879 
1880 #elif defined(CONFIG_64BIT)
1881 /*
1882  * Register pressure in the mixing function is an issue, particularly
1883  * on 32-bit x86, but almost any function requires one state value and
1884  * one temporary.  Instead, use a function designed for two state values
1885  * and no temporaries.
1886  *
1887  * This function cannot create a collision in only two iterations, so
1888  * we have two iterations to achieve avalanche.  In those two iterations,
1889  * we have six layers of mixing, which is enough to spread one bit's
1890  * influence out to 2^6 = 64 state bits.
1891  *
1892  * Rotate constants are scored by considering either 64 one-bit input
1893  * deltas or 64*63/2 = 2016 two-bit input deltas, and finding the
1894  * probability of that delta causing a change to each of the 128 output
1895  * bits, using a sample of random initial states.
1896  *
1897  * The Shannon entropy of the computed probabilities is then summed
1898  * to produce a score.  Ideally, any input change has a 50% chance of
1899  * toggling any given output bit.
1900  *
1901  * Mixing scores (in bits) for (12,45):
1902  * Input delta: 1-bit      2-bit
1903  * 1 round:     713.3    42542.6
1904  * 2 rounds:   2753.7   140389.8
1905  * 3 rounds:   5954.1   233458.2
1906  * 4 rounds:   7862.6   256672.2
1907  * Perfect:    8192     258048
1908  *            (64*128) (64*63/2 * 128)
1909  */
1910 #define HASH_MIX(x, y, a)	\
1911 	(	x ^= (a),	\
1912 	y ^= x,	x = rol64(x,12),\
1913 	x += y,	y = rol64(y,45),\
1914 	y *= 9			)
1915 
1916 /*
1917  * Fold two longs into one 32-bit hash value.  This must be fast, but
1918  * latency isn't quite as critical, as there is a fair bit of additional
1919  * work done before the hash value is used.
1920  */
1921 static inline unsigned int fold_hash(unsigned long x, unsigned long y)
1922 {
1923 	y ^= x * GOLDEN_RATIO_64;
1924 	y *= GOLDEN_RATIO_64;
1925 	return y >> 32;
1926 }
1927 
1928 #else	/* 32-bit case */
1929 
1930 /*
1931  * Mixing scores (in bits) for (7,20):
1932  * Input delta: 1-bit      2-bit
1933  * 1 round:     330.3     9201.6
1934  * 2 rounds:   1246.4    25475.4
1935  * 3 rounds:   1907.1    31295.1
1936  * 4 rounds:   2042.3    31718.6
1937  * Perfect:    2048      31744
1938  *            (32*64)   (32*31/2 * 64)
1939  */
1940 #define HASH_MIX(x, y, a)	\
1941 	(	x ^= (a),	\
1942 	y ^= x,	x = rol32(x, 7),\
1943 	x += y,	y = rol32(y,20),\
1944 	y *= 9			)
1945 
1946 static inline unsigned int fold_hash(unsigned long x, unsigned long y)
1947 {
1948 	/* Use arch-optimized multiply if one exists */
1949 	return __hash_32(y ^ __hash_32(x));
1950 }
1951 
1952 #endif
1953 
1954 /*
1955  * Return the hash of a string of known length.  This is carfully
1956  * designed to match hash_name(), which is the more critical function.
1957  * In particular, we must end by hashing a final word containing 0..7
1958  * payload bytes, to match the way that hash_name() iterates until it
1959  * finds the delimiter after the name.
1960  */
1961 unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
1962 {
1963 	unsigned long a, x = 0, y = (unsigned long)salt;
1964 
1965 	for (;;) {
1966 		if (!len)
1967 			goto done;
1968 		a = load_unaligned_zeropad(name);
1969 		if (len < sizeof(unsigned long))
1970 			break;
1971 		HASH_MIX(x, y, a);
1972 		name += sizeof(unsigned long);
1973 		len -= sizeof(unsigned long);
1974 	}
1975 	x ^= a & bytemask_from_count(len);
1976 done:
1977 	return fold_hash(x, y);
1978 }
1979 EXPORT_SYMBOL(full_name_hash);
1980 
1981 /* Return the "hash_len" (hash and length) of a null-terminated string */
1982 u64 hashlen_string(const void *salt, const char *name)
1983 {
1984 	unsigned long a = 0, x = 0, y = (unsigned long)salt;
1985 	unsigned long adata, mask, len;
1986 	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
1987 
1988 	len = 0;
1989 	goto inside;
1990 
1991 	do {
1992 		HASH_MIX(x, y, a);
1993 		len += sizeof(unsigned long);
1994 inside:
1995 		a = load_unaligned_zeropad(name+len);
1996 	} while (!has_zero(a, &adata, &constants));
1997 
1998 	adata = prep_zero_mask(a, adata, &constants);
1999 	mask = create_zero_mask(adata);
2000 	x ^= a & zero_bytemask(mask);
2001 
2002 	return hashlen_create(fold_hash(x, y), len + find_zero(mask));
2003 }
2004 EXPORT_SYMBOL(hashlen_string);
2005 
2006 /*
2007  * Calculate the length and hash of the path component, and
2008  * return the "hash_len" as the result.
2009  */
2010 static inline u64 hash_name(const void *salt, const char *name)
2011 {
2012 	unsigned long a = 0, b, x = 0, y = (unsigned long)salt;
2013 	unsigned long adata, bdata, mask, len;
2014 	const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
2015 
2016 	len = 0;
2017 	goto inside;
2018 
2019 	do {
2020 		HASH_MIX(x, y, a);
2021 		len += sizeof(unsigned long);
2022 inside:
2023 		a = load_unaligned_zeropad(name+len);
2024 		b = a ^ REPEAT_BYTE('/');
2025 	} while (!(has_zero(a, &adata, &constants) | has_zero(b, &bdata, &constants)));
2026 
2027 	adata = prep_zero_mask(a, adata, &constants);
2028 	bdata = prep_zero_mask(b, bdata, &constants);
2029 	mask = create_zero_mask(adata | bdata);
2030 	x ^= a & zero_bytemask(mask);
2031 
2032 	return hashlen_create(fold_hash(x, y), len + find_zero(mask));
2033 }
2034 
2035 #else	/* !CONFIG_DCACHE_WORD_ACCESS: Slow, byte-at-a-time version */
2036 
2037 /* Return the hash of a string of known length */
2038 unsigned int full_name_hash(const void *salt, const char *name, unsigned int len)
2039 {
2040 	unsigned long hash = init_name_hash(salt);
2041 	while (len--)
2042 		hash = partial_name_hash((unsigned char)*name++, hash);
2043 	return end_name_hash(hash);
2044 }
2045 EXPORT_SYMBOL(full_name_hash);
2046 
2047 /* Return the "hash_len" (hash and length) of a null-terminated string */
2048 u64 hashlen_string(const void *salt, const char *name)
2049 {
2050 	unsigned long hash = init_name_hash(salt);
2051 	unsigned long len = 0, c;
2052 
2053 	c = (unsigned char)*name;
2054 	while (c) {
2055 		len++;
2056 		hash = partial_name_hash(c, hash);
2057 		c = (unsigned char)name[len];
2058 	}
2059 	return hashlen_create(end_name_hash(hash), len);
2060 }
2061 EXPORT_SYMBOL(hashlen_string);
2062 
2063 /*
2064  * We know there's a real path component here of at least
2065  * one character.
2066  */
2067 static inline u64 hash_name(const void *salt, const char *name)
2068 {
2069 	unsigned long hash = init_name_hash(salt);
2070 	unsigned long len = 0, c;
2071 
2072 	c = (unsigned char)*name;
2073 	do {
2074 		len++;
2075 		hash = partial_name_hash(c, hash);
2076 		c = (unsigned char)name[len];
2077 	} while (c && c != '/');
2078 	return hashlen_create(end_name_hash(hash), len);
2079 }
2080 
2081 #endif
2082 
2083 /*
2084  * Name resolution.
2085  * This is the basic name resolution function, turning a pathname into
2086  * the final dentry. We expect 'base' to be positive and a directory.
2087  *
2088  * Returns 0 and nd will have valid dentry and mnt on success.
2089  * Returns error and drops reference to input namei data on failure.
2090  */
2091 static int link_path_walk(const char *name, struct nameidata *nd)
2092 {
2093 	int depth = 0; // depth <= nd->depth
2094 	int err;
2095 
2096 	nd->last_type = LAST_ROOT;
2097 	nd->flags |= LOOKUP_PARENT;
2098 	if (IS_ERR(name))
2099 		return PTR_ERR(name);
2100 	while (*name=='/')
2101 		name++;
2102 	if (!*name)
2103 		return 0;
2104 
2105 	/* At this point we know we have a real path component. */
2106 	for(;;) {
2107 		const char *link;
2108 		u64 hash_len;
2109 		int type;
2110 
2111 		err = may_lookup(nd);
2112 		if (err)
2113 			return err;
2114 
2115 		hash_len = hash_name(nd->path.dentry, name);
2116 
2117 		type = LAST_NORM;
2118 		if (name[0] == '.') switch (hashlen_len(hash_len)) {
2119 			case 2:
2120 				if (name[1] == '.') {
2121 					type = LAST_DOTDOT;
2122 					nd->flags |= LOOKUP_JUMPED;
2123 				}
2124 				break;
2125 			case 1:
2126 				type = LAST_DOT;
2127 		}
2128 		if (likely(type == LAST_NORM)) {
2129 			struct dentry *parent = nd->path.dentry;
2130 			nd->flags &= ~LOOKUP_JUMPED;
2131 			if (unlikely(parent->d_flags & DCACHE_OP_HASH)) {
2132 				struct qstr this = { { .hash_len = hash_len }, .name = name };
2133 				err = parent->d_op->d_hash(parent, &this);
2134 				if (err < 0)
2135 					return err;
2136 				hash_len = this.hash_len;
2137 				name = this.name;
2138 			}
2139 		}
2140 
2141 		nd->last.hash_len = hash_len;
2142 		nd->last.name = name;
2143 		nd->last_type = type;
2144 
2145 		name += hashlen_len(hash_len);
2146 		if (!*name)
2147 			goto OK;
2148 		/*
2149 		 * If it wasn't NUL, we know it was '/'. Skip that
2150 		 * slash, and continue until no more slashes.
2151 		 */
2152 		do {
2153 			name++;
2154 		} while (unlikely(*name == '/'));
2155 		if (unlikely(!*name)) {
2156 OK:
2157 			/* pathname or trailing symlink, done */
2158 			if (!depth) {
2159 				nd->dir_uid = nd->inode->i_uid;
2160 				nd->dir_mode = nd->inode->i_mode;
2161 				nd->flags &= ~LOOKUP_PARENT;
2162 				return 0;
2163 			}
2164 			/* last component of nested symlink */
2165 			name = nd->stack[--depth].name;
2166 			link = walk_component(nd, 0);
2167 		} else {
2168 			/* not the last component */
2169 			link = walk_component(nd, WALK_MORE);
2170 		}
2171 		if (unlikely(link)) {
2172 			if (IS_ERR(link))
2173 				return PTR_ERR(link);
2174 			/* a symlink to follow */
2175 			nd->stack[depth++].name = name;
2176 			name = link;
2177 			continue;
2178 		}
2179 		if (unlikely(!d_can_lookup(nd->path.dentry))) {
2180 			if (nd->flags & LOOKUP_RCU) {
2181 				if (unlazy_walk(nd))
2182 					return -ECHILD;
2183 			}
2184 			return -ENOTDIR;
2185 		}
2186 	}
2187 }
2188 
2189 /* must be paired with terminate_walk() */
2190 static const char *path_init(struct nameidata *nd, unsigned flags)
2191 {
2192 	int error;
2193 	const char *s = nd->name->name;
2194 
2195 	if (!*s)
2196 		flags &= ~LOOKUP_RCU;
2197 	if (flags & LOOKUP_RCU)
2198 		rcu_read_lock();
2199 
2200 	nd->flags = flags | LOOKUP_JUMPED;
2201 	nd->depth = 0;
2202 
2203 	nd->m_seq = __read_seqcount_begin(&mount_lock.seqcount);
2204 	nd->r_seq = __read_seqcount_begin(&rename_lock.seqcount);
2205 	smp_rmb();
2206 
2207 	if (flags & LOOKUP_ROOT) {
2208 		struct dentry *root = nd->root.dentry;
2209 		struct inode *inode = root->d_inode;
2210 		if (*s && unlikely(!d_can_lookup(root)))
2211 			return ERR_PTR(-ENOTDIR);
2212 		nd->path = nd->root;
2213 		nd->inode = inode;
2214 		if (flags & LOOKUP_RCU) {
2215 			nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
2216 			nd->root_seq = nd->seq;
2217 		} else {
2218 			path_get(&nd->path);
2219 		}
2220 		return s;
2221 	}
2222 
2223 	nd->root.mnt = NULL;
2224 	nd->path.mnt = NULL;
2225 	nd->path.dentry = NULL;
2226 
2227 	/* Absolute pathname -- fetch the root (LOOKUP_IN_ROOT uses nd->dfd). */
2228 	if (*s == '/' && !(flags & LOOKUP_IN_ROOT)) {
2229 		error = nd_jump_root(nd);
2230 		if (unlikely(error))
2231 			return ERR_PTR(error);
2232 		return s;
2233 	}
2234 
2235 	/* Relative pathname -- get the starting-point it is relative to. */
2236 	if (nd->dfd == AT_FDCWD) {
2237 		if (flags & LOOKUP_RCU) {
2238 			struct fs_struct *fs = current->fs;
2239 			unsigned seq;
2240 
2241 			do {
2242 				seq = read_seqcount_begin(&fs->seq);
2243 				nd->path = fs->pwd;
2244 				nd->inode = nd->path.dentry->d_inode;
2245 				nd->seq = __read_seqcount_begin(&nd->path.dentry->d_seq);
2246 			} while (read_seqcount_retry(&fs->seq, seq));
2247 		} else {
2248 			get_fs_pwd(current->fs, &nd->path);
2249 			nd->inode = nd->path.dentry->d_inode;
2250 		}
2251 	} else {
2252 		/* Caller must check execute permissions on the starting path component */
2253 		struct fd f = fdget_raw(nd->dfd);
2254 		struct dentry *dentry;
2255 
2256 		if (!f.file)
2257 			return ERR_PTR(-EBADF);
2258 
2259 		dentry = f.file->f_path.dentry;
2260 
2261 		if (*s && unlikely(!d_can_lookup(dentry))) {
2262 			fdput(f);
2263 			return ERR_PTR(-ENOTDIR);
2264 		}
2265 
2266 		nd->path = f.file->f_path;
2267 		if (flags & LOOKUP_RCU) {
2268 			nd->inode = nd->path.dentry->d_inode;
2269 			nd->seq = read_seqcount_begin(&nd->path.dentry->d_seq);
2270 		} else {
2271 			path_get(&nd->path);
2272 			nd->inode = nd->path.dentry->d_inode;
2273 		}
2274 		fdput(f);
2275 	}
2276 
2277 	/* For scoped-lookups we need to set the root to the dirfd as well. */
2278 	if (flags & LOOKUP_IS_SCOPED) {
2279 		nd->root = nd->path;
2280 		if (flags & LOOKUP_RCU) {
2281 			nd->root_seq = nd->seq;
2282 		} else {
2283 			path_get(&nd->root);
2284 			nd->flags |= LOOKUP_ROOT_GRABBED;
2285 		}
2286 	}
2287 	return s;
2288 }
2289 
2290 static inline const char *lookup_last(struct nameidata *nd)
2291 {
2292 	if (nd->last_type == LAST_NORM && nd->last.name[nd->last.len])
2293 		nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
2294 
2295 	return walk_component(nd, WALK_TRAILING);
2296 }
2297 
2298 static int handle_lookup_down(struct nameidata *nd)
2299 {
2300 	if (!(nd->flags & LOOKUP_RCU))
2301 		dget(nd->path.dentry);
2302 	return PTR_ERR(step_into(nd, WALK_NOFOLLOW,
2303 			nd->path.dentry, nd->inode, nd->seq));
2304 }
2305 
2306 /* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2307 static int path_lookupat(struct nameidata *nd, unsigned flags, struct path *path)
2308 {
2309 	const char *s = path_init(nd, flags);
2310 	int err;
2311 
2312 	if (unlikely(flags & LOOKUP_DOWN) && !IS_ERR(s)) {
2313 		err = handle_lookup_down(nd);
2314 		if (unlikely(err < 0))
2315 			s = ERR_PTR(err);
2316 	}
2317 
2318 	while (!(err = link_path_walk(s, nd)) &&
2319 	       (s = lookup_last(nd)) != NULL)
2320 		;
2321 	if (!err)
2322 		err = complete_walk(nd);
2323 
2324 	if (!err && nd->flags & LOOKUP_DIRECTORY)
2325 		if (!d_can_lookup(nd->path.dentry))
2326 			err = -ENOTDIR;
2327 	if (!err && unlikely(nd->flags & LOOKUP_MOUNTPOINT)) {
2328 		err = handle_lookup_down(nd);
2329 		nd->flags &= ~LOOKUP_JUMPED; // no d_weak_revalidate(), please...
2330 	}
2331 	if (!err) {
2332 		*path = nd->path;
2333 		nd->path.mnt = NULL;
2334 		nd->path.dentry = NULL;
2335 	}
2336 	terminate_walk(nd);
2337 	return err;
2338 }
2339 
2340 int filename_lookup(int dfd, struct filename *name, unsigned flags,
2341 		    struct path *path, struct path *root)
2342 {
2343 	int retval;
2344 	struct nameidata nd;
2345 	if (IS_ERR(name))
2346 		return PTR_ERR(name);
2347 	if (unlikely(root)) {
2348 		nd.root = *root;
2349 		flags |= LOOKUP_ROOT;
2350 	}
2351 	set_nameidata(&nd, dfd, name);
2352 	retval = path_lookupat(&nd, flags | LOOKUP_RCU, path);
2353 	if (unlikely(retval == -ECHILD))
2354 		retval = path_lookupat(&nd, flags, path);
2355 	if (unlikely(retval == -ESTALE))
2356 		retval = path_lookupat(&nd, flags | LOOKUP_REVAL, path);
2357 
2358 	if (likely(!retval))
2359 		audit_inode(name, path->dentry,
2360 			    flags & LOOKUP_MOUNTPOINT ? AUDIT_INODE_NOEVAL : 0);
2361 	restore_nameidata();
2362 	putname(name);
2363 	return retval;
2364 }
2365 
2366 /* Returns 0 and nd will be valid on success; Retuns error, otherwise. */
2367 static int path_parentat(struct nameidata *nd, unsigned flags,
2368 				struct path *parent)
2369 {
2370 	const char *s = path_init(nd, flags);
2371 	int err = link_path_walk(s, nd);
2372 	if (!err)
2373 		err = complete_walk(nd);
2374 	if (!err) {
2375 		*parent = nd->path;
2376 		nd->path.mnt = NULL;
2377 		nd->path.dentry = NULL;
2378 	}
2379 	terminate_walk(nd);
2380 	return err;
2381 }
2382 
2383 static struct filename *filename_parentat(int dfd, struct filename *name,
2384 				unsigned int flags, struct path *parent,
2385 				struct qstr *last, int *type)
2386 {
2387 	int retval;
2388 	struct nameidata nd;
2389 
2390 	if (IS_ERR(name))
2391 		return name;
2392 	set_nameidata(&nd, dfd, name);
2393 	retval = path_parentat(&nd, flags | LOOKUP_RCU, parent);
2394 	if (unlikely(retval == -ECHILD))
2395 		retval = path_parentat(&nd, flags, parent);
2396 	if (unlikely(retval == -ESTALE))
2397 		retval = path_parentat(&nd, flags | LOOKUP_REVAL, parent);
2398 	if (likely(!retval)) {
2399 		*last = nd.last;
2400 		*type = nd.last_type;
2401 		audit_inode(name, parent->dentry, AUDIT_INODE_PARENT);
2402 	} else {
2403 		putname(name);
2404 		name = ERR_PTR(retval);
2405 	}
2406 	restore_nameidata();
2407 	return name;
2408 }
2409 
2410 /* does lookup, returns the object with parent locked */
2411 struct dentry *kern_path_locked(const char *name, struct path *path)
2412 {
2413 	struct filename *filename;
2414 	struct dentry *d;
2415 	struct qstr last;
2416 	int type;
2417 
2418 	filename = filename_parentat(AT_FDCWD, getname_kernel(name), 0, path,
2419 				    &last, &type);
2420 	if (IS_ERR(filename))
2421 		return ERR_CAST(filename);
2422 	if (unlikely(type != LAST_NORM)) {
2423 		path_put(path);
2424 		putname(filename);
2425 		return ERR_PTR(-EINVAL);
2426 	}
2427 	inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
2428 	d = __lookup_hash(&last, path->dentry, 0);
2429 	if (IS_ERR(d)) {
2430 		inode_unlock(path->dentry->d_inode);
2431 		path_put(path);
2432 	}
2433 	putname(filename);
2434 	return d;
2435 }
2436 
2437 int kern_path(const char *name, unsigned int flags, struct path *path)
2438 {
2439 	return filename_lookup(AT_FDCWD, getname_kernel(name),
2440 			       flags, path, NULL);
2441 }
2442 EXPORT_SYMBOL(kern_path);
2443 
2444 /**
2445  * vfs_path_lookup - lookup a file path relative to a dentry-vfsmount pair
2446  * @dentry:  pointer to dentry of the base directory
2447  * @mnt: pointer to vfs mount of the base directory
2448  * @name: pointer to file name
2449  * @flags: lookup flags
2450  * @path: pointer to struct path to fill
2451  */
2452 int vfs_path_lookup(struct dentry *dentry, struct vfsmount *mnt,
2453 		    const char *name, unsigned int flags,
2454 		    struct path *path)
2455 {
2456 	struct path root = {.mnt = mnt, .dentry = dentry};
2457 	/* the first argument of filename_lookup() is ignored with root */
2458 	return filename_lookup(AT_FDCWD, getname_kernel(name),
2459 			       flags , path, &root);
2460 }
2461 EXPORT_SYMBOL(vfs_path_lookup);
2462 
2463 static int lookup_one_len_common(const char *name, struct dentry *base,
2464 				 int len, struct qstr *this)
2465 {
2466 	this->name = name;
2467 	this->len = len;
2468 	this->hash = full_name_hash(base, name, len);
2469 	if (!len)
2470 		return -EACCES;
2471 
2472 	if (unlikely(name[0] == '.')) {
2473 		if (len < 2 || (len == 2 && name[1] == '.'))
2474 			return -EACCES;
2475 	}
2476 
2477 	while (len--) {
2478 		unsigned int c = *(const unsigned char *)name++;
2479 		if (c == '/' || c == '\0')
2480 			return -EACCES;
2481 	}
2482 	/*
2483 	 * See if the low-level filesystem might want
2484 	 * to use its own hash..
2485 	 */
2486 	if (base->d_flags & DCACHE_OP_HASH) {
2487 		int err = base->d_op->d_hash(base, this);
2488 		if (err < 0)
2489 			return err;
2490 	}
2491 
2492 	return inode_permission(base->d_inode, MAY_EXEC);
2493 }
2494 
2495 /**
2496  * try_lookup_one_len - filesystem helper to lookup single pathname component
2497  * @name:	pathname component to lookup
2498  * @base:	base directory to lookup from
2499  * @len:	maximum length @len should be interpreted to
2500  *
2501  * Look up a dentry by name in the dcache, returning NULL if it does not
2502  * currently exist.  The function does not try to create a dentry.
2503  *
2504  * Note that this routine is purely a helper for filesystem usage and should
2505  * not be called by generic code.
2506  *
2507  * The caller must hold base->i_mutex.
2508  */
2509 struct dentry *try_lookup_one_len(const char *name, struct dentry *base, int len)
2510 {
2511 	struct qstr this;
2512 	int err;
2513 
2514 	WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2515 
2516 	err = lookup_one_len_common(name, base, len, &this);
2517 	if (err)
2518 		return ERR_PTR(err);
2519 
2520 	return lookup_dcache(&this, base, 0);
2521 }
2522 EXPORT_SYMBOL(try_lookup_one_len);
2523 
2524 /**
2525  * lookup_one_len - filesystem helper to lookup single pathname component
2526  * @name:	pathname component to lookup
2527  * @base:	base directory to lookup from
2528  * @len:	maximum length @len should be interpreted to
2529  *
2530  * Note that this routine is purely a helper for filesystem usage and should
2531  * not be called by generic code.
2532  *
2533  * The caller must hold base->i_mutex.
2534  */
2535 struct dentry *lookup_one_len(const char *name, struct dentry *base, int len)
2536 {
2537 	struct dentry *dentry;
2538 	struct qstr this;
2539 	int err;
2540 
2541 	WARN_ON_ONCE(!inode_is_locked(base->d_inode));
2542 
2543 	err = lookup_one_len_common(name, base, len, &this);
2544 	if (err)
2545 		return ERR_PTR(err);
2546 
2547 	dentry = lookup_dcache(&this, base, 0);
2548 	return dentry ? dentry : __lookup_slow(&this, base, 0);
2549 }
2550 EXPORT_SYMBOL(lookup_one_len);
2551 
2552 /**
2553  * lookup_one_len_unlocked - filesystem helper to lookup single pathname component
2554  * @name:	pathname component to lookup
2555  * @base:	base directory to lookup from
2556  * @len:	maximum length @len should be interpreted to
2557  *
2558  * Note that this routine is purely a helper for filesystem usage and should
2559  * not be called by generic code.
2560  *
2561  * Unlike lookup_one_len, it should be called without the parent
2562  * i_mutex held, and will take the i_mutex itself if necessary.
2563  */
2564 struct dentry *lookup_one_len_unlocked(const char *name,
2565 				       struct dentry *base, int len)
2566 {
2567 	struct qstr this;
2568 	int err;
2569 	struct dentry *ret;
2570 
2571 	err = lookup_one_len_common(name, base, len, &this);
2572 	if (err)
2573 		return ERR_PTR(err);
2574 
2575 	ret = lookup_dcache(&this, base, 0);
2576 	if (!ret)
2577 		ret = lookup_slow(&this, base, 0);
2578 	return ret;
2579 }
2580 EXPORT_SYMBOL(lookup_one_len_unlocked);
2581 
2582 /*
2583  * Like lookup_one_len_unlocked(), except that it yields ERR_PTR(-ENOENT)
2584  * on negatives.  Returns known positive or ERR_PTR(); that's what
2585  * most of the users want.  Note that pinned negative with unlocked parent
2586  * _can_ become positive at any time, so callers of lookup_one_len_unlocked()
2587  * need to be very careful; pinned positives have ->d_inode stable, so
2588  * this one avoids such problems.
2589  */
2590 struct dentry *lookup_positive_unlocked(const char *name,
2591 				       struct dentry *base, int len)
2592 {
2593 	struct dentry *ret = lookup_one_len_unlocked(name, base, len);
2594 	if (!IS_ERR(ret) && d_flags_negative(smp_load_acquire(&ret->d_flags))) {
2595 		dput(ret);
2596 		ret = ERR_PTR(-ENOENT);
2597 	}
2598 	return ret;
2599 }
2600 EXPORT_SYMBOL(lookup_positive_unlocked);
2601 
2602 #ifdef CONFIG_UNIX98_PTYS
2603 int path_pts(struct path *path)
2604 {
2605 	/* Find something mounted on "pts" in the same directory as
2606 	 * the input path.
2607 	 */
2608 	struct dentry *parent = dget_parent(path->dentry);
2609 	struct dentry *child;
2610 	struct qstr this = QSTR_INIT("pts", 3);
2611 
2612 	if (unlikely(!path_connected(path->mnt, parent))) {
2613 		dput(parent);
2614 		return -ENOENT;
2615 	}
2616 	dput(path->dentry);
2617 	path->dentry = parent;
2618 	child = d_hash_and_lookup(parent, &this);
2619 	if (!child)
2620 		return -ENOENT;
2621 
2622 	path->dentry = child;
2623 	dput(parent);
2624 	follow_down(path);
2625 	return 0;
2626 }
2627 #endif
2628 
2629 int user_path_at_empty(int dfd, const char __user *name, unsigned flags,
2630 		 struct path *path, int *empty)
2631 {
2632 	return filename_lookup(dfd, getname_flags(name, flags, empty),
2633 			       flags, path, NULL);
2634 }
2635 EXPORT_SYMBOL(user_path_at_empty);
2636 
2637 int __check_sticky(struct inode *dir, struct inode *inode)
2638 {
2639 	kuid_t fsuid = current_fsuid();
2640 
2641 	if (uid_eq(inode->i_uid, fsuid))
2642 		return 0;
2643 	if (uid_eq(dir->i_uid, fsuid))
2644 		return 0;
2645 	return !capable_wrt_inode_uidgid(inode, CAP_FOWNER);
2646 }
2647 EXPORT_SYMBOL(__check_sticky);
2648 
2649 /*
2650  *	Check whether we can remove a link victim from directory dir, check
2651  *  whether the type of victim is right.
2652  *  1. We can't do it if dir is read-only (done in permission())
2653  *  2. We should have write and exec permissions on dir
2654  *  3. We can't remove anything from append-only dir
2655  *  4. We can't do anything with immutable dir (done in permission())
2656  *  5. If the sticky bit on dir is set we should either
2657  *	a. be owner of dir, or
2658  *	b. be owner of victim, or
2659  *	c. have CAP_FOWNER capability
2660  *  6. If the victim is append-only or immutable we can't do antyhing with
2661  *     links pointing to it.
2662  *  7. If the victim has an unknown uid or gid we can't change the inode.
2663  *  8. If we were asked to remove a directory and victim isn't one - ENOTDIR.
2664  *  9. If we were asked to remove a non-directory and victim isn't one - EISDIR.
2665  * 10. We can't remove a root or mountpoint.
2666  * 11. We don't allow removal of NFS sillyrenamed files; it's handled by
2667  *     nfs_async_unlink().
2668  */
2669 static int may_delete(struct inode *dir, struct dentry *victim, bool isdir)
2670 {
2671 	struct inode *inode = d_backing_inode(victim);
2672 	int error;
2673 
2674 	if (d_is_negative(victim))
2675 		return -ENOENT;
2676 	BUG_ON(!inode);
2677 
2678 	BUG_ON(victim->d_parent->d_inode != dir);
2679 
2680 	/* Inode writeback is not safe when the uid or gid are invalid. */
2681 	if (!uid_valid(inode->i_uid) || !gid_valid(inode->i_gid))
2682 		return -EOVERFLOW;
2683 
2684 	audit_inode_child(dir, victim, AUDIT_TYPE_CHILD_DELETE);
2685 
2686 	error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
2687 	if (error)
2688 		return error;
2689 	if (IS_APPEND(dir))
2690 		return -EPERM;
2691 
2692 	if (check_sticky(dir, inode) || IS_APPEND(inode) ||
2693 	    IS_IMMUTABLE(inode) || IS_SWAPFILE(inode) || HAS_UNMAPPED_ID(inode))
2694 		return -EPERM;
2695 	if (isdir) {
2696 		if (!d_is_dir(victim))
2697 			return -ENOTDIR;
2698 		if (IS_ROOT(victim))
2699 			return -EBUSY;
2700 	} else if (d_is_dir(victim))
2701 		return -EISDIR;
2702 	if (IS_DEADDIR(dir))
2703 		return -ENOENT;
2704 	if (victim->d_flags & DCACHE_NFSFS_RENAMED)
2705 		return -EBUSY;
2706 	return 0;
2707 }
2708 
2709 /*	Check whether we can create an object with dentry child in directory
2710  *  dir.
2711  *  1. We can't do it if child already exists (open has special treatment for
2712  *     this case, but since we are inlined it's OK)
2713  *  2. We can't do it if dir is read-only (done in permission())
2714  *  3. We can't do it if the fs can't represent the fsuid or fsgid.
2715  *  4. We should have write and exec permissions on dir
2716  *  5. We can't do it if dir is immutable (done in permission())
2717  */
2718 static inline int may_create(struct inode *dir, struct dentry *child)
2719 {
2720 	struct user_namespace *s_user_ns;
2721 	audit_inode_child(dir, child, AUDIT_TYPE_CHILD_CREATE);
2722 	if (child->d_inode)
2723 		return -EEXIST;
2724 	if (IS_DEADDIR(dir))
2725 		return -ENOENT;
2726 	s_user_ns = dir->i_sb->s_user_ns;
2727 	if (!kuid_has_mapping(s_user_ns, current_fsuid()) ||
2728 	    !kgid_has_mapping(s_user_ns, current_fsgid()))
2729 		return -EOVERFLOW;
2730 	return inode_permission(dir, MAY_WRITE | MAY_EXEC);
2731 }
2732 
2733 /*
2734  * p1 and p2 should be directories on the same fs.
2735  */
2736 struct dentry *lock_rename(struct dentry *p1, struct dentry *p2)
2737 {
2738 	struct dentry *p;
2739 
2740 	if (p1 == p2) {
2741 		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2742 		return NULL;
2743 	}
2744 
2745 	mutex_lock(&p1->d_sb->s_vfs_rename_mutex);
2746 
2747 	p = d_ancestor(p2, p1);
2748 	if (p) {
2749 		inode_lock_nested(p2->d_inode, I_MUTEX_PARENT);
2750 		inode_lock_nested(p1->d_inode, I_MUTEX_CHILD);
2751 		return p;
2752 	}
2753 
2754 	p = d_ancestor(p1, p2);
2755 	if (p) {
2756 		inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2757 		inode_lock_nested(p2->d_inode, I_MUTEX_CHILD);
2758 		return p;
2759 	}
2760 
2761 	inode_lock_nested(p1->d_inode, I_MUTEX_PARENT);
2762 	inode_lock_nested(p2->d_inode, I_MUTEX_PARENT2);
2763 	return NULL;
2764 }
2765 EXPORT_SYMBOL(lock_rename);
2766 
2767 void unlock_rename(struct dentry *p1, struct dentry *p2)
2768 {
2769 	inode_unlock(p1->d_inode);
2770 	if (p1 != p2) {
2771 		inode_unlock(p2->d_inode);
2772 		mutex_unlock(&p1->d_sb->s_vfs_rename_mutex);
2773 	}
2774 }
2775 EXPORT_SYMBOL(unlock_rename);
2776 
2777 int vfs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2778 		bool want_excl)
2779 {
2780 	int error = may_create(dir, dentry);
2781 	if (error)
2782 		return error;
2783 
2784 	if (!dir->i_op->create)
2785 		return -EACCES;	/* shouldn't it be ENOSYS? */
2786 	mode &= S_IALLUGO;
2787 	mode |= S_IFREG;
2788 	error = security_inode_create(dir, dentry, mode);
2789 	if (error)
2790 		return error;
2791 	error = dir->i_op->create(dir, dentry, mode, want_excl);
2792 	if (!error)
2793 		fsnotify_create(dir, dentry);
2794 	return error;
2795 }
2796 EXPORT_SYMBOL(vfs_create);
2797 
2798 int vfs_mkobj(struct dentry *dentry, umode_t mode,
2799 		int (*f)(struct dentry *, umode_t, void *),
2800 		void *arg)
2801 {
2802 	struct inode *dir = dentry->d_parent->d_inode;
2803 	int error = may_create(dir, dentry);
2804 	if (error)
2805 		return error;
2806 
2807 	mode &= S_IALLUGO;
2808 	mode |= S_IFREG;
2809 	error = security_inode_create(dir, dentry, mode);
2810 	if (error)
2811 		return error;
2812 	error = f(dentry, mode, arg);
2813 	if (!error)
2814 		fsnotify_create(dir, dentry);
2815 	return error;
2816 }
2817 EXPORT_SYMBOL(vfs_mkobj);
2818 
2819 bool may_open_dev(const struct path *path)
2820 {
2821 	return !(path->mnt->mnt_flags & MNT_NODEV) &&
2822 		!(path->mnt->mnt_sb->s_iflags & SB_I_NODEV);
2823 }
2824 
2825 static int may_open(const struct path *path, int acc_mode, int flag)
2826 {
2827 	struct dentry *dentry = path->dentry;
2828 	struct inode *inode = dentry->d_inode;
2829 	int error;
2830 
2831 	if (!inode)
2832 		return -ENOENT;
2833 
2834 	switch (inode->i_mode & S_IFMT) {
2835 	case S_IFLNK:
2836 		return -ELOOP;
2837 	case S_IFDIR:
2838 		if (acc_mode & MAY_WRITE)
2839 			return -EISDIR;
2840 		break;
2841 	case S_IFBLK:
2842 	case S_IFCHR:
2843 		if (!may_open_dev(path))
2844 			return -EACCES;
2845 		/*FALLTHRU*/
2846 	case S_IFIFO:
2847 	case S_IFSOCK:
2848 		flag &= ~O_TRUNC;
2849 		break;
2850 	}
2851 
2852 	error = inode_permission(inode, MAY_OPEN | acc_mode);
2853 	if (error)
2854 		return error;
2855 
2856 	/*
2857 	 * An append-only file must be opened in append mode for writing.
2858 	 */
2859 	if (IS_APPEND(inode)) {
2860 		if  ((flag & O_ACCMODE) != O_RDONLY && !(flag & O_APPEND))
2861 			return -EPERM;
2862 		if (flag & O_TRUNC)
2863 			return -EPERM;
2864 	}
2865 
2866 	/* O_NOATIME can only be set by the owner or superuser */
2867 	if (flag & O_NOATIME && !inode_owner_or_capable(inode))
2868 		return -EPERM;
2869 
2870 	return 0;
2871 }
2872 
2873 static int handle_truncate(struct file *filp)
2874 {
2875 	const struct path *path = &filp->f_path;
2876 	struct inode *inode = path->dentry->d_inode;
2877 	int error = get_write_access(inode);
2878 	if (error)
2879 		return error;
2880 	/*
2881 	 * Refuse to truncate files with mandatory locks held on them.
2882 	 */
2883 	error = locks_verify_locked(filp);
2884 	if (!error)
2885 		error = security_path_truncate(path);
2886 	if (!error) {
2887 		error = do_truncate(path->dentry, 0,
2888 				    ATTR_MTIME|ATTR_CTIME|ATTR_OPEN,
2889 				    filp);
2890 	}
2891 	put_write_access(inode);
2892 	return error;
2893 }
2894 
2895 static inline int open_to_namei_flags(int flag)
2896 {
2897 	if ((flag & O_ACCMODE) == 3)
2898 		flag--;
2899 	return flag;
2900 }
2901 
2902 static int may_o_create(const struct path *dir, struct dentry *dentry, umode_t mode)
2903 {
2904 	struct user_namespace *s_user_ns;
2905 	int error = security_path_mknod(dir, dentry, mode, 0);
2906 	if (error)
2907 		return error;
2908 
2909 	s_user_ns = dir->dentry->d_sb->s_user_ns;
2910 	if (!kuid_has_mapping(s_user_ns, current_fsuid()) ||
2911 	    !kgid_has_mapping(s_user_ns, current_fsgid()))
2912 		return -EOVERFLOW;
2913 
2914 	error = inode_permission(dir->dentry->d_inode, MAY_WRITE | MAY_EXEC);
2915 	if (error)
2916 		return error;
2917 
2918 	return security_inode_create(dir->dentry->d_inode, dentry, mode);
2919 }
2920 
2921 /*
2922  * Attempt to atomically look up, create and open a file from a negative
2923  * dentry.
2924  *
2925  * Returns 0 if successful.  The file will have been created and attached to
2926  * @file by the filesystem calling finish_open().
2927  *
2928  * If the file was looked up only or didn't need creating, FMODE_OPENED won't
2929  * be set.  The caller will need to perform the open themselves.  @path will
2930  * have been updated to point to the new dentry.  This may be negative.
2931  *
2932  * Returns an error code otherwise.
2933  */
2934 static struct dentry *atomic_open(struct nameidata *nd, struct dentry *dentry,
2935 				  struct file *file,
2936 				  int open_flag, umode_t mode)
2937 {
2938 	struct dentry *const DENTRY_NOT_SET = (void *) -1UL;
2939 	struct inode *dir =  nd->path.dentry->d_inode;
2940 	int error;
2941 
2942 	if (nd->flags & LOOKUP_DIRECTORY)
2943 		open_flag |= O_DIRECTORY;
2944 
2945 	file->f_path.dentry = DENTRY_NOT_SET;
2946 	file->f_path.mnt = nd->path.mnt;
2947 	error = dir->i_op->atomic_open(dir, dentry, file,
2948 				       open_to_namei_flags(open_flag), mode);
2949 	d_lookup_done(dentry);
2950 	if (!error) {
2951 		if (file->f_mode & FMODE_OPENED) {
2952 			if (unlikely(dentry != file->f_path.dentry)) {
2953 				dput(dentry);
2954 				dentry = dget(file->f_path.dentry);
2955 			}
2956 		} else if (WARN_ON(file->f_path.dentry == DENTRY_NOT_SET)) {
2957 			error = -EIO;
2958 		} else {
2959 			if (file->f_path.dentry) {
2960 				dput(dentry);
2961 				dentry = file->f_path.dentry;
2962 			}
2963 			if (unlikely(d_is_negative(dentry)))
2964 				error = -ENOENT;
2965 		}
2966 	}
2967 	if (error) {
2968 		dput(dentry);
2969 		dentry = ERR_PTR(error);
2970 	}
2971 	return dentry;
2972 }
2973 
2974 /*
2975  * Look up and maybe create and open the last component.
2976  *
2977  * Must be called with parent locked (exclusive in O_CREAT case).
2978  *
2979  * Returns 0 on success, that is, if
2980  *  the file was successfully atomically created (if necessary) and opened, or
2981  *  the file was not completely opened at this time, though lookups and
2982  *  creations were performed.
2983  * These case are distinguished by presence of FMODE_OPENED on file->f_mode.
2984  * In the latter case dentry returned in @path might be negative if O_CREAT
2985  * hadn't been specified.
2986  *
2987  * An error code is returned on failure.
2988  */
2989 static struct dentry *lookup_open(struct nameidata *nd, struct file *file,
2990 				  const struct open_flags *op,
2991 				  bool got_write)
2992 {
2993 	struct dentry *dir = nd->path.dentry;
2994 	struct inode *dir_inode = dir->d_inode;
2995 	int open_flag = op->open_flag;
2996 	struct dentry *dentry;
2997 	int error, create_error = 0;
2998 	umode_t mode = op->mode;
2999 	DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
3000 
3001 	if (unlikely(IS_DEADDIR(dir_inode)))
3002 		return ERR_PTR(-ENOENT);
3003 
3004 	file->f_mode &= ~FMODE_CREATED;
3005 	dentry = d_lookup(dir, &nd->last);
3006 	for (;;) {
3007 		if (!dentry) {
3008 			dentry = d_alloc_parallel(dir, &nd->last, &wq);
3009 			if (IS_ERR(dentry))
3010 				return dentry;
3011 		}
3012 		if (d_in_lookup(dentry))
3013 			break;
3014 
3015 		error = d_revalidate(dentry, nd->flags);
3016 		if (likely(error > 0))
3017 			break;
3018 		if (error)
3019 			goto out_dput;
3020 		d_invalidate(dentry);
3021 		dput(dentry);
3022 		dentry = NULL;
3023 	}
3024 	if (dentry->d_inode) {
3025 		/* Cached positive dentry: will open in f_op->open */
3026 		return dentry;
3027 	}
3028 
3029 	/*
3030 	 * Checking write permission is tricky, bacuse we don't know if we are
3031 	 * going to actually need it: O_CREAT opens should work as long as the
3032 	 * file exists.  But checking existence breaks atomicity.  The trick is
3033 	 * to check access and if not granted clear O_CREAT from the flags.
3034 	 *
3035 	 * Another problem is returing the "right" error value (e.g. for an
3036 	 * O_EXCL open we want to return EEXIST not EROFS).
3037 	 */
3038 	if (unlikely(!got_write))
3039 		open_flag &= ~O_TRUNC;
3040 	if (open_flag & O_CREAT) {
3041 		if (open_flag & O_EXCL)
3042 			open_flag &= ~O_TRUNC;
3043 		if (!IS_POSIXACL(dir->d_inode))
3044 			mode &= ~current_umask();
3045 		if (likely(got_write))
3046 			create_error = may_o_create(&nd->path, dentry, mode);
3047 		else
3048 			create_error = -EROFS;
3049 	}
3050 	if (create_error)
3051 		open_flag &= ~O_CREAT;
3052 	if (dir_inode->i_op->atomic_open) {
3053 		dentry = atomic_open(nd, dentry, file, open_flag, mode);
3054 		if (unlikely(create_error) && dentry == ERR_PTR(-ENOENT))
3055 			dentry = ERR_PTR(create_error);
3056 		return dentry;
3057 	}
3058 
3059 	if (d_in_lookup(dentry)) {
3060 		struct dentry *res = dir_inode->i_op->lookup(dir_inode, dentry,
3061 							     nd->flags);
3062 		d_lookup_done(dentry);
3063 		if (unlikely(res)) {
3064 			if (IS_ERR(res)) {
3065 				error = PTR_ERR(res);
3066 				goto out_dput;
3067 			}
3068 			dput(dentry);
3069 			dentry = res;
3070 		}
3071 	}
3072 
3073 	/* Negative dentry, just create the file */
3074 	if (!dentry->d_inode && (open_flag & O_CREAT)) {
3075 		file->f_mode |= FMODE_CREATED;
3076 		audit_inode_child(dir_inode, dentry, AUDIT_TYPE_CHILD_CREATE);
3077 		if (!dir_inode->i_op->create) {
3078 			error = -EACCES;
3079 			goto out_dput;
3080 		}
3081 		error = dir_inode->i_op->create(dir_inode, dentry, mode,
3082 						open_flag & O_EXCL);
3083 		if (error)
3084 			goto out_dput;
3085 	}
3086 	if (unlikely(create_error) && !dentry->d_inode) {
3087 		error = create_error;
3088 		goto out_dput;
3089 	}
3090 	return dentry;
3091 
3092 out_dput:
3093 	dput(dentry);
3094 	return ERR_PTR(error);
3095 }
3096 
3097 static const char *open_last_lookups(struct nameidata *nd,
3098 		   struct file *file, const struct open_flags *op)
3099 {
3100 	struct dentry *dir = nd->path.dentry;
3101 	int open_flag = op->open_flag;
3102 	bool got_write = false;
3103 	unsigned seq;
3104 	struct inode *inode;
3105 	struct dentry *dentry;
3106 	const char *res;
3107 	int error;
3108 
3109 	nd->flags |= op->intent;
3110 
3111 	if (nd->last_type != LAST_NORM) {
3112 		if (nd->depth)
3113 			put_link(nd);
3114 		return handle_dots(nd, nd->last_type);
3115 	}
3116 
3117 	if (!(open_flag & O_CREAT)) {
3118 		if (nd->last.name[nd->last.len])
3119 			nd->flags |= LOOKUP_FOLLOW | LOOKUP_DIRECTORY;
3120 		/* we _can_ be in RCU mode here */
3121 		dentry = lookup_fast(nd, &inode, &seq);
3122 		if (IS_ERR(dentry))
3123 			return ERR_CAST(dentry);
3124 		if (likely(dentry))
3125 			goto finish_lookup;
3126 
3127 		BUG_ON(nd->flags & LOOKUP_RCU);
3128 	} else {
3129 		/* create side of things */
3130 		if (nd->flags & LOOKUP_RCU) {
3131 			error = unlazy_walk(nd);
3132 			if (unlikely(error))
3133 				return ERR_PTR(error);
3134 		}
3135 		audit_inode(nd->name, dir, AUDIT_INODE_PARENT);
3136 		/* trailing slashes? */
3137 		if (unlikely(nd->last.name[nd->last.len]))
3138 			return ERR_PTR(-EISDIR);
3139 	}
3140 
3141 	if (open_flag & (O_CREAT | O_TRUNC | O_WRONLY | O_RDWR)) {
3142 		error = mnt_want_write(nd->path.mnt);
3143 		if (!error)
3144 			got_write = true;
3145 		/*
3146 		 * do _not_ fail yet - we might not need that or fail with
3147 		 * a different error; let lookup_open() decide; we'll be
3148 		 * dropping this one anyway.
3149 		 */
3150 	}
3151 	if (open_flag & O_CREAT)
3152 		inode_lock(dir->d_inode);
3153 	else
3154 		inode_lock_shared(dir->d_inode);
3155 	dentry = lookup_open(nd, file, op, got_write);
3156 	if (!IS_ERR(dentry) && (file->f_mode & FMODE_CREATED))
3157 		fsnotify_create(dir->d_inode, dentry);
3158 	if (open_flag & O_CREAT)
3159 		inode_unlock(dir->d_inode);
3160 	else
3161 		inode_unlock_shared(dir->d_inode);
3162 
3163 	if (got_write)
3164 		mnt_drop_write(nd->path.mnt);
3165 
3166 	if (IS_ERR(dentry))
3167 		return ERR_CAST(dentry);
3168 
3169 	if (file->f_mode & (FMODE_OPENED | FMODE_CREATED)) {
3170 		dput(nd->path.dentry);
3171 		nd->path.dentry = dentry;
3172 		return NULL;
3173 	}
3174 
3175 finish_lookup:
3176 	if (nd->depth)
3177 		put_link(nd);
3178 	res = step_into(nd, WALK_TRAILING, dentry, inode, seq);
3179 	if (unlikely(res))
3180 		nd->flags &= ~(LOOKUP_OPEN|LOOKUP_CREATE|LOOKUP_EXCL);
3181 	return res;
3182 }
3183 
3184 /*
3185  * Handle the last step of open()
3186  */
3187 static int do_open(struct nameidata *nd,
3188 		   struct file *file, const struct open_flags *op)
3189 {
3190 	int open_flag = op->open_flag;
3191 	bool do_truncate;
3192 	int acc_mode;
3193 	int error;
3194 
3195 	if (!(file->f_mode & (FMODE_OPENED | FMODE_CREATED))) {
3196 		error = complete_walk(nd);
3197 		if (error)
3198 			return error;
3199 	}
3200 	if (!(file->f_mode & FMODE_CREATED))
3201 		audit_inode(nd->name, nd->path.dentry, 0);
3202 	if (open_flag & O_CREAT) {
3203 		if ((open_flag & O_EXCL) && !(file->f_mode & FMODE_CREATED))
3204 			return -EEXIST;
3205 		if (d_is_dir(nd->path.dentry))
3206 			return -EISDIR;
3207 		error = may_create_in_sticky(nd->dir_mode, nd->dir_uid,
3208 					     d_backing_inode(nd->path.dentry));
3209 		if (unlikely(error))
3210 			return error;
3211 	}
3212 	if ((nd->flags & LOOKUP_DIRECTORY) && !d_can_lookup(nd->path.dentry))
3213 		return -ENOTDIR;
3214 
3215 	do_truncate = false;
3216 	acc_mode = op->acc_mode;
3217 	if (file->f_mode & FMODE_CREATED) {
3218 		/* Don't check for write permission, don't truncate */
3219 		open_flag &= ~O_TRUNC;
3220 		acc_mode = 0;
3221 	} else if (d_is_reg(nd->path.dentry) && open_flag & O_TRUNC) {
3222 		error = mnt_want_write(nd->path.mnt);
3223 		if (error)
3224 			return error;
3225 		do_truncate = true;
3226 	}
3227 	error = may_open(&nd->path, acc_mode, open_flag);
3228 	if (!error && !(file->f_mode & FMODE_OPENED))
3229 		error = vfs_open(&nd->path, file);
3230 	if (!error)
3231 		error = ima_file_check(file, op->acc_mode);
3232 	if (!error && do_truncate)
3233 		error = handle_truncate(file);
3234 	if (unlikely(error > 0)) {
3235 		WARN_ON(1);
3236 		error = -EINVAL;
3237 	}
3238 	if (do_truncate)
3239 		mnt_drop_write(nd->path.mnt);
3240 	return error;
3241 }
3242 
3243 struct dentry *vfs_tmpfile(struct dentry *dentry, umode_t mode, int open_flag)
3244 {
3245 	struct dentry *child = NULL;
3246 	struct inode *dir = dentry->d_inode;
3247 	struct inode *inode;
3248 	int error;
3249 
3250 	/* we want directory to be writable */
3251 	error = inode_permission(dir, MAY_WRITE | MAY_EXEC);
3252 	if (error)
3253 		goto out_err;
3254 	error = -EOPNOTSUPP;
3255 	if (!dir->i_op->tmpfile)
3256 		goto out_err;
3257 	error = -ENOMEM;
3258 	child = d_alloc(dentry, &slash_name);
3259 	if (unlikely(!child))
3260 		goto out_err;
3261 	error = dir->i_op->tmpfile(dir, child, mode);
3262 	if (error)
3263 		goto out_err;
3264 	error = -ENOENT;
3265 	inode = child->d_inode;
3266 	if (unlikely(!inode))
3267 		goto out_err;
3268 	if (!(open_flag & O_EXCL)) {
3269 		spin_lock(&inode->i_lock);
3270 		inode->i_state |= I_LINKABLE;
3271 		spin_unlock(&inode->i_lock);
3272 	}
3273 	ima_post_create_tmpfile(inode);
3274 	return child;
3275 
3276 out_err:
3277 	dput(child);
3278 	return ERR_PTR(error);
3279 }
3280 EXPORT_SYMBOL(vfs_tmpfile);
3281 
3282 static int do_tmpfile(struct nameidata *nd, unsigned flags,
3283 		const struct open_flags *op,
3284 		struct file *file)
3285 {
3286 	struct dentry *child;
3287 	struct path path;
3288 	int error = path_lookupat(nd, flags | LOOKUP_DIRECTORY, &path);
3289 	if (unlikely(error))
3290 		return error;
3291 	error = mnt_want_write(path.mnt);
3292 	if (unlikely(error))
3293 		goto out;
3294 	child = vfs_tmpfile(path.dentry, op->mode, op->open_flag);
3295 	error = PTR_ERR(child);
3296 	if (IS_ERR(child))
3297 		goto out2;
3298 	dput(path.dentry);
3299 	path.dentry = child;
3300 	audit_inode(nd->name, child, 0);
3301 	/* Don't check for other permissions, the inode was just created */
3302 	error = may_open(&path, 0, op->open_flag);
3303 	if (error)
3304 		goto out2;
3305 	file->f_path.mnt = path.mnt;
3306 	error = finish_open(file, child, NULL);
3307 out2:
3308 	mnt_drop_write(path.mnt);
3309 out:
3310 	path_put(&path);
3311 	return error;
3312 }
3313 
3314 static int do_o_path(struct nameidata *nd, unsigned flags, struct file *file)
3315 {
3316 	struct path path;
3317 	int error = path_lookupat(nd, flags, &path);
3318 	if (!error) {
3319 		audit_inode(nd->name, path.dentry, 0);
3320 		error = vfs_open(&path, file);
3321 		path_put(&path);
3322 	}
3323 	return error;
3324 }
3325 
3326 static struct file *path_openat(struct nameidata *nd,
3327 			const struct open_flags *op, unsigned flags)
3328 {
3329 	struct file *file;
3330 	int error;
3331 
3332 	file = alloc_empty_file(op->open_flag, current_cred());
3333 	if (IS_ERR(file))
3334 		return file;
3335 
3336 	if (unlikely(file->f_flags & __O_TMPFILE)) {
3337 		error = do_tmpfile(nd, flags, op, file);
3338 	} else if (unlikely(file->f_flags & O_PATH)) {
3339 		error = do_o_path(nd, flags, file);
3340 	} else {
3341 		const char *s = path_init(nd, flags);
3342 		while (!(error = link_path_walk(s, nd)) &&
3343 		       (s = open_last_lookups(nd, file, op)) != NULL)
3344 			;
3345 		if (!error)
3346 			error = do_open(nd, file, op);
3347 		terminate_walk(nd);
3348 	}
3349 	if (likely(!error)) {
3350 		if (likely(file->f_mode & FMODE_OPENED))
3351 			return file;
3352 		WARN_ON(1);
3353 		error = -EINVAL;
3354 	}
3355 	fput(file);
3356 	if (error == -EOPENSTALE) {
3357 		if (flags & LOOKUP_RCU)
3358 			error = -ECHILD;
3359 		else
3360 			error = -ESTALE;
3361 	}
3362 	return ERR_PTR(error);
3363 }
3364 
3365 struct file *do_filp_open(int dfd, struct filename *pathname,
3366 		const struct open_flags *op)
3367 {
3368 	struct nameidata nd;
3369 	int flags = op->lookup_flags;
3370 	struct file *filp;
3371 
3372 	set_nameidata(&nd, dfd, pathname);
3373 	filp = path_openat(&nd, op, flags | LOOKUP_RCU);
3374 	if (unlikely(filp == ERR_PTR(-ECHILD)))
3375 		filp = path_openat(&nd, op, flags);
3376 	if (unlikely(filp == ERR_PTR(-ESTALE)))
3377 		filp = path_openat(&nd, op, flags | LOOKUP_REVAL);
3378 	restore_nameidata();
3379 	return filp;
3380 }
3381 
3382 struct file *do_file_open_root(struct dentry *dentry, struct vfsmount *mnt,
3383 		const char *name, const struct open_flags *op)
3384 {
3385 	struct nameidata nd;
3386 	struct file *file;
3387 	struct filename *filename;
3388 	int flags = op->lookup_flags | LOOKUP_ROOT;
3389 
3390 	nd.root.mnt = mnt;
3391 	nd.root.dentry = dentry;
3392 
3393 	if (d_is_symlink(dentry) && op->intent & LOOKUP_OPEN)
3394 		return ERR_PTR(-ELOOP);
3395 
3396 	filename = getname_kernel(name);
3397 	if (IS_ERR(filename))
3398 		return ERR_CAST(filename);
3399 
3400 	set_nameidata(&nd, -1, filename);
3401 	file = path_openat(&nd, op, flags | LOOKUP_RCU);
3402 	if (unlikely(file == ERR_PTR(-ECHILD)))
3403 		file = path_openat(&nd, op, flags);
3404 	if (unlikely(file == ERR_PTR(-ESTALE)))
3405 		file = path_openat(&nd, op, flags | LOOKUP_REVAL);
3406 	restore_nameidata();
3407 	putname(filename);
3408 	return file;
3409 }
3410 
3411 static struct dentry *filename_create(int dfd, struct filename *name,
3412 				struct path *path, unsigned int lookup_flags)
3413 {
3414 	struct dentry *dentry = ERR_PTR(-EEXIST);
3415 	struct qstr last;
3416 	int type;
3417 	int err2;
3418 	int error;
3419 	bool is_dir = (lookup_flags & LOOKUP_DIRECTORY);
3420 
3421 	/*
3422 	 * Note that only LOOKUP_REVAL and LOOKUP_DIRECTORY matter here. Any
3423 	 * other flags passed in are ignored!
3424 	 */
3425 	lookup_flags &= LOOKUP_REVAL;
3426 
3427 	name = filename_parentat(dfd, name, lookup_flags, path, &last, &type);
3428 	if (IS_ERR(name))
3429 		return ERR_CAST(name);
3430 
3431 	/*
3432 	 * Yucky last component or no last component at all?
3433 	 * (foo/., foo/.., /////)
3434 	 */
3435 	if (unlikely(type != LAST_NORM))
3436 		goto out;
3437 
3438 	/* don't fail immediately if it's r/o, at least try to report other errors */
3439 	err2 = mnt_want_write(path->mnt);
3440 	/*
3441 	 * Do the final lookup.
3442 	 */
3443 	lookup_flags |= LOOKUP_CREATE | LOOKUP_EXCL;
3444 	inode_lock_nested(path->dentry->d_inode, I_MUTEX_PARENT);
3445 	dentry = __lookup_hash(&last, path->dentry, lookup_flags);
3446 	if (IS_ERR(dentry))
3447 		goto unlock;
3448 
3449 	error = -EEXIST;
3450 	if (d_is_positive(dentry))
3451 		goto fail;
3452 
3453 	/*
3454 	 * Special case - lookup gave negative, but... we had foo/bar/
3455 	 * From the vfs_mknod() POV we just have a negative dentry -
3456 	 * all is fine. Let's be bastards - you had / on the end, you've
3457 	 * been asking for (non-existent) directory. -ENOENT for you.
3458 	 */
3459 	if (unlikely(!is_dir && last.name[last.len])) {
3460 		error = -ENOENT;
3461 		goto fail;
3462 	}
3463 	if (unlikely(err2)) {
3464 		error = err2;
3465 		goto fail;
3466 	}
3467 	putname(name);
3468 	return dentry;
3469 fail:
3470 	dput(dentry);
3471 	dentry = ERR_PTR(error);
3472 unlock:
3473 	inode_unlock(path->dentry->d_inode);
3474 	if (!err2)
3475 		mnt_drop_write(path->mnt);
3476 out:
3477 	path_put(path);
3478 	putname(name);
3479 	return dentry;
3480 }
3481 
3482 struct dentry *kern_path_create(int dfd, const char *pathname,
3483 				struct path *path, unsigned int lookup_flags)
3484 {
3485 	return filename_create(dfd, getname_kernel(pathname),
3486 				path, lookup_flags);
3487 }
3488 EXPORT_SYMBOL(kern_path_create);
3489 
3490 void done_path_create(struct path *path, struct dentry *dentry)
3491 {
3492 	dput(dentry);
3493 	inode_unlock(path->dentry->d_inode);
3494 	mnt_drop_write(path->mnt);
3495 	path_put(path);
3496 }
3497 EXPORT_SYMBOL(done_path_create);
3498 
3499 inline struct dentry *user_path_create(int dfd, const char __user *pathname,
3500 				struct path *path, unsigned int lookup_flags)
3501 {
3502 	return filename_create(dfd, getname(pathname), path, lookup_flags);
3503 }
3504 EXPORT_SYMBOL(user_path_create);
3505 
3506 int vfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
3507 {
3508 	int error = may_create(dir, dentry);
3509 
3510 	if (error)
3511 		return error;
3512 
3513 	if ((S_ISCHR(mode) || S_ISBLK(mode)) && !capable(CAP_MKNOD))
3514 		return -EPERM;
3515 
3516 	if (!dir->i_op->mknod)
3517 		return -EPERM;
3518 
3519 	error = devcgroup_inode_mknod(mode, dev);
3520 	if (error)
3521 		return error;
3522 
3523 	error = security_inode_mknod(dir, dentry, mode, dev);
3524 	if (error)
3525 		return error;
3526 
3527 	error = dir->i_op->mknod(dir, dentry, mode, dev);
3528 	if (!error)
3529 		fsnotify_create(dir, dentry);
3530 	return error;
3531 }
3532 EXPORT_SYMBOL(vfs_mknod);
3533 
3534 static int may_mknod(umode_t mode)
3535 {
3536 	switch (mode & S_IFMT) {
3537 	case S_IFREG:
3538 	case S_IFCHR:
3539 	case S_IFBLK:
3540 	case S_IFIFO:
3541 	case S_IFSOCK:
3542 	case 0: /* zero mode translates to S_IFREG */
3543 		return 0;
3544 	case S_IFDIR:
3545 		return -EPERM;
3546 	default:
3547 		return -EINVAL;
3548 	}
3549 }
3550 
3551 long do_mknodat(int dfd, const char __user *filename, umode_t mode,
3552 		unsigned int dev)
3553 {
3554 	struct dentry *dentry;
3555 	struct path path;
3556 	int error;
3557 	unsigned int lookup_flags = 0;
3558 
3559 	error = may_mknod(mode);
3560 	if (error)
3561 		return error;
3562 retry:
3563 	dentry = user_path_create(dfd, filename, &path, lookup_flags);
3564 	if (IS_ERR(dentry))
3565 		return PTR_ERR(dentry);
3566 
3567 	if (!IS_POSIXACL(path.dentry->d_inode))
3568 		mode &= ~current_umask();
3569 	error = security_path_mknod(&path, dentry, mode, dev);
3570 	if (error)
3571 		goto out;
3572 	switch (mode & S_IFMT) {
3573 		case 0: case S_IFREG:
3574 			error = vfs_create(path.dentry->d_inode,dentry,mode,true);
3575 			if (!error)
3576 				ima_post_path_mknod(dentry);
3577 			break;
3578 		case S_IFCHR: case S_IFBLK:
3579 			error = vfs_mknod(path.dentry->d_inode,dentry,mode,
3580 					new_decode_dev(dev));
3581 			break;
3582 		case S_IFIFO: case S_IFSOCK:
3583 			error = vfs_mknod(path.dentry->d_inode,dentry,mode,0);
3584 			break;
3585 	}
3586 out:
3587 	done_path_create(&path, dentry);
3588 	if (retry_estale(error, lookup_flags)) {
3589 		lookup_flags |= LOOKUP_REVAL;
3590 		goto retry;
3591 	}
3592 	return error;
3593 }
3594 
3595 SYSCALL_DEFINE4(mknodat, int, dfd, const char __user *, filename, umode_t, mode,
3596 		unsigned int, dev)
3597 {
3598 	return do_mknodat(dfd, filename, mode, dev);
3599 }
3600 
3601 SYSCALL_DEFINE3(mknod, const char __user *, filename, umode_t, mode, unsigned, dev)
3602 {
3603 	return do_mknodat(AT_FDCWD, filename, mode, dev);
3604 }
3605 
3606 int vfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
3607 {
3608 	int error = may_create(dir, dentry);
3609 	unsigned max_links = dir->i_sb->s_max_links;
3610 
3611 	if (error)
3612 		return error;
3613 
3614 	if (!dir->i_op->mkdir)
3615 		return -EPERM;
3616 
3617 	mode &= (S_IRWXUGO|S_ISVTX);
3618 	error = security_inode_mkdir(dir, dentry, mode);
3619 	if (error)
3620 		return error;
3621 
3622 	if (max_links && dir->i_nlink >= max_links)
3623 		return -EMLINK;
3624 
3625 	error = dir->i_op->mkdir(dir, dentry, mode);
3626 	if (!error)
3627 		fsnotify_mkdir(dir, dentry);
3628 	return error;
3629 }
3630 EXPORT_SYMBOL(vfs_mkdir);
3631 
3632 long do_mkdirat(int dfd, const char __user *pathname, umode_t mode)
3633 {
3634 	struct dentry *dentry;
3635 	struct path path;
3636 	int error;
3637 	unsigned int lookup_flags = LOOKUP_DIRECTORY;
3638 
3639 retry:
3640 	dentry = user_path_create(dfd, pathname, &path, lookup_flags);
3641 	if (IS_ERR(dentry))
3642 		return PTR_ERR(dentry);
3643 
3644 	if (!IS_POSIXACL(path.dentry->d_inode))
3645 		mode &= ~current_umask();
3646 	error = security_path_mkdir(&path, dentry, mode);
3647 	if (!error)
3648 		error = vfs_mkdir(path.dentry->d_inode, dentry, mode);
3649 	done_path_create(&path, dentry);
3650 	if (retry_estale(error, lookup_flags)) {
3651 		lookup_flags |= LOOKUP_REVAL;
3652 		goto retry;
3653 	}
3654 	return error;
3655 }
3656 
3657 SYSCALL_DEFINE3(mkdirat, int, dfd, const char __user *, pathname, umode_t, mode)
3658 {
3659 	return do_mkdirat(dfd, pathname, mode);
3660 }
3661 
3662 SYSCALL_DEFINE2(mkdir, const char __user *, pathname, umode_t, mode)
3663 {
3664 	return do_mkdirat(AT_FDCWD, pathname, mode);
3665 }
3666 
3667 int vfs_rmdir(struct inode *dir, struct dentry *dentry)
3668 {
3669 	int error = may_delete(dir, dentry, 1);
3670 
3671 	if (error)
3672 		return error;
3673 
3674 	if (!dir->i_op->rmdir)
3675 		return -EPERM;
3676 
3677 	dget(dentry);
3678 	inode_lock(dentry->d_inode);
3679 
3680 	error = -EBUSY;
3681 	if (is_local_mountpoint(dentry))
3682 		goto out;
3683 
3684 	error = security_inode_rmdir(dir, dentry);
3685 	if (error)
3686 		goto out;
3687 
3688 	error = dir->i_op->rmdir(dir, dentry);
3689 	if (error)
3690 		goto out;
3691 
3692 	shrink_dcache_parent(dentry);
3693 	dentry->d_inode->i_flags |= S_DEAD;
3694 	dont_mount(dentry);
3695 	detach_mounts(dentry);
3696 	fsnotify_rmdir(dir, dentry);
3697 
3698 out:
3699 	inode_unlock(dentry->d_inode);
3700 	dput(dentry);
3701 	if (!error)
3702 		d_delete(dentry);
3703 	return error;
3704 }
3705 EXPORT_SYMBOL(vfs_rmdir);
3706 
3707 long do_rmdir(int dfd, const char __user *pathname)
3708 {
3709 	int error = 0;
3710 	struct filename *name;
3711 	struct dentry *dentry;
3712 	struct path path;
3713 	struct qstr last;
3714 	int type;
3715 	unsigned int lookup_flags = 0;
3716 retry:
3717 	name = filename_parentat(dfd, getname(pathname), lookup_flags,
3718 				&path, &last, &type);
3719 	if (IS_ERR(name))
3720 		return PTR_ERR(name);
3721 
3722 	switch (type) {
3723 	case LAST_DOTDOT:
3724 		error = -ENOTEMPTY;
3725 		goto exit1;
3726 	case LAST_DOT:
3727 		error = -EINVAL;
3728 		goto exit1;
3729 	case LAST_ROOT:
3730 		error = -EBUSY;
3731 		goto exit1;
3732 	}
3733 
3734 	error = mnt_want_write(path.mnt);
3735 	if (error)
3736 		goto exit1;
3737 
3738 	inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
3739 	dentry = __lookup_hash(&last, path.dentry, lookup_flags);
3740 	error = PTR_ERR(dentry);
3741 	if (IS_ERR(dentry))
3742 		goto exit2;
3743 	if (!dentry->d_inode) {
3744 		error = -ENOENT;
3745 		goto exit3;
3746 	}
3747 	error = security_path_rmdir(&path, dentry);
3748 	if (error)
3749 		goto exit3;
3750 	error = vfs_rmdir(path.dentry->d_inode, dentry);
3751 exit3:
3752 	dput(dentry);
3753 exit2:
3754 	inode_unlock(path.dentry->d_inode);
3755 	mnt_drop_write(path.mnt);
3756 exit1:
3757 	path_put(&path);
3758 	putname(name);
3759 	if (retry_estale(error, lookup_flags)) {
3760 		lookup_flags |= LOOKUP_REVAL;
3761 		goto retry;
3762 	}
3763 	return error;
3764 }
3765 
3766 SYSCALL_DEFINE1(rmdir, const char __user *, pathname)
3767 {
3768 	return do_rmdir(AT_FDCWD, pathname);
3769 }
3770 
3771 /**
3772  * vfs_unlink - unlink a filesystem object
3773  * @dir:	parent directory
3774  * @dentry:	victim
3775  * @delegated_inode: returns victim inode, if the inode is delegated.
3776  *
3777  * The caller must hold dir->i_mutex.
3778  *
3779  * If vfs_unlink discovers a delegation, it will return -EWOULDBLOCK and
3780  * return a reference to the inode in delegated_inode.  The caller
3781  * should then break the delegation on that inode and retry.  Because
3782  * breaking a delegation may take a long time, the caller should drop
3783  * dir->i_mutex before doing so.
3784  *
3785  * Alternatively, a caller may pass NULL for delegated_inode.  This may
3786  * be appropriate for callers that expect the underlying filesystem not
3787  * to be NFS exported.
3788  */
3789 int vfs_unlink(struct inode *dir, struct dentry *dentry, struct inode **delegated_inode)
3790 {
3791 	struct inode *target = dentry->d_inode;
3792 	int error = may_delete(dir, dentry, 0);
3793 
3794 	if (error)
3795 		return error;
3796 
3797 	if (!dir->i_op->unlink)
3798 		return -EPERM;
3799 
3800 	inode_lock(target);
3801 	if (is_local_mountpoint(dentry))
3802 		error = -EBUSY;
3803 	else {
3804 		error = security_inode_unlink(dir, dentry);
3805 		if (!error) {
3806 			error = try_break_deleg(target, delegated_inode);
3807 			if (error)
3808 				goto out;
3809 			error = dir->i_op->unlink(dir, dentry);
3810 			if (!error) {
3811 				dont_mount(dentry);
3812 				detach_mounts(dentry);
3813 				fsnotify_unlink(dir, dentry);
3814 			}
3815 		}
3816 	}
3817 out:
3818 	inode_unlock(target);
3819 
3820 	/* We don't d_delete() NFS sillyrenamed files--they still exist. */
3821 	if (!error && !(dentry->d_flags & DCACHE_NFSFS_RENAMED)) {
3822 		fsnotify_link_count(target);
3823 		d_delete(dentry);
3824 	}
3825 
3826 	return error;
3827 }
3828 EXPORT_SYMBOL(vfs_unlink);
3829 
3830 /*
3831  * Make sure that the actual truncation of the file will occur outside its
3832  * directory's i_mutex.  Truncate can take a long time if there is a lot of
3833  * writeout happening, and we don't want to prevent access to the directory
3834  * while waiting on the I/O.
3835  */
3836 long do_unlinkat(int dfd, struct filename *name)
3837 {
3838 	int error;
3839 	struct dentry *dentry;
3840 	struct path path;
3841 	struct qstr last;
3842 	int type;
3843 	struct inode *inode = NULL;
3844 	struct inode *delegated_inode = NULL;
3845 	unsigned int lookup_flags = 0;
3846 retry:
3847 	name = filename_parentat(dfd, name, lookup_flags, &path, &last, &type);
3848 	if (IS_ERR(name))
3849 		return PTR_ERR(name);
3850 
3851 	error = -EISDIR;
3852 	if (type != LAST_NORM)
3853 		goto exit1;
3854 
3855 	error = mnt_want_write(path.mnt);
3856 	if (error)
3857 		goto exit1;
3858 retry_deleg:
3859 	inode_lock_nested(path.dentry->d_inode, I_MUTEX_PARENT);
3860 	dentry = __lookup_hash(&last, path.dentry, lookup_flags);
3861 	error = PTR_ERR(dentry);
3862 	if (!IS_ERR(dentry)) {
3863 		/* Why not before? Because we want correct error value */
3864 		if (last.name[last.len])
3865 			goto slashes;
3866 		inode = dentry->d_inode;
3867 		if (d_is_negative(dentry))
3868 			goto slashes;
3869 		ihold(inode);
3870 		error = security_path_unlink(&path, dentry);
3871 		if (error)
3872 			goto exit2;
3873 		error = vfs_unlink(path.dentry->d_inode, dentry, &delegated_inode);
3874 exit2:
3875 		dput(dentry);
3876 	}
3877 	inode_unlock(path.dentry->d_inode);
3878 	if (inode)
3879 		iput(inode);	/* truncate the inode here */
3880 	inode = NULL;
3881 	if (delegated_inode) {
3882 		error = break_deleg_wait(&delegated_inode);
3883 		if (!error)
3884 			goto retry_deleg;
3885 	}
3886 	mnt_drop_write(path.mnt);
3887 exit1:
3888 	path_put(&path);
3889 	if (retry_estale(error, lookup_flags)) {
3890 		lookup_flags |= LOOKUP_REVAL;
3891 		inode = NULL;
3892 		goto retry;
3893 	}
3894 	putname(name);
3895 	return error;
3896 
3897 slashes:
3898 	if (d_is_negative(dentry))
3899 		error = -ENOENT;
3900 	else if (d_is_dir(dentry))
3901 		error = -EISDIR;
3902 	else
3903 		error = -ENOTDIR;
3904 	goto exit2;
3905 }
3906 
3907 SYSCALL_DEFINE3(unlinkat, int, dfd, const char __user *, pathname, int, flag)
3908 {
3909 	if ((flag & ~AT_REMOVEDIR) != 0)
3910 		return -EINVAL;
3911 
3912 	if (flag & AT_REMOVEDIR)
3913 		return do_rmdir(dfd, pathname);
3914 
3915 	return do_unlinkat(dfd, getname(pathname));
3916 }
3917 
3918 SYSCALL_DEFINE1(unlink, const char __user *, pathname)
3919 {
3920 	return do_unlinkat(AT_FDCWD, getname(pathname));
3921 }
3922 
3923 int vfs_symlink(struct inode *dir, struct dentry *dentry, const char *oldname)
3924 {
3925 	int error = may_create(dir, dentry);
3926 
3927 	if (error)
3928 		return error;
3929 
3930 	if (!dir->i_op->symlink)
3931 		return -EPERM;
3932 
3933 	error = security_inode_symlink(dir, dentry, oldname);
3934 	if (error)
3935 		return error;
3936 
3937 	error = dir->i_op->symlink(dir, dentry, oldname);
3938 	if (!error)
3939 		fsnotify_create(dir, dentry);
3940 	return error;
3941 }
3942 EXPORT_SYMBOL(vfs_symlink);
3943 
3944 long do_symlinkat(const char __user *oldname, int newdfd,
3945 		  const char __user *newname)
3946 {
3947 	int error;
3948 	struct filename *from;
3949 	struct dentry *dentry;
3950 	struct path path;
3951 	unsigned int lookup_flags = 0;
3952 
3953 	from = getname(oldname);
3954 	if (IS_ERR(from))
3955 		return PTR_ERR(from);
3956 retry:
3957 	dentry = user_path_create(newdfd, newname, &path, lookup_flags);
3958 	error = PTR_ERR(dentry);
3959 	if (IS_ERR(dentry))
3960 		goto out_putname;
3961 
3962 	error = security_path_symlink(&path, dentry, from->name);
3963 	if (!error)
3964 		error = vfs_symlink(path.dentry->d_inode, dentry, from->name);
3965 	done_path_create(&path, dentry);
3966 	if (retry_estale(error, lookup_flags)) {
3967 		lookup_flags |= LOOKUP_REVAL;
3968 		goto retry;
3969 	}
3970 out_putname:
3971 	putname(from);
3972 	return error;
3973 }
3974 
3975 SYSCALL_DEFINE3(symlinkat, const char __user *, oldname,
3976 		int, newdfd, const char __user *, newname)
3977 {
3978 	return do_symlinkat(oldname, newdfd, newname);
3979 }
3980 
3981 SYSCALL_DEFINE2(symlink, const char __user *, oldname, const char __user *, newname)
3982 {
3983 	return do_symlinkat(oldname, AT_FDCWD, newname);
3984 }
3985 
3986 /**
3987  * vfs_link - create a new link
3988  * @old_dentry:	object to be linked
3989  * @dir:	new parent
3990  * @new_dentry:	where to create the new link
3991  * @delegated_inode: returns inode needing a delegation break
3992  *
3993  * The caller must hold dir->i_mutex
3994  *
3995  * If vfs_link discovers a delegation on the to-be-linked file in need
3996  * of breaking, it will return -EWOULDBLOCK and return a reference to the
3997  * inode in delegated_inode.  The caller should then break the delegation
3998  * and retry.  Because breaking a delegation may take a long time, the
3999  * caller should drop the i_mutex before doing so.
4000  *
4001  * Alternatively, a caller may pass NULL for delegated_inode.  This may
4002  * be appropriate for callers that expect the underlying filesystem not
4003  * to be NFS exported.
4004  */
4005 int vfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry, struct inode **delegated_inode)
4006 {
4007 	struct inode *inode = old_dentry->d_inode;
4008 	unsigned max_links = dir->i_sb->s_max_links;
4009 	int error;
4010 
4011 	if (!inode)
4012 		return -ENOENT;
4013 
4014 	error = may_create(dir, new_dentry);
4015 	if (error)
4016 		return error;
4017 
4018 	if (dir->i_sb != inode->i_sb)
4019 		return -EXDEV;
4020 
4021 	/*
4022 	 * A link to an append-only or immutable file cannot be created.
4023 	 */
4024 	if (IS_APPEND(inode) || IS_IMMUTABLE(inode))
4025 		return -EPERM;
4026 	/*
4027 	 * Updating the link count will likely cause i_uid and i_gid to
4028 	 * be writen back improperly if their true value is unknown to
4029 	 * the vfs.
4030 	 */
4031 	if (HAS_UNMAPPED_ID(inode))
4032 		return -EPERM;
4033 	if (!dir->i_op->link)
4034 		return -EPERM;
4035 	if (S_ISDIR(inode->i_mode))
4036 		return -EPERM;
4037 
4038 	error = security_inode_link(old_dentry, dir, new_dentry);
4039 	if (error)
4040 		return error;
4041 
4042 	inode_lock(inode);
4043 	/* Make sure we don't allow creating hardlink to an unlinked file */
4044 	if (inode->i_nlink == 0 && !(inode->i_state & I_LINKABLE))
4045 		error =  -ENOENT;
4046 	else if (max_links && inode->i_nlink >= max_links)
4047 		error = -EMLINK;
4048 	else {
4049 		error = try_break_deleg(inode, delegated_inode);
4050 		if (!error)
4051 			error = dir->i_op->link(old_dentry, dir, new_dentry);
4052 	}
4053 
4054 	if (!error && (inode->i_state & I_LINKABLE)) {
4055 		spin_lock(&inode->i_lock);
4056 		inode->i_state &= ~I_LINKABLE;
4057 		spin_unlock(&inode->i_lock);
4058 	}
4059 	inode_unlock(inode);
4060 	if (!error)
4061 		fsnotify_link(dir, inode, new_dentry);
4062 	return error;
4063 }
4064 EXPORT_SYMBOL(vfs_link);
4065 
4066 /*
4067  * Hardlinks are often used in delicate situations.  We avoid
4068  * security-related surprises by not following symlinks on the
4069  * newname.  --KAB
4070  *
4071  * We don't follow them on the oldname either to be compatible
4072  * with linux 2.0, and to avoid hard-linking to directories
4073  * and other special files.  --ADM
4074  */
4075 int do_linkat(int olddfd, const char __user *oldname, int newdfd,
4076 	      const char __user *newname, int flags)
4077 {
4078 	struct dentry *new_dentry;
4079 	struct path old_path, new_path;
4080 	struct inode *delegated_inode = NULL;
4081 	int how = 0;
4082 	int error;
4083 
4084 	if ((flags & ~(AT_SYMLINK_FOLLOW | AT_EMPTY_PATH)) != 0)
4085 		return -EINVAL;
4086 	/*
4087 	 * To use null names we require CAP_DAC_READ_SEARCH
4088 	 * This ensures that not everyone will be able to create
4089 	 * handlink using the passed filedescriptor.
4090 	 */
4091 	if (flags & AT_EMPTY_PATH) {
4092 		if (!capable(CAP_DAC_READ_SEARCH))
4093 			return -ENOENT;
4094 		how = LOOKUP_EMPTY;
4095 	}
4096 
4097 	if (flags & AT_SYMLINK_FOLLOW)
4098 		how |= LOOKUP_FOLLOW;
4099 retry:
4100 	error = user_path_at(olddfd, oldname, how, &old_path);
4101 	if (error)
4102 		return error;
4103 
4104 	new_dentry = user_path_create(newdfd, newname, &new_path,
4105 					(how & LOOKUP_REVAL));
4106 	error = PTR_ERR(new_dentry);
4107 	if (IS_ERR(new_dentry))
4108 		goto out;
4109 
4110 	error = -EXDEV;
4111 	if (old_path.mnt != new_path.mnt)
4112 		goto out_dput;
4113 	error = may_linkat(&old_path);
4114 	if (unlikely(error))
4115 		goto out_dput;
4116 	error = security_path_link(old_path.dentry, &new_path, new_dentry);
4117 	if (error)
4118 		goto out_dput;
4119 	error = vfs_link(old_path.dentry, new_path.dentry->d_inode, new_dentry, &delegated_inode);
4120 out_dput:
4121 	done_path_create(&new_path, new_dentry);
4122 	if (delegated_inode) {
4123 		error = break_deleg_wait(&delegated_inode);
4124 		if (!error) {
4125 			path_put(&old_path);
4126 			goto retry;
4127 		}
4128 	}
4129 	if (retry_estale(error, how)) {
4130 		path_put(&old_path);
4131 		how |= LOOKUP_REVAL;
4132 		goto retry;
4133 	}
4134 out:
4135 	path_put(&old_path);
4136 
4137 	return error;
4138 }
4139 
4140 SYSCALL_DEFINE5(linkat, int, olddfd, const char __user *, oldname,
4141 		int, newdfd, const char __user *, newname, int, flags)
4142 {
4143 	return do_linkat(olddfd, oldname, newdfd, newname, flags);
4144 }
4145 
4146 SYSCALL_DEFINE2(link, const char __user *, oldname, const char __user *, newname)
4147 {
4148 	return do_linkat(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
4149 }
4150 
4151 /**
4152  * vfs_rename - rename a filesystem object
4153  * @old_dir:	parent of source
4154  * @old_dentry:	source
4155  * @new_dir:	parent of destination
4156  * @new_dentry:	destination
4157  * @delegated_inode: returns an inode needing a delegation break
4158  * @flags:	rename flags
4159  *
4160  * The caller must hold multiple mutexes--see lock_rename()).
4161  *
4162  * If vfs_rename discovers a delegation in need of breaking at either
4163  * the source or destination, it will return -EWOULDBLOCK and return a
4164  * reference to the inode in delegated_inode.  The caller should then
4165  * break the delegation and retry.  Because breaking a delegation may
4166  * take a long time, the caller should drop all locks before doing
4167  * so.
4168  *
4169  * Alternatively, a caller may pass NULL for delegated_inode.  This may
4170  * be appropriate for callers that expect the underlying filesystem not
4171  * to be NFS exported.
4172  *
4173  * The worst of all namespace operations - renaming directory. "Perverted"
4174  * doesn't even start to describe it. Somebody in UCB had a heck of a trip...
4175  * Problems:
4176  *
4177  *	a) we can get into loop creation.
4178  *	b) race potential - two innocent renames can create a loop together.
4179  *	   That's where 4.4 screws up. Current fix: serialization on
4180  *	   sb->s_vfs_rename_mutex. We might be more accurate, but that's another
4181  *	   story.
4182  *	c) we have to lock _four_ objects - parents and victim (if it exists),
4183  *	   and source (if it is not a directory).
4184  *	   And that - after we got ->i_mutex on parents (until then we don't know
4185  *	   whether the target exists).  Solution: try to be smart with locking
4186  *	   order for inodes.  We rely on the fact that tree topology may change
4187  *	   only under ->s_vfs_rename_mutex _and_ that parent of the object we
4188  *	   move will be locked.  Thus we can rank directories by the tree
4189  *	   (ancestors first) and rank all non-directories after them.
4190  *	   That works since everybody except rename does "lock parent, lookup,
4191  *	   lock child" and rename is under ->s_vfs_rename_mutex.
4192  *	   HOWEVER, it relies on the assumption that any object with ->lookup()
4193  *	   has no more than 1 dentry.  If "hybrid" objects will ever appear,
4194  *	   we'd better make sure that there's no link(2) for them.
4195  *	d) conversion from fhandle to dentry may come in the wrong moment - when
4196  *	   we are removing the target. Solution: we will have to grab ->i_mutex
4197  *	   in the fhandle_to_dentry code. [FIXME - current nfsfh.c relies on
4198  *	   ->i_mutex on parents, which works but leads to some truly excessive
4199  *	   locking].
4200  */
4201 int vfs_rename(struct inode *old_dir, struct dentry *old_dentry,
4202 	       struct inode *new_dir, struct dentry *new_dentry,
4203 	       struct inode **delegated_inode, unsigned int flags)
4204 {
4205 	int error;
4206 	bool is_dir = d_is_dir(old_dentry);
4207 	struct inode *source = old_dentry->d_inode;
4208 	struct inode *target = new_dentry->d_inode;
4209 	bool new_is_dir = false;
4210 	unsigned max_links = new_dir->i_sb->s_max_links;
4211 	struct name_snapshot old_name;
4212 
4213 	if (source == target)
4214 		return 0;
4215 
4216 	error = may_delete(old_dir, old_dentry, is_dir);
4217 	if (error)
4218 		return error;
4219 
4220 	if (!target) {
4221 		error = may_create(new_dir, new_dentry);
4222 	} else {
4223 		new_is_dir = d_is_dir(new_dentry);
4224 
4225 		if (!(flags & RENAME_EXCHANGE))
4226 			error = may_delete(new_dir, new_dentry, is_dir);
4227 		else
4228 			error = may_delete(new_dir, new_dentry, new_is_dir);
4229 	}
4230 	if (error)
4231 		return error;
4232 
4233 	if (!old_dir->i_op->rename)
4234 		return -EPERM;
4235 
4236 	/*
4237 	 * If we are going to change the parent - check write permissions,
4238 	 * we'll need to flip '..'.
4239 	 */
4240 	if (new_dir != old_dir) {
4241 		if (is_dir) {
4242 			error = inode_permission(source, MAY_WRITE);
4243 			if (error)
4244 				return error;
4245 		}
4246 		if ((flags & RENAME_EXCHANGE) && new_is_dir) {
4247 			error = inode_permission(target, MAY_WRITE);
4248 			if (error)
4249 				return error;
4250 		}
4251 	}
4252 
4253 	error = security_inode_rename(old_dir, old_dentry, new_dir, new_dentry,
4254 				      flags);
4255 	if (error)
4256 		return error;
4257 
4258 	take_dentry_name_snapshot(&old_name, old_dentry);
4259 	dget(new_dentry);
4260 	if (!is_dir || (flags & RENAME_EXCHANGE))
4261 		lock_two_nondirectories(source, target);
4262 	else if (target)
4263 		inode_lock(target);
4264 
4265 	error = -EBUSY;
4266 	if (is_local_mountpoint(old_dentry) || is_local_mountpoint(new_dentry))
4267 		goto out;
4268 
4269 	if (max_links && new_dir != old_dir) {
4270 		error = -EMLINK;
4271 		if (is_dir && !new_is_dir && new_dir->i_nlink >= max_links)
4272 			goto out;
4273 		if ((flags & RENAME_EXCHANGE) && !is_dir && new_is_dir &&
4274 		    old_dir->i_nlink >= max_links)
4275 			goto out;
4276 	}
4277 	if (!is_dir) {
4278 		error = try_break_deleg(source, delegated_inode);
4279 		if (error)
4280 			goto out;
4281 	}
4282 	if (target && !new_is_dir) {
4283 		error = try_break_deleg(target, delegated_inode);
4284 		if (error)
4285 			goto out;
4286 	}
4287 	error = old_dir->i_op->rename(old_dir, old_dentry,
4288 				       new_dir, new_dentry, flags);
4289 	if (error)
4290 		goto out;
4291 
4292 	if (!(flags & RENAME_EXCHANGE) && target) {
4293 		if (is_dir) {
4294 			shrink_dcache_parent(new_dentry);
4295 			target->i_flags |= S_DEAD;
4296 		}
4297 		dont_mount(new_dentry);
4298 		detach_mounts(new_dentry);
4299 	}
4300 	if (!(old_dir->i_sb->s_type->fs_flags & FS_RENAME_DOES_D_MOVE)) {
4301 		if (!(flags & RENAME_EXCHANGE))
4302 			d_move(old_dentry, new_dentry);
4303 		else
4304 			d_exchange(old_dentry, new_dentry);
4305 	}
4306 out:
4307 	if (!is_dir || (flags & RENAME_EXCHANGE))
4308 		unlock_two_nondirectories(source, target);
4309 	else if (target)
4310 		inode_unlock(target);
4311 	dput(new_dentry);
4312 	if (!error) {
4313 		fsnotify_move(old_dir, new_dir, &old_name.name, is_dir,
4314 			      !(flags & RENAME_EXCHANGE) ? target : NULL, old_dentry);
4315 		if (flags & RENAME_EXCHANGE) {
4316 			fsnotify_move(new_dir, old_dir, &old_dentry->d_name,
4317 				      new_is_dir, NULL, new_dentry);
4318 		}
4319 	}
4320 	release_dentry_name_snapshot(&old_name);
4321 
4322 	return error;
4323 }
4324 EXPORT_SYMBOL(vfs_rename);
4325 
4326 static int do_renameat2(int olddfd, const char __user *oldname, int newdfd,
4327 			const char __user *newname, unsigned int flags)
4328 {
4329 	struct dentry *old_dentry, *new_dentry;
4330 	struct dentry *trap;
4331 	struct path old_path, new_path;
4332 	struct qstr old_last, new_last;
4333 	int old_type, new_type;
4334 	struct inode *delegated_inode = NULL;
4335 	struct filename *from;
4336 	struct filename *to;
4337 	unsigned int lookup_flags = 0, target_flags = LOOKUP_RENAME_TARGET;
4338 	bool should_retry = false;
4339 	int error;
4340 
4341 	if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
4342 		return -EINVAL;
4343 
4344 	if ((flags & (RENAME_NOREPLACE | RENAME_WHITEOUT)) &&
4345 	    (flags & RENAME_EXCHANGE))
4346 		return -EINVAL;
4347 
4348 	if ((flags & RENAME_WHITEOUT) && !capable(CAP_MKNOD))
4349 		return -EPERM;
4350 
4351 	if (flags & RENAME_EXCHANGE)
4352 		target_flags = 0;
4353 
4354 retry:
4355 	from = filename_parentat(olddfd, getname(oldname), lookup_flags,
4356 				&old_path, &old_last, &old_type);
4357 	if (IS_ERR(from)) {
4358 		error = PTR_ERR(from);
4359 		goto exit;
4360 	}
4361 
4362 	to = filename_parentat(newdfd, getname(newname), lookup_flags,
4363 				&new_path, &new_last, &new_type);
4364 	if (IS_ERR(to)) {
4365 		error = PTR_ERR(to);
4366 		goto exit1;
4367 	}
4368 
4369 	error = -EXDEV;
4370 	if (old_path.mnt != new_path.mnt)
4371 		goto exit2;
4372 
4373 	error = -EBUSY;
4374 	if (old_type != LAST_NORM)
4375 		goto exit2;
4376 
4377 	if (flags & RENAME_NOREPLACE)
4378 		error = -EEXIST;
4379 	if (new_type != LAST_NORM)
4380 		goto exit2;
4381 
4382 	error = mnt_want_write(old_path.mnt);
4383 	if (error)
4384 		goto exit2;
4385 
4386 retry_deleg:
4387 	trap = lock_rename(new_path.dentry, old_path.dentry);
4388 
4389 	old_dentry = __lookup_hash(&old_last, old_path.dentry, lookup_flags);
4390 	error = PTR_ERR(old_dentry);
4391 	if (IS_ERR(old_dentry))
4392 		goto exit3;
4393 	/* source must exist */
4394 	error = -ENOENT;
4395 	if (d_is_negative(old_dentry))
4396 		goto exit4;
4397 	new_dentry = __lookup_hash(&new_last, new_path.dentry, lookup_flags | target_flags);
4398 	error = PTR_ERR(new_dentry);
4399 	if (IS_ERR(new_dentry))
4400 		goto exit4;
4401 	error = -EEXIST;
4402 	if ((flags & RENAME_NOREPLACE) && d_is_positive(new_dentry))
4403 		goto exit5;
4404 	if (flags & RENAME_EXCHANGE) {
4405 		error = -ENOENT;
4406 		if (d_is_negative(new_dentry))
4407 			goto exit5;
4408 
4409 		if (!d_is_dir(new_dentry)) {
4410 			error = -ENOTDIR;
4411 			if (new_last.name[new_last.len])
4412 				goto exit5;
4413 		}
4414 	}
4415 	/* unless the source is a directory trailing slashes give -ENOTDIR */
4416 	if (!d_is_dir(old_dentry)) {
4417 		error = -ENOTDIR;
4418 		if (old_last.name[old_last.len])
4419 			goto exit5;
4420 		if (!(flags & RENAME_EXCHANGE) && new_last.name[new_last.len])
4421 			goto exit5;
4422 	}
4423 	/* source should not be ancestor of target */
4424 	error = -EINVAL;
4425 	if (old_dentry == trap)
4426 		goto exit5;
4427 	/* target should not be an ancestor of source */
4428 	if (!(flags & RENAME_EXCHANGE))
4429 		error = -ENOTEMPTY;
4430 	if (new_dentry == trap)
4431 		goto exit5;
4432 
4433 	error = security_path_rename(&old_path, old_dentry,
4434 				     &new_path, new_dentry, flags);
4435 	if (error)
4436 		goto exit5;
4437 	error = vfs_rename(old_path.dentry->d_inode, old_dentry,
4438 			   new_path.dentry->d_inode, new_dentry,
4439 			   &delegated_inode, flags);
4440 exit5:
4441 	dput(new_dentry);
4442 exit4:
4443 	dput(old_dentry);
4444 exit3:
4445 	unlock_rename(new_path.dentry, old_path.dentry);
4446 	if (delegated_inode) {
4447 		error = break_deleg_wait(&delegated_inode);
4448 		if (!error)
4449 			goto retry_deleg;
4450 	}
4451 	mnt_drop_write(old_path.mnt);
4452 exit2:
4453 	if (retry_estale(error, lookup_flags))
4454 		should_retry = true;
4455 	path_put(&new_path);
4456 	putname(to);
4457 exit1:
4458 	path_put(&old_path);
4459 	putname(from);
4460 	if (should_retry) {
4461 		should_retry = false;
4462 		lookup_flags |= LOOKUP_REVAL;
4463 		goto retry;
4464 	}
4465 exit:
4466 	return error;
4467 }
4468 
4469 SYSCALL_DEFINE5(renameat2, int, olddfd, const char __user *, oldname,
4470 		int, newdfd, const char __user *, newname, unsigned int, flags)
4471 {
4472 	return do_renameat2(olddfd, oldname, newdfd, newname, flags);
4473 }
4474 
4475 SYSCALL_DEFINE4(renameat, int, olddfd, const char __user *, oldname,
4476 		int, newdfd, const char __user *, newname)
4477 {
4478 	return do_renameat2(olddfd, oldname, newdfd, newname, 0);
4479 }
4480 
4481 SYSCALL_DEFINE2(rename, const char __user *, oldname, const char __user *, newname)
4482 {
4483 	return do_renameat2(AT_FDCWD, oldname, AT_FDCWD, newname, 0);
4484 }
4485 
4486 int vfs_whiteout(struct inode *dir, struct dentry *dentry)
4487 {
4488 	int error = may_create(dir, dentry);
4489 	if (error)
4490 		return error;
4491 
4492 	if (!dir->i_op->mknod)
4493 		return -EPERM;
4494 
4495 	return dir->i_op->mknod(dir, dentry,
4496 				S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
4497 }
4498 EXPORT_SYMBOL(vfs_whiteout);
4499 
4500 int readlink_copy(char __user *buffer, int buflen, const char *link)
4501 {
4502 	int len = PTR_ERR(link);
4503 	if (IS_ERR(link))
4504 		goto out;
4505 
4506 	len = strlen(link);
4507 	if (len > (unsigned) buflen)
4508 		len = buflen;
4509 	if (copy_to_user(buffer, link, len))
4510 		len = -EFAULT;
4511 out:
4512 	return len;
4513 }
4514 
4515 /**
4516  * vfs_readlink - copy symlink body into userspace buffer
4517  * @dentry: dentry on which to get symbolic link
4518  * @buffer: user memory pointer
4519  * @buflen: size of buffer
4520  *
4521  * Does not touch atime.  That's up to the caller if necessary
4522  *
4523  * Does not call security hook.
4524  */
4525 int vfs_readlink(struct dentry *dentry, char __user *buffer, int buflen)
4526 {
4527 	struct inode *inode = d_inode(dentry);
4528 	DEFINE_DELAYED_CALL(done);
4529 	const char *link;
4530 	int res;
4531 
4532 	if (unlikely(!(inode->i_opflags & IOP_DEFAULT_READLINK))) {
4533 		if (unlikely(inode->i_op->readlink))
4534 			return inode->i_op->readlink(dentry, buffer, buflen);
4535 
4536 		if (!d_is_symlink(dentry))
4537 			return -EINVAL;
4538 
4539 		spin_lock(&inode->i_lock);
4540 		inode->i_opflags |= IOP_DEFAULT_READLINK;
4541 		spin_unlock(&inode->i_lock);
4542 	}
4543 
4544 	link = READ_ONCE(inode->i_link);
4545 	if (!link) {
4546 		link = inode->i_op->get_link(dentry, inode, &done);
4547 		if (IS_ERR(link))
4548 			return PTR_ERR(link);
4549 	}
4550 	res = readlink_copy(buffer, buflen, link);
4551 	do_delayed_call(&done);
4552 	return res;
4553 }
4554 EXPORT_SYMBOL(vfs_readlink);
4555 
4556 /**
4557  * vfs_get_link - get symlink body
4558  * @dentry: dentry on which to get symbolic link
4559  * @done: caller needs to free returned data with this
4560  *
4561  * Calls security hook and i_op->get_link() on the supplied inode.
4562  *
4563  * It does not touch atime.  That's up to the caller if necessary.
4564  *
4565  * Does not work on "special" symlinks like /proc/$$/fd/N
4566  */
4567 const char *vfs_get_link(struct dentry *dentry, struct delayed_call *done)
4568 {
4569 	const char *res = ERR_PTR(-EINVAL);
4570 	struct inode *inode = d_inode(dentry);
4571 
4572 	if (d_is_symlink(dentry)) {
4573 		res = ERR_PTR(security_inode_readlink(dentry));
4574 		if (!res)
4575 			res = inode->i_op->get_link(dentry, inode, done);
4576 	}
4577 	return res;
4578 }
4579 EXPORT_SYMBOL(vfs_get_link);
4580 
4581 /* get the link contents into pagecache */
4582 const char *page_get_link(struct dentry *dentry, struct inode *inode,
4583 			  struct delayed_call *callback)
4584 {
4585 	char *kaddr;
4586 	struct page *page;
4587 	struct address_space *mapping = inode->i_mapping;
4588 
4589 	if (!dentry) {
4590 		page = find_get_page(mapping, 0);
4591 		if (!page)
4592 			return ERR_PTR(-ECHILD);
4593 		if (!PageUptodate(page)) {
4594 			put_page(page);
4595 			return ERR_PTR(-ECHILD);
4596 		}
4597 	} else {
4598 		page = read_mapping_page(mapping, 0, NULL);
4599 		if (IS_ERR(page))
4600 			return (char*)page;
4601 	}
4602 	set_delayed_call(callback, page_put_link, page);
4603 	BUG_ON(mapping_gfp_mask(mapping) & __GFP_HIGHMEM);
4604 	kaddr = page_address(page);
4605 	nd_terminate_link(kaddr, inode->i_size, PAGE_SIZE - 1);
4606 	return kaddr;
4607 }
4608 
4609 EXPORT_SYMBOL(page_get_link);
4610 
4611 void page_put_link(void *arg)
4612 {
4613 	put_page(arg);
4614 }
4615 EXPORT_SYMBOL(page_put_link);
4616 
4617 int page_readlink(struct dentry *dentry, char __user *buffer, int buflen)
4618 {
4619 	DEFINE_DELAYED_CALL(done);
4620 	int res = readlink_copy(buffer, buflen,
4621 				page_get_link(dentry, d_inode(dentry),
4622 					      &done));
4623 	do_delayed_call(&done);
4624 	return res;
4625 }
4626 EXPORT_SYMBOL(page_readlink);
4627 
4628 /*
4629  * The nofs argument instructs pagecache_write_begin to pass AOP_FLAG_NOFS
4630  */
4631 int __page_symlink(struct inode *inode, const char *symname, int len, int nofs)
4632 {
4633 	struct address_space *mapping = inode->i_mapping;
4634 	struct page *page;
4635 	void *fsdata;
4636 	int err;
4637 	unsigned int flags = 0;
4638 	if (nofs)
4639 		flags |= AOP_FLAG_NOFS;
4640 
4641 retry:
4642 	err = pagecache_write_begin(NULL, mapping, 0, len-1,
4643 				flags, &page, &fsdata);
4644 	if (err)
4645 		goto fail;
4646 
4647 	memcpy(page_address(page), symname, len-1);
4648 
4649 	err = pagecache_write_end(NULL, mapping, 0, len-1, len-1,
4650 							page, fsdata);
4651 	if (err < 0)
4652 		goto fail;
4653 	if (err < len-1)
4654 		goto retry;
4655 
4656 	mark_inode_dirty(inode);
4657 	return 0;
4658 fail:
4659 	return err;
4660 }
4661 EXPORT_SYMBOL(__page_symlink);
4662 
4663 int page_symlink(struct inode *inode, const char *symname, int len)
4664 {
4665 	return __page_symlink(inode, symname, len,
4666 			!mapping_gfp_constraint(inode->i_mapping, __GFP_FS));
4667 }
4668 EXPORT_SYMBOL(page_symlink);
4669 
4670 const struct inode_operations page_symlink_inode_operations = {
4671 	.get_link	= page_get_link,
4672 };
4673 EXPORT_SYMBOL(page_symlink_inode_operations);
4674