xref: /openbmc/linux/fs/kernfs/dir.c (revision e2f1cf25)
1 /*
2  * fs/kernfs/dir.c - kernfs directory implementation
3  *
4  * Copyright (c) 2001-3 Patrick Mochel
5  * Copyright (c) 2007 SUSE Linux Products GmbH
6  * Copyright (c) 2007, 2013 Tejun Heo <tj@kernel.org>
7  *
8  * This file is released under the GPLv2.
9  */
10 
11 #include <linux/sched.h>
12 #include <linux/fs.h>
13 #include <linux/namei.h>
14 #include <linux/idr.h>
15 #include <linux/slab.h>
16 #include <linux/security.h>
17 #include <linux/hash.h>
18 
19 #include "kernfs-internal.h"
20 
21 DEFINE_MUTEX(kernfs_mutex);
22 static DEFINE_SPINLOCK(kernfs_rename_lock);	/* kn->parent and ->name */
23 static char kernfs_pr_cont_buf[PATH_MAX];	/* protected by rename_lock */
24 
25 #define rb_to_kn(X) rb_entry((X), struct kernfs_node, rb)
26 
27 static bool kernfs_active(struct kernfs_node *kn)
28 {
29 	lockdep_assert_held(&kernfs_mutex);
30 	return atomic_read(&kn->active) >= 0;
31 }
32 
33 static bool kernfs_lockdep(struct kernfs_node *kn)
34 {
35 #ifdef CONFIG_DEBUG_LOCK_ALLOC
36 	return kn->flags & KERNFS_LOCKDEP;
37 #else
38 	return false;
39 #endif
40 }
41 
42 static int kernfs_name_locked(struct kernfs_node *kn, char *buf, size_t buflen)
43 {
44 	return strlcpy(buf, kn->parent ? kn->name : "/", buflen);
45 }
46 
47 static char * __must_check kernfs_path_locked(struct kernfs_node *kn, char *buf,
48 					      size_t buflen)
49 {
50 	char *p = buf + buflen;
51 	int len;
52 
53 	*--p = '\0';
54 
55 	do {
56 		len = strlen(kn->name);
57 		if (p - buf < len + 1) {
58 			buf[0] = '\0';
59 			p = NULL;
60 			break;
61 		}
62 		p -= len;
63 		memcpy(p, kn->name, len);
64 		*--p = '/';
65 		kn = kn->parent;
66 	} while (kn && kn->parent);
67 
68 	return p;
69 }
70 
71 /**
72  * kernfs_name - obtain the name of a given node
73  * @kn: kernfs_node of interest
74  * @buf: buffer to copy @kn's name into
75  * @buflen: size of @buf
76  *
77  * Copies the name of @kn into @buf of @buflen bytes.  The behavior is
78  * similar to strlcpy().  It returns the length of @kn's name and if @buf
79  * isn't long enough, it's filled upto @buflen-1 and nul terminated.
80  *
81  * This function can be called from any context.
82  */
83 int kernfs_name(struct kernfs_node *kn, char *buf, size_t buflen)
84 {
85 	unsigned long flags;
86 	int ret;
87 
88 	spin_lock_irqsave(&kernfs_rename_lock, flags);
89 	ret = kernfs_name_locked(kn, buf, buflen);
90 	spin_unlock_irqrestore(&kernfs_rename_lock, flags);
91 	return ret;
92 }
93 
94 /**
95  * kernfs_path - build full path of a given node
96  * @kn: kernfs_node of interest
97  * @buf: buffer to copy @kn's name into
98  * @buflen: size of @buf
99  *
100  * Builds and returns the full path of @kn in @buf of @buflen bytes.  The
101  * path is built from the end of @buf so the returned pointer usually
102  * doesn't match @buf.  If @buf isn't long enough, @buf is nul terminated
103  * and %NULL is returned.
104  */
105 char *kernfs_path(struct kernfs_node *kn, char *buf, size_t buflen)
106 {
107 	unsigned long flags;
108 	char *p;
109 
110 	spin_lock_irqsave(&kernfs_rename_lock, flags);
111 	p = kernfs_path_locked(kn, buf, buflen);
112 	spin_unlock_irqrestore(&kernfs_rename_lock, flags);
113 	return p;
114 }
115 EXPORT_SYMBOL_GPL(kernfs_path);
116 
117 /**
118  * pr_cont_kernfs_name - pr_cont name of a kernfs_node
119  * @kn: kernfs_node of interest
120  *
121  * This function can be called from any context.
122  */
123 void pr_cont_kernfs_name(struct kernfs_node *kn)
124 {
125 	unsigned long flags;
126 
127 	spin_lock_irqsave(&kernfs_rename_lock, flags);
128 
129 	kernfs_name_locked(kn, kernfs_pr_cont_buf, sizeof(kernfs_pr_cont_buf));
130 	pr_cont("%s", kernfs_pr_cont_buf);
131 
132 	spin_unlock_irqrestore(&kernfs_rename_lock, flags);
133 }
134 
135 /**
136  * pr_cont_kernfs_path - pr_cont path of a kernfs_node
137  * @kn: kernfs_node of interest
138  *
139  * This function can be called from any context.
140  */
141 void pr_cont_kernfs_path(struct kernfs_node *kn)
142 {
143 	unsigned long flags;
144 	char *p;
145 
146 	spin_lock_irqsave(&kernfs_rename_lock, flags);
147 
148 	p = kernfs_path_locked(kn, kernfs_pr_cont_buf,
149 			       sizeof(kernfs_pr_cont_buf));
150 	if (p)
151 		pr_cont("%s", p);
152 	else
153 		pr_cont("<name too long>");
154 
155 	spin_unlock_irqrestore(&kernfs_rename_lock, flags);
156 }
157 
158 /**
159  * kernfs_get_parent - determine the parent node and pin it
160  * @kn: kernfs_node of interest
161  *
162  * Determines @kn's parent, pins and returns it.  This function can be
163  * called from any context.
164  */
165 struct kernfs_node *kernfs_get_parent(struct kernfs_node *kn)
166 {
167 	struct kernfs_node *parent;
168 	unsigned long flags;
169 
170 	spin_lock_irqsave(&kernfs_rename_lock, flags);
171 	parent = kn->parent;
172 	kernfs_get(parent);
173 	spin_unlock_irqrestore(&kernfs_rename_lock, flags);
174 
175 	return parent;
176 }
177 
178 /**
179  *	kernfs_name_hash
180  *	@name: Null terminated string to hash
181  *	@ns:   Namespace tag to hash
182  *
183  *	Returns 31 bit hash of ns + name (so it fits in an off_t )
184  */
185 static unsigned int kernfs_name_hash(const char *name, const void *ns)
186 {
187 	unsigned long hash = init_name_hash();
188 	unsigned int len = strlen(name);
189 	while (len--)
190 		hash = partial_name_hash(*name++, hash);
191 	hash = (end_name_hash(hash) ^ hash_ptr((void *)ns, 31));
192 	hash &= 0x7fffffffU;
193 	/* Reserve hash numbers 0, 1 and INT_MAX for magic directory entries */
194 	if (hash < 2)
195 		hash += 2;
196 	if (hash >= INT_MAX)
197 		hash = INT_MAX - 1;
198 	return hash;
199 }
200 
201 static int kernfs_name_compare(unsigned int hash, const char *name,
202 			       const void *ns, const struct kernfs_node *kn)
203 {
204 	if (hash < kn->hash)
205 		return -1;
206 	if (hash > kn->hash)
207 		return 1;
208 	if (ns < kn->ns)
209 		return -1;
210 	if (ns > kn->ns)
211 		return 1;
212 	return strcmp(name, kn->name);
213 }
214 
215 static int kernfs_sd_compare(const struct kernfs_node *left,
216 			     const struct kernfs_node *right)
217 {
218 	return kernfs_name_compare(left->hash, left->name, left->ns, right);
219 }
220 
221 /**
222  *	kernfs_link_sibling - link kernfs_node into sibling rbtree
223  *	@kn: kernfs_node of interest
224  *
225  *	Link @kn into its sibling rbtree which starts from
226  *	@kn->parent->dir.children.
227  *
228  *	Locking:
229  *	mutex_lock(kernfs_mutex)
230  *
231  *	RETURNS:
232  *	0 on susccess -EEXIST on failure.
233  */
234 static int kernfs_link_sibling(struct kernfs_node *kn)
235 {
236 	struct rb_node **node = &kn->parent->dir.children.rb_node;
237 	struct rb_node *parent = NULL;
238 
239 	while (*node) {
240 		struct kernfs_node *pos;
241 		int result;
242 
243 		pos = rb_to_kn(*node);
244 		parent = *node;
245 		result = kernfs_sd_compare(kn, pos);
246 		if (result < 0)
247 			node = &pos->rb.rb_left;
248 		else if (result > 0)
249 			node = &pos->rb.rb_right;
250 		else
251 			return -EEXIST;
252 	}
253 
254 	/* add new node and rebalance the tree */
255 	rb_link_node(&kn->rb, parent, node);
256 	rb_insert_color(&kn->rb, &kn->parent->dir.children);
257 
258 	/* successfully added, account subdir number */
259 	if (kernfs_type(kn) == KERNFS_DIR)
260 		kn->parent->dir.subdirs++;
261 
262 	return 0;
263 }
264 
265 /**
266  *	kernfs_unlink_sibling - unlink kernfs_node from sibling rbtree
267  *	@kn: kernfs_node of interest
268  *
269  *	Try to unlink @kn from its sibling rbtree which starts from
270  *	kn->parent->dir.children.  Returns %true if @kn was actually
271  *	removed, %false if @kn wasn't on the rbtree.
272  *
273  *	Locking:
274  *	mutex_lock(kernfs_mutex)
275  */
276 static bool kernfs_unlink_sibling(struct kernfs_node *kn)
277 {
278 	if (RB_EMPTY_NODE(&kn->rb))
279 		return false;
280 
281 	if (kernfs_type(kn) == KERNFS_DIR)
282 		kn->parent->dir.subdirs--;
283 
284 	rb_erase(&kn->rb, &kn->parent->dir.children);
285 	RB_CLEAR_NODE(&kn->rb);
286 	return true;
287 }
288 
289 /**
290  *	kernfs_get_active - get an active reference to kernfs_node
291  *	@kn: kernfs_node to get an active reference to
292  *
293  *	Get an active reference of @kn.  This function is noop if @kn
294  *	is NULL.
295  *
296  *	RETURNS:
297  *	Pointer to @kn on success, NULL on failure.
298  */
299 struct kernfs_node *kernfs_get_active(struct kernfs_node *kn)
300 {
301 	if (unlikely(!kn))
302 		return NULL;
303 
304 	if (!atomic_inc_unless_negative(&kn->active))
305 		return NULL;
306 
307 	if (kernfs_lockdep(kn))
308 		rwsem_acquire_read(&kn->dep_map, 0, 1, _RET_IP_);
309 	return kn;
310 }
311 
312 /**
313  *	kernfs_put_active - put an active reference to kernfs_node
314  *	@kn: kernfs_node to put an active reference to
315  *
316  *	Put an active reference to @kn.  This function is noop if @kn
317  *	is NULL.
318  */
319 void kernfs_put_active(struct kernfs_node *kn)
320 {
321 	struct kernfs_root *root = kernfs_root(kn);
322 	int v;
323 
324 	if (unlikely(!kn))
325 		return;
326 
327 	if (kernfs_lockdep(kn))
328 		rwsem_release(&kn->dep_map, 1, _RET_IP_);
329 	v = atomic_dec_return(&kn->active);
330 	if (likely(v != KN_DEACTIVATED_BIAS))
331 		return;
332 
333 	wake_up_all(&root->deactivate_waitq);
334 }
335 
336 /**
337  * kernfs_drain - drain kernfs_node
338  * @kn: kernfs_node to drain
339  *
340  * Drain existing usages and nuke all existing mmaps of @kn.  Mutiple
341  * removers may invoke this function concurrently on @kn and all will
342  * return after draining is complete.
343  */
344 static void kernfs_drain(struct kernfs_node *kn)
345 	__releases(&kernfs_mutex) __acquires(&kernfs_mutex)
346 {
347 	struct kernfs_root *root = kernfs_root(kn);
348 
349 	lockdep_assert_held(&kernfs_mutex);
350 	WARN_ON_ONCE(kernfs_active(kn));
351 
352 	mutex_unlock(&kernfs_mutex);
353 
354 	if (kernfs_lockdep(kn)) {
355 		rwsem_acquire(&kn->dep_map, 0, 0, _RET_IP_);
356 		if (atomic_read(&kn->active) != KN_DEACTIVATED_BIAS)
357 			lock_contended(&kn->dep_map, _RET_IP_);
358 	}
359 
360 	/* but everyone should wait for draining */
361 	wait_event(root->deactivate_waitq,
362 		   atomic_read(&kn->active) == KN_DEACTIVATED_BIAS);
363 
364 	if (kernfs_lockdep(kn)) {
365 		lock_acquired(&kn->dep_map, _RET_IP_);
366 		rwsem_release(&kn->dep_map, 1, _RET_IP_);
367 	}
368 
369 	kernfs_unmap_bin_file(kn);
370 
371 	mutex_lock(&kernfs_mutex);
372 }
373 
374 /**
375  * kernfs_get - get a reference count on a kernfs_node
376  * @kn: the target kernfs_node
377  */
378 void kernfs_get(struct kernfs_node *kn)
379 {
380 	if (kn) {
381 		WARN_ON(!atomic_read(&kn->count));
382 		atomic_inc(&kn->count);
383 	}
384 }
385 EXPORT_SYMBOL_GPL(kernfs_get);
386 
387 /**
388  * kernfs_put - put a reference count on a kernfs_node
389  * @kn: the target kernfs_node
390  *
391  * Put a reference count of @kn and destroy it if it reached zero.
392  */
393 void kernfs_put(struct kernfs_node *kn)
394 {
395 	struct kernfs_node *parent;
396 	struct kernfs_root *root;
397 
398 	if (!kn || !atomic_dec_and_test(&kn->count))
399 		return;
400 	root = kernfs_root(kn);
401  repeat:
402 	/*
403 	 * Moving/renaming is always done while holding reference.
404 	 * kn->parent won't change beneath us.
405 	 */
406 	parent = kn->parent;
407 
408 	WARN_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS,
409 		  "kernfs_put: %s/%s: released with incorrect active_ref %d\n",
410 		  parent ? parent->name : "", kn->name, atomic_read(&kn->active));
411 
412 	if (kernfs_type(kn) == KERNFS_LINK)
413 		kernfs_put(kn->symlink.target_kn);
414 
415 	kfree_const(kn->name);
416 
417 	if (kn->iattr) {
418 		if (kn->iattr->ia_secdata)
419 			security_release_secctx(kn->iattr->ia_secdata,
420 						kn->iattr->ia_secdata_len);
421 		simple_xattrs_free(&kn->iattr->xattrs);
422 	}
423 	kfree(kn->iattr);
424 	ida_simple_remove(&root->ino_ida, kn->ino);
425 	kmem_cache_free(kernfs_node_cache, kn);
426 
427 	kn = parent;
428 	if (kn) {
429 		if (atomic_dec_and_test(&kn->count))
430 			goto repeat;
431 	} else {
432 		/* just released the root kn, free @root too */
433 		ida_destroy(&root->ino_ida);
434 		kfree(root);
435 	}
436 }
437 EXPORT_SYMBOL_GPL(kernfs_put);
438 
439 static int kernfs_dop_revalidate(struct dentry *dentry, unsigned int flags)
440 {
441 	struct kernfs_node *kn;
442 
443 	if (flags & LOOKUP_RCU)
444 		return -ECHILD;
445 
446 	/* Always perform fresh lookup for negatives */
447 	if (d_really_is_negative(dentry))
448 		goto out_bad_unlocked;
449 
450 	kn = dentry->d_fsdata;
451 	mutex_lock(&kernfs_mutex);
452 
453 	/* The kernfs node has been deactivated */
454 	if (!kernfs_active(kn))
455 		goto out_bad;
456 
457 	/* The kernfs node has been moved? */
458 	if (dentry->d_parent->d_fsdata != kn->parent)
459 		goto out_bad;
460 
461 	/* The kernfs node has been renamed */
462 	if (strcmp(dentry->d_name.name, kn->name) != 0)
463 		goto out_bad;
464 
465 	/* The kernfs node has been moved to a different namespace */
466 	if (kn->parent && kernfs_ns_enabled(kn->parent) &&
467 	    kernfs_info(dentry->d_sb)->ns != kn->ns)
468 		goto out_bad;
469 
470 	mutex_unlock(&kernfs_mutex);
471 	return 1;
472 out_bad:
473 	mutex_unlock(&kernfs_mutex);
474 out_bad_unlocked:
475 	return 0;
476 }
477 
478 static void kernfs_dop_release(struct dentry *dentry)
479 {
480 	kernfs_put(dentry->d_fsdata);
481 }
482 
483 const struct dentry_operations kernfs_dops = {
484 	.d_revalidate	= kernfs_dop_revalidate,
485 	.d_release	= kernfs_dop_release,
486 };
487 
488 /**
489  * kernfs_node_from_dentry - determine kernfs_node associated with a dentry
490  * @dentry: the dentry in question
491  *
492  * Return the kernfs_node associated with @dentry.  If @dentry is not a
493  * kernfs one, %NULL is returned.
494  *
495  * While the returned kernfs_node will stay accessible as long as @dentry
496  * is accessible, the returned node can be in any state and the caller is
497  * fully responsible for determining what's accessible.
498  */
499 struct kernfs_node *kernfs_node_from_dentry(struct dentry *dentry)
500 {
501 	if (dentry->d_sb->s_op == &kernfs_sops)
502 		return dentry->d_fsdata;
503 	return NULL;
504 }
505 
506 static struct kernfs_node *__kernfs_new_node(struct kernfs_root *root,
507 					     const char *name, umode_t mode,
508 					     unsigned flags)
509 {
510 	struct kernfs_node *kn;
511 	int ret;
512 
513 	name = kstrdup_const(name, GFP_KERNEL);
514 	if (!name)
515 		return NULL;
516 
517 	kn = kmem_cache_zalloc(kernfs_node_cache, GFP_KERNEL);
518 	if (!kn)
519 		goto err_out1;
520 
521 	/*
522 	 * If the ino of the sysfs entry created for a kmem cache gets
523 	 * allocated from an ida layer, which is accounted to the memcg that
524 	 * owns the cache, the memcg will get pinned forever. So do not account
525 	 * ino ida allocations.
526 	 */
527 	ret = ida_simple_get(&root->ino_ida, 1, 0,
528 			     GFP_KERNEL | __GFP_NOACCOUNT);
529 	if (ret < 0)
530 		goto err_out2;
531 	kn->ino = ret;
532 
533 	atomic_set(&kn->count, 1);
534 	atomic_set(&kn->active, KN_DEACTIVATED_BIAS);
535 	RB_CLEAR_NODE(&kn->rb);
536 
537 	kn->name = name;
538 	kn->mode = mode;
539 	kn->flags = flags;
540 
541 	return kn;
542 
543  err_out2:
544 	kmem_cache_free(kernfs_node_cache, kn);
545  err_out1:
546 	kfree_const(name);
547 	return NULL;
548 }
549 
550 struct kernfs_node *kernfs_new_node(struct kernfs_node *parent,
551 				    const char *name, umode_t mode,
552 				    unsigned flags)
553 {
554 	struct kernfs_node *kn;
555 
556 	kn = __kernfs_new_node(kernfs_root(parent), name, mode, flags);
557 	if (kn) {
558 		kernfs_get(parent);
559 		kn->parent = parent;
560 	}
561 	return kn;
562 }
563 
564 /**
565  *	kernfs_add_one - add kernfs_node to parent without warning
566  *	@kn: kernfs_node to be added
567  *
568  *	The caller must already have initialized @kn->parent.  This
569  *	function increments nlink of the parent's inode if @kn is a
570  *	directory and link into the children list of the parent.
571  *
572  *	RETURNS:
573  *	0 on success, -EEXIST if entry with the given name already
574  *	exists.
575  */
576 int kernfs_add_one(struct kernfs_node *kn)
577 {
578 	struct kernfs_node *parent = kn->parent;
579 	struct kernfs_iattrs *ps_iattr;
580 	bool has_ns;
581 	int ret;
582 
583 	mutex_lock(&kernfs_mutex);
584 
585 	ret = -EINVAL;
586 	has_ns = kernfs_ns_enabled(parent);
587 	if (WARN(has_ns != (bool)kn->ns, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",
588 		 has_ns ? "required" : "invalid", parent->name, kn->name))
589 		goto out_unlock;
590 
591 	if (kernfs_type(parent) != KERNFS_DIR)
592 		goto out_unlock;
593 
594 	ret = -ENOENT;
595 	if (parent->flags & KERNFS_EMPTY_DIR)
596 		goto out_unlock;
597 
598 	if ((parent->flags & KERNFS_ACTIVATED) && !kernfs_active(parent))
599 		goto out_unlock;
600 
601 	kn->hash = kernfs_name_hash(kn->name, kn->ns);
602 
603 	ret = kernfs_link_sibling(kn);
604 	if (ret)
605 		goto out_unlock;
606 
607 	/* Update timestamps on the parent */
608 	ps_iattr = parent->iattr;
609 	if (ps_iattr) {
610 		struct iattr *ps_iattrs = &ps_iattr->ia_iattr;
611 		ps_iattrs->ia_ctime = ps_iattrs->ia_mtime = CURRENT_TIME;
612 	}
613 
614 	mutex_unlock(&kernfs_mutex);
615 
616 	/*
617 	 * Activate the new node unless CREATE_DEACTIVATED is requested.
618 	 * If not activated here, the kernfs user is responsible for
619 	 * activating the node with kernfs_activate().  A node which hasn't
620 	 * been activated is not visible to userland and its removal won't
621 	 * trigger deactivation.
622 	 */
623 	if (!(kernfs_root(kn)->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
624 		kernfs_activate(kn);
625 	return 0;
626 
627 out_unlock:
628 	mutex_unlock(&kernfs_mutex);
629 	return ret;
630 }
631 
632 /**
633  * kernfs_find_ns - find kernfs_node with the given name
634  * @parent: kernfs_node to search under
635  * @name: name to look for
636  * @ns: the namespace tag to use
637  *
638  * Look for kernfs_node with name @name under @parent.  Returns pointer to
639  * the found kernfs_node on success, %NULL on failure.
640  */
641 static struct kernfs_node *kernfs_find_ns(struct kernfs_node *parent,
642 					  const unsigned char *name,
643 					  const void *ns)
644 {
645 	struct rb_node *node = parent->dir.children.rb_node;
646 	bool has_ns = kernfs_ns_enabled(parent);
647 	unsigned int hash;
648 
649 	lockdep_assert_held(&kernfs_mutex);
650 
651 	if (has_ns != (bool)ns) {
652 		WARN(1, KERN_WARNING "kernfs: ns %s in '%s' for '%s'\n",
653 		     has_ns ? "required" : "invalid", parent->name, name);
654 		return NULL;
655 	}
656 
657 	hash = kernfs_name_hash(name, ns);
658 	while (node) {
659 		struct kernfs_node *kn;
660 		int result;
661 
662 		kn = rb_to_kn(node);
663 		result = kernfs_name_compare(hash, name, ns, kn);
664 		if (result < 0)
665 			node = node->rb_left;
666 		else if (result > 0)
667 			node = node->rb_right;
668 		else
669 			return kn;
670 	}
671 	return NULL;
672 }
673 
674 /**
675  * kernfs_find_and_get_ns - find and get kernfs_node with the given name
676  * @parent: kernfs_node to search under
677  * @name: name to look for
678  * @ns: the namespace tag to use
679  *
680  * Look for kernfs_node with name @name under @parent and get a reference
681  * if found.  This function may sleep and returns pointer to the found
682  * kernfs_node on success, %NULL on failure.
683  */
684 struct kernfs_node *kernfs_find_and_get_ns(struct kernfs_node *parent,
685 					   const char *name, const void *ns)
686 {
687 	struct kernfs_node *kn;
688 
689 	mutex_lock(&kernfs_mutex);
690 	kn = kernfs_find_ns(parent, name, ns);
691 	kernfs_get(kn);
692 	mutex_unlock(&kernfs_mutex);
693 
694 	return kn;
695 }
696 EXPORT_SYMBOL_GPL(kernfs_find_and_get_ns);
697 
698 /**
699  * kernfs_create_root - create a new kernfs hierarchy
700  * @scops: optional syscall operations for the hierarchy
701  * @flags: KERNFS_ROOT_* flags
702  * @priv: opaque data associated with the new directory
703  *
704  * Returns the root of the new hierarchy on success, ERR_PTR() value on
705  * failure.
706  */
707 struct kernfs_root *kernfs_create_root(struct kernfs_syscall_ops *scops,
708 				       unsigned int flags, void *priv)
709 {
710 	struct kernfs_root *root;
711 	struct kernfs_node *kn;
712 
713 	root = kzalloc(sizeof(*root), GFP_KERNEL);
714 	if (!root)
715 		return ERR_PTR(-ENOMEM);
716 
717 	ida_init(&root->ino_ida);
718 	INIT_LIST_HEAD(&root->supers);
719 
720 	kn = __kernfs_new_node(root, "", S_IFDIR | S_IRUGO | S_IXUGO,
721 			       KERNFS_DIR);
722 	if (!kn) {
723 		ida_destroy(&root->ino_ida);
724 		kfree(root);
725 		return ERR_PTR(-ENOMEM);
726 	}
727 
728 	kn->priv = priv;
729 	kn->dir.root = root;
730 
731 	root->syscall_ops = scops;
732 	root->flags = flags;
733 	root->kn = kn;
734 	init_waitqueue_head(&root->deactivate_waitq);
735 
736 	if (!(root->flags & KERNFS_ROOT_CREATE_DEACTIVATED))
737 		kernfs_activate(kn);
738 
739 	return root;
740 }
741 
742 /**
743  * kernfs_destroy_root - destroy a kernfs hierarchy
744  * @root: root of the hierarchy to destroy
745  *
746  * Destroy the hierarchy anchored at @root by removing all existing
747  * directories and destroying @root.
748  */
749 void kernfs_destroy_root(struct kernfs_root *root)
750 {
751 	kernfs_remove(root->kn);	/* will also free @root */
752 }
753 
754 /**
755  * kernfs_create_dir_ns - create a directory
756  * @parent: parent in which to create a new directory
757  * @name: name of the new directory
758  * @mode: mode of the new directory
759  * @priv: opaque data associated with the new directory
760  * @ns: optional namespace tag of the directory
761  *
762  * Returns the created node on success, ERR_PTR() value on failure.
763  */
764 struct kernfs_node *kernfs_create_dir_ns(struct kernfs_node *parent,
765 					 const char *name, umode_t mode,
766 					 void *priv, const void *ns)
767 {
768 	struct kernfs_node *kn;
769 	int rc;
770 
771 	/* allocate */
772 	kn = kernfs_new_node(parent, name, mode | S_IFDIR, KERNFS_DIR);
773 	if (!kn)
774 		return ERR_PTR(-ENOMEM);
775 
776 	kn->dir.root = parent->dir.root;
777 	kn->ns = ns;
778 	kn->priv = priv;
779 
780 	/* link in */
781 	rc = kernfs_add_one(kn);
782 	if (!rc)
783 		return kn;
784 
785 	kernfs_put(kn);
786 	return ERR_PTR(rc);
787 }
788 
789 /**
790  * kernfs_create_empty_dir - create an always empty directory
791  * @parent: parent in which to create a new directory
792  * @name: name of the new directory
793  *
794  * Returns the created node on success, ERR_PTR() value on failure.
795  */
796 struct kernfs_node *kernfs_create_empty_dir(struct kernfs_node *parent,
797 					    const char *name)
798 {
799 	struct kernfs_node *kn;
800 	int rc;
801 
802 	/* allocate */
803 	kn = kernfs_new_node(parent, name, S_IRUGO|S_IXUGO|S_IFDIR, KERNFS_DIR);
804 	if (!kn)
805 		return ERR_PTR(-ENOMEM);
806 
807 	kn->flags |= KERNFS_EMPTY_DIR;
808 	kn->dir.root = parent->dir.root;
809 	kn->ns = NULL;
810 	kn->priv = NULL;
811 
812 	/* link in */
813 	rc = kernfs_add_one(kn);
814 	if (!rc)
815 		return kn;
816 
817 	kernfs_put(kn);
818 	return ERR_PTR(rc);
819 }
820 
821 static struct dentry *kernfs_iop_lookup(struct inode *dir,
822 					struct dentry *dentry,
823 					unsigned int flags)
824 {
825 	struct dentry *ret;
826 	struct kernfs_node *parent = dentry->d_parent->d_fsdata;
827 	struct kernfs_node *kn;
828 	struct inode *inode;
829 	const void *ns = NULL;
830 
831 	mutex_lock(&kernfs_mutex);
832 
833 	if (kernfs_ns_enabled(parent))
834 		ns = kernfs_info(dir->i_sb)->ns;
835 
836 	kn = kernfs_find_ns(parent, dentry->d_name.name, ns);
837 
838 	/* no such entry */
839 	if (!kn || !kernfs_active(kn)) {
840 		ret = NULL;
841 		goto out_unlock;
842 	}
843 	kernfs_get(kn);
844 	dentry->d_fsdata = kn;
845 
846 	/* attach dentry and inode */
847 	inode = kernfs_get_inode(dir->i_sb, kn);
848 	if (!inode) {
849 		ret = ERR_PTR(-ENOMEM);
850 		goto out_unlock;
851 	}
852 
853 	/* instantiate and hash dentry */
854 	ret = d_splice_alias(inode, dentry);
855  out_unlock:
856 	mutex_unlock(&kernfs_mutex);
857 	return ret;
858 }
859 
860 static int kernfs_iop_mkdir(struct inode *dir, struct dentry *dentry,
861 			    umode_t mode)
862 {
863 	struct kernfs_node *parent = dir->i_private;
864 	struct kernfs_syscall_ops *scops = kernfs_root(parent)->syscall_ops;
865 	int ret;
866 
867 	if (!scops || !scops->mkdir)
868 		return -EPERM;
869 
870 	if (!kernfs_get_active(parent))
871 		return -ENODEV;
872 
873 	ret = scops->mkdir(parent, dentry->d_name.name, mode);
874 
875 	kernfs_put_active(parent);
876 	return ret;
877 }
878 
879 static int kernfs_iop_rmdir(struct inode *dir, struct dentry *dentry)
880 {
881 	struct kernfs_node *kn  = dentry->d_fsdata;
882 	struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;
883 	int ret;
884 
885 	if (!scops || !scops->rmdir)
886 		return -EPERM;
887 
888 	if (!kernfs_get_active(kn))
889 		return -ENODEV;
890 
891 	ret = scops->rmdir(kn);
892 
893 	kernfs_put_active(kn);
894 	return ret;
895 }
896 
897 static int kernfs_iop_rename(struct inode *old_dir, struct dentry *old_dentry,
898 			     struct inode *new_dir, struct dentry *new_dentry)
899 {
900 	struct kernfs_node *kn  = old_dentry->d_fsdata;
901 	struct kernfs_node *new_parent = new_dir->i_private;
902 	struct kernfs_syscall_ops *scops = kernfs_root(kn)->syscall_ops;
903 	int ret;
904 
905 	if (!scops || !scops->rename)
906 		return -EPERM;
907 
908 	if (!kernfs_get_active(kn))
909 		return -ENODEV;
910 
911 	if (!kernfs_get_active(new_parent)) {
912 		kernfs_put_active(kn);
913 		return -ENODEV;
914 	}
915 
916 	ret = scops->rename(kn, new_parent, new_dentry->d_name.name);
917 
918 	kernfs_put_active(new_parent);
919 	kernfs_put_active(kn);
920 	return ret;
921 }
922 
923 const struct inode_operations kernfs_dir_iops = {
924 	.lookup		= kernfs_iop_lookup,
925 	.permission	= kernfs_iop_permission,
926 	.setattr	= kernfs_iop_setattr,
927 	.getattr	= kernfs_iop_getattr,
928 	.setxattr	= kernfs_iop_setxattr,
929 	.removexattr	= kernfs_iop_removexattr,
930 	.getxattr	= kernfs_iop_getxattr,
931 	.listxattr	= kernfs_iop_listxattr,
932 
933 	.mkdir		= kernfs_iop_mkdir,
934 	.rmdir		= kernfs_iop_rmdir,
935 	.rename		= kernfs_iop_rename,
936 };
937 
938 static struct kernfs_node *kernfs_leftmost_descendant(struct kernfs_node *pos)
939 {
940 	struct kernfs_node *last;
941 
942 	while (true) {
943 		struct rb_node *rbn;
944 
945 		last = pos;
946 
947 		if (kernfs_type(pos) != KERNFS_DIR)
948 			break;
949 
950 		rbn = rb_first(&pos->dir.children);
951 		if (!rbn)
952 			break;
953 
954 		pos = rb_to_kn(rbn);
955 	}
956 
957 	return last;
958 }
959 
960 /**
961  * kernfs_next_descendant_post - find the next descendant for post-order walk
962  * @pos: the current position (%NULL to initiate traversal)
963  * @root: kernfs_node whose descendants to walk
964  *
965  * Find the next descendant to visit for post-order traversal of @root's
966  * descendants.  @root is included in the iteration and the last node to be
967  * visited.
968  */
969 static struct kernfs_node *kernfs_next_descendant_post(struct kernfs_node *pos,
970 						       struct kernfs_node *root)
971 {
972 	struct rb_node *rbn;
973 
974 	lockdep_assert_held(&kernfs_mutex);
975 
976 	/* if first iteration, visit leftmost descendant which may be root */
977 	if (!pos)
978 		return kernfs_leftmost_descendant(root);
979 
980 	/* if we visited @root, we're done */
981 	if (pos == root)
982 		return NULL;
983 
984 	/* if there's an unvisited sibling, visit its leftmost descendant */
985 	rbn = rb_next(&pos->rb);
986 	if (rbn)
987 		return kernfs_leftmost_descendant(rb_to_kn(rbn));
988 
989 	/* no sibling left, visit parent */
990 	return pos->parent;
991 }
992 
993 /**
994  * kernfs_activate - activate a node which started deactivated
995  * @kn: kernfs_node whose subtree is to be activated
996  *
997  * If the root has KERNFS_ROOT_CREATE_DEACTIVATED set, a newly created node
998  * needs to be explicitly activated.  A node which hasn't been activated
999  * isn't visible to userland and deactivation is skipped during its
1000  * removal.  This is useful to construct atomic init sequences where
1001  * creation of multiple nodes should either succeed or fail atomically.
1002  *
1003  * The caller is responsible for ensuring that this function is not called
1004  * after kernfs_remove*() is invoked on @kn.
1005  */
1006 void kernfs_activate(struct kernfs_node *kn)
1007 {
1008 	struct kernfs_node *pos;
1009 
1010 	mutex_lock(&kernfs_mutex);
1011 
1012 	pos = NULL;
1013 	while ((pos = kernfs_next_descendant_post(pos, kn))) {
1014 		if (!pos || (pos->flags & KERNFS_ACTIVATED))
1015 			continue;
1016 
1017 		WARN_ON_ONCE(pos->parent && RB_EMPTY_NODE(&pos->rb));
1018 		WARN_ON_ONCE(atomic_read(&pos->active) != KN_DEACTIVATED_BIAS);
1019 
1020 		atomic_sub(KN_DEACTIVATED_BIAS, &pos->active);
1021 		pos->flags |= KERNFS_ACTIVATED;
1022 	}
1023 
1024 	mutex_unlock(&kernfs_mutex);
1025 }
1026 
1027 static void __kernfs_remove(struct kernfs_node *kn)
1028 {
1029 	struct kernfs_node *pos;
1030 
1031 	lockdep_assert_held(&kernfs_mutex);
1032 
1033 	/*
1034 	 * Short-circuit if non-root @kn has already finished removal.
1035 	 * This is for kernfs_remove_self() which plays with active ref
1036 	 * after removal.
1037 	 */
1038 	if (!kn || (kn->parent && RB_EMPTY_NODE(&kn->rb)))
1039 		return;
1040 
1041 	pr_debug("kernfs %s: removing\n", kn->name);
1042 
1043 	/* prevent any new usage under @kn by deactivating all nodes */
1044 	pos = NULL;
1045 	while ((pos = kernfs_next_descendant_post(pos, kn)))
1046 		if (kernfs_active(pos))
1047 			atomic_add(KN_DEACTIVATED_BIAS, &pos->active);
1048 
1049 	/* deactivate and unlink the subtree node-by-node */
1050 	do {
1051 		pos = kernfs_leftmost_descendant(kn);
1052 
1053 		/*
1054 		 * kernfs_drain() drops kernfs_mutex temporarily and @pos's
1055 		 * base ref could have been put by someone else by the time
1056 		 * the function returns.  Make sure it doesn't go away
1057 		 * underneath us.
1058 		 */
1059 		kernfs_get(pos);
1060 
1061 		/*
1062 		 * Drain iff @kn was activated.  This avoids draining and
1063 		 * its lockdep annotations for nodes which have never been
1064 		 * activated and allows embedding kernfs_remove() in create
1065 		 * error paths without worrying about draining.
1066 		 */
1067 		if (kn->flags & KERNFS_ACTIVATED)
1068 			kernfs_drain(pos);
1069 		else
1070 			WARN_ON_ONCE(atomic_read(&kn->active) != KN_DEACTIVATED_BIAS);
1071 
1072 		/*
1073 		 * kernfs_unlink_sibling() succeeds once per node.  Use it
1074 		 * to decide who's responsible for cleanups.
1075 		 */
1076 		if (!pos->parent || kernfs_unlink_sibling(pos)) {
1077 			struct kernfs_iattrs *ps_iattr =
1078 				pos->parent ? pos->parent->iattr : NULL;
1079 
1080 			/* update timestamps on the parent */
1081 			if (ps_iattr) {
1082 				ps_iattr->ia_iattr.ia_ctime = CURRENT_TIME;
1083 				ps_iattr->ia_iattr.ia_mtime = CURRENT_TIME;
1084 			}
1085 
1086 			kernfs_put(pos);
1087 		}
1088 
1089 		kernfs_put(pos);
1090 	} while (pos != kn);
1091 }
1092 
1093 /**
1094  * kernfs_remove - remove a kernfs_node recursively
1095  * @kn: the kernfs_node to remove
1096  *
1097  * Remove @kn along with all its subdirectories and files.
1098  */
1099 void kernfs_remove(struct kernfs_node *kn)
1100 {
1101 	mutex_lock(&kernfs_mutex);
1102 	__kernfs_remove(kn);
1103 	mutex_unlock(&kernfs_mutex);
1104 }
1105 
1106 /**
1107  * kernfs_break_active_protection - break out of active protection
1108  * @kn: the self kernfs_node
1109  *
1110  * The caller must be running off of a kernfs operation which is invoked
1111  * with an active reference - e.g. one of kernfs_ops.  Each invocation of
1112  * this function must also be matched with an invocation of
1113  * kernfs_unbreak_active_protection().
1114  *
1115  * This function releases the active reference of @kn the caller is
1116  * holding.  Once this function is called, @kn may be removed at any point
1117  * and the caller is solely responsible for ensuring that the objects it
1118  * dereferences are accessible.
1119  */
1120 void kernfs_break_active_protection(struct kernfs_node *kn)
1121 {
1122 	/*
1123 	 * Take out ourself out of the active ref dependency chain.  If
1124 	 * we're called without an active ref, lockdep will complain.
1125 	 */
1126 	kernfs_put_active(kn);
1127 }
1128 
1129 /**
1130  * kernfs_unbreak_active_protection - undo kernfs_break_active_protection()
1131  * @kn: the self kernfs_node
1132  *
1133  * If kernfs_break_active_protection() was called, this function must be
1134  * invoked before finishing the kernfs operation.  Note that while this
1135  * function restores the active reference, it doesn't and can't actually
1136  * restore the active protection - @kn may already or be in the process of
1137  * being removed.  Once kernfs_break_active_protection() is invoked, that
1138  * protection is irreversibly gone for the kernfs operation instance.
1139  *
1140  * While this function may be called at any point after
1141  * kernfs_break_active_protection() is invoked, its most useful location
1142  * would be right before the enclosing kernfs operation returns.
1143  */
1144 void kernfs_unbreak_active_protection(struct kernfs_node *kn)
1145 {
1146 	/*
1147 	 * @kn->active could be in any state; however, the increment we do
1148 	 * here will be undone as soon as the enclosing kernfs operation
1149 	 * finishes and this temporary bump can't break anything.  If @kn
1150 	 * is alive, nothing changes.  If @kn is being deactivated, the
1151 	 * soon-to-follow put will either finish deactivation or restore
1152 	 * deactivated state.  If @kn is already removed, the temporary
1153 	 * bump is guaranteed to be gone before @kn is released.
1154 	 */
1155 	atomic_inc(&kn->active);
1156 	if (kernfs_lockdep(kn))
1157 		rwsem_acquire(&kn->dep_map, 0, 1, _RET_IP_);
1158 }
1159 
1160 /**
1161  * kernfs_remove_self - remove a kernfs_node from its own method
1162  * @kn: the self kernfs_node to remove
1163  *
1164  * The caller must be running off of a kernfs operation which is invoked
1165  * with an active reference - e.g. one of kernfs_ops.  This can be used to
1166  * implement a file operation which deletes itself.
1167  *
1168  * For example, the "delete" file for a sysfs device directory can be
1169  * implemented by invoking kernfs_remove_self() on the "delete" file
1170  * itself.  This function breaks the circular dependency of trying to
1171  * deactivate self while holding an active ref itself.  It isn't necessary
1172  * to modify the usual removal path to use kernfs_remove_self().  The
1173  * "delete" implementation can simply invoke kernfs_remove_self() on self
1174  * before proceeding with the usual removal path.  kernfs will ignore later
1175  * kernfs_remove() on self.
1176  *
1177  * kernfs_remove_self() can be called multiple times concurrently on the
1178  * same kernfs_node.  Only the first one actually performs removal and
1179  * returns %true.  All others will wait until the kernfs operation which
1180  * won self-removal finishes and return %false.  Note that the losers wait
1181  * for the completion of not only the winning kernfs_remove_self() but also
1182  * the whole kernfs_ops which won the arbitration.  This can be used to
1183  * guarantee, for example, all concurrent writes to a "delete" file to
1184  * finish only after the whole operation is complete.
1185  */
1186 bool kernfs_remove_self(struct kernfs_node *kn)
1187 {
1188 	bool ret;
1189 
1190 	mutex_lock(&kernfs_mutex);
1191 	kernfs_break_active_protection(kn);
1192 
1193 	/*
1194 	 * SUICIDAL is used to arbitrate among competing invocations.  Only
1195 	 * the first one will actually perform removal.  When the removal
1196 	 * is complete, SUICIDED is set and the active ref is restored
1197 	 * while holding kernfs_mutex.  The ones which lost arbitration
1198 	 * waits for SUICDED && drained which can happen only after the
1199 	 * enclosing kernfs operation which executed the winning instance
1200 	 * of kernfs_remove_self() finished.
1201 	 */
1202 	if (!(kn->flags & KERNFS_SUICIDAL)) {
1203 		kn->flags |= KERNFS_SUICIDAL;
1204 		__kernfs_remove(kn);
1205 		kn->flags |= KERNFS_SUICIDED;
1206 		ret = true;
1207 	} else {
1208 		wait_queue_head_t *waitq = &kernfs_root(kn)->deactivate_waitq;
1209 		DEFINE_WAIT(wait);
1210 
1211 		while (true) {
1212 			prepare_to_wait(waitq, &wait, TASK_UNINTERRUPTIBLE);
1213 
1214 			if ((kn->flags & KERNFS_SUICIDED) &&
1215 			    atomic_read(&kn->active) == KN_DEACTIVATED_BIAS)
1216 				break;
1217 
1218 			mutex_unlock(&kernfs_mutex);
1219 			schedule();
1220 			mutex_lock(&kernfs_mutex);
1221 		}
1222 		finish_wait(waitq, &wait);
1223 		WARN_ON_ONCE(!RB_EMPTY_NODE(&kn->rb));
1224 		ret = false;
1225 	}
1226 
1227 	/*
1228 	 * This must be done while holding kernfs_mutex; otherwise, waiting
1229 	 * for SUICIDED && deactivated could finish prematurely.
1230 	 */
1231 	kernfs_unbreak_active_protection(kn);
1232 
1233 	mutex_unlock(&kernfs_mutex);
1234 	return ret;
1235 }
1236 
1237 /**
1238  * kernfs_remove_by_name_ns - find a kernfs_node by name and remove it
1239  * @parent: parent of the target
1240  * @name: name of the kernfs_node to remove
1241  * @ns: namespace tag of the kernfs_node to remove
1242  *
1243  * Look for the kernfs_node with @name and @ns under @parent and remove it.
1244  * Returns 0 on success, -ENOENT if such entry doesn't exist.
1245  */
1246 int kernfs_remove_by_name_ns(struct kernfs_node *parent, const char *name,
1247 			     const void *ns)
1248 {
1249 	struct kernfs_node *kn;
1250 
1251 	if (!parent) {
1252 		WARN(1, KERN_WARNING "kernfs: can not remove '%s', no directory\n",
1253 			name);
1254 		return -ENOENT;
1255 	}
1256 
1257 	mutex_lock(&kernfs_mutex);
1258 
1259 	kn = kernfs_find_ns(parent, name, ns);
1260 	if (kn)
1261 		__kernfs_remove(kn);
1262 
1263 	mutex_unlock(&kernfs_mutex);
1264 
1265 	if (kn)
1266 		return 0;
1267 	else
1268 		return -ENOENT;
1269 }
1270 
1271 /**
1272  * kernfs_rename_ns - move and rename a kernfs_node
1273  * @kn: target node
1274  * @new_parent: new parent to put @sd under
1275  * @new_name: new name
1276  * @new_ns: new namespace tag
1277  */
1278 int kernfs_rename_ns(struct kernfs_node *kn, struct kernfs_node *new_parent,
1279 		     const char *new_name, const void *new_ns)
1280 {
1281 	struct kernfs_node *old_parent;
1282 	const char *old_name = NULL;
1283 	int error;
1284 
1285 	/* can't move or rename root */
1286 	if (!kn->parent)
1287 		return -EINVAL;
1288 
1289 	mutex_lock(&kernfs_mutex);
1290 
1291 	error = -ENOENT;
1292 	if (!kernfs_active(kn) || !kernfs_active(new_parent) ||
1293 	    (new_parent->flags & KERNFS_EMPTY_DIR))
1294 		goto out;
1295 
1296 	error = 0;
1297 	if ((kn->parent == new_parent) && (kn->ns == new_ns) &&
1298 	    (strcmp(kn->name, new_name) == 0))
1299 		goto out;	/* nothing to rename */
1300 
1301 	error = -EEXIST;
1302 	if (kernfs_find_ns(new_parent, new_name, new_ns))
1303 		goto out;
1304 
1305 	/* rename kernfs_node */
1306 	if (strcmp(kn->name, new_name) != 0) {
1307 		error = -ENOMEM;
1308 		new_name = kstrdup_const(new_name, GFP_KERNEL);
1309 		if (!new_name)
1310 			goto out;
1311 	} else {
1312 		new_name = NULL;
1313 	}
1314 
1315 	/*
1316 	 * Move to the appropriate place in the appropriate directories rbtree.
1317 	 */
1318 	kernfs_unlink_sibling(kn);
1319 	kernfs_get(new_parent);
1320 
1321 	/* rename_lock protects ->parent and ->name accessors */
1322 	spin_lock_irq(&kernfs_rename_lock);
1323 
1324 	old_parent = kn->parent;
1325 	kn->parent = new_parent;
1326 
1327 	kn->ns = new_ns;
1328 	if (new_name) {
1329 		old_name = kn->name;
1330 		kn->name = new_name;
1331 	}
1332 
1333 	spin_unlock_irq(&kernfs_rename_lock);
1334 
1335 	kn->hash = kernfs_name_hash(kn->name, kn->ns);
1336 	kernfs_link_sibling(kn);
1337 
1338 	kernfs_put(old_parent);
1339 	kfree_const(old_name);
1340 
1341 	error = 0;
1342  out:
1343 	mutex_unlock(&kernfs_mutex);
1344 	return error;
1345 }
1346 
1347 /* Relationship between s_mode and the DT_xxx types */
1348 static inline unsigned char dt_type(struct kernfs_node *kn)
1349 {
1350 	return (kn->mode >> 12) & 15;
1351 }
1352 
1353 static int kernfs_dir_fop_release(struct inode *inode, struct file *filp)
1354 {
1355 	kernfs_put(filp->private_data);
1356 	return 0;
1357 }
1358 
1359 static struct kernfs_node *kernfs_dir_pos(const void *ns,
1360 	struct kernfs_node *parent, loff_t hash, struct kernfs_node *pos)
1361 {
1362 	if (pos) {
1363 		int valid = kernfs_active(pos) &&
1364 			pos->parent == parent && hash == pos->hash;
1365 		kernfs_put(pos);
1366 		if (!valid)
1367 			pos = NULL;
1368 	}
1369 	if (!pos && (hash > 1) && (hash < INT_MAX)) {
1370 		struct rb_node *node = parent->dir.children.rb_node;
1371 		while (node) {
1372 			pos = rb_to_kn(node);
1373 
1374 			if (hash < pos->hash)
1375 				node = node->rb_left;
1376 			else if (hash > pos->hash)
1377 				node = node->rb_right;
1378 			else
1379 				break;
1380 		}
1381 	}
1382 	/* Skip over entries which are dying/dead or in the wrong namespace */
1383 	while (pos && (!kernfs_active(pos) || pos->ns != ns)) {
1384 		struct rb_node *node = rb_next(&pos->rb);
1385 		if (!node)
1386 			pos = NULL;
1387 		else
1388 			pos = rb_to_kn(node);
1389 	}
1390 	return pos;
1391 }
1392 
1393 static struct kernfs_node *kernfs_dir_next_pos(const void *ns,
1394 	struct kernfs_node *parent, ino_t ino, struct kernfs_node *pos)
1395 {
1396 	pos = kernfs_dir_pos(ns, parent, ino, pos);
1397 	if (pos) {
1398 		do {
1399 			struct rb_node *node = rb_next(&pos->rb);
1400 			if (!node)
1401 				pos = NULL;
1402 			else
1403 				pos = rb_to_kn(node);
1404 		} while (pos && (!kernfs_active(pos) || pos->ns != ns));
1405 	}
1406 	return pos;
1407 }
1408 
1409 static int kernfs_fop_readdir(struct file *file, struct dir_context *ctx)
1410 {
1411 	struct dentry *dentry = file->f_path.dentry;
1412 	struct kernfs_node *parent = dentry->d_fsdata;
1413 	struct kernfs_node *pos = file->private_data;
1414 	const void *ns = NULL;
1415 
1416 	if (!dir_emit_dots(file, ctx))
1417 		return 0;
1418 	mutex_lock(&kernfs_mutex);
1419 
1420 	if (kernfs_ns_enabled(parent))
1421 		ns = kernfs_info(dentry->d_sb)->ns;
1422 
1423 	for (pos = kernfs_dir_pos(ns, parent, ctx->pos, pos);
1424 	     pos;
1425 	     pos = kernfs_dir_next_pos(ns, parent, ctx->pos, pos)) {
1426 		const char *name = pos->name;
1427 		unsigned int type = dt_type(pos);
1428 		int len = strlen(name);
1429 		ino_t ino = pos->ino;
1430 
1431 		ctx->pos = pos->hash;
1432 		file->private_data = pos;
1433 		kernfs_get(pos);
1434 
1435 		mutex_unlock(&kernfs_mutex);
1436 		if (!dir_emit(ctx, name, len, ino, type))
1437 			return 0;
1438 		mutex_lock(&kernfs_mutex);
1439 	}
1440 	mutex_unlock(&kernfs_mutex);
1441 	file->private_data = NULL;
1442 	ctx->pos = INT_MAX;
1443 	return 0;
1444 }
1445 
1446 static loff_t kernfs_dir_fop_llseek(struct file *file, loff_t offset,
1447 				    int whence)
1448 {
1449 	struct inode *inode = file_inode(file);
1450 	loff_t ret;
1451 
1452 	mutex_lock(&inode->i_mutex);
1453 	ret = generic_file_llseek(file, offset, whence);
1454 	mutex_unlock(&inode->i_mutex);
1455 
1456 	return ret;
1457 }
1458 
1459 const struct file_operations kernfs_dir_fops = {
1460 	.read		= generic_read_dir,
1461 	.iterate	= kernfs_fop_readdir,
1462 	.release	= kernfs_dir_fop_release,
1463 	.llseek		= kernfs_dir_fop_llseek,
1464 };
1465