1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * fs/dcache.c
41da177e4SLinus Torvalds *
51da177e4SLinus Torvalds * Complete reimplementation
61da177e4SLinus Torvalds * (C) 1997 Thomas Schoebel-Theuer,
71da177e4SLinus Torvalds * with heavy changes by Linus Torvalds
81da177e4SLinus Torvalds */
91da177e4SLinus Torvalds
101da177e4SLinus Torvalds /*
111da177e4SLinus Torvalds * Notes on the allocation strategy:
121da177e4SLinus Torvalds *
131da177e4SLinus Torvalds * The dcache is a master of the icache - whenever a dcache entry
141da177e4SLinus Torvalds * exists, the inode will always exist. "iput()" is done either when
151da177e4SLinus Torvalds * the dcache entry is deleted or garbage collected.
161da177e4SLinus Torvalds */
171da177e4SLinus Torvalds
187a5cf791SAl Viro #include <linux/ratelimit.h>
191da177e4SLinus Torvalds #include <linux/string.h>
201da177e4SLinus Torvalds #include <linux/mm.h>
211da177e4SLinus Torvalds #include <linux/fs.h>
220bf3d5c1SEric Biggers #include <linux/fscrypt.h>
237a91bf7fSJohn McCutchan #include <linux/fsnotify.h>
241da177e4SLinus Torvalds #include <linux/slab.h>
251da177e4SLinus Torvalds #include <linux/init.h>
261da177e4SLinus Torvalds #include <linux/hash.h>
271da177e4SLinus Torvalds #include <linux/cache.h>
28630d9c47SPaul Gortmaker #include <linux/export.h>
291da177e4SLinus Torvalds #include <linux/security.h>
301da177e4SLinus Torvalds #include <linux/seqlock.h>
3157c8a661SMike Rapoport #include <linux/memblock.h>
32ceb5bdc2SNick Piggin #include <linux/bit_spinlock.h>
33ceb5bdc2SNick Piggin #include <linux/rculist_bl.h>
34f6041567SDave Chinner #include <linux/list_lru.h>
3507f3f05cSDavid Howells #include "internal.h"
36b2dba1afSAl Viro #include "mount.h"
371da177e4SLinus Torvalds
38789680d1SNick Piggin /*
39789680d1SNick Piggin * Usage:
40873feea0SNick Piggin * dcache->d_inode->i_lock protects:
41946e51f2SAl Viro * - i_dentry, d_u.d_alias, d_inode of aliases
42ceb5bdc2SNick Piggin * dcache_hash_bucket lock protects:
43ceb5bdc2SNick Piggin * - the dcache hash table
44f1ee6162SNeilBrown * s_roots bl list spinlock protects:
45f1ee6162SNeilBrown * - the s_roots list (see __d_drop)
4619156840SDave Chinner * dentry->d_sb->s_dentry_lru_lock protects:
4723044507SNick Piggin * - the dcache lru lists and counters
4823044507SNick Piggin * d_lock protects:
4923044507SNick Piggin * - d_flags
5023044507SNick Piggin * - d_name
5123044507SNick Piggin * - d_lru
52b7ab39f6SNick Piggin * - d_count
53da502956SNick Piggin * - d_unhashed()
542fd6b7f5SNick Piggin * - d_parent and d_subdirs
552fd6b7f5SNick Piggin * - childrens' d_child and d_parent
56946e51f2SAl Viro * - d_u.d_alias, d_inode
57789680d1SNick Piggin *
58789680d1SNick Piggin * Ordering:
59873feea0SNick Piggin * dentry->d_inode->i_lock
60789680d1SNick Piggin * dentry->d_lock
6119156840SDave Chinner * dentry->d_sb->s_dentry_lru_lock
62ceb5bdc2SNick Piggin * dcache_hash_bucket lock
63f1ee6162SNeilBrown * s_roots lock
64789680d1SNick Piggin *
65da502956SNick Piggin * If there is an ancestor relationship:
66da502956SNick Piggin * dentry->d_parent->...->d_parent->d_lock
67da502956SNick Piggin * ...
68da502956SNick Piggin * dentry->d_parent->d_lock
69da502956SNick Piggin * dentry->d_lock
70da502956SNick Piggin *
71da502956SNick Piggin * If no ancestor relationship:
72076515fcSAl Viro * arbitrary, since it's serialized on rename_lock
73789680d1SNick Piggin */
74fa3536ccSEric Dumazet int sysctl_vfs_cache_pressure __read_mostly = 100;
751da177e4SLinus Torvalds EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure);
761da177e4SLinus Torvalds
7774c3cbe3SAl Viro __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock);
781da177e4SLinus Torvalds
79949854d0SNick Piggin EXPORT_SYMBOL(rename_lock);
801da177e4SLinus Torvalds
81e18b890bSChristoph Lameter static struct kmem_cache *dentry_cache __read_mostly;
821da177e4SLinus Torvalds
83cdf01226SDavid Howells const struct qstr empty_name = QSTR_INIT("", 0);
84cdf01226SDavid Howells EXPORT_SYMBOL(empty_name);
85cdf01226SDavid Howells const struct qstr slash_name = QSTR_INIT("/", 1);
86cdf01226SDavid Howells EXPORT_SYMBOL(slash_name);
8780e5d1ffSAl Viro const struct qstr dotdot_name = QSTR_INIT("..", 2);
8880e5d1ffSAl Viro EXPORT_SYMBOL(dotdot_name);
89cdf01226SDavid Howells
901da177e4SLinus Torvalds /*
911da177e4SLinus Torvalds * This is the single most critical data structure when it comes
921da177e4SLinus Torvalds * to the dcache: the hashtable for lookups. Somebody should try
931da177e4SLinus Torvalds * to make this good - I've just made it work.
941da177e4SLinus Torvalds *
951da177e4SLinus Torvalds * This hash-function tries to avoid losing too many bits of hash
961da177e4SLinus Torvalds * information, yet avoid using a prime hash-size or similar.
971da177e4SLinus Torvalds */
981da177e4SLinus Torvalds
99fa3536ccSEric Dumazet static unsigned int d_hash_shift __read_mostly;
100ceb5bdc2SNick Piggin
101b07ad996SLinus Torvalds static struct hlist_bl_head *dentry_hashtable __read_mostly;
102ceb5bdc2SNick Piggin
d_hash(unsigned int hash)1038387ff25SLinus Torvalds static inline struct hlist_bl_head *d_hash(unsigned int hash)
104ceb5bdc2SNick Piggin {
105854d3e63SAlexey Dobriyan return dentry_hashtable + (hash >> d_hash_shift);
106ceb5bdc2SNick Piggin }
107ceb5bdc2SNick Piggin
10894bdd655SAl Viro #define IN_LOOKUP_SHIFT 10
10994bdd655SAl Viro static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT];
11094bdd655SAl Viro
in_lookup_hash(const struct dentry * parent,unsigned int hash)11194bdd655SAl Viro static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent,
11294bdd655SAl Viro unsigned int hash)
11394bdd655SAl Viro {
11494bdd655SAl Viro hash += (unsigned long) parent / L1_CACHE_BYTES;
11594bdd655SAl Viro return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT);
11694bdd655SAl Viro }
11794bdd655SAl Viro
118c8c0c239SLuis Chamberlain struct dentry_stat_t {
119c8c0c239SLuis Chamberlain long nr_dentry;
120c8c0c239SLuis Chamberlain long nr_unused;
121c8c0c239SLuis Chamberlain long age_limit; /* age in seconds */
122c8c0c239SLuis Chamberlain long want_pages; /* pages requested by system */
123c8c0c239SLuis Chamberlain long nr_negative; /* # of unused negative dentries */
124c8c0c239SLuis Chamberlain long dummy; /* Reserved for future use */
1251da177e4SLinus Torvalds };
1261da177e4SLinus Torvalds
1273942c07cSGlauber Costa static DEFINE_PER_CPU(long, nr_dentry);
12862d36c77SDave Chinner static DEFINE_PER_CPU(long, nr_dentry_unused);
129af0c9af1SWaiman Long static DEFINE_PER_CPU(long, nr_dentry_negative);
130312d3ca8SChristoph Hellwig
131312d3ca8SChristoph Hellwig #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS)
132c8c0c239SLuis Chamberlain /* Statistics gathering. */
133c8c0c239SLuis Chamberlain static struct dentry_stat_t dentry_stat = {
134c8c0c239SLuis Chamberlain .age_limit = 45,
135c8c0c239SLuis Chamberlain };
13662d36c77SDave Chinner
13762d36c77SDave Chinner /*
13862d36c77SDave Chinner * Here we resort to our own counters instead of using generic per-cpu counters
13962d36c77SDave Chinner * for consistency with what the vfs inode code does. We are expected to harvest
14062d36c77SDave Chinner * better code and performance by having our own specialized counters.
14162d36c77SDave Chinner *
14262d36c77SDave Chinner * Please note that the loop is done over all possible CPUs, not over all online
14362d36c77SDave Chinner * CPUs. The reason for this is that we don't want to play games with CPUs going
14462d36c77SDave Chinner * on and off. If one of them goes off, we will just keep their counters.
14562d36c77SDave Chinner *
14662d36c77SDave Chinner * glommer: See cffbc8a for details, and if you ever intend to change this,
14762d36c77SDave Chinner * please update all vfs counters to match.
14862d36c77SDave Chinner */
get_nr_dentry(void)1493942c07cSGlauber Costa static long get_nr_dentry(void)
1503e880fb5SNick Piggin {
1513e880fb5SNick Piggin int i;
1523942c07cSGlauber Costa long sum = 0;
1533e880fb5SNick Piggin for_each_possible_cpu(i)
1543e880fb5SNick Piggin sum += per_cpu(nr_dentry, i);
1553e880fb5SNick Piggin return sum < 0 ? 0 : sum;
1563e880fb5SNick Piggin }
1573e880fb5SNick Piggin
get_nr_dentry_unused(void)15862d36c77SDave Chinner static long get_nr_dentry_unused(void)
15962d36c77SDave Chinner {
16062d36c77SDave Chinner int i;
16162d36c77SDave Chinner long sum = 0;
16262d36c77SDave Chinner for_each_possible_cpu(i)
16362d36c77SDave Chinner sum += per_cpu(nr_dentry_unused, i);
16462d36c77SDave Chinner return sum < 0 ? 0 : sum;
16562d36c77SDave Chinner }
16662d36c77SDave Chinner
get_nr_dentry_negative(void)167af0c9af1SWaiman Long static long get_nr_dentry_negative(void)
168af0c9af1SWaiman Long {
169af0c9af1SWaiman Long int i;
170af0c9af1SWaiman Long long sum = 0;
171af0c9af1SWaiman Long
172af0c9af1SWaiman Long for_each_possible_cpu(i)
173af0c9af1SWaiman Long sum += per_cpu(nr_dentry_negative, i);
174af0c9af1SWaiman Long return sum < 0 ? 0 : sum;
175af0c9af1SWaiman Long }
176af0c9af1SWaiman Long
proc_nr_dentry(struct ctl_table * table,int write,void * buffer,size_t * lenp,loff_t * ppos)177c8c0c239SLuis Chamberlain static int proc_nr_dentry(struct ctl_table *table, int write, void *buffer,
178312d3ca8SChristoph Hellwig size_t *lenp, loff_t *ppos)
179312d3ca8SChristoph Hellwig {
1803e880fb5SNick Piggin dentry_stat.nr_dentry = get_nr_dentry();
18162d36c77SDave Chinner dentry_stat.nr_unused = get_nr_dentry_unused();
182af0c9af1SWaiman Long dentry_stat.nr_negative = get_nr_dentry_negative();
1833942c07cSGlauber Costa return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);
184312d3ca8SChristoph Hellwig }
185c8c0c239SLuis Chamberlain
186c8c0c239SLuis Chamberlain static struct ctl_table fs_dcache_sysctls[] = {
187c8c0c239SLuis Chamberlain {
188c8c0c239SLuis Chamberlain .procname = "dentry-state",
189c8c0c239SLuis Chamberlain .data = &dentry_stat,
190c8c0c239SLuis Chamberlain .maxlen = 6*sizeof(long),
191c8c0c239SLuis Chamberlain .mode = 0444,
192c8c0c239SLuis Chamberlain .proc_handler = proc_nr_dentry,
193c8c0c239SLuis Chamberlain },
194c8c0c239SLuis Chamberlain { }
195c8c0c239SLuis Chamberlain };
196c8c0c239SLuis Chamberlain
init_fs_dcache_sysctls(void)197c8c0c239SLuis Chamberlain static int __init init_fs_dcache_sysctls(void)
198c8c0c239SLuis Chamberlain {
199c8c0c239SLuis Chamberlain register_sysctl_init("fs", fs_dcache_sysctls);
200c8c0c239SLuis Chamberlain return 0;
201c8c0c239SLuis Chamberlain }
202c8c0c239SLuis Chamberlain fs_initcall(init_fs_dcache_sysctls);
203312d3ca8SChristoph Hellwig #endif
204312d3ca8SChristoph Hellwig
2055483f18eSLinus Torvalds /*
2065483f18eSLinus Torvalds * Compare 2 name strings, return 0 if they match, otherwise non-zero.
2075483f18eSLinus Torvalds * The strings are both count bytes long, and count is non-zero.
2085483f18eSLinus Torvalds */
209e419b4ccSLinus Torvalds #ifdef CONFIG_DCACHE_WORD_ACCESS
210e419b4ccSLinus Torvalds
211e419b4ccSLinus Torvalds #include <asm/word-at-a-time.h>
212e419b4ccSLinus Torvalds /*
213e419b4ccSLinus Torvalds * NOTE! 'cs' and 'scount' come from a dentry, so it has a
214e419b4ccSLinus Torvalds * aligned allocation for this particular component. We don't
215e419b4ccSLinus Torvalds * strictly need the load_unaligned_zeropad() safety, but it
216e419b4ccSLinus Torvalds * doesn't hurt either.
217e419b4ccSLinus Torvalds *
218e419b4ccSLinus Torvalds * In contrast, 'ct' and 'tcount' can be from a pathname, and do
219e419b4ccSLinus Torvalds * need the careful unaligned handling.
220e419b4ccSLinus Torvalds */
dentry_string_cmp(const unsigned char * cs,const unsigned char * ct,unsigned tcount)22194753db5SLinus Torvalds static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
2225483f18eSLinus Torvalds {
223bfcfaa77SLinus Torvalds unsigned long a,b,mask;
224bfcfaa77SLinus Torvalds
225bfcfaa77SLinus Torvalds for (;;) {
226bfe7aa6cSAndrey Ryabinin a = read_word_at_a_time(cs);
227e419b4ccSLinus Torvalds b = load_unaligned_zeropad(ct);
228bfcfaa77SLinus Torvalds if (tcount < sizeof(unsigned long))
229bfcfaa77SLinus Torvalds break;
230bfcfaa77SLinus Torvalds if (unlikely(a != b))
231bfcfaa77SLinus Torvalds return 1;
232bfcfaa77SLinus Torvalds cs += sizeof(unsigned long);
233bfcfaa77SLinus Torvalds ct += sizeof(unsigned long);
234bfcfaa77SLinus Torvalds tcount -= sizeof(unsigned long);
235bfcfaa77SLinus Torvalds if (!tcount)
236bfcfaa77SLinus Torvalds return 0;
237bfcfaa77SLinus Torvalds }
238a5c21dceSWill Deacon mask = bytemask_from_count(tcount);
239bfcfaa77SLinus Torvalds return unlikely(!!((a ^ b) & mask));
240e419b4ccSLinus Torvalds }
241e419b4ccSLinus Torvalds
242bfcfaa77SLinus Torvalds #else
243e419b4ccSLinus Torvalds
dentry_string_cmp(const unsigned char * cs,const unsigned char * ct,unsigned tcount)24494753db5SLinus Torvalds static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount)
245e419b4ccSLinus Torvalds {
2465483f18eSLinus Torvalds do {
2475483f18eSLinus Torvalds if (*cs != *ct)
2485483f18eSLinus Torvalds return 1;
2495483f18eSLinus Torvalds cs++;
2505483f18eSLinus Torvalds ct++;
2515483f18eSLinus Torvalds tcount--;
2525483f18eSLinus Torvalds } while (tcount);
2535483f18eSLinus Torvalds return 0;
2545483f18eSLinus Torvalds }
2555483f18eSLinus Torvalds
256e419b4ccSLinus Torvalds #endif
257e419b4ccSLinus Torvalds
dentry_cmp(const struct dentry * dentry,const unsigned char * ct,unsigned tcount)25894753db5SLinus Torvalds static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount)
25994753db5SLinus Torvalds {
26094753db5SLinus Torvalds /*
26194753db5SLinus Torvalds * Be careful about RCU walk racing with rename:
262506458efSWill Deacon * use 'READ_ONCE' to fetch the name pointer.
26394753db5SLinus Torvalds *
26494753db5SLinus Torvalds * NOTE! Even if a rename will mean that the length
26594753db5SLinus Torvalds * was not loaded atomically, we don't care. The
26694753db5SLinus Torvalds * RCU walk will check the sequence count eventually,
26794753db5SLinus Torvalds * and catch it. And we won't overrun the buffer,
26894753db5SLinus Torvalds * because we're reading the name pointer atomically,
26994753db5SLinus Torvalds * and a dentry name is guaranteed to be properly
27094753db5SLinus Torvalds * terminated with a NUL byte.
27194753db5SLinus Torvalds *
27294753db5SLinus Torvalds * End result: even if 'len' is wrong, we'll exit
27394753db5SLinus Torvalds * early because the data cannot match (there can
27494753db5SLinus Torvalds * be no NUL in the ct/tcount data)
27594753db5SLinus Torvalds */
276506458efSWill Deacon const unsigned char *cs = READ_ONCE(dentry->d_name.name);
277ae0a843cSHe Kuang
2786326c71fSLinus Torvalds return dentry_string_cmp(cs, ct, tcount);
27994753db5SLinus Torvalds }
28094753db5SLinus Torvalds
2818d85b484SAl Viro struct external_name {
2828d85b484SAl Viro union {
2838d85b484SAl Viro atomic_t count;
2848d85b484SAl Viro struct rcu_head head;
2858d85b484SAl Viro } u;
2868d85b484SAl Viro unsigned char name[];
2878d85b484SAl Viro };
2888d85b484SAl Viro
external_name(struct dentry * dentry)2898d85b484SAl Viro static inline struct external_name *external_name(struct dentry *dentry)
2908d85b484SAl Viro {
2918d85b484SAl Viro return container_of(dentry->d_name.name, struct external_name, name[0]);
2928d85b484SAl Viro }
2938d85b484SAl Viro
__d_free(struct rcu_head * head)2949c82ab9cSChristoph Hellwig static void __d_free(struct rcu_head *head)
2951da177e4SLinus Torvalds {
2969c82ab9cSChristoph Hellwig struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
2979c82ab9cSChristoph Hellwig
2988d85b484SAl Viro kmem_cache_free(dentry_cache, dentry);
2998d85b484SAl Viro }
3008d85b484SAl Viro
__d_free_external(struct rcu_head * head)3018d85b484SAl Viro static void __d_free_external(struct rcu_head *head)
3028d85b484SAl Viro {
3038d85b484SAl Viro struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu);
3042e03b4bcSVlastimil Babka kfree(external_name(dentry));
3051da177e4SLinus Torvalds kmem_cache_free(dentry_cache, dentry);
3061da177e4SLinus Torvalds }
3071da177e4SLinus Torvalds
dname_external(const struct dentry * dentry)308810bb172SAl Viro static inline int dname_external(const struct dentry *dentry)
309810bb172SAl Viro {
310810bb172SAl Viro return dentry->d_name.name != dentry->d_iname;
311810bb172SAl Viro }
312810bb172SAl Viro
take_dentry_name_snapshot(struct name_snapshot * name,struct dentry * dentry)31349d31c2fSAl Viro void take_dentry_name_snapshot(struct name_snapshot *name, struct dentry *dentry)
31449d31c2fSAl Viro {
31549d31c2fSAl Viro spin_lock(&dentry->d_lock);
316230c6402SAl Viro name->name = dentry->d_name;
31749d31c2fSAl Viro if (unlikely(dname_external(dentry))) {
318230c6402SAl Viro atomic_inc(&external_name(dentry)->u.count);
31949d31c2fSAl Viro } else {
3206cd00a01STetsuo Handa memcpy(name->inline_name, dentry->d_iname,
3216cd00a01STetsuo Handa dentry->d_name.len + 1);
322230c6402SAl Viro name->name.name = name->inline_name;
32349d31c2fSAl Viro }
324230c6402SAl Viro spin_unlock(&dentry->d_lock);
32549d31c2fSAl Viro }
32649d31c2fSAl Viro EXPORT_SYMBOL(take_dentry_name_snapshot);
32749d31c2fSAl Viro
release_dentry_name_snapshot(struct name_snapshot * name)32849d31c2fSAl Viro void release_dentry_name_snapshot(struct name_snapshot *name)
32949d31c2fSAl Viro {
330230c6402SAl Viro if (unlikely(name->name.name != name->inline_name)) {
33149d31c2fSAl Viro struct external_name *p;
332230c6402SAl Viro p = container_of(name->name.name, struct external_name, name[0]);
33349d31c2fSAl Viro if (unlikely(atomic_dec_and_test(&p->u.count)))
3342e03b4bcSVlastimil Babka kfree_rcu(p, u.head);
33549d31c2fSAl Viro }
33649d31c2fSAl Viro }
33749d31c2fSAl Viro EXPORT_SYMBOL(release_dentry_name_snapshot);
33849d31c2fSAl Viro
__d_set_inode_and_type(struct dentry * dentry,struct inode * inode,unsigned type_flags)3394bf46a27SDavid Howells static inline void __d_set_inode_and_type(struct dentry *dentry,
3404bf46a27SDavid Howells struct inode *inode,
3414bf46a27SDavid Howells unsigned type_flags)
3424bf46a27SDavid Howells {
3434bf46a27SDavid Howells unsigned flags;
3444bf46a27SDavid Howells
3454bf46a27SDavid Howells dentry->d_inode = inode;
3464bf46a27SDavid Howells flags = READ_ONCE(dentry->d_flags);
3474bf46a27SDavid Howells flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
3484bf46a27SDavid Howells flags |= type_flags;
3492fa6b1e0SAl Viro smp_store_release(&dentry->d_flags, flags);
3504bf46a27SDavid Howells }
3514bf46a27SDavid Howells
__d_clear_type_and_inode(struct dentry * dentry)3524bf46a27SDavid Howells static inline void __d_clear_type_and_inode(struct dentry *dentry)
3534bf46a27SDavid Howells {
3544bf46a27SDavid Howells unsigned flags = READ_ONCE(dentry->d_flags);
3554bf46a27SDavid Howells
3564bf46a27SDavid Howells flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU);
3574bf46a27SDavid Howells WRITE_ONCE(dentry->d_flags, flags);
3584bf46a27SDavid Howells dentry->d_inode = NULL;
359af0c9af1SWaiman Long if (dentry->d_flags & DCACHE_LRU_LIST)
360af0c9af1SWaiman Long this_cpu_inc(nr_dentry_negative);
3614bf46a27SDavid Howells }
3624bf46a27SDavid Howells
dentry_free(struct dentry * dentry)363b4f0354eSAl Viro static void dentry_free(struct dentry *dentry)
364b4f0354eSAl Viro {
365946e51f2SAl Viro WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias));
3668d85b484SAl Viro if (unlikely(dname_external(dentry))) {
3678d85b484SAl Viro struct external_name *p = external_name(dentry);
3688d85b484SAl Viro if (likely(atomic_dec_and_test(&p->u.count))) {
3698d85b484SAl Viro call_rcu(&dentry->d_u.d_rcu, __d_free_external);
3708d85b484SAl Viro return;
3718d85b484SAl Viro }
3728d85b484SAl Viro }
373b4f0354eSAl Viro /* if dentry was never visible to RCU, immediate free is OK */
3745467a68cSAl Viro if (dentry->d_flags & DCACHE_NORCU)
375b4f0354eSAl Viro __d_free(&dentry->d_u.d_rcu);
376b4f0354eSAl Viro else
377b4f0354eSAl Viro call_rcu(&dentry->d_u.d_rcu, __d_free);
378b4f0354eSAl Viro }
379b4f0354eSAl Viro
3801da177e4SLinus Torvalds /*
3811da177e4SLinus Torvalds * Release the dentry's inode, using the filesystem
382550dce01SAl Viro * d_iput() operation if defined.
38331e6b01fSNick Piggin */
dentry_unlink_inode(struct dentry * dentry)38431e6b01fSNick Piggin static void dentry_unlink_inode(struct dentry * dentry)
38531e6b01fSNick Piggin __releases(dentry->d_lock)
386873feea0SNick Piggin __releases(dentry->d_inode->i_lock)
38731e6b01fSNick Piggin {
38831e6b01fSNick Piggin struct inode *inode = dentry->d_inode;
389a528aca7SAl Viro
390a528aca7SAl Viro raw_write_seqcount_begin(&dentry->d_seq);
3914bf46a27SDavid Howells __d_clear_type_and_inode(dentry);
392946e51f2SAl Viro hlist_del_init(&dentry->d_u.d_alias);
393a528aca7SAl Viro raw_write_seqcount_end(&dentry->d_seq);
39431e6b01fSNick Piggin spin_unlock(&dentry->d_lock);
395873feea0SNick Piggin spin_unlock(&inode->i_lock);
39631e6b01fSNick Piggin if (!inode->i_nlink)
39731e6b01fSNick Piggin fsnotify_inoderemove(inode);
39831e6b01fSNick Piggin if (dentry->d_op && dentry->d_op->d_iput)
39931e6b01fSNick Piggin dentry->d_op->d_iput(dentry, inode);
40031e6b01fSNick Piggin else
40131e6b01fSNick Piggin iput(inode);
40231e6b01fSNick Piggin }
40331e6b01fSNick Piggin
40431e6b01fSNick Piggin /*
40589dc77bcSLinus Torvalds * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry
40689dc77bcSLinus Torvalds * is in use - which includes both the "real" per-superblock
40789dc77bcSLinus Torvalds * LRU list _and_ the DCACHE_SHRINK_LIST use.
40889dc77bcSLinus Torvalds *
40989dc77bcSLinus Torvalds * The DCACHE_SHRINK_LIST bit is set whenever the dentry is
41089dc77bcSLinus Torvalds * on the shrink list (ie not on the superblock LRU list).
41189dc77bcSLinus Torvalds *
41289dc77bcSLinus Torvalds * The per-cpu "nr_dentry_unused" counters are updated with
41389dc77bcSLinus Torvalds * the DCACHE_LRU_LIST bit.
41489dc77bcSLinus Torvalds *
415af0c9af1SWaiman Long * The per-cpu "nr_dentry_negative" counters are only updated
416af0c9af1SWaiman Long * when deleted from or added to the per-superblock LRU list, not
417af0c9af1SWaiman Long * from/to the shrink list. That is to avoid an unneeded dec/inc
418af0c9af1SWaiman Long * pair when moving from LRU to shrink list in select_collect().
419af0c9af1SWaiman Long *
42089dc77bcSLinus Torvalds * These helper functions make sure we always follow the
42189dc77bcSLinus Torvalds * rules. d_lock must be held by the caller.
42289dc77bcSLinus Torvalds */
42389dc77bcSLinus Torvalds #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x))
d_lru_add(struct dentry * dentry)42489dc77bcSLinus Torvalds static void d_lru_add(struct dentry *dentry)
42589dc77bcSLinus Torvalds {
42689dc77bcSLinus Torvalds D_FLAG_VERIFY(dentry, 0);
42789dc77bcSLinus Torvalds dentry->d_flags |= DCACHE_LRU_LIST;
42889dc77bcSLinus Torvalds this_cpu_inc(nr_dentry_unused);
429af0c9af1SWaiman Long if (d_is_negative(dentry))
430af0c9af1SWaiman Long this_cpu_inc(nr_dentry_negative);
43189dc77bcSLinus Torvalds WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
43289dc77bcSLinus Torvalds }
43389dc77bcSLinus Torvalds
d_lru_del(struct dentry * dentry)43489dc77bcSLinus Torvalds static void d_lru_del(struct dentry *dentry)
43589dc77bcSLinus Torvalds {
43689dc77bcSLinus Torvalds D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
43789dc77bcSLinus Torvalds dentry->d_flags &= ~DCACHE_LRU_LIST;
43889dc77bcSLinus Torvalds this_cpu_dec(nr_dentry_unused);
439af0c9af1SWaiman Long if (d_is_negative(dentry))
440af0c9af1SWaiman Long this_cpu_dec(nr_dentry_negative);
44189dc77bcSLinus Torvalds WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru));
44289dc77bcSLinus Torvalds }
44389dc77bcSLinus Torvalds
d_shrink_del(struct dentry * dentry)44489dc77bcSLinus Torvalds static void d_shrink_del(struct dentry *dentry)
44589dc77bcSLinus Torvalds {
44689dc77bcSLinus Torvalds D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
44789dc77bcSLinus Torvalds list_del_init(&dentry->d_lru);
44889dc77bcSLinus Torvalds dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST);
44989dc77bcSLinus Torvalds this_cpu_dec(nr_dentry_unused);
45089dc77bcSLinus Torvalds }
45189dc77bcSLinus Torvalds
d_shrink_add(struct dentry * dentry,struct list_head * list)45289dc77bcSLinus Torvalds static void d_shrink_add(struct dentry *dentry, struct list_head *list)
45389dc77bcSLinus Torvalds {
45489dc77bcSLinus Torvalds D_FLAG_VERIFY(dentry, 0);
45589dc77bcSLinus Torvalds list_add(&dentry->d_lru, list);
45689dc77bcSLinus Torvalds dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST;
45789dc77bcSLinus Torvalds this_cpu_inc(nr_dentry_unused);
45889dc77bcSLinus Torvalds }
45989dc77bcSLinus Torvalds
46089dc77bcSLinus Torvalds /*
46189dc77bcSLinus Torvalds * These can only be called under the global LRU lock, ie during the
46289dc77bcSLinus Torvalds * callback for freeing the LRU list. "isolate" removes it from the
46389dc77bcSLinus Torvalds * LRU lists entirely, while shrink_move moves it to the indicated
46489dc77bcSLinus Torvalds * private list.
46589dc77bcSLinus Torvalds */
d_lru_isolate(struct list_lru_one * lru,struct dentry * dentry)4663f97b163SVladimir Davydov static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry)
46789dc77bcSLinus Torvalds {
46889dc77bcSLinus Torvalds D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
46989dc77bcSLinus Torvalds dentry->d_flags &= ~DCACHE_LRU_LIST;
47089dc77bcSLinus Torvalds this_cpu_dec(nr_dentry_unused);
471af0c9af1SWaiman Long if (d_is_negative(dentry))
472af0c9af1SWaiman Long this_cpu_dec(nr_dentry_negative);
4733f97b163SVladimir Davydov list_lru_isolate(lru, &dentry->d_lru);
47489dc77bcSLinus Torvalds }
47589dc77bcSLinus Torvalds
d_lru_shrink_move(struct list_lru_one * lru,struct dentry * dentry,struct list_head * list)4763f97b163SVladimir Davydov static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry,
4773f97b163SVladimir Davydov struct list_head *list)
47889dc77bcSLinus Torvalds {
47989dc77bcSLinus Torvalds D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST);
48089dc77bcSLinus Torvalds dentry->d_flags |= DCACHE_SHRINK_LIST;
481af0c9af1SWaiman Long if (d_is_negative(dentry))
482af0c9af1SWaiman Long this_cpu_dec(nr_dentry_negative);
4833f97b163SVladimir Davydov list_lru_isolate_move(lru, &dentry->d_lru, list);
48489dc77bcSLinus Torvalds }
48589dc77bcSLinus Torvalds
___d_drop(struct dentry * dentry)48661647823SNeilBrown static void ___d_drop(struct dentry *dentry)
487789680d1SNick Piggin {
488b61625d2SAl Viro struct hlist_bl_head *b;
4897632e465SJ. Bruce Fields /*
4907632e465SJ. Bruce Fields * Hashed dentries are normally on the dentry hashtable,
4917632e465SJ. Bruce Fields * with the exception of those newly allocated by
492f1ee6162SNeilBrown * d_obtain_root, which are always IS_ROOT:
4937632e465SJ. Bruce Fields */
4947632e465SJ. Bruce Fields if (unlikely(IS_ROOT(dentry)))
495f1ee6162SNeilBrown b = &dentry->d_sb->s_roots;
496b61625d2SAl Viro else
4978387ff25SLinus Torvalds b = d_hash(dentry->d_name.hash);
498b61625d2SAl Viro
499b61625d2SAl Viro hlist_bl_lock(b);
500b61625d2SAl Viro __hlist_bl_del(&dentry->d_hash);
501b61625d2SAl Viro hlist_bl_unlock(b);
502789680d1SNick Piggin }
50361647823SNeilBrown
__d_drop(struct dentry * dentry)50461647823SNeilBrown void __d_drop(struct dentry *dentry)
50561647823SNeilBrown {
5060632a9acSAl Viro if (!d_unhashed(dentry)) {
50761647823SNeilBrown ___d_drop(dentry);
50861647823SNeilBrown dentry->d_hash.pprev = NULL;
5090632a9acSAl Viro write_seqcount_invalidate(&dentry->d_seq);
5100632a9acSAl Viro }
51161647823SNeilBrown }
512789680d1SNick Piggin EXPORT_SYMBOL(__d_drop);
513789680d1SNick Piggin
514961f3c89SMauro Carvalho Chehab /**
515961f3c89SMauro Carvalho Chehab * d_drop - drop a dentry
516961f3c89SMauro Carvalho Chehab * @dentry: dentry to drop
517961f3c89SMauro Carvalho Chehab *
518961f3c89SMauro Carvalho Chehab * d_drop() unhashes the entry from the parent dentry hashes, so that it won't
519961f3c89SMauro Carvalho Chehab * be found through a VFS lookup any more. Note that this is different from
520961f3c89SMauro Carvalho Chehab * deleting the dentry - d_delete will try to mark the dentry negative if
521961f3c89SMauro Carvalho Chehab * possible, giving a successful _negative_ lookup, while d_drop will
522961f3c89SMauro Carvalho Chehab * just make the cache lookup fail.
523961f3c89SMauro Carvalho Chehab *
524961f3c89SMauro Carvalho Chehab * d_drop() is used mainly for stuff that wants to invalidate a dentry for some
525961f3c89SMauro Carvalho Chehab * reason (NFS timeouts or autofs deletes).
526961f3c89SMauro Carvalho Chehab *
527961f3c89SMauro Carvalho Chehab * __d_drop requires dentry->d_lock
528961f3c89SMauro Carvalho Chehab *
529961f3c89SMauro Carvalho Chehab * ___d_drop doesn't mark dentry as "unhashed"
530961f3c89SMauro Carvalho Chehab * (dentry->d_hash.pprev will be LIST_POISON2, not NULL).
531961f3c89SMauro Carvalho Chehab */
d_drop(struct dentry * dentry)532789680d1SNick Piggin void d_drop(struct dentry *dentry)
533789680d1SNick Piggin {
534789680d1SNick Piggin spin_lock(&dentry->d_lock);
535789680d1SNick Piggin __d_drop(dentry);
536789680d1SNick Piggin spin_unlock(&dentry->d_lock);
537789680d1SNick Piggin }
538789680d1SNick Piggin EXPORT_SYMBOL(d_drop);
539789680d1SNick Piggin
dentry_unlist(struct dentry * dentry,struct dentry * parent)540ba65dc5eSAl Viro static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent)
541ba65dc5eSAl Viro {
542ba65dc5eSAl Viro struct dentry *next;
543ba65dc5eSAl Viro /*
544ba65dc5eSAl Viro * Inform d_walk() and shrink_dentry_list() that we are no longer
545ba65dc5eSAl Viro * attached to the dentry tree
546ba65dc5eSAl Viro */
547ba65dc5eSAl Viro dentry->d_flags |= DCACHE_DENTRY_KILLED;
548ba65dc5eSAl Viro if (unlikely(list_empty(&dentry->d_child)))
549ba65dc5eSAl Viro return;
550ba65dc5eSAl Viro __list_del_entry(&dentry->d_child);
551ba65dc5eSAl Viro /*
552ba65dc5eSAl Viro * Cursors can move around the list of children. While we'd been
553ba65dc5eSAl Viro * a normal list member, it didn't matter - ->d_child.next would've
554ba65dc5eSAl Viro * been updated. However, from now on it won't be and for the
555ba65dc5eSAl Viro * things like d_walk() it might end up with a nasty surprise.
556ba65dc5eSAl Viro * Normally d_walk() doesn't care about cursors moving around -
557ba65dc5eSAl Viro * ->d_lock on parent prevents that and since a cursor has no children
558ba65dc5eSAl Viro * of its own, we get through it without ever unlocking the parent.
559ba65dc5eSAl Viro * There is one exception, though - if we ascend from a child that
560ba65dc5eSAl Viro * gets killed as soon as we unlock it, the next sibling is found
561ba65dc5eSAl Viro * using the value left in its ->d_child.next. And if _that_
562ba65dc5eSAl Viro * pointed to a cursor, and cursor got moved (e.g. by lseek())
563ba65dc5eSAl Viro * before d_walk() regains parent->d_lock, we'll end up skipping
564ba65dc5eSAl Viro * everything the cursor had been moved past.
565ba65dc5eSAl Viro *
566ba65dc5eSAl Viro * Solution: make sure that the pointer left behind in ->d_child.next
567ba65dc5eSAl Viro * points to something that won't be moving around. I.e. skip the
568ba65dc5eSAl Viro * cursors.
569ba65dc5eSAl Viro */
570ba65dc5eSAl Viro while (dentry->d_child.next != &parent->d_subdirs) {
571ba65dc5eSAl Viro next = list_entry(dentry->d_child.next, struct dentry, d_child);
572ba65dc5eSAl Viro if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR)))
573ba65dc5eSAl Viro break;
574ba65dc5eSAl Viro dentry->d_child.next = next->d_child.next;
575ba65dc5eSAl Viro }
576ba65dc5eSAl Viro }
577ba65dc5eSAl Viro
__dentry_kill(struct dentry * dentry)578e55fd011SAl Viro static void __dentry_kill(struct dentry *dentry)
57977812a1eSNick Piggin {
58041edf278SAl Viro struct dentry *parent = NULL;
58141edf278SAl Viro bool can_free = true;
58241edf278SAl Viro if (!IS_ROOT(dentry))
58377812a1eSNick Piggin parent = dentry->d_parent;
58431e6b01fSNick Piggin
5850d98439eSLinus Torvalds /*
5860d98439eSLinus Torvalds * The dentry is now unrecoverably dead to the world.
5870d98439eSLinus Torvalds */
5880d98439eSLinus Torvalds lockref_mark_dead(&dentry->d_lockref);
5890d98439eSLinus Torvalds
590f0023bc6SSage Weil /*
591f0023bc6SSage Weil * inform the fs via d_prune that this dentry is about to be
592f0023bc6SSage Weil * unhashed and destroyed.
593f0023bc6SSage Weil */
59429266201SAl Viro if (dentry->d_flags & DCACHE_OP_PRUNE)
59561572bb1SYan, Zheng dentry->d_op->d_prune(dentry);
59661572bb1SYan, Zheng
59701b60351SAl Viro if (dentry->d_flags & DCACHE_LRU_LIST) {
59801b60351SAl Viro if (!(dentry->d_flags & DCACHE_SHRINK_LIST))
59901b60351SAl Viro d_lru_del(dentry);
60001b60351SAl Viro }
60177812a1eSNick Piggin /* if it was on the hash then remove it */
60277812a1eSNick Piggin __d_drop(dentry);
603ba65dc5eSAl Viro dentry_unlist(dentry, parent);
60403b3b889SAl Viro if (parent)
60503b3b889SAl Viro spin_unlock(&parent->d_lock);
606550dce01SAl Viro if (dentry->d_inode)
607550dce01SAl Viro dentry_unlink_inode(dentry);
608550dce01SAl Viro else
609550dce01SAl Viro spin_unlock(&dentry->d_lock);
61003b3b889SAl Viro this_cpu_dec(nr_dentry);
61103b3b889SAl Viro if (dentry->d_op && dentry->d_op->d_release)
61203b3b889SAl Viro dentry->d_op->d_release(dentry);
61303b3b889SAl Viro
61441edf278SAl Viro spin_lock(&dentry->d_lock);
61541edf278SAl Viro if (dentry->d_flags & DCACHE_SHRINK_LIST) {
61641edf278SAl Viro dentry->d_flags |= DCACHE_MAY_FREE;
61741edf278SAl Viro can_free = false;
61841edf278SAl Viro }
61941edf278SAl Viro spin_unlock(&dentry->d_lock);
62041edf278SAl Viro if (likely(can_free))
621b4f0354eSAl Viro dentry_free(dentry);
6229c5f1d30SAl Viro cond_resched();
623e55fd011SAl Viro }
624e55fd011SAl Viro
__lock_parent(struct dentry * dentry)6258b987a46SAl Viro static struct dentry *__lock_parent(struct dentry *dentry)
626046b961bSAl Viro {
6278b987a46SAl Viro struct dentry *parent;
628046b961bSAl Viro rcu_read_lock();
629c2338f2dSAl Viro spin_unlock(&dentry->d_lock);
630046b961bSAl Viro again:
63166702eb5SMark Rutland parent = READ_ONCE(dentry->d_parent);
632046b961bSAl Viro spin_lock(&parent->d_lock);
633046b961bSAl Viro /*
634046b961bSAl Viro * We can't blindly lock dentry until we are sure
635046b961bSAl Viro * that we won't violate the locking order.
636046b961bSAl Viro * Any changes of dentry->d_parent must have
637046b961bSAl Viro * been done with parent->d_lock held, so
638046b961bSAl Viro * spin_lock() above is enough of a barrier
639046b961bSAl Viro * for checking if it's still our child.
640046b961bSAl Viro */
641046b961bSAl Viro if (unlikely(parent != dentry->d_parent)) {
642046b961bSAl Viro spin_unlock(&parent->d_lock);
643046b961bSAl Viro goto again;
644046b961bSAl Viro }
6453b821409SAl Viro rcu_read_unlock();
64665d8eb5aSAl Viro if (parent != dentry)
64765d8eb5aSAl Viro spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
64865d8eb5aSAl Viro else
64965d8eb5aSAl Viro parent = NULL;
650046b961bSAl Viro return parent;
651046b961bSAl Viro }
652046b961bSAl Viro
lock_parent(struct dentry * dentry)6538b987a46SAl Viro static inline struct dentry *lock_parent(struct dentry *dentry)
6548b987a46SAl Viro {
6558b987a46SAl Viro struct dentry *parent = dentry->d_parent;
6568b987a46SAl Viro if (IS_ROOT(dentry))
6578b987a46SAl Viro return NULL;
6588b987a46SAl Viro if (likely(spin_trylock(&parent->d_lock)))
6598b987a46SAl Viro return parent;
6608b987a46SAl Viro return __lock_parent(dentry);
6618b987a46SAl Viro }
6628b987a46SAl Viro
retain_dentry(struct dentry * dentry)663a338579fSAl Viro static inline bool retain_dentry(struct dentry *dentry)
664a338579fSAl Viro {
665a338579fSAl Viro WARN_ON(d_in_lookup(dentry));
666a338579fSAl Viro
667a338579fSAl Viro /* Unreachable? Get rid of it */
668a338579fSAl Viro if (unlikely(d_unhashed(dentry)))
669a338579fSAl Viro return false;
670a338579fSAl Viro
671a338579fSAl Viro if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED))
672a338579fSAl Viro return false;
673a338579fSAl Viro
674a338579fSAl Viro if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) {
675a338579fSAl Viro if (dentry->d_op->d_delete(dentry))
676a338579fSAl Viro return false;
677a338579fSAl Viro }
6782c567af4SIra Weiny
6792c567af4SIra Weiny if (unlikely(dentry->d_flags & DCACHE_DONTCACHE))
6802c567af4SIra Weiny return false;
6812c567af4SIra Weiny
68262d9956cSAl Viro /* retain; LRU fodder */
68362d9956cSAl Viro dentry->d_lockref.count--;
68462d9956cSAl Viro if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST)))
68562d9956cSAl Viro d_lru_add(dentry);
68662d9956cSAl Viro else if (unlikely(!(dentry->d_flags & DCACHE_REFERENCED)))
68762d9956cSAl Viro dentry->d_flags |= DCACHE_REFERENCED;
688a338579fSAl Viro return true;
689a338579fSAl Viro }
690a338579fSAl Viro
d_mark_dontcache(struct inode * inode)6912c567af4SIra Weiny void d_mark_dontcache(struct inode *inode)
6922c567af4SIra Weiny {
6932c567af4SIra Weiny struct dentry *de;
6942c567af4SIra Weiny
6952c567af4SIra Weiny spin_lock(&inode->i_lock);
6962c567af4SIra Weiny hlist_for_each_entry(de, &inode->i_dentry, d_u.d_alias) {
6972c567af4SIra Weiny spin_lock(&de->d_lock);
6982c567af4SIra Weiny de->d_flags |= DCACHE_DONTCACHE;
6992c567af4SIra Weiny spin_unlock(&de->d_lock);
7002c567af4SIra Weiny }
7012c567af4SIra Weiny inode->i_state |= I_DONTCACHE;
7022c567af4SIra Weiny spin_unlock(&inode->i_lock);
7032c567af4SIra Weiny }
7042c567af4SIra Weiny EXPORT_SYMBOL(d_mark_dontcache);
7052c567af4SIra Weiny
70677812a1eSNick Piggin /*
707c1d0c1a2SJohn Ogness * Finish off a dentry we've decided to kill.
708c1d0c1a2SJohn Ogness * dentry->d_lock must be held, returns with it unlocked.
709c1d0c1a2SJohn Ogness * Returns dentry requiring refcount drop, or NULL if we're done.
710c1d0c1a2SJohn Ogness */
dentry_kill(struct dentry * dentry)711c1d0c1a2SJohn Ogness static struct dentry *dentry_kill(struct dentry *dentry)
712c1d0c1a2SJohn Ogness __releases(dentry->d_lock)
713c1d0c1a2SJohn Ogness {
714c1d0c1a2SJohn Ogness struct inode *inode = dentry->d_inode;
715c1d0c1a2SJohn Ogness struct dentry *parent = NULL;
716c1d0c1a2SJohn Ogness
717c1d0c1a2SJohn Ogness if (inode && unlikely(!spin_trylock(&inode->i_lock)))
718f657a666SAl Viro goto slow_positive;
719c1d0c1a2SJohn Ogness
720c1d0c1a2SJohn Ogness if (!IS_ROOT(dentry)) {
721c1d0c1a2SJohn Ogness parent = dentry->d_parent;
722c1d0c1a2SJohn Ogness if (unlikely(!spin_trylock(&parent->d_lock))) {
723f657a666SAl Viro parent = __lock_parent(dentry);
724f657a666SAl Viro if (likely(inode || !dentry->d_inode))
725f657a666SAl Viro goto got_locks;
726f657a666SAl Viro /* negative that became positive */
727f657a666SAl Viro if (parent)
728f657a666SAl Viro spin_unlock(&parent->d_lock);
729f657a666SAl Viro inode = dentry->d_inode;
730f657a666SAl Viro goto slow_positive;
731c1d0c1a2SJohn Ogness }
732c1d0c1a2SJohn Ogness }
733c1d0c1a2SJohn Ogness __dentry_kill(dentry);
734c1d0c1a2SJohn Ogness return parent;
735c1d0c1a2SJohn Ogness
736f657a666SAl Viro slow_positive:
737c1d0c1a2SJohn Ogness spin_unlock(&dentry->d_lock);
738f657a666SAl Viro spin_lock(&inode->i_lock);
739f657a666SAl Viro spin_lock(&dentry->d_lock);
740f657a666SAl Viro parent = lock_parent(dentry);
741f657a666SAl Viro got_locks:
742f657a666SAl Viro if (unlikely(dentry->d_lockref.count != 1)) {
743f657a666SAl Viro dentry->d_lockref.count--;
744f657a666SAl Viro } else if (likely(!retain_dentry(dentry))) {
745f657a666SAl Viro __dentry_kill(dentry);
746f657a666SAl Viro return parent;
747f657a666SAl Viro }
748f657a666SAl Viro /* we are keeping it, after all */
749f657a666SAl Viro if (inode)
750f657a666SAl Viro spin_unlock(&inode->i_lock);
751f657a666SAl Viro if (parent)
752f657a666SAl Viro spin_unlock(&parent->d_lock);
753f657a666SAl Viro spin_unlock(&dentry->d_lock);
754f657a666SAl Viro return NULL;
755c1d0c1a2SJohn Ogness }
756c1d0c1a2SJohn Ogness
757c1d0c1a2SJohn Ogness /*
758360f5479SLinus Torvalds * Try to do a lockless dput(), and return whether that was successful.
759360f5479SLinus Torvalds *
760360f5479SLinus Torvalds * If unsuccessful, we return false, having already taken the dentry lock.
761360f5479SLinus Torvalds *
762360f5479SLinus Torvalds * The caller needs to hold the RCU read lock, so that the dentry is
763360f5479SLinus Torvalds * guaranteed to stay around even if the refcount goes down to zero!
764360f5479SLinus Torvalds */
fast_dput(struct dentry * dentry)765360f5479SLinus Torvalds static inline bool fast_dput(struct dentry *dentry)
766360f5479SLinus Torvalds {
767360f5479SLinus Torvalds int ret;
768360f5479SLinus Torvalds unsigned int d_flags;
769360f5479SLinus Torvalds
770360f5479SLinus Torvalds /*
771360f5479SLinus Torvalds * If we have a d_op->d_delete() operation, we sould not
77275a6f82aSAl Viro * let the dentry count go to zero, so use "put_or_lock".
773360f5479SLinus Torvalds */
774360f5479SLinus Torvalds if (unlikely(dentry->d_flags & DCACHE_OP_DELETE))
775360f5479SLinus Torvalds return lockref_put_or_lock(&dentry->d_lockref);
776360f5479SLinus Torvalds
777360f5479SLinus Torvalds /*
778360f5479SLinus Torvalds * .. otherwise, we can try to just decrement the
779360f5479SLinus Torvalds * lockref optimistically.
780360f5479SLinus Torvalds */
781360f5479SLinus Torvalds ret = lockref_put_return(&dentry->d_lockref);
782360f5479SLinus Torvalds
783360f5479SLinus Torvalds /*
784360f5479SLinus Torvalds * If the lockref_put_return() failed due to the lock being held
785360f5479SLinus Torvalds * by somebody else, the fast path has failed. We will need to
786360f5479SLinus Torvalds * get the lock, and then check the count again.
787360f5479SLinus Torvalds */
788360f5479SLinus Torvalds if (unlikely(ret < 0)) {
789360f5479SLinus Torvalds spin_lock(&dentry->d_lock);
790*408f4c8eSAl Viro if (WARN_ON_ONCE(dentry->d_lockref.count <= 0)) {
791360f5479SLinus Torvalds spin_unlock(&dentry->d_lock);
7927964410fSGustavo A. R. Silva return true;
793360f5479SLinus Torvalds }
794*408f4c8eSAl Viro dentry->d_lockref.count--;
795*408f4c8eSAl Viro goto locked;
796360f5479SLinus Torvalds }
797360f5479SLinus Torvalds
798360f5479SLinus Torvalds /*
799360f5479SLinus Torvalds * If we weren't the last ref, we're done.
800360f5479SLinus Torvalds */
801360f5479SLinus Torvalds if (ret)
8027964410fSGustavo A. R. Silva return true;
803360f5479SLinus Torvalds
804360f5479SLinus Torvalds /*
805360f5479SLinus Torvalds * Careful, careful. The reference count went down
806360f5479SLinus Torvalds * to zero, but we don't hold the dentry lock, so
807360f5479SLinus Torvalds * somebody else could get it again, and do another
808360f5479SLinus Torvalds * dput(), and we need to not race with that.
809360f5479SLinus Torvalds *
810360f5479SLinus Torvalds * However, there is a very special and common case
811360f5479SLinus Torvalds * where we don't care, because there is nothing to
812360f5479SLinus Torvalds * do: the dentry is still hashed, it does not have
813360f5479SLinus Torvalds * a 'delete' op, and it's referenced and already on
814360f5479SLinus Torvalds * the LRU list.
815360f5479SLinus Torvalds *
816360f5479SLinus Torvalds * NOTE! Since we aren't locked, these values are
817360f5479SLinus Torvalds * not "stable". However, it is sufficient that at
818360f5479SLinus Torvalds * some point after we dropped the reference the
819360f5479SLinus Torvalds * dentry was hashed and the flags had the proper
820360f5479SLinus Torvalds * value. Other dentry users may have re-gotten
821360f5479SLinus Torvalds * a reference to the dentry and change that, but
822360f5479SLinus Torvalds * our work is done - we can leave the dentry
823360f5479SLinus Torvalds * around with a zero refcount.
82477573fa3SHao Li *
82577573fa3SHao Li * Nevertheless, there are two cases that we should kill
82677573fa3SHao Li * the dentry anyway.
82777573fa3SHao Li * 1. free disconnected dentries as soon as their refcount
82877573fa3SHao Li * reached zero.
82977573fa3SHao Li * 2. free dentries if they should not be cached.
830360f5479SLinus Torvalds */
831360f5479SLinus Torvalds smp_rmb();
83266702eb5SMark Rutland d_flags = READ_ONCE(dentry->d_flags);
83377573fa3SHao Li d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST |
83477573fa3SHao Li DCACHE_DISCONNECTED | DCACHE_DONTCACHE;
835360f5479SLinus Torvalds
836360f5479SLinus Torvalds /* Nothing to do? Dropping the reference was all we needed? */
837360f5479SLinus Torvalds if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry))
8387964410fSGustavo A. R. Silva return true;
839360f5479SLinus Torvalds
840360f5479SLinus Torvalds /*
841360f5479SLinus Torvalds * Not the fast normal case? Get the lock. We've already decremented
842360f5479SLinus Torvalds * the refcount, but we'll need to re-check the situation after
843360f5479SLinus Torvalds * getting the lock.
844360f5479SLinus Torvalds */
845360f5479SLinus Torvalds spin_lock(&dentry->d_lock);
846360f5479SLinus Torvalds
847360f5479SLinus Torvalds /*
848360f5479SLinus Torvalds * Did somebody else grab a reference to it in the meantime, and
849360f5479SLinus Torvalds * we're no longer the last user after all? Alternatively, somebody
850360f5479SLinus Torvalds * else could have killed it and marked it dead. Either way, we
851360f5479SLinus Torvalds * don't need to do anything else.
852360f5479SLinus Torvalds */
853*408f4c8eSAl Viro locked:
854360f5479SLinus Torvalds if (dentry->d_lockref.count) {
855360f5479SLinus Torvalds spin_unlock(&dentry->d_lock);
8567964410fSGustavo A. R. Silva return true;
857360f5479SLinus Torvalds }
858360f5479SLinus Torvalds
859360f5479SLinus Torvalds /*
860360f5479SLinus Torvalds * Re-get the reference we optimistically dropped. We hold the
861360f5479SLinus Torvalds * lock, and we just tested that it was zero, so we can just
862360f5479SLinus Torvalds * set it to 1.
863360f5479SLinus Torvalds */
864360f5479SLinus Torvalds dentry->d_lockref.count = 1;
8657964410fSGustavo A. R. Silva return false;
866360f5479SLinus Torvalds }
867360f5479SLinus Torvalds
868360f5479SLinus Torvalds
869360f5479SLinus Torvalds /*
8701da177e4SLinus Torvalds * This is dput
8711da177e4SLinus Torvalds *
8721da177e4SLinus Torvalds * This is complicated by the fact that we do not want to put
8731da177e4SLinus Torvalds * dentries that are no longer on any hash chain on the unused
8741da177e4SLinus Torvalds * list: we'd much rather just get rid of them immediately.
8751da177e4SLinus Torvalds *
8761da177e4SLinus Torvalds * However, that implies that we have to traverse the dentry
8771da177e4SLinus Torvalds * tree upwards to the parents which might _also_ now be
8781da177e4SLinus Torvalds * scheduled for deletion (it may have been only waiting for
8791da177e4SLinus Torvalds * its last child to go away).
8801da177e4SLinus Torvalds *
8811da177e4SLinus Torvalds * This tail recursion is done by hand as we don't want to depend
8821da177e4SLinus Torvalds * on the compiler to always get this right (gcc generally doesn't).
8831da177e4SLinus Torvalds * Real recursion would eat up our stack space.
8841da177e4SLinus Torvalds */
8851da177e4SLinus Torvalds
8861da177e4SLinus Torvalds /*
8871da177e4SLinus Torvalds * dput - release a dentry
8881da177e4SLinus Torvalds * @dentry: dentry to release
8891da177e4SLinus Torvalds *
8901da177e4SLinus Torvalds * Release a dentry. This will drop the usage count and if appropriate
8911da177e4SLinus Torvalds * call the dentry unlink method as well as removing it from the queues and
8921da177e4SLinus Torvalds * releasing its resources. If the parent dentries were scheduled for release
8931da177e4SLinus Torvalds * they too may now get deleted.
8941da177e4SLinus Torvalds */
dput(struct dentry * dentry)8951da177e4SLinus Torvalds void dput(struct dentry *dentry)
8961da177e4SLinus Torvalds {
8971088a640SAl Viro while (dentry) {
89847be6184SWei Fang might_sleep();
89947be6184SWei Fang
900360f5479SLinus Torvalds rcu_read_lock();
901360f5479SLinus Torvalds if (likely(fast_dput(dentry))) {
902360f5479SLinus Torvalds rcu_read_unlock();
9031da177e4SLinus Torvalds return;
904360f5479SLinus Torvalds }
905360f5479SLinus Torvalds
906360f5479SLinus Torvalds /* Slow case: now with the dentry lock held */
907360f5479SLinus Torvalds rcu_read_unlock();
9081da177e4SLinus Torvalds
909a338579fSAl Viro if (likely(retain_dentry(dentry))) {
9101da177e4SLinus Torvalds spin_unlock(&dentry->d_lock);
9111da177e4SLinus Torvalds return;
912a338579fSAl Viro }
9131da177e4SLinus Torvalds
9148cbf74daSAl Viro dentry = dentry_kill(dentry);
9151da177e4SLinus Torvalds }
91647be6184SWei Fang }
917ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(dput);
9181da177e4SLinus Torvalds
__dput_to_list(struct dentry * dentry,struct list_head * list)9199bdebc2bSAl Viro static void __dput_to_list(struct dentry *dentry, struct list_head *list)
9209bdebc2bSAl Viro __must_hold(&dentry->d_lock)
9219bdebc2bSAl Viro {
9229bdebc2bSAl Viro if (dentry->d_flags & DCACHE_SHRINK_LIST) {
9239bdebc2bSAl Viro /* let the owner of the list it's on deal with it */
9249bdebc2bSAl Viro --dentry->d_lockref.count;
9259bdebc2bSAl Viro } else {
9269bdebc2bSAl Viro if (dentry->d_flags & DCACHE_LRU_LIST)
9279bdebc2bSAl Viro d_lru_del(dentry);
9289bdebc2bSAl Viro if (!--dentry->d_lockref.count)
9299bdebc2bSAl Viro d_shrink_add(dentry, list);
9309bdebc2bSAl Viro }
9319bdebc2bSAl Viro }
9329bdebc2bSAl Viro
dput_to_list(struct dentry * dentry,struct list_head * list)9339bdebc2bSAl Viro void dput_to_list(struct dentry *dentry, struct list_head *list)
9349bdebc2bSAl Viro {
9359bdebc2bSAl Viro rcu_read_lock();
9369bdebc2bSAl Viro if (likely(fast_dput(dentry))) {
9379bdebc2bSAl Viro rcu_read_unlock();
9389bdebc2bSAl Viro return;
9399bdebc2bSAl Viro }
9409bdebc2bSAl Viro rcu_read_unlock();
9419bdebc2bSAl Viro if (!retain_dentry(dentry))
9429bdebc2bSAl Viro __dput_to_list(dentry, list);
9439bdebc2bSAl Viro spin_unlock(&dentry->d_lock);
9449bdebc2bSAl Viro }
9451da177e4SLinus Torvalds
946b5c84bf6SNick Piggin /* This must be called with d_lock held */
__dget_dlock(struct dentry * dentry)947dc0474beSNick Piggin static inline void __dget_dlock(struct dentry *dentry)
9481da177e4SLinus Torvalds {
94998474236SWaiman Long dentry->d_lockref.count++;
9501da177e4SLinus Torvalds }
9511da177e4SLinus Torvalds
__dget(struct dentry * dentry)952dc0474beSNick Piggin static inline void __dget(struct dentry *dentry)
95323044507SNick Piggin {
95498474236SWaiman Long lockref_get(&dentry->d_lockref);
95523044507SNick Piggin }
95623044507SNick Piggin
dget_parent(struct dentry * dentry)957b7ab39f6SNick Piggin struct dentry *dget_parent(struct dentry *dentry)
958b7ab39f6SNick Piggin {
959df3d0bbcSWaiman Long int gotref;
960b7ab39f6SNick Piggin struct dentry *ret;
961e8400933SAl Viro unsigned seq;
962b7ab39f6SNick Piggin
963df3d0bbcSWaiman Long /*
964df3d0bbcSWaiman Long * Do optimistic parent lookup without any
965df3d0bbcSWaiman Long * locking.
966df3d0bbcSWaiman Long */
967df3d0bbcSWaiman Long rcu_read_lock();
968e8400933SAl Viro seq = raw_seqcount_begin(&dentry->d_seq);
96966702eb5SMark Rutland ret = READ_ONCE(dentry->d_parent);
970df3d0bbcSWaiman Long gotref = lockref_get_not_zero(&ret->d_lockref);
971df3d0bbcSWaiman Long rcu_read_unlock();
972df3d0bbcSWaiman Long if (likely(gotref)) {
973e8400933SAl Viro if (!read_seqcount_retry(&dentry->d_seq, seq))
974df3d0bbcSWaiman Long return ret;
975df3d0bbcSWaiman Long dput(ret);
976df3d0bbcSWaiman Long }
977df3d0bbcSWaiman Long
978b7ab39f6SNick Piggin repeat:
979a734eb45SNick Piggin /*
980a734eb45SNick Piggin * Don't need rcu_dereference because we re-check it was correct under
981a734eb45SNick Piggin * the lock.
982a734eb45SNick Piggin */
983a734eb45SNick Piggin rcu_read_lock();
984b7ab39f6SNick Piggin ret = dentry->d_parent;
985a734eb45SNick Piggin spin_lock(&ret->d_lock);
986a734eb45SNick Piggin if (unlikely(ret != dentry->d_parent)) {
987a734eb45SNick Piggin spin_unlock(&ret->d_lock);
988a734eb45SNick Piggin rcu_read_unlock();
989b7ab39f6SNick Piggin goto repeat;
990b7ab39f6SNick Piggin }
991a734eb45SNick Piggin rcu_read_unlock();
99298474236SWaiman Long BUG_ON(!ret->d_lockref.count);
99398474236SWaiman Long ret->d_lockref.count++;
994b7ab39f6SNick Piggin spin_unlock(&ret->d_lock);
995b7ab39f6SNick Piggin return ret;
996b7ab39f6SNick Piggin }
997b7ab39f6SNick Piggin EXPORT_SYMBOL(dget_parent);
998b7ab39f6SNick Piggin
__d_find_any_alias(struct inode * inode)99961fec493SAl Viro static struct dentry * __d_find_any_alias(struct inode *inode)
100061fec493SAl Viro {
100161fec493SAl Viro struct dentry *alias;
100261fec493SAl Viro
100361fec493SAl Viro if (hlist_empty(&inode->i_dentry))
100461fec493SAl Viro return NULL;
100561fec493SAl Viro alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias);
100661fec493SAl Viro __dget(alias);
100761fec493SAl Viro return alias;
100861fec493SAl Viro }
100961fec493SAl Viro
101061fec493SAl Viro /**
101161fec493SAl Viro * d_find_any_alias - find any alias for a given inode
101261fec493SAl Viro * @inode: inode to find an alias for
101361fec493SAl Viro *
101461fec493SAl Viro * If any aliases exist for the given inode, take and return a
101561fec493SAl Viro * reference for one of them. If no aliases exist, return %NULL.
101661fec493SAl Viro */
d_find_any_alias(struct inode * inode)101761fec493SAl Viro struct dentry *d_find_any_alias(struct inode *inode)
101861fec493SAl Viro {
101961fec493SAl Viro struct dentry *de;
102061fec493SAl Viro
102161fec493SAl Viro spin_lock(&inode->i_lock);
102261fec493SAl Viro de = __d_find_any_alias(inode);
102361fec493SAl Viro spin_unlock(&inode->i_lock);
102461fec493SAl Viro return de;
102561fec493SAl Viro }
102661fec493SAl Viro EXPORT_SYMBOL(d_find_any_alias);
102761fec493SAl Viro
__d_find_alias(struct inode * inode)102852ed46f0SJ. Bruce Fields static struct dentry *__d_find_alias(struct inode *inode)
10291da177e4SLinus Torvalds {
103061fec493SAl Viro struct dentry *alias;
10311da177e4SLinus Torvalds
103261fec493SAl Viro if (S_ISDIR(inode->i_mode))
103361fec493SAl Viro return __d_find_any_alias(inode);
103461fec493SAl Viro
1035946e51f2SAl Viro hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
1036da502956SNick Piggin spin_lock(&alias->d_lock);
103761fec493SAl Viro if (!d_unhashed(alias)) {
1038dc0474beSNick Piggin __dget_dlock(alias);
1039da502956SNick Piggin spin_unlock(&alias->d_lock);
1040da502956SNick Piggin return alias;
1041da502956SNick Piggin }
1042da502956SNick Piggin spin_unlock(&alias->d_lock);
1043da502956SNick Piggin }
1044da502956SNick Piggin return NULL;
10451da177e4SLinus Torvalds }
10461da177e4SLinus Torvalds
1047961f3c89SMauro Carvalho Chehab /**
1048961f3c89SMauro Carvalho Chehab * d_find_alias - grab a hashed alias of inode
1049961f3c89SMauro Carvalho Chehab * @inode: inode in question
1050961f3c89SMauro Carvalho Chehab *
1051961f3c89SMauro Carvalho Chehab * If inode has a hashed alias, or is a directory and has any alias,
1052961f3c89SMauro Carvalho Chehab * acquire the reference to alias and return it. Otherwise return NULL.
1053961f3c89SMauro Carvalho Chehab * Notice that if inode is a directory there can be only one alias and
1054961f3c89SMauro Carvalho Chehab * it can be unhashed only if it has no children, or if it is the root
1055961f3c89SMauro Carvalho Chehab * of a filesystem, or if the directory was renamed and d_revalidate
1056961f3c89SMauro Carvalho Chehab * was the first vfs operation to notice.
1057961f3c89SMauro Carvalho Chehab *
1058961f3c89SMauro Carvalho Chehab * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer
1059961f3c89SMauro Carvalho Chehab * any other hashed alias over that one.
1060961f3c89SMauro Carvalho Chehab */
d_find_alias(struct inode * inode)10611da177e4SLinus Torvalds struct dentry *d_find_alias(struct inode *inode)
10621da177e4SLinus Torvalds {
1063214fda1fSDavid Howells struct dentry *de = NULL;
1064214fda1fSDavid Howells
1065b3d9b7a3SAl Viro if (!hlist_empty(&inode->i_dentry)) {
1066873feea0SNick Piggin spin_lock(&inode->i_lock);
106752ed46f0SJ. Bruce Fields de = __d_find_alias(inode);
1068873feea0SNick Piggin spin_unlock(&inode->i_lock);
1069214fda1fSDavid Howells }
10701da177e4SLinus Torvalds return de;
10711da177e4SLinus Torvalds }
1072ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_find_alias);
10731da177e4SLinus Torvalds
10741da177e4SLinus Torvalds /*
1075bca585d2SAl Viro * Caller MUST be holding rcu_read_lock() and be guaranteed
1076bca585d2SAl Viro * that inode won't get freed until rcu_read_unlock().
1077bca585d2SAl Viro */
d_find_alias_rcu(struct inode * inode)1078bca585d2SAl Viro struct dentry *d_find_alias_rcu(struct inode *inode)
1079bca585d2SAl Viro {
1080bca585d2SAl Viro struct hlist_head *l = &inode->i_dentry;
1081bca585d2SAl Viro struct dentry *de = NULL;
1082bca585d2SAl Viro
1083bca585d2SAl Viro spin_lock(&inode->i_lock);
1084bca585d2SAl Viro // ->i_dentry and ->i_rcu are colocated, but the latter won't be
1085bca585d2SAl Viro // used without having I_FREEING set, which means no aliases left
1086bca585d2SAl Viro if (likely(!(inode->i_state & I_FREEING) && !hlist_empty(l))) {
1087bca585d2SAl Viro if (S_ISDIR(inode->i_mode)) {
1088bca585d2SAl Viro de = hlist_entry(l->first, struct dentry, d_u.d_alias);
1089bca585d2SAl Viro } else {
1090bca585d2SAl Viro hlist_for_each_entry(de, l, d_u.d_alias)
1091bca585d2SAl Viro if (!d_unhashed(de))
1092bca585d2SAl Viro break;
1093bca585d2SAl Viro }
1094bca585d2SAl Viro }
1095bca585d2SAl Viro spin_unlock(&inode->i_lock);
1096bca585d2SAl Viro return de;
1097bca585d2SAl Viro }
1098bca585d2SAl Viro
1099bca585d2SAl Viro /*
11001da177e4SLinus Torvalds * Try to kill dentries associated with this inode.
11011da177e4SLinus Torvalds * WARNING: you must own a reference to inode.
11021da177e4SLinus Torvalds */
d_prune_aliases(struct inode * inode)11031da177e4SLinus Torvalds void d_prune_aliases(struct inode *inode)
11041da177e4SLinus Torvalds {
11050cdca3f9SDomen Puncer struct dentry *dentry;
11061da177e4SLinus Torvalds restart:
1107873feea0SNick Piggin spin_lock(&inode->i_lock);
1108946e51f2SAl Viro hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) {
11091da177e4SLinus Torvalds spin_lock(&dentry->d_lock);
111098474236SWaiman Long if (!dentry->d_lockref.count) {
111129355c39SAl Viro struct dentry *parent = lock_parent(dentry);
111229355c39SAl Viro if (likely(!dentry->d_lockref.count)) {
111329355c39SAl Viro __dentry_kill(dentry);
11144a7795d3SYan, Zheng dput(parent);
11151da177e4SLinus Torvalds goto restart;
11161da177e4SLinus Torvalds }
111729355c39SAl Viro if (parent)
111829355c39SAl Viro spin_unlock(&parent->d_lock);
111929355c39SAl Viro }
11201da177e4SLinus Torvalds spin_unlock(&dentry->d_lock);
11211da177e4SLinus Torvalds }
1122873feea0SNick Piggin spin_unlock(&inode->i_lock);
11231da177e4SLinus Torvalds }
1124ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_prune_aliases);
11251da177e4SLinus Torvalds
11263b3f09f4SAl Viro /*
11273b3f09f4SAl Viro * Lock a dentry from shrink list.
11288f04da2aSJohn Ogness * Called under rcu_read_lock() and dentry->d_lock; the former
11298f04da2aSJohn Ogness * guarantees that nothing we access will be freed under us.
11303b3f09f4SAl Viro * Note that dentry is *not* protected from concurrent dentry_kill(),
11318f04da2aSJohn Ogness * d_delete(), etc.
11328f04da2aSJohn Ogness *
11333b3f09f4SAl Viro * Return false if dentry has been disrupted or grabbed, leaving
11343b3f09f4SAl Viro * the caller to kick it off-list. Otherwise, return true and have
11353b3f09f4SAl Viro * that dentry's inode and parent both locked.
11363b3f09f4SAl Viro */
shrink_lock_dentry(struct dentry * dentry)11373b3f09f4SAl Viro static bool shrink_lock_dentry(struct dentry *dentry)
11381da177e4SLinus Torvalds {
1139ff2fde99SAl Viro struct inode *inode;
11403b3f09f4SAl Viro struct dentry *parent;
1141046b961bSAl Viro
11423b3f09f4SAl Viro if (dentry->d_lockref.count)
11433b3f09f4SAl Viro return false;
1144dd1f6b2eSDave Chinner
11453b3f09f4SAl Viro inode = dentry->d_inode;
11463b3f09f4SAl Viro if (inode && unlikely(!spin_trylock(&inode->i_lock))) {
11471da177e4SLinus Torvalds spin_unlock(&dentry->d_lock);
11483b3f09f4SAl Viro spin_lock(&inode->i_lock);
11493b3f09f4SAl Viro spin_lock(&dentry->d_lock);
11503b3f09f4SAl Viro if (unlikely(dentry->d_lockref.count))
11513b3f09f4SAl Viro goto out;
11523b3f09f4SAl Viro /* changed inode means that somebody had grabbed it */
11533b3f09f4SAl Viro if (unlikely(inode != dentry->d_inode))
11543b3f09f4SAl Viro goto out;
11551da177e4SLinus Torvalds }
115677812a1eSNick Piggin
11573b3f09f4SAl Viro parent = dentry->d_parent;
11583b3f09f4SAl Viro if (IS_ROOT(dentry) || likely(spin_trylock(&parent->d_lock)))
11593b3f09f4SAl Viro return true;
116064fd72e0SAl Viro
116164fd72e0SAl Viro spin_unlock(&dentry->d_lock);
11623b3f09f4SAl Viro spin_lock(&parent->d_lock);
11633b3f09f4SAl Viro if (unlikely(parent != dentry->d_parent)) {
1164046b961bSAl Viro spin_unlock(&parent->d_lock);
11653b3f09f4SAl Viro spin_lock(&dentry->d_lock);
11663b3f09f4SAl Viro goto out;
11673b3f09f4SAl Viro }
11683b3f09f4SAl Viro spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
11698f04da2aSJohn Ogness if (likely(!dentry->d_lockref.count))
11703b3f09f4SAl Viro return true;
11713b3f09f4SAl Viro spin_unlock(&parent->d_lock);
11723b3f09f4SAl Viro out:
11733b3f09f4SAl Viro if (inode)
11743b3f09f4SAl Viro spin_unlock(&inode->i_lock);
11753b3f09f4SAl Viro return false;
11763b3f09f4SAl Viro }
11773b3f09f4SAl Viro
shrink_dentry_list(struct list_head * list)11789bdebc2bSAl Viro void shrink_dentry_list(struct list_head *list)
11793b3f09f4SAl Viro {
11803b3f09f4SAl Viro while (!list_empty(list)) {
11813b3f09f4SAl Viro struct dentry *dentry, *parent;
11823b3f09f4SAl Viro
11833b3f09f4SAl Viro dentry = list_entry(list->prev, struct dentry, d_lru);
11843b3f09f4SAl Viro spin_lock(&dentry->d_lock);
11858f04da2aSJohn Ogness rcu_read_lock();
11863b3f09f4SAl Viro if (!shrink_lock_dentry(dentry)) {
11873b3f09f4SAl Viro bool can_free = false;
11888f04da2aSJohn Ogness rcu_read_unlock();
11893b3f09f4SAl Viro d_shrink_del(dentry);
11903b3f09f4SAl Viro if (dentry->d_lockref.count < 0)
11913b3f09f4SAl Viro can_free = dentry->d_flags & DCACHE_MAY_FREE;
11923b3f09f4SAl Viro spin_unlock(&dentry->d_lock);
119364fd72e0SAl Viro if (can_free)
119464fd72e0SAl Viro dentry_free(dentry);
119564fd72e0SAl Viro continue;
119664fd72e0SAl Viro }
11978f04da2aSJohn Ogness rcu_read_unlock();
11983b3f09f4SAl Viro d_shrink_del(dentry);
11993b3f09f4SAl Viro parent = dentry->d_parent;
12009bdebc2bSAl Viro if (parent != dentry)
12019bdebc2bSAl Viro __dput_to_list(parent, list);
1202ff2fde99SAl Viro __dentry_kill(dentry);
12030feae5c4SNeilBrown }
12043049cfe2SChristoph Hellwig }
12053049cfe2SChristoph Hellwig
dentry_lru_isolate(struct list_head * item,struct list_lru_one * lru,spinlock_t * lru_lock,void * arg)12063f97b163SVladimir Davydov static enum lru_status dentry_lru_isolate(struct list_head *item,
12073f97b163SVladimir Davydov struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1208f6041567SDave Chinner {
1209f6041567SDave Chinner struct list_head *freeable = arg;
1210f6041567SDave Chinner struct dentry *dentry = container_of(item, struct dentry, d_lru);
1211f6041567SDave Chinner
1212f6041567SDave Chinner
1213f6041567SDave Chinner /*
1214f6041567SDave Chinner * we are inverting the lru lock/dentry->d_lock here,
1215f6041567SDave Chinner * so use a trylock. If we fail to get the lock, just skip
1216f6041567SDave Chinner * it
1217f6041567SDave Chinner */
1218f6041567SDave Chinner if (!spin_trylock(&dentry->d_lock))
1219f6041567SDave Chinner return LRU_SKIP;
1220f6041567SDave Chinner
1221f6041567SDave Chinner /*
1222f6041567SDave Chinner * Referenced dentries are still in use. If they have active
1223f6041567SDave Chinner * counts, just remove them from the LRU. Otherwise give them
1224f6041567SDave Chinner * another pass through the LRU.
1225f6041567SDave Chinner */
1226f6041567SDave Chinner if (dentry->d_lockref.count) {
12273f97b163SVladimir Davydov d_lru_isolate(lru, dentry);
1228f6041567SDave Chinner spin_unlock(&dentry->d_lock);
1229f6041567SDave Chinner return LRU_REMOVED;
1230f6041567SDave Chinner }
1231f6041567SDave Chinner
1232f6041567SDave Chinner if (dentry->d_flags & DCACHE_REFERENCED) {
1233f6041567SDave Chinner dentry->d_flags &= ~DCACHE_REFERENCED;
1234f6041567SDave Chinner spin_unlock(&dentry->d_lock);
1235f6041567SDave Chinner
1236f6041567SDave Chinner /*
1237f6041567SDave Chinner * The list move itself will be made by the common LRU code. At
1238f6041567SDave Chinner * this point, we've dropped the dentry->d_lock but keep the
1239f6041567SDave Chinner * lru lock. This is safe to do, since every list movement is
1240f6041567SDave Chinner * protected by the lru lock even if both locks are held.
1241f6041567SDave Chinner *
1242f6041567SDave Chinner * This is guaranteed by the fact that all LRU management
1243f6041567SDave Chinner * functions are intermediated by the LRU API calls like
1244f6041567SDave Chinner * list_lru_add and list_lru_del. List movement in this file
1245f6041567SDave Chinner * only ever occur through this functions or through callbacks
1246f6041567SDave Chinner * like this one, that are called from the LRU API.
1247f6041567SDave Chinner *
1248f6041567SDave Chinner * The only exceptions to this are functions like
1249f6041567SDave Chinner * shrink_dentry_list, and code that first checks for the
1250f6041567SDave Chinner * DCACHE_SHRINK_LIST flag. Those are guaranteed to be
1251f6041567SDave Chinner * operating only with stack provided lists after they are
1252f6041567SDave Chinner * properly isolated from the main list. It is thus, always a
1253f6041567SDave Chinner * local access.
1254f6041567SDave Chinner */
1255f6041567SDave Chinner return LRU_ROTATE;
1256f6041567SDave Chinner }
1257f6041567SDave Chinner
12583f97b163SVladimir Davydov d_lru_shrink_move(lru, dentry, freeable);
1259f6041567SDave Chinner spin_unlock(&dentry->d_lock);
1260f6041567SDave Chinner
1261f6041567SDave Chinner return LRU_REMOVED;
1262f6041567SDave Chinner }
1263f6041567SDave Chinner
12643049cfe2SChristoph Hellwig /**
1265b48f03b3SDave Chinner * prune_dcache_sb - shrink the dcache
1266b48f03b3SDave Chinner * @sb: superblock
1267503c358cSVladimir Davydov * @sc: shrink control, passed to list_lru_shrink_walk()
12683049cfe2SChristoph Hellwig *
1269503c358cSVladimir Davydov * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This
1270503c358cSVladimir Davydov * is done when we need more memory and called from the superblock shrinker
1271b48f03b3SDave Chinner * function.
1272b48f03b3SDave Chinner *
1273b48f03b3SDave Chinner * This function may fail to free any resources if all the dentries are in
1274b48f03b3SDave Chinner * use.
12753049cfe2SChristoph Hellwig */
prune_dcache_sb(struct super_block * sb,struct shrink_control * sc)1276503c358cSVladimir Davydov long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc)
12773049cfe2SChristoph Hellwig {
1278f6041567SDave Chinner LIST_HEAD(dispose);
1279f6041567SDave Chinner long freed;
12803049cfe2SChristoph Hellwig
1281503c358cSVladimir Davydov freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc,
1282503c358cSVladimir Davydov dentry_lru_isolate, &dispose);
1283f6041567SDave Chinner shrink_dentry_list(&dispose);
12840a234c6dSDave Chinner return freed;
128523044507SNick Piggin }
128623044507SNick Piggin
dentry_lru_isolate_shrink(struct list_head * item,struct list_lru_one * lru,spinlock_t * lru_lock,void * arg)12874e717f5cSGlauber Costa static enum lru_status dentry_lru_isolate_shrink(struct list_head *item,
12883f97b163SVladimir Davydov struct list_lru_one *lru, spinlock_t *lru_lock, void *arg)
1289dd1f6b2eSDave Chinner {
12904e717f5cSGlauber Costa struct list_head *freeable = arg;
12914e717f5cSGlauber Costa struct dentry *dentry = container_of(item, struct dentry, d_lru);
1292dd1f6b2eSDave Chinner
12934e717f5cSGlauber Costa /*
12944e717f5cSGlauber Costa * we are inverting the lru lock/dentry->d_lock here,
12954e717f5cSGlauber Costa * so use a trylock. If we fail to get the lock, just skip
12964e717f5cSGlauber Costa * it
12974e717f5cSGlauber Costa */
12984e717f5cSGlauber Costa if (!spin_trylock(&dentry->d_lock))
12994e717f5cSGlauber Costa return LRU_SKIP;
13004e717f5cSGlauber Costa
13013f97b163SVladimir Davydov d_lru_shrink_move(lru, dentry, freeable);
130223044507SNick Piggin spin_unlock(&dentry->d_lock);
1303ec33679dSNick Piggin
13044e717f5cSGlauber Costa return LRU_REMOVED;
13051da177e4SLinus Torvalds }
13061da177e4SLinus Torvalds
1307dd1f6b2eSDave Chinner
1308da3bbdd4SKentaro Makita /**
13091da177e4SLinus Torvalds * shrink_dcache_sb - shrink dcache for a superblock
13101da177e4SLinus Torvalds * @sb: superblock
13111da177e4SLinus Torvalds *
13123049cfe2SChristoph Hellwig * Shrink the dcache for the specified super block. This is used to free
13133049cfe2SChristoph Hellwig * the dcache before unmounting a file system.
13141da177e4SLinus Torvalds */
shrink_dcache_sb(struct super_block * sb)13151da177e4SLinus Torvalds void shrink_dcache_sb(struct super_block *sb)
13161da177e4SLinus Torvalds {
13174e717f5cSGlauber Costa do {
13184e717f5cSGlauber Costa LIST_HEAD(dispose);
13194e717f5cSGlauber Costa
13201dbd449cSWaiman Long list_lru_walk(&sb->s_dentry_lru,
1321b17c070fSSahitya Tummala dentry_lru_isolate_shrink, &dispose, 1024);
13224e717f5cSGlauber Costa shrink_dentry_list(&dispose);
1323b17c070fSSahitya Tummala } while (list_lru_count(&sb->s_dentry_lru) > 0);
13241da177e4SLinus Torvalds }
1325ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(shrink_dcache_sb);
13261da177e4SLinus Torvalds
1327db14fc3aSMiklos Szeredi /**
1328db14fc3aSMiklos Szeredi * enum d_walk_ret - action to talke during tree walk
1329db14fc3aSMiklos Szeredi * @D_WALK_CONTINUE: contrinue walk
1330db14fc3aSMiklos Szeredi * @D_WALK_QUIT: quit walk
1331db14fc3aSMiklos Szeredi * @D_WALK_NORETRY: quit when retry is needed
1332db14fc3aSMiklos Szeredi * @D_WALK_SKIP: skip this dentry and its children
13331da177e4SLinus Torvalds */
1334db14fc3aSMiklos Szeredi enum d_walk_ret {
1335db14fc3aSMiklos Szeredi D_WALK_CONTINUE,
1336db14fc3aSMiklos Szeredi D_WALK_QUIT,
1337db14fc3aSMiklos Szeredi D_WALK_NORETRY,
1338db14fc3aSMiklos Szeredi D_WALK_SKIP,
1339db14fc3aSMiklos Szeredi };
13401da177e4SLinus Torvalds
13411da177e4SLinus Torvalds /**
1342db14fc3aSMiklos Szeredi * d_walk - walk the dentry tree
1343db14fc3aSMiklos Szeredi * @parent: start of walk
1344db14fc3aSMiklos Szeredi * @data: data passed to @enter() and @finish()
1345db14fc3aSMiklos Szeredi * @enter: callback when first entering the dentry
13461da177e4SLinus Torvalds *
13473a8e3611SAl Viro * The @enter() callbacks are called with d_lock held.
13481da177e4SLinus Torvalds */
d_walk(struct dentry * parent,void * data,enum d_walk_ret (* enter)(void *,struct dentry *))1349db14fc3aSMiklos Szeredi static void d_walk(struct dentry *parent, void *data,
13503a8e3611SAl Viro enum d_walk_ret (*enter)(void *, struct dentry *))
13511da177e4SLinus Torvalds {
1352949854d0SNick Piggin struct dentry *this_parent;
13531da177e4SLinus Torvalds struct list_head *next;
135448f5ec21SAl Viro unsigned seq = 0;
1355db14fc3aSMiklos Szeredi enum d_walk_ret ret;
1356db14fc3aSMiklos Szeredi bool retry = true;
1357949854d0SNick Piggin
135858db63d0SNick Piggin again:
135948f5ec21SAl Viro read_seqbegin_or_lock(&rename_lock, &seq);
136058db63d0SNick Piggin this_parent = parent;
13612fd6b7f5SNick Piggin spin_lock(&this_parent->d_lock);
1362db14fc3aSMiklos Szeredi
1363db14fc3aSMiklos Szeredi ret = enter(data, this_parent);
1364db14fc3aSMiklos Szeredi switch (ret) {
1365db14fc3aSMiklos Szeredi case D_WALK_CONTINUE:
1366db14fc3aSMiklos Szeredi break;
1367db14fc3aSMiklos Szeredi case D_WALK_QUIT:
1368db14fc3aSMiklos Szeredi case D_WALK_SKIP:
1369db14fc3aSMiklos Szeredi goto out_unlock;
1370db14fc3aSMiklos Szeredi case D_WALK_NORETRY:
1371db14fc3aSMiklos Szeredi retry = false;
1372db14fc3aSMiklos Szeredi break;
1373db14fc3aSMiklos Szeredi }
13741da177e4SLinus Torvalds repeat:
13751da177e4SLinus Torvalds next = this_parent->d_subdirs.next;
13761da177e4SLinus Torvalds resume:
13771da177e4SLinus Torvalds while (next != &this_parent->d_subdirs) {
13781da177e4SLinus Torvalds struct list_head *tmp = next;
1379946e51f2SAl Viro struct dentry *dentry = list_entry(tmp, struct dentry, d_child);
13801da177e4SLinus Torvalds next = tmp->next;
13812fd6b7f5SNick Piggin
1382ba65dc5eSAl Viro if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR))
1383ba65dc5eSAl Viro continue;
1384ba65dc5eSAl Viro
13852fd6b7f5SNick Piggin spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
1386db14fc3aSMiklos Szeredi
1387db14fc3aSMiklos Szeredi ret = enter(data, dentry);
1388db14fc3aSMiklos Szeredi switch (ret) {
1389db14fc3aSMiklos Szeredi case D_WALK_CONTINUE:
1390db14fc3aSMiklos Szeredi break;
1391db14fc3aSMiklos Szeredi case D_WALK_QUIT:
13922fd6b7f5SNick Piggin spin_unlock(&dentry->d_lock);
1393db14fc3aSMiklos Szeredi goto out_unlock;
1394db14fc3aSMiklos Szeredi case D_WALK_NORETRY:
1395db14fc3aSMiklos Szeredi retry = false;
1396db14fc3aSMiklos Szeredi break;
1397db14fc3aSMiklos Szeredi case D_WALK_SKIP:
1398db14fc3aSMiklos Szeredi spin_unlock(&dentry->d_lock);
1399db14fc3aSMiklos Szeredi continue;
14002fd6b7f5SNick Piggin }
1401db14fc3aSMiklos Szeredi
14021da177e4SLinus Torvalds if (!list_empty(&dentry->d_subdirs)) {
14032fd6b7f5SNick Piggin spin_unlock(&this_parent->d_lock);
14045facae4fSQian Cai spin_release(&dentry->d_lock.dep_map, _RET_IP_);
14051da177e4SLinus Torvalds this_parent = dentry;
14062fd6b7f5SNick Piggin spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_);
14071da177e4SLinus Torvalds goto repeat;
14081da177e4SLinus Torvalds }
14092fd6b7f5SNick Piggin spin_unlock(&dentry->d_lock);
14101da177e4SLinus Torvalds }
14111da177e4SLinus Torvalds /*
14121da177e4SLinus Torvalds * All done at this level ... ascend and resume the search.
14131da177e4SLinus Torvalds */
1414ca5358efSAl Viro rcu_read_lock();
1415ca5358efSAl Viro ascend:
14161da177e4SLinus Torvalds if (this_parent != parent) {
1417c826cb7dSLinus Torvalds struct dentry *child = this_parent;
141831dec132SAl Viro this_parent = child->d_parent;
141931dec132SAl Viro
142031dec132SAl Viro spin_unlock(&child->d_lock);
142131dec132SAl Viro spin_lock(&this_parent->d_lock);
142231dec132SAl Viro
1423ca5358efSAl Viro /* might go back up the wrong parent if we have had a rename. */
1424ca5358efSAl Viro if (need_seqretry(&rename_lock, seq))
1425949854d0SNick Piggin goto rename_retry;
14262159184eSAl Viro /* go into the first sibling still alive */
14272159184eSAl Viro do {
1428ca5358efSAl Viro next = child->d_child.next;
1429ca5358efSAl Viro if (next == &this_parent->d_subdirs)
1430ca5358efSAl Viro goto ascend;
1431ca5358efSAl Viro child = list_entry(next, struct dentry, d_child);
14322159184eSAl Viro } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED));
143331dec132SAl Viro rcu_read_unlock();
14341da177e4SLinus Torvalds goto resume;
14351da177e4SLinus Torvalds }
1436ca5358efSAl Viro if (need_seqretry(&rename_lock, seq))
1437949854d0SNick Piggin goto rename_retry;
1438ca5358efSAl Viro rcu_read_unlock();
1439db14fc3aSMiklos Szeredi
1440db14fc3aSMiklos Szeredi out_unlock:
1441db14fc3aSMiklos Szeredi spin_unlock(&this_parent->d_lock);
144248f5ec21SAl Viro done_seqretry(&rename_lock, seq);
1443db14fc3aSMiklos Szeredi return;
144458db63d0SNick Piggin
144558db63d0SNick Piggin rename_retry:
1446ca5358efSAl Viro spin_unlock(&this_parent->d_lock);
1447ca5358efSAl Viro rcu_read_unlock();
1448ca5358efSAl Viro BUG_ON(seq & 1);
1449db14fc3aSMiklos Szeredi if (!retry)
1450db14fc3aSMiklos Szeredi return;
145148f5ec21SAl Viro seq = 1;
145258db63d0SNick Piggin goto again;
14531da177e4SLinus Torvalds }
1454db14fc3aSMiklos Szeredi
145501619491SIan Kent struct check_mount {
145601619491SIan Kent struct vfsmount *mnt;
145701619491SIan Kent unsigned int mounted;
145801619491SIan Kent };
145901619491SIan Kent
path_check_mount(void * data,struct dentry * dentry)146001619491SIan Kent static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry)
146101619491SIan Kent {
146201619491SIan Kent struct check_mount *info = data;
146301619491SIan Kent struct path path = { .mnt = info->mnt, .dentry = dentry };
146401619491SIan Kent
146501619491SIan Kent if (likely(!d_mountpoint(dentry)))
146601619491SIan Kent return D_WALK_CONTINUE;
146701619491SIan Kent if (__path_is_mountpoint(&path)) {
146801619491SIan Kent info->mounted = 1;
146901619491SIan Kent return D_WALK_QUIT;
147001619491SIan Kent }
147101619491SIan Kent return D_WALK_CONTINUE;
147201619491SIan Kent }
147301619491SIan Kent
147401619491SIan Kent /**
147501619491SIan Kent * path_has_submounts - check for mounts over a dentry in the
147601619491SIan Kent * current namespace.
147701619491SIan Kent * @parent: path to check.
147801619491SIan Kent *
147901619491SIan Kent * Return true if the parent or its subdirectories contain
148001619491SIan Kent * a mount point in the current namespace.
148101619491SIan Kent */
path_has_submounts(const struct path * parent)148201619491SIan Kent int path_has_submounts(const struct path *parent)
148301619491SIan Kent {
148401619491SIan Kent struct check_mount data = { .mnt = parent->mnt, .mounted = 0 };
148501619491SIan Kent
148601619491SIan Kent read_seqlock_excl(&mount_lock);
14873a8e3611SAl Viro d_walk(parent->dentry, &data, path_check_mount);
148801619491SIan Kent read_sequnlock_excl(&mount_lock);
148901619491SIan Kent
149001619491SIan Kent return data.mounted;
149101619491SIan Kent }
149201619491SIan Kent EXPORT_SYMBOL(path_has_submounts);
149301619491SIan Kent
14941da177e4SLinus Torvalds /*
1495eed81007SMiklos Szeredi * Called by mount code to set a mountpoint and check if the mountpoint is
1496eed81007SMiklos Szeredi * reachable (e.g. NFS can unhash a directory dentry and then the complete
1497eed81007SMiklos Szeredi * subtree can become unreachable).
1498eed81007SMiklos Szeredi *
14991ffe46d1SEric W. Biederman * Only one of d_invalidate() and d_set_mounted() must succeed. For
1500eed81007SMiklos Szeredi * this reason take rename_lock and d_lock on dentry and ancestors.
1501eed81007SMiklos Szeredi */
d_set_mounted(struct dentry * dentry)1502eed81007SMiklos Szeredi int d_set_mounted(struct dentry *dentry)
1503eed81007SMiklos Szeredi {
1504eed81007SMiklos Szeredi struct dentry *p;
1505eed81007SMiklos Szeredi int ret = -ENOENT;
1506eed81007SMiklos Szeredi write_seqlock(&rename_lock);
1507eed81007SMiklos Szeredi for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) {
15081ffe46d1SEric W. Biederman /* Need exclusion wrt. d_invalidate() */
1509eed81007SMiklos Szeredi spin_lock(&p->d_lock);
1510eed81007SMiklos Szeredi if (unlikely(d_unhashed(p))) {
1511eed81007SMiklos Szeredi spin_unlock(&p->d_lock);
1512eed81007SMiklos Szeredi goto out;
1513eed81007SMiklos Szeredi }
1514eed81007SMiklos Szeredi spin_unlock(&p->d_lock);
1515eed81007SMiklos Szeredi }
1516eed81007SMiklos Szeredi spin_lock(&dentry->d_lock);
1517eed81007SMiklos Szeredi if (!d_unlinked(dentry)) {
15183895dbf8SEric W. Biederman ret = -EBUSY;
15193895dbf8SEric W. Biederman if (!d_mountpoint(dentry)) {
1520eed81007SMiklos Szeredi dentry->d_flags |= DCACHE_MOUNTED;
1521eed81007SMiklos Szeredi ret = 0;
1522eed81007SMiklos Szeredi }
15233895dbf8SEric W. Biederman }
1524eed81007SMiklos Szeredi spin_unlock(&dentry->d_lock);
1525eed81007SMiklos Szeredi out:
1526eed81007SMiklos Szeredi write_sequnlock(&rename_lock);
1527eed81007SMiklos Szeredi return ret;
1528eed81007SMiklos Szeredi }
1529eed81007SMiklos Szeredi
1530eed81007SMiklos Szeredi /*
1531fd517909SJ. Bruce Fields * Search the dentry child list of the specified parent,
15321da177e4SLinus Torvalds * and move any unused dentries to the end of the unused
15331da177e4SLinus Torvalds * list for prune_dcache(). We descend to the next level
15341da177e4SLinus Torvalds * whenever the d_subdirs list is non-empty and continue
15351da177e4SLinus Torvalds * searching.
15361da177e4SLinus Torvalds *
15371da177e4SLinus Torvalds * It returns zero iff there are no unused children,
15381da177e4SLinus Torvalds * otherwise it returns the number of children moved to
15391da177e4SLinus Torvalds * the end of the unused list. This may not be the total
15401da177e4SLinus Torvalds * number of unused children, because select_parent can
15411da177e4SLinus Torvalds * drop the lock and return early due to latency
15421da177e4SLinus Torvalds * constraints.
15431da177e4SLinus Torvalds */
1544db14fc3aSMiklos Szeredi
1545db14fc3aSMiklos Szeredi struct select_data {
1546db14fc3aSMiklos Szeredi struct dentry *start;
15479bdebc2bSAl Viro union {
15489bdebc2bSAl Viro long found;
15499bdebc2bSAl Viro struct dentry *victim;
15509bdebc2bSAl Viro };
1551db14fc3aSMiklos Szeredi struct list_head dispose;
1552db14fc3aSMiklos Szeredi };
1553db14fc3aSMiklos Szeredi
select_collect(void * _data,struct dentry * dentry)1554db14fc3aSMiklos Szeredi static enum d_walk_ret select_collect(void *_data, struct dentry *dentry)
15551da177e4SLinus Torvalds {
1556db14fc3aSMiklos Szeredi struct select_data *data = _data;
1557db14fc3aSMiklos Szeredi enum d_walk_ret ret = D_WALK_CONTINUE;
15581da177e4SLinus Torvalds
1559db14fc3aSMiklos Szeredi if (data->start == dentry)
1560db14fc3aSMiklos Szeredi goto out;
156123044507SNick Piggin
1562fe91522aSAl Viro if (dentry->d_flags & DCACHE_SHRINK_LIST) {
1563fe91522aSAl Viro data->found++;
1564fe91522aSAl Viro } else {
1565fe91522aSAl Viro if (dentry->d_flags & DCACHE_LRU_LIST)
156689dc77bcSLinus Torvalds d_lru_del(dentry);
1567fe91522aSAl Viro if (!dentry->d_lockref.count) {
156889dc77bcSLinus Torvalds d_shrink_add(dentry, &data->dispose);
1569db14fc3aSMiklos Szeredi data->found++;
1570fe91522aSAl Viro }
15711da177e4SLinus Torvalds }
15721da177e4SLinus Torvalds /*
15731da177e4SLinus Torvalds * We can return to the caller if we have found some (this
15741da177e4SLinus Torvalds * ensures forward progress). We'll be coming back to find
15751da177e4SLinus Torvalds * the rest.
15761da177e4SLinus Torvalds */
1577fe91522aSAl Viro if (!list_empty(&data->dispose))
1578fe91522aSAl Viro ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
15791da177e4SLinus Torvalds out:
1580db14fc3aSMiklos Szeredi return ret;
15811da177e4SLinus Torvalds }
15821da177e4SLinus Torvalds
select_collect2(void * _data,struct dentry * dentry)15839bdebc2bSAl Viro static enum d_walk_ret select_collect2(void *_data, struct dentry *dentry)
15849bdebc2bSAl Viro {
15859bdebc2bSAl Viro struct select_data *data = _data;
15869bdebc2bSAl Viro enum d_walk_ret ret = D_WALK_CONTINUE;
15879bdebc2bSAl Viro
15889bdebc2bSAl Viro if (data->start == dentry)
15899bdebc2bSAl Viro goto out;
15909bdebc2bSAl Viro
15919bdebc2bSAl Viro if (dentry->d_flags & DCACHE_SHRINK_LIST) {
15929bdebc2bSAl Viro if (!dentry->d_lockref.count) {
15939bdebc2bSAl Viro rcu_read_lock();
15949bdebc2bSAl Viro data->victim = dentry;
15959bdebc2bSAl Viro return D_WALK_QUIT;
15969bdebc2bSAl Viro }
15979bdebc2bSAl Viro } else {
15989bdebc2bSAl Viro if (dentry->d_flags & DCACHE_LRU_LIST)
15999bdebc2bSAl Viro d_lru_del(dentry);
16009bdebc2bSAl Viro if (!dentry->d_lockref.count)
16019bdebc2bSAl Viro d_shrink_add(dentry, &data->dispose);
16029bdebc2bSAl Viro }
16039bdebc2bSAl Viro /*
16049bdebc2bSAl Viro * We can return to the caller if we have found some (this
16059bdebc2bSAl Viro * ensures forward progress). We'll be coming back to find
16069bdebc2bSAl Viro * the rest.
16079bdebc2bSAl Viro */
16089bdebc2bSAl Viro if (!list_empty(&data->dispose))
16099bdebc2bSAl Viro ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY;
16109bdebc2bSAl Viro out:
16119bdebc2bSAl Viro return ret;
16129bdebc2bSAl Viro }
16139bdebc2bSAl Viro
16141da177e4SLinus Torvalds /**
16151da177e4SLinus Torvalds * shrink_dcache_parent - prune dcache
16161da177e4SLinus Torvalds * @parent: parent of entries to prune
16171da177e4SLinus Torvalds *
16181da177e4SLinus Torvalds * Prune the dcache to remove unused children of the parent dentry.
16191da177e4SLinus Torvalds */
shrink_dcache_parent(struct dentry * parent)16201da177e4SLinus Torvalds void shrink_dcache_parent(struct dentry *parent)
16211da177e4SLinus Torvalds {
1622db14fc3aSMiklos Szeredi for (;;) {
16239bdebc2bSAl Viro struct select_data data = {.start = parent};
16241da177e4SLinus Torvalds
1625db14fc3aSMiklos Szeredi INIT_LIST_HEAD(&data.dispose);
16263a8e3611SAl Viro d_walk(parent, &data, select_collect);
16274fb48871SAl Viro
16284fb48871SAl Viro if (!list_empty(&data.dispose)) {
16294fb48871SAl Viro shrink_dentry_list(&data.dispose);
16304fb48871SAl Viro continue;
16314fb48871SAl Viro }
16324fb48871SAl Viro
16334fb48871SAl Viro cond_resched();
1634db14fc3aSMiklos Szeredi if (!data.found)
1635db14fc3aSMiklos Szeredi break;
16369bdebc2bSAl Viro data.victim = NULL;
16379bdebc2bSAl Viro d_walk(parent, &data, select_collect2);
16389bdebc2bSAl Viro if (data.victim) {
16399bdebc2bSAl Viro struct dentry *parent;
16409bdebc2bSAl Viro spin_lock(&data.victim->d_lock);
16419bdebc2bSAl Viro if (!shrink_lock_dentry(data.victim)) {
16429bdebc2bSAl Viro spin_unlock(&data.victim->d_lock);
16439bdebc2bSAl Viro rcu_read_unlock();
16449bdebc2bSAl Viro } else {
16459bdebc2bSAl Viro rcu_read_unlock();
16469bdebc2bSAl Viro parent = data.victim->d_parent;
16479bdebc2bSAl Viro if (parent != data.victim)
16489bdebc2bSAl Viro __dput_to_list(parent, &data.dispose);
16499bdebc2bSAl Viro __dentry_kill(data.victim);
16509bdebc2bSAl Viro }
16519bdebc2bSAl Viro }
16529bdebc2bSAl Viro if (!list_empty(&data.dispose))
16539bdebc2bSAl Viro shrink_dentry_list(&data.dispose);
1654421348f1SGreg Thelen }
16551da177e4SLinus Torvalds }
1656ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(shrink_dcache_parent);
16571da177e4SLinus Torvalds
umount_check(void * _data,struct dentry * dentry)16589c8c10e2SAl Viro static enum d_walk_ret umount_check(void *_data, struct dentry *dentry)
165942c32608SAl Viro {
16609c8c10e2SAl Viro /* it has busy descendents; complain about those instead */
16619c8c10e2SAl Viro if (!list_empty(&dentry->d_subdirs))
16629c8c10e2SAl Viro return D_WALK_CONTINUE;
166342c32608SAl Viro
16649c8c10e2SAl Viro /* root with refcount 1 is fine */
16659c8c10e2SAl Viro if (dentry == _data && dentry->d_lockref.count == 1)
16669c8c10e2SAl Viro return D_WALK_CONTINUE;
16679c8c10e2SAl Viro
16688c8e7dbaSAnh Tuan Phan WARN(1, "BUG: Dentry %p{i=%lx,n=%pd} "
16699c8c10e2SAl Viro " still in use (%d) [unmount of %s %s]\n",
167042c32608SAl Viro dentry,
167142c32608SAl Viro dentry->d_inode ?
167242c32608SAl Viro dentry->d_inode->i_ino : 0UL,
16739c8c10e2SAl Viro dentry,
167442c32608SAl Viro dentry->d_lockref.count,
167542c32608SAl Viro dentry->d_sb->s_type->name,
167642c32608SAl Viro dentry->d_sb->s_id);
16779c8c10e2SAl Viro return D_WALK_CONTINUE;
167842c32608SAl Viro }
16799c8c10e2SAl Viro
do_one_tree(struct dentry * dentry)16809c8c10e2SAl Viro static void do_one_tree(struct dentry *dentry)
16819c8c10e2SAl Viro {
16829c8c10e2SAl Viro shrink_dcache_parent(dentry);
16833a8e3611SAl Viro d_walk(dentry, dentry, umount_check);
16849c8c10e2SAl Viro d_drop(dentry);
16859c8c10e2SAl Viro dput(dentry);
168642c32608SAl Viro }
168742c32608SAl Viro
168842c32608SAl Viro /*
168942c32608SAl Viro * destroy the dentries attached to a superblock on unmounting
169042c32608SAl Viro */
shrink_dcache_for_umount(struct super_block * sb)169142c32608SAl Viro void shrink_dcache_for_umount(struct super_block *sb)
169242c32608SAl Viro {
169342c32608SAl Viro struct dentry *dentry;
169442c32608SAl Viro
16959c8c10e2SAl Viro WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked");
169642c32608SAl Viro
169742c32608SAl Viro dentry = sb->s_root;
169842c32608SAl Viro sb->s_root = NULL;
16999c8c10e2SAl Viro do_one_tree(dentry);
170042c32608SAl Viro
1701f1ee6162SNeilBrown while (!hlist_bl_empty(&sb->s_roots)) {
1702f1ee6162SNeilBrown dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_roots), struct dentry, d_hash));
17039c8c10e2SAl Viro do_one_tree(dentry);
170442c32608SAl Viro }
170542c32608SAl Viro }
170642c32608SAl Viro
find_submount(void * _data,struct dentry * dentry)1707ff17fa56SAl Viro static enum d_walk_ret find_submount(void *_data, struct dentry *dentry)
1708848ac114SMiklos Szeredi {
1709ff17fa56SAl Viro struct dentry **victim = _data;
1710848ac114SMiklos Szeredi if (d_mountpoint(dentry)) {
17118ed936b5SEric W. Biederman __dget_dlock(dentry);
1712ff17fa56SAl Viro *victim = dentry;
1713848ac114SMiklos Szeredi return D_WALK_QUIT;
1714848ac114SMiklos Szeredi }
1715ff17fa56SAl Viro return D_WALK_CONTINUE;
1716848ac114SMiklos Szeredi }
1717848ac114SMiklos Szeredi
1718848ac114SMiklos Szeredi /**
17191ffe46d1SEric W. Biederman * d_invalidate - detach submounts, prune dcache, and drop
17201ffe46d1SEric W. Biederman * @dentry: dentry to invalidate (aka detach, prune and drop)
1721848ac114SMiklos Szeredi */
d_invalidate(struct dentry * dentry)17225542aa2fSEric W. Biederman void d_invalidate(struct dentry *dentry)
1723848ac114SMiklos Szeredi {
1724ff17fa56SAl Viro bool had_submounts = false;
17251ffe46d1SEric W. Biederman spin_lock(&dentry->d_lock);
17261ffe46d1SEric W. Biederman if (d_unhashed(dentry)) {
17271ffe46d1SEric W. Biederman spin_unlock(&dentry->d_lock);
17285542aa2fSEric W. Biederman return;
17291ffe46d1SEric W. Biederman }
1730ff17fa56SAl Viro __d_drop(dentry);
17311ffe46d1SEric W. Biederman spin_unlock(&dentry->d_lock);
17321ffe46d1SEric W. Biederman
1733848ac114SMiklos Szeredi /* Negative dentries can be dropped without further checks */
1734ff17fa56SAl Viro if (!dentry->d_inode)
17355542aa2fSEric W. Biederman return;
1736848ac114SMiklos Szeredi
1737ff17fa56SAl Viro shrink_dcache_parent(dentry);
1738848ac114SMiklos Szeredi for (;;) {
1739ff17fa56SAl Viro struct dentry *victim = NULL;
17403a8e3611SAl Viro d_walk(dentry, &victim, find_submount);
1741ff17fa56SAl Viro if (!victim) {
1742ff17fa56SAl Viro if (had_submounts)
1743ff17fa56SAl Viro shrink_dcache_parent(dentry);
174481be24d2SAl Viro return;
17458ed936b5SEric W. Biederman }
1746ff17fa56SAl Viro had_submounts = true;
1747ff17fa56SAl Viro detach_mounts(victim);
1748ff17fa56SAl Viro dput(victim);
1749848ac114SMiklos Szeredi }
1750848ac114SMiklos Szeredi }
17511ffe46d1SEric W. Biederman EXPORT_SYMBOL(d_invalidate);
1752848ac114SMiklos Szeredi
17531da177e4SLinus Torvalds /**
1754a4464dbcSAl Viro * __d_alloc - allocate a dcache entry
1755a4464dbcSAl Viro * @sb: filesystem it will belong to
17561da177e4SLinus Torvalds * @name: qstr of the name
17571da177e4SLinus Torvalds *
17581da177e4SLinus Torvalds * Allocates a dentry. It returns %NULL if there is insufficient memory
17591da177e4SLinus Torvalds * available. On a success the dentry is returned. The name passed in is
17601da177e4SLinus Torvalds * copied and the copy passed in may be reused after this call.
17611da177e4SLinus Torvalds */
17621da177e4SLinus Torvalds
__d_alloc(struct super_block * sb,const struct qstr * name)17635c8b0dfcSAl Viro static struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name)
17641da177e4SLinus Torvalds {
17651da177e4SLinus Torvalds struct dentry *dentry;
17661da177e4SLinus Torvalds char *dname;
1767285b102dSMiklos Szeredi int err;
17681da177e4SLinus Torvalds
1769f53bf711SMuchun Song dentry = kmem_cache_alloc_lru(dentry_cache, &sb->s_dentry_lru,
1770f53bf711SMuchun Song GFP_KERNEL);
17711da177e4SLinus Torvalds if (!dentry)
17721da177e4SLinus Torvalds return NULL;
17731da177e4SLinus Torvalds
17746326c71fSLinus Torvalds /*
17756326c71fSLinus Torvalds * We guarantee that the inline name is always NUL-terminated.
17766326c71fSLinus Torvalds * This way the memcpy() done by the name switching in rename
17776326c71fSLinus Torvalds * will still always have a NUL at the end, even if we might
17786326c71fSLinus Torvalds * be overwriting an internal NUL character
17796326c71fSLinus Torvalds */
17806326c71fSLinus Torvalds dentry->d_iname[DNAME_INLINE_LEN-1] = 0;
1781798434bdSAl Viro if (unlikely(!name)) {
1782cdf01226SDavid Howells name = &slash_name;
1783798434bdSAl Viro dname = dentry->d_iname;
1784798434bdSAl Viro } else if (name->len > DNAME_INLINE_LEN-1) {
17858d85b484SAl Viro size_t size = offsetof(struct external_name, name[1]);
17862e03b4bcSVlastimil Babka struct external_name *p = kmalloc(size + name->len,
17872e03b4bcSVlastimil Babka GFP_KERNEL_ACCOUNT |
17882e03b4bcSVlastimil Babka __GFP_RECLAIMABLE);
17892e03b4bcSVlastimil Babka if (!p) {
17901da177e4SLinus Torvalds kmem_cache_free(dentry_cache, dentry);
17911da177e4SLinus Torvalds return NULL;
17921da177e4SLinus Torvalds }
17932e03b4bcSVlastimil Babka atomic_set(&p->u.count, 1);
17942e03b4bcSVlastimil Babka dname = p->name;
17951da177e4SLinus Torvalds } else {
17961da177e4SLinus Torvalds dname = dentry->d_iname;
17971da177e4SLinus Torvalds }
17981da177e4SLinus Torvalds
17991da177e4SLinus Torvalds dentry->d_name.len = name->len;
18001da177e4SLinus Torvalds dentry->d_name.hash = name->hash;
18011da177e4SLinus Torvalds memcpy(dname, name->name, name->len);
18021da177e4SLinus Torvalds dname[name->len] = 0;
18031da177e4SLinus Torvalds
18046326c71fSLinus Torvalds /* Make sure we always see the terminating NUL character */
18057088efa9SPaul E. McKenney smp_store_release(&dentry->d_name.name, dname); /* ^^^ */
18066326c71fSLinus Torvalds
180798474236SWaiman Long dentry->d_lockref.count = 1;
1808dea3667bSLinus Torvalds dentry->d_flags = 0;
18091da177e4SLinus Torvalds spin_lock_init(&dentry->d_lock);
181026475371SAhmed S. Darwish seqcount_spinlock_init(&dentry->d_seq, &dentry->d_lock);
18111da177e4SLinus Torvalds dentry->d_inode = NULL;
1812a4464dbcSAl Viro dentry->d_parent = dentry;
1813a4464dbcSAl Viro dentry->d_sb = sb;
18141da177e4SLinus Torvalds dentry->d_op = NULL;
18151da177e4SLinus Torvalds dentry->d_fsdata = NULL;
1816ceb5bdc2SNick Piggin INIT_HLIST_BL_NODE(&dentry->d_hash);
18171da177e4SLinus Torvalds INIT_LIST_HEAD(&dentry->d_lru);
18181da177e4SLinus Torvalds INIT_LIST_HEAD(&dentry->d_subdirs);
1819946e51f2SAl Viro INIT_HLIST_NODE(&dentry->d_u.d_alias);
1820946e51f2SAl Viro INIT_LIST_HEAD(&dentry->d_child);
1821a4464dbcSAl Viro d_set_d_op(dentry, dentry->d_sb->s_d_op);
18221da177e4SLinus Torvalds
1823285b102dSMiklos Szeredi if (dentry->d_op && dentry->d_op->d_init) {
1824285b102dSMiklos Szeredi err = dentry->d_op->d_init(dentry);
1825285b102dSMiklos Szeredi if (err) {
1826285b102dSMiklos Szeredi if (dname_external(dentry))
1827285b102dSMiklos Szeredi kfree(external_name(dentry));
1828285b102dSMiklos Szeredi kmem_cache_free(dentry_cache, dentry);
1829285b102dSMiklos Szeredi return NULL;
1830285b102dSMiklos Szeredi }
1831285b102dSMiklos Szeredi }
1832285b102dSMiklos Szeredi
1833a4464dbcSAl Viro this_cpu_inc(nr_dentry);
1834a4464dbcSAl Viro
1835a4464dbcSAl Viro return dentry;
1836a4464dbcSAl Viro }
1837a4464dbcSAl Viro
1838a4464dbcSAl Viro /**
1839a4464dbcSAl Viro * d_alloc - allocate a dcache entry
1840a4464dbcSAl Viro * @parent: parent of entry to allocate
1841a4464dbcSAl Viro * @name: qstr of the name
1842a4464dbcSAl Viro *
1843a4464dbcSAl Viro * Allocates a dentry. It returns %NULL if there is insufficient memory
1844a4464dbcSAl Viro * available. On a success the dentry is returned. The name passed in is
1845a4464dbcSAl Viro * copied and the copy passed in may be reused after this call.
1846a4464dbcSAl Viro */
d_alloc(struct dentry * parent,const struct qstr * name)1847a4464dbcSAl Viro struct dentry *d_alloc(struct dentry * parent, const struct qstr *name)
1848a4464dbcSAl Viro {
1849a4464dbcSAl Viro struct dentry *dentry = __d_alloc(parent->d_sb, name);
1850a4464dbcSAl Viro if (!dentry)
1851a4464dbcSAl Viro return NULL;
18522fd6b7f5SNick Piggin spin_lock(&parent->d_lock);
185389ad485fSNick Piggin /*
185489ad485fSNick Piggin * don't need child lock because it is not subject
185589ad485fSNick Piggin * to concurrency here
185689ad485fSNick Piggin */
1857dc0474beSNick Piggin __dget_dlock(parent);
1858dc0474beSNick Piggin dentry->d_parent = parent;
1859946e51f2SAl Viro list_add(&dentry->d_child, &parent->d_subdirs);
18602fd6b7f5SNick Piggin spin_unlock(&parent->d_lock);
1861312d3ca8SChristoph Hellwig
18621da177e4SLinus Torvalds return dentry;
18631da177e4SLinus Torvalds }
1864ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_alloc);
18651da177e4SLinus Torvalds
d_alloc_anon(struct super_block * sb)1866f9c34674SMiklos Szeredi struct dentry *d_alloc_anon(struct super_block *sb)
1867f9c34674SMiklos Szeredi {
1868f9c34674SMiklos Szeredi return __d_alloc(sb, NULL);
1869f9c34674SMiklos Szeredi }
1870f9c34674SMiklos Szeredi EXPORT_SYMBOL(d_alloc_anon);
1871f9c34674SMiklos Szeredi
d_alloc_cursor(struct dentry * parent)1872ba65dc5eSAl Viro struct dentry *d_alloc_cursor(struct dentry * parent)
1873ba65dc5eSAl Viro {
1874f9c34674SMiklos Szeredi struct dentry *dentry = d_alloc_anon(parent->d_sb);
1875ba65dc5eSAl Viro if (dentry) {
18765467a68cSAl Viro dentry->d_flags |= DCACHE_DENTRY_CURSOR;
1877ba65dc5eSAl Viro dentry->d_parent = dget(parent);
1878ba65dc5eSAl Viro }
1879ba65dc5eSAl Viro return dentry;
1880ba65dc5eSAl Viro }
1881ba65dc5eSAl Viro
1882e1a24bb0SJ. Bruce Fields /**
1883e1a24bb0SJ. Bruce Fields * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems)
1884e1a24bb0SJ. Bruce Fields * @sb: the superblock
1885e1a24bb0SJ. Bruce Fields * @name: qstr of the name
1886e1a24bb0SJ. Bruce Fields *
1887e1a24bb0SJ. Bruce Fields * For a filesystem that just pins its dentries in memory and never
1888e1a24bb0SJ. Bruce Fields * performs lookups at all, return an unhashed IS_ROOT dentry.
18895467a68cSAl Viro * This is used for pipes, sockets et.al. - the stuff that should
18905467a68cSAl Viro * never be anyone's children or parents. Unlike all other
18915467a68cSAl Viro * dentries, these will not have RCU delay between dropping the
18925467a68cSAl Viro * last reference and freeing them.
1893ab1152ddSAl Viro *
1894ab1152ddSAl Viro * The only user is alloc_file_pseudo() and that's what should
1895ab1152ddSAl Viro * be considered a public interface. Don't use directly.
1896e1a24bb0SJ. Bruce Fields */
d_alloc_pseudo(struct super_block * sb,const struct qstr * name)18974b936885SNick Piggin struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name)
18984b936885SNick Piggin {
18995467a68cSAl Viro struct dentry *dentry = __d_alloc(sb, name);
19005467a68cSAl Viro if (likely(dentry))
19015467a68cSAl Viro dentry->d_flags |= DCACHE_NORCU;
19025467a68cSAl Viro return dentry;
19034b936885SNick Piggin }
19044b936885SNick Piggin
d_alloc_name(struct dentry * parent,const char * name)19051da177e4SLinus Torvalds struct dentry *d_alloc_name(struct dentry *parent, const char *name)
19061da177e4SLinus Torvalds {
19071da177e4SLinus Torvalds struct qstr q;
19081da177e4SLinus Torvalds
19091da177e4SLinus Torvalds q.name = name;
19108387ff25SLinus Torvalds q.hash_len = hashlen_string(parent, name);
19111da177e4SLinus Torvalds return d_alloc(parent, &q);
19121da177e4SLinus Torvalds }
1913ef26ca97SH Hartley Sweeten EXPORT_SYMBOL(d_alloc_name);
19141da177e4SLinus Torvalds
d_set_d_op(struct dentry * dentry,const struct dentry_operations * op)1915fb045adbSNick Piggin void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op)
1916fb045adbSNick Piggin {
19176f7f7caaSLinus Torvalds WARN_ON_ONCE(dentry->d_op);
19186f7f7caaSLinus Torvalds WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH |
1919fb045adbSNick Piggin DCACHE_OP_COMPARE |
1920fb045adbSNick Piggin DCACHE_OP_REVALIDATE |
1921ecf3d1f1SJeff Layton DCACHE_OP_WEAK_REVALIDATE |
19224bacc9c9SDavid Howells DCACHE_OP_DELETE |
1923d101a125SMiklos Szeredi DCACHE_OP_REAL));
1924fb045adbSNick Piggin dentry->d_op = op;
1925fb045adbSNick Piggin if (!op)
1926fb045adbSNick Piggin return;
1927fb045adbSNick Piggin if (op->d_hash)
1928fb045adbSNick Piggin dentry->d_flags |= DCACHE_OP_HASH;
1929fb045adbSNick Piggin if (op->d_compare)
1930fb045adbSNick Piggin dentry->d_flags |= DCACHE_OP_COMPARE;
1931fb045adbSNick Piggin if (op->d_revalidate)
1932fb045adbSNick Piggin dentry->d_flags |= DCACHE_OP_REVALIDATE;
1933ecf3d1f1SJeff Layton if (op->d_weak_revalidate)
1934ecf3d1f1SJeff Layton dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE;
1935fb045adbSNick Piggin if (op->d_delete)
1936fb045adbSNick Piggin dentry->d_flags |= DCACHE_OP_DELETE;
1937f0023bc6SSage Weil if (op->d_prune)
1938f0023bc6SSage Weil dentry->d_flags |= DCACHE_OP_PRUNE;
1939d101a125SMiklos Szeredi if (op->d_real)
1940d101a125SMiklos Szeredi dentry->d_flags |= DCACHE_OP_REAL;
1941fb045adbSNick Piggin
1942fb045adbSNick Piggin }
1943fb045adbSNick Piggin EXPORT_SYMBOL(d_set_d_op);
1944fb045adbSNick Piggin
1945df1a085aSDavid Howells
1946df1a085aSDavid Howells /*
1947df1a085aSDavid Howells * d_set_fallthru - Mark a dentry as falling through to a lower layer
1948df1a085aSDavid Howells * @dentry - The dentry to mark
1949df1a085aSDavid Howells *
1950df1a085aSDavid Howells * Mark a dentry as falling through to the lower layer (as set with
1951df1a085aSDavid Howells * d_pin_lower()). This flag may be recorded on the medium.
1952df1a085aSDavid Howells */
d_set_fallthru(struct dentry * dentry)1953df1a085aSDavid Howells void d_set_fallthru(struct dentry *dentry)
1954df1a085aSDavid Howells {
1955df1a085aSDavid Howells spin_lock(&dentry->d_lock);
1956df1a085aSDavid Howells dentry->d_flags |= DCACHE_FALLTHRU;
1957df1a085aSDavid Howells spin_unlock(&dentry->d_lock);
1958df1a085aSDavid Howells }
1959df1a085aSDavid Howells EXPORT_SYMBOL(d_set_fallthru);
1960df1a085aSDavid Howells
d_flags_for_inode(struct inode * inode)1961b18825a7SDavid Howells static unsigned d_flags_for_inode(struct inode *inode)
1962b18825a7SDavid Howells {
196344bdb5e5SDavid Howells unsigned add_flags = DCACHE_REGULAR_TYPE;
1964b18825a7SDavid Howells
1965b18825a7SDavid Howells if (!inode)
1966b18825a7SDavid Howells return DCACHE_MISS_TYPE;
1967b18825a7SDavid Howells
1968b18825a7SDavid Howells if (S_ISDIR(inode->i_mode)) {
1969b18825a7SDavid Howells add_flags = DCACHE_DIRECTORY_TYPE;
1970b18825a7SDavid Howells if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) {
1971b18825a7SDavid Howells if (unlikely(!inode->i_op->lookup))
1972b18825a7SDavid Howells add_flags = DCACHE_AUTODIR_TYPE;
1973b18825a7SDavid Howells else
1974b18825a7SDavid Howells inode->i_opflags |= IOP_LOOKUP;
1975b18825a7SDavid Howells }
197644bdb5e5SDavid Howells goto type_determined;
197744bdb5e5SDavid Howells }
197844bdb5e5SDavid Howells
197944bdb5e5SDavid Howells if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) {
19806b255391SAl Viro if (unlikely(inode->i_op->get_link)) {
1981b18825a7SDavid Howells add_flags = DCACHE_SYMLINK_TYPE;
198244bdb5e5SDavid Howells goto type_determined;
198344bdb5e5SDavid Howells }
1984b18825a7SDavid Howells inode->i_opflags |= IOP_NOFOLLOW;
1985b18825a7SDavid Howells }
1986b18825a7SDavid Howells
198744bdb5e5SDavid Howells if (unlikely(!S_ISREG(inode->i_mode)))
198844bdb5e5SDavid Howells add_flags = DCACHE_SPECIAL_TYPE;
198944bdb5e5SDavid Howells
199044bdb5e5SDavid Howells type_determined:
1991b18825a7SDavid Howells if (unlikely(IS_AUTOMOUNT(inode)))
1992b18825a7SDavid Howells add_flags |= DCACHE_NEED_AUTOMOUNT;
1993b18825a7SDavid Howells return add_flags;
1994b18825a7SDavid Howells }
1995b18825a7SDavid Howells
__d_instantiate(struct dentry * dentry,struct inode * inode)1996360da900SOGAWA Hirofumi static void __d_instantiate(struct dentry *dentry, struct inode *inode)
1997360da900SOGAWA Hirofumi {
1998b18825a7SDavid Howells unsigned add_flags = d_flags_for_inode(inode);
199985c7f810SAl Viro WARN_ON(d_in_lookup(dentry));
2000b18825a7SDavid Howells
2001b23fb0a6SNick Piggin spin_lock(&dentry->d_lock);
2002af0c9af1SWaiman Long /*
2003af0c9af1SWaiman Long * Decrement negative dentry count if it was in the LRU list.
2004af0c9af1SWaiman Long */
2005af0c9af1SWaiman Long if (dentry->d_flags & DCACHE_LRU_LIST)
2006af0c9af1SWaiman Long this_cpu_dec(nr_dentry_negative);
2007946e51f2SAl Viro hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2008a528aca7SAl Viro raw_write_seqcount_begin(&dentry->d_seq);
20094bf46a27SDavid Howells __d_set_inode_and_type(dentry, inode, add_flags);
2010a528aca7SAl Viro raw_write_seqcount_end(&dentry->d_seq);
2011affda484SAl Viro fsnotify_update_flags(dentry);
2012b23fb0a6SNick Piggin spin_unlock(&dentry->d_lock);
2013360da900SOGAWA Hirofumi }
2014360da900SOGAWA Hirofumi
20151da177e4SLinus Torvalds /**
20161da177e4SLinus Torvalds * d_instantiate - fill in inode information for a dentry
20171da177e4SLinus Torvalds * @entry: dentry to complete
20181da177e4SLinus Torvalds * @inode: inode to attach to this dentry
20191da177e4SLinus Torvalds *
20201da177e4SLinus Torvalds * Fill in inode information in the entry.
20211da177e4SLinus Torvalds *
20221da177e4SLinus Torvalds * This turns negative dentries into productive full members
20231da177e4SLinus Torvalds * of society.
20241da177e4SLinus Torvalds *
20251da177e4SLinus Torvalds * NOTE! This assumes that the inode count has been incremented
20261da177e4SLinus Torvalds * (or otherwise set) by the caller to indicate that it is now
20271da177e4SLinus Torvalds * in use by the dcache.
20281da177e4SLinus Torvalds */
20291da177e4SLinus Torvalds
d_instantiate(struct dentry * entry,struct inode * inode)20301da177e4SLinus Torvalds void d_instantiate(struct dentry *entry, struct inode * inode)
20311da177e4SLinus Torvalds {
2032946e51f2SAl Viro BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
2033de689f5eSAl Viro if (inode) {
2034b9680917SAl Viro security_d_instantiate(entry, inode);
2035873feea0SNick Piggin spin_lock(&inode->i_lock);
2036360da900SOGAWA Hirofumi __d_instantiate(entry, inode);
2037873feea0SNick Piggin spin_unlock(&inode->i_lock);
2038de689f5eSAl Viro }
20391da177e4SLinus Torvalds }
2040ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_instantiate);
20411da177e4SLinus Torvalds
20421e2e547aSAl Viro /*
20431e2e547aSAl Viro * This should be equivalent to d_instantiate() + unlock_new_inode(),
20441e2e547aSAl Viro * with lockdep-related part of unlock_new_inode() done before
20451e2e547aSAl Viro * anything else. Use that instead of open-coding d_instantiate()/
20461e2e547aSAl Viro * unlock_new_inode() combinations.
20471e2e547aSAl Viro */
d_instantiate_new(struct dentry * entry,struct inode * inode)20481e2e547aSAl Viro void d_instantiate_new(struct dentry *entry, struct inode *inode)
20491e2e547aSAl Viro {
20501e2e547aSAl Viro BUG_ON(!hlist_unhashed(&entry->d_u.d_alias));
20511e2e547aSAl Viro BUG_ON(!inode);
20521e2e547aSAl Viro lockdep_annotate_inode_mutex_key(inode);
20531e2e547aSAl Viro security_d_instantiate(entry, inode);
20541e2e547aSAl Viro spin_lock(&inode->i_lock);
20551e2e547aSAl Viro __d_instantiate(entry, inode);
20561e2e547aSAl Viro WARN_ON(!(inode->i_state & I_NEW));
2057c2b6d621SAl Viro inode->i_state &= ~I_NEW & ~I_CREATING;
20581e2e547aSAl Viro smp_mb();
20591e2e547aSAl Viro wake_up_bit(&inode->i_state, __I_NEW);
20601e2e547aSAl Viro spin_unlock(&inode->i_lock);
20611e2e547aSAl Viro }
20621e2e547aSAl Viro EXPORT_SYMBOL(d_instantiate_new);
20631e2e547aSAl Viro
d_make_root(struct inode * root_inode)2064adc0e91aSAl Viro struct dentry *d_make_root(struct inode *root_inode)
2065adc0e91aSAl Viro {
2066adc0e91aSAl Viro struct dentry *res = NULL;
2067adc0e91aSAl Viro
2068adc0e91aSAl Viro if (root_inode) {
2069f9c34674SMiklos Szeredi res = d_alloc_anon(root_inode->i_sb);
20705467a68cSAl Viro if (res)
2071adc0e91aSAl Viro d_instantiate(res, root_inode);
20725467a68cSAl Viro else
2073adc0e91aSAl Viro iput(root_inode);
2074adc0e91aSAl Viro }
2075adc0e91aSAl Viro return res;
2076adc0e91aSAl Viro }
2077adc0e91aSAl Viro EXPORT_SYMBOL(d_make_root);
2078adc0e91aSAl Viro
__d_instantiate_anon(struct dentry * dentry,struct inode * inode,bool disconnected)2079f9c34674SMiklos Szeredi static struct dentry *__d_instantiate_anon(struct dentry *dentry,
2080f9c34674SMiklos Szeredi struct inode *inode,
2081f9c34674SMiklos Szeredi bool disconnected)
2082f9c34674SMiklos Szeredi {
2083f9c34674SMiklos Szeredi struct dentry *res;
2084f9c34674SMiklos Szeredi unsigned add_flags;
2085f9c34674SMiklos Szeredi
2086f9c34674SMiklos Szeredi security_d_instantiate(dentry, inode);
2087f9c34674SMiklos Szeredi spin_lock(&inode->i_lock);
2088f9c34674SMiklos Szeredi res = __d_find_any_alias(inode);
2089f9c34674SMiklos Szeredi if (res) {
2090f9c34674SMiklos Szeredi spin_unlock(&inode->i_lock);
2091f9c34674SMiklos Szeredi dput(dentry);
2092f9c34674SMiklos Szeredi goto out_iput;
2093f9c34674SMiklos Szeredi }
2094f9c34674SMiklos Szeredi
2095f9c34674SMiklos Szeredi /* attach a disconnected dentry */
2096f9c34674SMiklos Szeredi add_flags = d_flags_for_inode(inode);
2097f9c34674SMiklos Szeredi
2098f9c34674SMiklos Szeredi if (disconnected)
2099f9c34674SMiklos Szeredi add_flags |= DCACHE_DISCONNECTED;
2100f9c34674SMiklos Szeredi
2101f9c34674SMiklos Szeredi spin_lock(&dentry->d_lock);
2102f9c34674SMiklos Szeredi __d_set_inode_and_type(dentry, inode, add_flags);
2103f9c34674SMiklos Szeredi hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
2104139351f1SLinus Torvalds if (!disconnected) {
2105139351f1SLinus Torvalds hlist_bl_lock(&dentry->d_sb->s_roots);
2106139351f1SLinus Torvalds hlist_bl_add_head(&dentry->d_hash, &dentry->d_sb->s_roots);
2107139351f1SLinus Torvalds hlist_bl_unlock(&dentry->d_sb->s_roots);
2108139351f1SLinus Torvalds }
2109f9c34674SMiklos Szeredi spin_unlock(&dentry->d_lock);
2110f9c34674SMiklos Szeredi spin_unlock(&inode->i_lock);
2111f9c34674SMiklos Szeredi
2112f9c34674SMiklos Szeredi return dentry;
2113f9c34674SMiklos Szeredi
2114f9c34674SMiklos Szeredi out_iput:
2115f9c34674SMiklos Szeredi iput(inode);
2116f9c34674SMiklos Szeredi return res;
2117f9c34674SMiklos Szeredi }
2118f9c34674SMiklos Szeredi
d_instantiate_anon(struct dentry * dentry,struct inode * inode)2119f9c34674SMiklos Szeredi struct dentry *d_instantiate_anon(struct dentry *dentry, struct inode *inode)
2120f9c34674SMiklos Szeredi {
2121f9c34674SMiklos Szeredi return __d_instantiate_anon(dentry, inode, true);
2122f9c34674SMiklos Szeredi }
2123f9c34674SMiklos Szeredi EXPORT_SYMBOL(d_instantiate_anon);
2124f9c34674SMiklos Szeredi
__d_obtain_alias(struct inode * inode,bool disconnected)2125f9c34674SMiklos Szeredi static struct dentry *__d_obtain_alias(struct inode *inode, bool disconnected)
21264ea3ada2SChristoph Hellwig {
21279308a612SChristoph Hellwig struct dentry *tmp;
21289308a612SChristoph Hellwig struct dentry *res;
21294ea3ada2SChristoph Hellwig
21304ea3ada2SChristoph Hellwig if (!inode)
213144003728SChristoph Hellwig return ERR_PTR(-ESTALE);
21324ea3ada2SChristoph Hellwig if (IS_ERR(inode))
21334ea3ada2SChristoph Hellwig return ERR_CAST(inode);
21344ea3ada2SChristoph Hellwig
2135d891eedbSJ. Bruce Fields res = d_find_any_alias(inode);
21369308a612SChristoph Hellwig if (res)
21379308a612SChristoph Hellwig goto out_iput;
21389308a612SChristoph Hellwig
2139f9c34674SMiklos Szeredi tmp = d_alloc_anon(inode->i_sb);
21409308a612SChristoph Hellwig if (!tmp) {
21419308a612SChristoph Hellwig res = ERR_PTR(-ENOMEM);
21429308a612SChristoph Hellwig goto out_iput;
21434ea3ada2SChristoph Hellwig }
2144b5c84bf6SNick Piggin
2145f9c34674SMiklos Szeredi return __d_instantiate_anon(tmp, inode, disconnected);
21469308a612SChristoph Hellwig
21479308a612SChristoph Hellwig out_iput:
21489308a612SChristoph Hellwig iput(inode);
21499308a612SChristoph Hellwig return res;
21504ea3ada2SChristoph Hellwig }
21511a0a397eSJ. Bruce Fields
21521a0a397eSJ. Bruce Fields /**
21531a0a397eSJ. Bruce Fields * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode
21541a0a397eSJ. Bruce Fields * @inode: inode to allocate the dentry for
21551a0a397eSJ. Bruce Fields *
21561a0a397eSJ. Bruce Fields * Obtain a dentry for an inode resulting from NFS filehandle conversion or
21571a0a397eSJ. Bruce Fields * similar open by handle operations. The returned dentry may be anonymous,
21581a0a397eSJ. Bruce Fields * or may have a full name (if the inode was already in the cache).
21591a0a397eSJ. Bruce Fields *
21601a0a397eSJ. Bruce Fields * When called on a directory inode, we must ensure that the inode only ever
21611a0a397eSJ. Bruce Fields * has one dentry. If a dentry is found, that is returned instead of
21621a0a397eSJ. Bruce Fields * allocating a new one.
21631a0a397eSJ. Bruce Fields *
21641a0a397eSJ. Bruce Fields * On successful return, the reference to the inode has been transferred
21651a0a397eSJ. Bruce Fields * to the dentry. In case of an error the reference on the inode is released.
21661a0a397eSJ. Bruce Fields * To make it easier to use in export operations a %NULL or IS_ERR inode may
21671a0a397eSJ. Bruce Fields * be passed in and the error will be propagated to the return value,
21681a0a397eSJ. Bruce Fields * with a %NULL @inode replaced by ERR_PTR(-ESTALE).
21691a0a397eSJ. Bruce Fields */
d_obtain_alias(struct inode * inode)21701a0a397eSJ. Bruce Fields struct dentry *d_obtain_alias(struct inode *inode)
21711a0a397eSJ. Bruce Fields {
2172f9c34674SMiklos Szeredi return __d_obtain_alias(inode, true);
21731a0a397eSJ. Bruce Fields }
2174adc48720SBenny Halevy EXPORT_SYMBOL(d_obtain_alias);
21751da177e4SLinus Torvalds
21761da177e4SLinus Torvalds /**
21771a0a397eSJ. Bruce Fields * d_obtain_root - find or allocate a dentry for a given inode
21781a0a397eSJ. Bruce Fields * @inode: inode to allocate the dentry for
21791a0a397eSJ. Bruce Fields *
21801a0a397eSJ. Bruce Fields * Obtain an IS_ROOT dentry for the root of a filesystem.
21811a0a397eSJ. Bruce Fields *
21821a0a397eSJ. Bruce Fields * We must ensure that directory inodes only ever have one dentry. If a
21831a0a397eSJ. Bruce Fields * dentry is found, that is returned instead of allocating a new one.
21841a0a397eSJ. Bruce Fields *
21851a0a397eSJ. Bruce Fields * On successful return, the reference to the inode has been transferred
21861a0a397eSJ. Bruce Fields * to the dentry. In case of an error the reference on the inode is
21871a0a397eSJ. Bruce Fields * released. A %NULL or IS_ERR inode may be passed in and will be the
21881a0a397eSJ. Bruce Fields * error will be propagate to the return value, with a %NULL @inode
21891a0a397eSJ. Bruce Fields * replaced by ERR_PTR(-ESTALE).
21901a0a397eSJ. Bruce Fields */
d_obtain_root(struct inode * inode)21911a0a397eSJ. Bruce Fields struct dentry *d_obtain_root(struct inode *inode)
21921a0a397eSJ. Bruce Fields {
2193f9c34674SMiklos Szeredi return __d_obtain_alias(inode, false);
21941a0a397eSJ. Bruce Fields }
21951a0a397eSJ. Bruce Fields EXPORT_SYMBOL(d_obtain_root);
21961a0a397eSJ. Bruce Fields
21971a0a397eSJ. Bruce Fields /**
21989403540cSBarry Naujok * d_add_ci - lookup or allocate new dentry with case-exact name
21999403540cSBarry Naujok * @inode: the inode case-insensitive lookup has found
22009403540cSBarry Naujok * @dentry: the negative dentry that was passed to the parent's lookup func
22019403540cSBarry Naujok * @name: the case-exact name to be associated with the returned dentry
22029403540cSBarry Naujok *
22039403540cSBarry Naujok * This is to avoid filling the dcache with case-insensitive names to the
22049403540cSBarry Naujok * same inode, only the actual correct case is stored in the dcache for
22059403540cSBarry Naujok * case-insensitive filesystems.
22069403540cSBarry Naujok *
22073d742d4bSRandy Dunlap * For a case-insensitive lookup match and if the case-exact dentry
22083d742d4bSRandy Dunlap * already exists in the dcache, use it and return it.
22099403540cSBarry Naujok *
22109403540cSBarry Naujok * If no entry exists with the exact case name, allocate new dentry with
22119403540cSBarry Naujok * the exact case, and return the spliced entry.
22129403540cSBarry Naujok */
d_add_ci(struct dentry * dentry,struct inode * inode,struct qstr * name)2213e45b590bSChristoph Hellwig struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode,
22149403540cSBarry Naujok struct qstr *name)
22159403540cSBarry Naujok {
2216d9171b93SAl Viro struct dentry *found, *res;
22179403540cSBarry Naujok
2218b6520c81SChristoph Hellwig /*
2219b6520c81SChristoph Hellwig * First check if a dentry matching the name already exists,
2220b6520c81SChristoph Hellwig * if not go ahead and create it now.
2221b6520c81SChristoph Hellwig */
22229403540cSBarry Naujok found = d_hash_and_lookup(dentry->d_parent, name);
22239403540cSBarry Naujok if (found) {
2224d9171b93SAl Viro iput(inode);
22259403540cSBarry Naujok return found;
22269403540cSBarry Naujok }
2227d9171b93SAl Viro if (d_in_lookup(dentry)) {
2228d9171b93SAl Viro found = d_alloc_parallel(dentry->d_parent, name,
2229d9171b93SAl Viro dentry->d_wait);
2230d9171b93SAl Viro if (IS_ERR(found) || !d_in_lookup(found)) {
22319403540cSBarry Naujok iput(inode);
22329403540cSBarry Naujok return found;
22339403540cSBarry Naujok }
2234d9171b93SAl Viro } else {
2235d9171b93SAl Viro found = d_alloc(dentry->d_parent, name);
2236d9171b93SAl Viro if (!found) {
2237d9171b93SAl Viro iput(inode);
2238d9171b93SAl Viro return ERR_PTR(-ENOMEM);
2239d9171b93SAl Viro }
2240d9171b93SAl Viro }
2241d9171b93SAl Viro res = d_splice_alias(inode, found);
2242d9171b93SAl Viro if (res) {
224340a3cb0dSAl Viro d_lookup_done(found);
2244d9171b93SAl Viro dput(found);
2245d9171b93SAl Viro return res;
2246d9171b93SAl Viro }
2247d9171b93SAl Viro return found;
2248d9171b93SAl Viro }
2249ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_add_ci);
22501da177e4SLinus Torvalds
22514f48d5daSXiubo Li /**
22524f48d5daSXiubo Li * d_same_name - compare dentry name with case-exact name
22534f48d5daSXiubo Li * @parent: parent dentry
22544f48d5daSXiubo Li * @dentry: the negative dentry that was passed to the parent's lookup func
22554f48d5daSXiubo Li * @name: the case-exact name to be associated with the returned dentry
22564f48d5daSXiubo Li *
22574f48d5daSXiubo Li * Return: true if names are same, or false
22584f48d5daSXiubo Li */
d_same_name(const struct dentry * dentry,const struct dentry * parent,const struct qstr * name)22594f48d5daSXiubo Li bool d_same_name(const struct dentry *dentry, const struct dentry *parent,
226012f8ad4bSLinus Torvalds const struct qstr *name)
226112f8ad4bSLinus Torvalds {
2262d4c91a8fSAl Viro if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) {
2263d4c91a8fSAl Viro if (dentry->d_name.len != name->len)
2264d4c91a8fSAl Viro return false;
2265d4c91a8fSAl Viro return dentry_cmp(dentry, name->name, name->len) == 0;
226612f8ad4bSLinus Torvalds }
22676fa67e70SAl Viro return parent->d_op->d_compare(dentry,
2268d4c91a8fSAl Viro dentry->d_name.len, dentry->d_name.name,
2269d4c91a8fSAl Viro name) == 0;
227012f8ad4bSLinus Torvalds }
22714f48d5daSXiubo Li EXPORT_SYMBOL_GPL(d_same_name);
227212f8ad4bSLinus Torvalds
2273ae2a8236SLinus Torvalds /*
2274ae2a8236SLinus Torvalds * This is __d_lookup_rcu() when the parent dentry has
2275ae2a8236SLinus Torvalds * DCACHE_OP_COMPARE, which makes things much nastier.
2276ae2a8236SLinus Torvalds */
__d_lookup_rcu_op_compare(const struct dentry * parent,const struct qstr * name,unsigned * seqp)2277ae2a8236SLinus Torvalds static noinline struct dentry *__d_lookup_rcu_op_compare(
2278ae2a8236SLinus Torvalds const struct dentry *parent,
2279ae2a8236SLinus Torvalds const struct qstr *name,
2280ae2a8236SLinus Torvalds unsigned *seqp)
2281ae2a8236SLinus Torvalds {
2282ae2a8236SLinus Torvalds u64 hashlen = name->hash_len;
2283ae2a8236SLinus Torvalds struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2284ae2a8236SLinus Torvalds struct hlist_bl_node *node;
2285ae2a8236SLinus Torvalds struct dentry *dentry;
2286ae2a8236SLinus Torvalds
2287ae2a8236SLinus Torvalds hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
2288ae2a8236SLinus Torvalds int tlen;
2289ae2a8236SLinus Torvalds const char *tname;
2290ae2a8236SLinus Torvalds unsigned seq;
2291ae2a8236SLinus Torvalds
2292ae2a8236SLinus Torvalds seqretry:
2293ae2a8236SLinus Torvalds seq = raw_seqcount_begin(&dentry->d_seq);
2294ae2a8236SLinus Torvalds if (dentry->d_parent != parent)
2295ae2a8236SLinus Torvalds continue;
2296ae2a8236SLinus Torvalds if (d_unhashed(dentry))
2297ae2a8236SLinus Torvalds continue;
2298ae2a8236SLinus Torvalds if (dentry->d_name.hash != hashlen_hash(hashlen))
2299ae2a8236SLinus Torvalds continue;
2300ae2a8236SLinus Torvalds tlen = dentry->d_name.len;
2301ae2a8236SLinus Torvalds tname = dentry->d_name.name;
2302ae2a8236SLinus Torvalds /* we want a consistent (name,len) pair */
2303ae2a8236SLinus Torvalds if (read_seqcount_retry(&dentry->d_seq, seq)) {
2304ae2a8236SLinus Torvalds cpu_relax();
2305ae2a8236SLinus Torvalds goto seqretry;
2306ae2a8236SLinus Torvalds }
2307ae2a8236SLinus Torvalds if (parent->d_op->d_compare(dentry, tlen, tname, name) != 0)
2308ae2a8236SLinus Torvalds continue;
2309ae2a8236SLinus Torvalds *seqp = seq;
2310ae2a8236SLinus Torvalds return dentry;
2311ae2a8236SLinus Torvalds }
2312ae2a8236SLinus Torvalds return NULL;
2313ae2a8236SLinus Torvalds }
2314ae2a8236SLinus Torvalds
23151da177e4SLinus Torvalds /**
231631e6b01fSNick Piggin * __d_lookup_rcu - search for a dentry (racy, store-free)
231731e6b01fSNick Piggin * @parent: parent dentry
231831e6b01fSNick Piggin * @name: qstr of name we wish to find
23191f1e6e52SRandy Dunlap * @seqp: returns d_seq value at the point where the dentry was found
232031e6b01fSNick Piggin * Returns: dentry, or NULL
232131e6b01fSNick Piggin *
232231e6b01fSNick Piggin * __d_lookup_rcu is the dcache lookup function for rcu-walk name
232331e6b01fSNick Piggin * resolution (store-free path walking) design described in
232431e6b01fSNick Piggin * Documentation/filesystems/path-lookup.txt.
232531e6b01fSNick Piggin *
232631e6b01fSNick Piggin * This is not to be used outside core vfs.
232731e6b01fSNick Piggin *
232831e6b01fSNick Piggin * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock
232931e6b01fSNick Piggin * held, and rcu_read_lock held. The returned dentry must not be stored into
233031e6b01fSNick Piggin * without taking d_lock and checking d_seq sequence count against @seq
233131e6b01fSNick Piggin * returned here.
233231e6b01fSNick Piggin *
233315570086SLinus Torvalds * A refcount may be taken on the found dentry with the d_rcu_to_refcount
233431e6b01fSNick Piggin * function.
233531e6b01fSNick Piggin *
233631e6b01fSNick Piggin * Alternatively, __d_lookup_rcu may be called again to look up the child of
233731e6b01fSNick Piggin * the returned dentry, so long as its parent's seqlock is checked after the
233831e6b01fSNick Piggin * child is looked up. Thus, an interlocking stepping of sequence lock checks
233931e6b01fSNick Piggin * is formed, giving integrity down the path walk.
234012f8ad4bSLinus Torvalds *
234112f8ad4bSLinus Torvalds * NOTE! The caller *has* to check the resulting dentry against the sequence
234212f8ad4bSLinus Torvalds * number we've returned before using any of the resulting dentry state!
234331e6b01fSNick Piggin */
__d_lookup_rcu(const struct dentry * parent,const struct qstr * name,unsigned * seqp)23448966be90SLinus Torvalds struct dentry *__d_lookup_rcu(const struct dentry *parent,
23458966be90SLinus Torvalds const struct qstr *name,
2346da53be12SLinus Torvalds unsigned *seqp)
234731e6b01fSNick Piggin {
234826fe5750SLinus Torvalds u64 hashlen = name->hash_len;
234931e6b01fSNick Piggin const unsigned char *str = name->name;
23508387ff25SLinus Torvalds struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen));
2351ceb5bdc2SNick Piggin struct hlist_bl_node *node;
235231e6b01fSNick Piggin struct dentry *dentry;
235331e6b01fSNick Piggin
235431e6b01fSNick Piggin /*
235531e6b01fSNick Piggin * Note: There is significant duplication with __d_lookup_rcu which is
235631e6b01fSNick Piggin * required to prevent single threaded performance regressions
235731e6b01fSNick Piggin * especially on architectures where smp_rmb (in seqcounts) are costly.
235831e6b01fSNick Piggin * Keep the two functions in sync.
235931e6b01fSNick Piggin */
236031e6b01fSNick Piggin
2361ae2a8236SLinus Torvalds if (unlikely(parent->d_flags & DCACHE_OP_COMPARE))
2362ae2a8236SLinus Torvalds return __d_lookup_rcu_op_compare(parent, name, seqp);
2363ae2a8236SLinus Torvalds
236431e6b01fSNick Piggin /*
236531e6b01fSNick Piggin * The hash list is protected using RCU.
236631e6b01fSNick Piggin *
236731e6b01fSNick Piggin * Carefully use d_seq when comparing a candidate dentry, to avoid
236831e6b01fSNick Piggin * races with d_move().
236931e6b01fSNick Piggin *
237031e6b01fSNick Piggin * It is possible that concurrent renames can mess up our list
237131e6b01fSNick Piggin * walk here and result in missing our dentry, resulting in the
237231e6b01fSNick Piggin * false-negative result. d_lookup() protects against concurrent
237331e6b01fSNick Piggin * renames using rename_lock seqlock.
237431e6b01fSNick Piggin *
2375b0a4bb83SNamhyung Kim * See Documentation/filesystems/path-lookup.txt for more details.
237631e6b01fSNick Piggin */
2377b07ad996SLinus Torvalds hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
23788966be90SLinus Torvalds unsigned seq;
237931e6b01fSNick Piggin
238012f8ad4bSLinus Torvalds /*
238112f8ad4bSLinus Torvalds * The dentry sequence count protects us from concurrent
2382da53be12SLinus Torvalds * renames, and thus protects parent and name fields.
238312f8ad4bSLinus Torvalds *
238412f8ad4bSLinus Torvalds * The caller must perform a seqcount check in order
2385da53be12SLinus Torvalds * to do anything useful with the returned dentry.
238612f8ad4bSLinus Torvalds *
238712f8ad4bSLinus Torvalds * NOTE! We do a "raw" seqcount_begin here. That means that
238812f8ad4bSLinus Torvalds * we don't wait for the sequence count to stabilize if it
238912f8ad4bSLinus Torvalds * is in the middle of a sequence change. If we do the slow
239012f8ad4bSLinus Torvalds * dentry compare, we will do seqretries until it is stable,
239112f8ad4bSLinus Torvalds * and if we end up with a successful lookup, we actually
239212f8ad4bSLinus Torvalds * want to exit RCU lookup anyway.
2393d4c91a8fSAl Viro *
2394d4c91a8fSAl Viro * Note that raw_seqcount_begin still *does* smp_rmb(), so
2395d4c91a8fSAl Viro * we are still guaranteed NUL-termination of ->d_name.name.
239612f8ad4bSLinus Torvalds */
239712f8ad4bSLinus Torvalds seq = raw_seqcount_begin(&dentry->d_seq);
239831e6b01fSNick Piggin if (dentry->d_parent != parent)
239931e6b01fSNick Piggin continue;
24002e321806SLinus Torvalds if (d_unhashed(dentry))
24012e321806SLinus Torvalds continue;
240226fe5750SLinus Torvalds if (dentry->d_name.hash_len != hashlen)
2403ee983e89SLinus Torvalds continue;
2404d4c91a8fSAl Viro if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0)
2405d4c91a8fSAl Viro continue;
2406da53be12SLinus Torvalds *seqp = seq;
240731e6b01fSNick Piggin return dentry;
240831e6b01fSNick Piggin }
240931e6b01fSNick Piggin return NULL;
241031e6b01fSNick Piggin }
241131e6b01fSNick Piggin
241231e6b01fSNick Piggin /**
24131da177e4SLinus Torvalds * d_lookup - search for a dentry
24141da177e4SLinus Torvalds * @parent: parent dentry
24151da177e4SLinus Torvalds * @name: qstr of name we wish to find
2416b04f784eSNick Piggin * Returns: dentry, or NULL
24171da177e4SLinus Torvalds *
2418b04f784eSNick Piggin * d_lookup searches the children of the parent dentry for the name in
2419b04f784eSNick Piggin * question. If the dentry is found its reference count is incremented and the
2420b04f784eSNick Piggin * dentry is returned. The caller must use dput to free the entry when it has
2421b04f784eSNick Piggin * finished using it. %NULL is returned if the dentry does not exist.
24221da177e4SLinus Torvalds */
d_lookup(const struct dentry * parent,const struct qstr * name)2423da2d8455SAl Viro struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name)
24241da177e4SLinus Torvalds {
242531e6b01fSNick Piggin struct dentry *dentry;
2426949854d0SNick Piggin unsigned seq;
24271da177e4SLinus Torvalds
24281da177e4SLinus Torvalds do {
24291da177e4SLinus Torvalds seq = read_seqbegin(&rename_lock);
24301da177e4SLinus Torvalds dentry = __d_lookup(parent, name);
24311da177e4SLinus Torvalds if (dentry)
24321da177e4SLinus Torvalds break;
24331da177e4SLinus Torvalds } while (read_seqretry(&rename_lock, seq));
24341da177e4SLinus Torvalds return dentry;
24351da177e4SLinus Torvalds }
2436ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_lookup);
24371da177e4SLinus Torvalds
243831e6b01fSNick Piggin /**
2439b04f784eSNick Piggin * __d_lookup - search for a dentry (racy)
2440b04f784eSNick Piggin * @parent: parent dentry
2441b04f784eSNick Piggin * @name: qstr of name we wish to find
2442b04f784eSNick Piggin * Returns: dentry, or NULL
2443b04f784eSNick Piggin *
2444b04f784eSNick Piggin * __d_lookup is like d_lookup, however it may (rarely) return a
2445b04f784eSNick Piggin * false-negative result due to unrelated rename activity.
2446b04f784eSNick Piggin *
2447b04f784eSNick Piggin * __d_lookup is slightly faster by avoiding rename_lock read seqlock,
2448b04f784eSNick Piggin * however it must be used carefully, eg. with a following d_lookup in
2449b04f784eSNick Piggin * the case of failure.
2450b04f784eSNick Piggin *
2451b04f784eSNick Piggin * __d_lookup callers must be commented.
2452b04f784eSNick Piggin */
__d_lookup(const struct dentry * parent,const struct qstr * name)2453a713ca2aSAl Viro struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name)
24541da177e4SLinus Torvalds {
24551da177e4SLinus Torvalds unsigned int hash = name->hash;
24568387ff25SLinus Torvalds struct hlist_bl_head *b = d_hash(hash);
2457ceb5bdc2SNick Piggin struct hlist_bl_node *node;
245831e6b01fSNick Piggin struct dentry *found = NULL;
2459665a7583SPaul E. McKenney struct dentry *dentry;
24601da177e4SLinus Torvalds
2461b04f784eSNick Piggin /*
246231e6b01fSNick Piggin * Note: There is significant duplication with __d_lookup_rcu which is
246331e6b01fSNick Piggin * required to prevent single threaded performance regressions
246431e6b01fSNick Piggin * especially on architectures where smp_rmb (in seqcounts) are costly.
246531e6b01fSNick Piggin * Keep the two functions in sync.
246631e6b01fSNick Piggin */
246731e6b01fSNick Piggin
246831e6b01fSNick Piggin /*
2469b04f784eSNick Piggin * The hash list is protected using RCU.
2470b04f784eSNick Piggin *
2471b04f784eSNick Piggin * Take d_lock when comparing a candidate dentry, to avoid races
2472b04f784eSNick Piggin * with d_move().
2473b04f784eSNick Piggin *
2474b04f784eSNick Piggin * It is possible that concurrent renames can mess up our list
2475b04f784eSNick Piggin * walk here and result in missing our dentry, resulting in the
2476b04f784eSNick Piggin * false-negative result. d_lookup() protects against concurrent
2477b04f784eSNick Piggin * renames using rename_lock seqlock.
2478b04f784eSNick Piggin *
2479b0a4bb83SNamhyung Kim * See Documentation/filesystems/path-lookup.txt for more details.
2480b04f784eSNick Piggin */
24811da177e4SLinus Torvalds rcu_read_lock();
24821da177e4SLinus Torvalds
2483b07ad996SLinus Torvalds hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) {
24841da177e4SLinus Torvalds
24851da177e4SLinus Torvalds if (dentry->d_name.hash != hash)
24861da177e4SLinus Torvalds continue;
24871da177e4SLinus Torvalds
24881da177e4SLinus Torvalds spin_lock(&dentry->d_lock);
24891da177e4SLinus Torvalds if (dentry->d_parent != parent)
24901da177e4SLinus Torvalds goto next;
2491d0185c08SLinus Torvalds if (d_unhashed(dentry))
2492d0185c08SLinus Torvalds goto next;
2493d0185c08SLinus Torvalds
2494d4c91a8fSAl Viro if (!d_same_name(dentry, parent, name))
24951da177e4SLinus Torvalds goto next;
24961da177e4SLinus Torvalds
249798474236SWaiman Long dentry->d_lockref.count++;
24981da177e4SLinus Torvalds found = dentry;
24991da177e4SLinus Torvalds spin_unlock(&dentry->d_lock);
25001da177e4SLinus Torvalds break;
25011da177e4SLinus Torvalds next:
25021da177e4SLinus Torvalds spin_unlock(&dentry->d_lock);
25031da177e4SLinus Torvalds }
25041da177e4SLinus Torvalds rcu_read_unlock();
25051da177e4SLinus Torvalds
25061da177e4SLinus Torvalds return found;
25071da177e4SLinus Torvalds }
25081da177e4SLinus Torvalds
25091da177e4SLinus Torvalds /**
25103e7e241fSEric W. Biederman * d_hash_and_lookup - hash the qstr then search for a dentry
25113e7e241fSEric W. Biederman * @dir: Directory to search in
25123e7e241fSEric W. Biederman * @name: qstr of name we wish to find
25133e7e241fSEric W. Biederman *
25144f522a24SAl Viro * On lookup failure NULL is returned; on bad name - ERR_PTR(-error)
25153e7e241fSEric W. Biederman */
d_hash_and_lookup(struct dentry * dir,struct qstr * name)25163e7e241fSEric W. Biederman struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name)
25173e7e241fSEric W. Biederman {
25183e7e241fSEric W. Biederman /*
25193e7e241fSEric W. Biederman * Check for a fs-specific hash function. Note that we must
25203e7e241fSEric W. Biederman * calculate the standard hash first, as the d_op->d_hash()
25213e7e241fSEric W. Biederman * routine may choose to leave the hash value unchanged.
25223e7e241fSEric W. Biederman */
25238387ff25SLinus Torvalds name->hash = full_name_hash(dir, name->name, name->len);
2524fb045adbSNick Piggin if (dir->d_flags & DCACHE_OP_HASH) {
2525da53be12SLinus Torvalds int err = dir->d_op->d_hash(dir, name);
25264f522a24SAl Viro if (unlikely(err < 0))
25274f522a24SAl Viro return ERR_PTR(err);
25283e7e241fSEric W. Biederman }
25294f522a24SAl Viro return d_lookup(dir, name);
25303e7e241fSEric W. Biederman }
25314f522a24SAl Viro EXPORT_SYMBOL(d_hash_and_lookup);
25323e7e241fSEric W. Biederman
25331da177e4SLinus Torvalds /*
25341da177e4SLinus Torvalds * When a file is deleted, we have two options:
25351da177e4SLinus Torvalds * - turn this dentry into a negative dentry
25361da177e4SLinus Torvalds * - unhash this dentry and free it.
25371da177e4SLinus Torvalds *
25381da177e4SLinus Torvalds * Usually, we want to just turn this into
25391da177e4SLinus Torvalds * a negative dentry, but if anybody else is
25401da177e4SLinus Torvalds * currently using the dentry or the inode
25411da177e4SLinus Torvalds * we can't do that and we fall back on removing
25421da177e4SLinus Torvalds * it from the hash queues and waiting for
25431da177e4SLinus Torvalds * it to be deleted later when it has no users
25441da177e4SLinus Torvalds */
25451da177e4SLinus Torvalds
25461da177e4SLinus Torvalds /**
25471da177e4SLinus Torvalds * d_delete - delete a dentry
25481da177e4SLinus Torvalds * @dentry: The dentry to delete
25491da177e4SLinus Torvalds *
25501da177e4SLinus Torvalds * Turn the dentry into a negative dentry if possible, otherwise
25511da177e4SLinus Torvalds * remove it from the hash queues so it can be deleted later
25521da177e4SLinus Torvalds */
25531da177e4SLinus Torvalds
d_delete(struct dentry * dentry)25541da177e4SLinus Torvalds void d_delete(struct dentry * dentry)
25551da177e4SLinus Torvalds {
2556c19457f0SAl Viro struct inode *inode = dentry->d_inode;
2557c19457f0SAl Viro
2558c19457f0SAl Viro spin_lock(&inode->i_lock);
2559c19457f0SAl Viro spin_lock(&dentry->d_lock);
25601da177e4SLinus Torvalds /*
25611da177e4SLinus Torvalds * Are we the only user?
25621da177e4SLinus Torvalds */
256398474236SWaiman Long if (dentry->d_lockref.count == 1) {
256413e3c5e5SAl Viro dentry->d_flags &= ~DCACHE_CANT_MOUNT;
256531e6b01fSNick Piggin dentry_unlink_inode(dentry);
2566c19457f0SAl Viro } else {
25671da177e4SLinus Torvalds __d_drop(dentry);
25681da177e4SLinus Torvalds spin_unlock(&dentry->d_lock);
2569c19457f0SAl Viro spin_unlock(&inode->i_lock);
2570c19457f0SAl Viro }
25711da177e4SLinus Torvalds }
2572ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_delete);
25731da177e4SLinus Torvalds
__d_rehash(struct dentry * entry)257415d3c589SAl Viro static void __d_rehash(struct dentry *entry)
25751da177e4SLinus Torvalds {
257615d3c589SAl Viro struct hlist_bl_head *b = d_hash(entry->d_name.hash);
257761647823SNeilBrown
25781879fd6aSChristoph Hellwig hlist_bl_lock(b);
2579b07ad996SLinus Torvalds hlist_bl_add_head_rcu(&entry->d_hash, b);
25801879fd6aSChristoph Hellwig hlist_bl_unlock(b);
25811da177e4SLinus Torvalds }
25821da177e4SLinus Torvalds
25831da177e4SLinus Torvalds /**
25841da177e4SLinus Torvalds * d_rehash - add an entry back to the hash
25851da177e4SLinus Torvalds * @entry: dentry to add to the hash
25861da177e4SLinus Torvalds *
25871da177e4SLinus Torvalds * Adds a dentry to the hash according to its name.
25881da177e4SLinus Torvalds */
25891da177e4SLinus Torvalds
d_rehash(struct dentry * entry)25901da177e4SLinus Torvalds void d_rehash(struct dentry * entry)
25911da177e4SLinus Torvalds {
25921da177e4SLinus Torvalds spin_lock(&entry->d_lock);
259315d3c589SAl Viro __d_rehash(entry);
25941da177e4SLinus Torvalds spin_unlock(&entry->d_lock);
25951da177e4SLinus Torvalds }
2596ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_rehash);
25971da177e4SLinus Torvalds
start_dir_add(struct inode * dir)259884e710daSAl Viro static inline unsigned start_dir_add(struct inode *dir)
259984e710daSAl Viro {
260093f6d4e1SThomas Gleixner preempt_disable_nested();
260184e710daSAl Viro for (;;) {
260284e710daSAl Viro unsigned n = dir->i_dir_seq;
260384e710daSAl Viro if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n)
260484e710daSAl Viro return n;
260584e710daSAl Viro cpu_relax();
260684e710daSAl Viro }
260784e710daSAl Viro }
260884e710daSAl Viro
end_dir_add(struct inode * dir,unsigned int n,wait_queue_head_t * d_wait)260950417d22SSebastian Andrzej Siewior static inline void end_dir_add(struct inode *dir, unsigned int n,
261050417d22SSebastian Andrzej Siewior wait_queue_head_t *d_wait)
261184e710daSAl Viro {
261284e710daSAl Viro smp_store_release(&dir->i_dir_seq, n + 2);
261393f6d4e1SThomas Gleixner preempt_enable_nested();
261450417d22SSebastian Andrzej Siewior wake_up_all(d_wait);
261584e710daSAl Viro }
261684e710daSAl Viro
d_wait_lookup(struct dentry * dentry)2617d9171b93SAl Viro static void d_wait_lookup(struct dentry *dentry)
2618d9171b93SAl Viro {
2619d9171b93SAl Viro if (d_in_lookup(dentry)) {
2620d9171b93SAl Viro DECLARE_WAITQUEUE(wait, current);
2621d9171b93SAl Viro add_wait_queue(dentry->d_wait, &wait);
2622d9171b93SAl Viro do {
2623d9171b93SAl Viro set_current_state(TASK_UNINTERRUPTIBLE);
2624d9171b93SAl Viro spin_unlock(&dentry->d_lock);
2625d9171b93SAl Viro schedule();
2626d9171b93SAl Viro spin_lock(&dentry->d_lock);
2627d9171b93SAl Viro } while (d_in_lookup(dentry));
2628d9171b93SAl Viro }
2629d9171b93SAl Viro }
2630d9171b93SAl Viro
d_alloc_parallel(struct dentry * parent,const struct qstr * name,wait_queue_head_t * wq)263194bdd655SAl Viro struct dentry *d_alloc_parallel(struct dentry *parent,
2632d9171b93SAl Viro const struct qstr *name,
2633d9171b93SAl Viro wait_queue_head_t *wq)
263494bdd655SAl Viro {
263594bdd655SAl Viro unsigned int hash = name->hash;
263694bdd655SAl Viro struct hlist_bl_head *b = in_lookup_hash(parent, hash);
263794bdd655SAl Viro struct hlist_bl_node *node;
263894bdd655SAl Viro struct dentry *new = d_alloc(parent, name);
263994bdd655SAl Viro struct dentry *dentry;
264094bdd655SAl Viro unsigned seq, r_seq, d_seq;
264194bdd655SAl Viro
264294bdd655SAl Viro if (unlikely(!new))
264394bdd655SAl Viro return ERR_PTR(-ENOMEM);
264494bdd655SAl Viro
264594bdd655SAl Viro retry:
264694bdd655SAl Viro rcu_read_lock();
2647015555fdSWill Deacon seq = smp_load_acquire(&parent->d_inode->i_dir_seq);
264894bdd655SAl Viro r_seq = read_seqbegin(&rename_lock);
264994bdd655SAl Viro dentry = __d_lookup_rcu(parent, name, &d_seq);
265094bdd655SAl Viro if (unlikely(dentry)) {
265194bdd655SAl Viro if (!lockref_get_not_dead(&dentry->d_lockref)) {
265294bdd655SAl Viro rcu_read_unlock();
265394bdd655SAl Viro goto retry;
265494bdd655SAl Viro }
265594bdd655SAl Viro if (read_seqcount_retry(&dentry->d_seq, d_seq)) {
265694bdd655SAl Viro rcu_read_unlock();
265794bdd655SAl Viro dput(dentry);
265894bdd655SAl Viro goto retry;
265994bdd655SAl Viro }
266094bdd655SAl Viro rcu_read_unlock();
266194bdd655SAl Viro dput(new);
266294bdd655SAl Viro return dentry;
266394bdd655SAl Viro }
266494bdd655SAl Viro if (unlikely(read_seqretry(&rename_lock, r_seq))) {
266594bdd655SAl Viro rcu_read_unlock();
266694bdd655SAl Viro goto retry;
266794bdd655SAl Viro }
2668015555fdSWill Deacon
2669015555fdSWill Deacon if (unlikely(seq & 1)) {
2670015555fdSWill Deacon rcu_read_unlock();
2671015555fdSWill Deacon goto retry;
2672015555fdSWill Deacon }
2673015555fdSWill Deacon
267494bdd655SAl Viro hlist_bl_lock(b);
26758cc07c80SWill Deacon if (unlikely(READ_ONCE(parent->d_inode->i_dir_seq) != seq)) {
267694bdd655SAl Viro hlist_bl_unlock(b);
267794bdd655SAl Viro rcu_read_unlock();
267894bdd655SAl Viro goto retry;
267994bdd655SAl Viro }
268094bdd655SAl Viro /*
268194bdd655SAl Viro * No changes for the parent since the beginning of d_lookup().
268294bdd655SAl Viro * Since all removals from the chain happen with hlist_bl_lock(),
268394bdd655SAl Viro * any potential in-lookup matches are going to stay here until
268494bdd655SAl Viro * we unlock the chain. All fields are stable in everything
268594bdd655SAl Viro * we encounter.
268694bdd655SAl Viro */
268794bdd655SAl Viro hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) {
268894bdd655SAl Viro if (dentry->d_name.hash != hash)
268994bdd655SAl Viro continue;
269094bdd655SAl Viro if (dentry->d_parent != parent)
269194bdd655SAl Viro continue;
2692d4c91a8fSAl Viro if (!d_same_name(dentry, parent, name))
269394bdd655SAl Viro continue;
269494bdd655SAl Viro hlist_bl_unlock(b);
2695e7d6ef97SAl Viro /* now we can try to grab a reference */
2696e7d6ef97SAl Viro if (!lockref_get_not_dead(&dentry->d_lockref)) {
2697e7d6ef97SAl Viro rcu_read_unlock();
2698e7d6ef97SAl Viro goto retry;
2699e7d6ef97SAl Viro }
2700e7d6ef97SAl Viro
2701e7d6ef97SAl Viro rcu_read_unlock();
2702e7d6ef97SAl Viro /*
2703e7d6ef97SAl Viro * somebody is likely to be still doing lookup for it;
2704e7d6ef97SAl Viro * wait for them to finish
2705e7d6ef97SAl Viro */
2706d9171b93SAl Viro spin_lock(&dentry->d_lock);
2707d9171b93SAl Viro d_wait_lookup(dentry);
2708d9171b93SAl Viro /*
2709d9171b93SAl Viro * it's not in-lookup anymore; in principle we should repeat
2710d9171b93SAl Viro * everything from dcache lookup, but it's likely to be what
2711d9171b93SAl Viro * d_lookup() would've found anyway. If it is, just return it;
2712d9171b93SAl Viro * otherwise we really have to repeat the whole thing.
2713d9171b93SAl Viro */
2714d9171b93SAl Viro if (unlikely(dentry->d_name.hash != hash))
2715d9171b93SAl Viro goto mismatch;
2716d9171b93SAl Viro if (unlikely(dentry->d_parent != parent))
2717d9171b93SAl Viro goto mismatch;
2718d9171b93SAl Viro if (unlikely(d_unhashed(dentry)))
2719d9171b93SAl Viro goto mismatch;
2720d4c91a8fSAl Viro if (unlikely(!d_same_name(dentry, parent, name)))
2721d9171b93SAl Viro goto mismatch;
2722d9171b93SAl Viro /* OK, it *is* a hashed match; return it */
2723d9171b93SAl Viro spin_unlock(&dentry->d_lock);
272494bdd655SAl Viro dput(new);
272594bdd655SAl Viro return dentry;
272694bdd655SAl Viro }
2727e7d6ef97SAl Viro rcu_read_unlock();
272894bdd655SAl Viro /* we can't take ->d_lock here; it's OK, though. */
272994bdd655SAl Viro new->d_flags |= DCACHE_PAR_LOOKUP;
2730d9171b93SAl Viro new->d_wait = wq;
273194bdd655SAl Viro hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b);
273294bdd655SAl Viro hlist_bl_unlock(b);
273394bdd655SAl Viro return new;
2734d9171b93SAl Viro mismatch:
2735d9171b93SAl Viro spin_unlock(&dentry->d_lock);
2736d9171b93SAl Viro dput(dentry);
2737d9171b93SAl Viro goto retry;
273894bdd655SAl Viro }
273994bdd655SAl Viro EXPORT_SYMBOL(d_alloc_parallel);
274094bdd655SAl Viro
274145f78b0aSSebastian Andrzej Siewior /*
274245f78b0aSSebastian Andrzej Siewior * - Unhash the dentry
274345f78b0aSSebastian Andrzej Siewior * - Retrieve and clear the waitqueue head in dentry
274445f78b0aSSebastian Andrzej Siewior * - Return the waitqueue head
274545f78b0aSSebastian Andrzej Siewior */
__d_lookup_unhash(struct dentry * dentry)274645f78b0aSSebastian Andrzej Siewior static wait_queue_head_t *__d_lookup_unhash(struct dentry *dentry)
274785c7f810SAl Viro {
274845f78b0aSSebastian Andrzej Siewior wait_queue_head_t *d_wait;
274945f78b0aSSebastian Andrzej Siewior struct hlist_bl_head *b;
275045f78b0aSSebastian Andrzej Siewior
275145f78b0aSSebastian Andrzej Siewior lockdep_assert_held(&dentry->d_lock);
275245f78b0aSSebastian Andrzej Siewior
275345f78b0aSSebastian Andrzej Siewior b = in_lookup_hash(dentry->d_parent, dentry->d_name.hash);
275494bdd655SAl Viro hlist_bl_lock(b);
275585c7f810SAl Viro dentry->d_flags &= ~DCACHE_PAR_LOOKUP;
275694bdd655SAl Viro __hlist_bl_del(&dentry->d_u.d_in_lookup_hash);
275745f78b0aSSebastian Andrzej Siewior d_wait = dentry->d_wait;
2758d9171b93SAl Viro dentry->d_wait = NULL;
275994bdd655SAl Viro hlist_bl_unlock(b);
276094bdd655SAl Viro INIT_HLIST_NODE(&dentry->d_u.d_alias);
2761d9171b93SAl Viro INIT_LIST_HEAD(&dentry->d_lru);
276245f78b0aSSebastian Andrzej Siewior return d_wait;
276385c7f810SAl Viro }
276445f78b0aSSebastian Andrzej Siewior
__d_lookup_unhash_wake(struct dentry * dentry)276545f78b0aSSebastian Andrzej Siewior void __d_lookup_unhash_wake(struct dentry *dentry)
276645f78b0aSSebastian Andrzej Siewior {
276745f78b0aSSebastian Andrzej Siewior spin_lock(&dentry->d_lock);
276845f78b0aSSebastian Andrzej Siewior wake_up_all(__d_lookup_unhash(dentry));
276945f78b0aSSebastian Andrzej Siewior spin_unlock(&dentry->d_lock);
277045f78b0aSSebastian Andrzej Siewior }
277145f78b0aSSebastian Andrzej Siewior EXPORT_SYMBOL(__d_lookup_unhash_wake);
2772ed782b5aSAl Viro
2773ed782b5aSAl Viro /* inode->i_lock held if inode is non-NULL */
2774ed782b5aSAl Viro
__d_add(struct dentry * dentry,struct inode * inode)2775ed782b5aSAl Viro static inline void __d_add(struct dentry *dentry, struct inode *inode)
2776ed782b5aSAl Viro {
277745f78b0aSSebastian Andrzej Siewior wait_queue_head_t *d_wait;
277884e710daSAl Viro struct inode *dir = NULL;
277984e710daSAl Viro unsigned n;
27800568d705SAl Viro spin_lock(&dentry->d_lock);
278184e710daSAl Viro if (unlikely(d_in_lookup(dentry))) {
278284e710daSAl Viro dir = dentry->d_parent->d_inode;
278384e710daSAl Viro n = start_dir_add(dir);
278445f78b0aSSebastian Andrzej Siewior d_wait = __d_lookup_unhash(dentry);
278584e710daSAl Viro }
2786ed782b5aSAl Viro if (inode) {
27870568d705SAl Viro unsigned add_flags = d_flags_for_inode(inode);
27880568d705SAl Viro hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry);
27890568d705SAl Viro raw_write_seqcount_begin(&dentry->d_seq);
27900568d705SAl Viro __d_set_inode_and_type(dentry, inode, add_flags);
27910568d705SAl Viro raw_write_seqcount_end(&dentry->d_seq);
2792affda484SAl Viro fsnotify_update_flags(dentry);
2793ed782b5aSAl Viro }
279415d3c589SAl Viro __d_rehash(dentry);
279584e710daSAl Viro if (dir)
279650417d22SSebastian Andrzej Siewior end_dir_add(dir, n, d_wait);
27970568d705SAl Viro spin_unlock(&dentry->d_lock);
27980568d705SAl Viro if (inode)
27990568d705SAl Viro spin_unlock(&inode->i_lock);
2800ed782b5aSAl Viro }
2801ed782b5aSAl Viro
2802fb2d5b86SNick Piggin /**
280334d0d19dSAl Viro * d_add - add dentry to hash queues
280434d0d19dSAl Viro * @entry: dentry to add
280534d0d19dSAl Viro * @inode: The inode to attach to this dentry
280634d0d19dSAl Viro *
280734d0d19dSAl Viro * This adds the entry to the hash queues and initializes @inode.
280834d0d19dSAl Viro * The entry was actually filled in earlier during d_alloc().
280934d0d19dSAl Viro */
281034d0d19dSAl Viro
d_add(struct dentry * entry,struct inode * inode)281134d0d19dSAl Viro void d_add(struct dentry *entry, struct inode *inode)
281234d0d19dSAl Viro {
2813b9680917SAl Viro if (inode) {
2814b9680917SAl Viro security_d_instantiate(entry, inode);
2815ed782b5aSAl Viro spin_lock(&inode->i_lock);
2816b9680917SAl Viro }
2817ed782b5aSAl Viro __d_add(entry, inode);
281834d0d19dSAl Viro }
281934d0d19dSAl Viro EXPORT_SYMBOL(d_add);
282034d0d19dSAl Viro
282134d0d19dSAl Viro /**
2822668d0cd5SAl Viro * d_exact_alias - find and hash an exact unhashed alias
2823668d0cd5SAl Viro * @entry: dentry to add
2824668d0cd5SAl Viro * @inode: The inode to go with this dentry
2825668d0cd5SAl Viro *
2826668d0cd5SAl Viro * If an unhashed dentry with the same name/parent and desired
2827668d0cd5SAl Viro * inode already exists, hash and return it. Otherwise, return
2828668d0cd5SAl Viro * NULL.
2829668d0cd5SAl Viro *
2830668d0cd5SAl Viro * Parent directory should be locked.
2831668d0cd5SAl Viro */
d_exact_alias(struct dentry * entry,struct inode * inode)2832668d0cd5SAl Viro struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode)
2833668d0cd5SAl Viro {
2834668d0cd5SAl Viro struct dentry *alias;
2835668d0cd5SAl Viro unsigned int hash = entry->d_name.hash;
2836668d0cd5SAl Viro
2837668d0cd5SAl Viro spin_lock(&inode->i_lock);
2838668d0cd5SAl Viro hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) {
2839668d0cd5SAl Viro /*
2840668d0cd5SAl Viro * Don't need alias->d_lock here, because aliases with
2841668d0cd5SAl Viro * d_parent == entry->d_parent are not subject to name or
2842668d0cd5SAl Viro * parent changes, because the parent inode i_mutex is held.
2843668d0cd5SAl Viro */
2844668d0cd5SAl Viro if (alias->d_name.hash != hash)
2845668d0cd5SAl Viro continue;
2846668d0cd5SAl Viro if (alias->d_parent != entry->d_parent)
2847668d0cd5SAl Viro continue;
2848d4c91a8fSAl Viro if (!d_same_name(alias, entry->d_parent, &entry->d_name))
2849668d0cd5SAl Viro continue;
2850668d0cd5SAl Viro spin_lock(&alias->d_lock);
2851668d0cd5SAl Viro if (!d_unhashed(alias)) {
2852668d0cd5SAl Viro spin_unlock(&alias->d_lock);
2853668d0cd5SAl Viro alias = NULL;
2854668d0cd5SAl Viro } else {
2855668d0cd5SAl Viro __dget_dlock(alias);
285615d3c589SAl Viro __d_rehash(alias);
2857668d0cd5SAl Viro spin_unlock(&alias->d_lock);
2858668d0cd5SAl Viro }
2859668d0cd5SAl Viro spin_unlock(&inode->i_lock);
2860668d0cd5SAl Viro return alias;
2861668d0cd5SAl Viro }
2862668d0cd5SAl Viro spin_unlock(&inode->i_lock);
2863668d0cd5SAl Viro return NULL;
2864668d0cd5SAl Viro }
2865668d0cd5SAl Viro EXPORT_SYMBOL(d_exact_alias);
2866668d0cd5SAl Viro
swap_names(struct dentry * dentry,struct dentry * target)28678d85b484SAl Viro static void swap_names(struct dentry *dentry, struct dentry *target)
28681da177e4SLinus Torvalds {
28698d85b484SAl Viro if (unlikely(dname_external(target))) {
28708d85b484SAl Viro if (unlikely(dname_external(dentry))) {
28711da177e4SLinus Torvalds /*
28721da177e4SLinus Torvalds * Both external: swap the pointers
28731da177e4SLinus Torvalds */
28749a8d5bb4SWu Fengguang swap(target->d_name.name, dentry->d_name.name);
28751da177e4SLinus Torvalds } else {
28761da177e4SLinus Torvalds /*
28771da177e4SLinus Torvalds * dentry:internal, target:external. Steal target's
28781da177e4SLinus Torvalds * storage and make target internal.
28791da177e4SLinus Torvalds */
2880321bcf92SJ. Bruce Fields memcpy(target->d_iname, dentry->d_name.name,
2881321bcf92SJ. Bruce Fields dentry->d_name.len + 1);
28821da177e4SLinus Torvalds dentry->d_name.name = target->d_name.name;
28831da177e4SLinus Torvalds target->d_name.name = target->d_iname;
28841da177e4SLinus Torvalds }
28851da177e4SLinus Torvalds } else {
28868d85b484SAl Viro if (unlikely(dname_external(dentry))) {
28871da177e4SLinus Torvalds /*
28881da177e4SLinus Torvalds * dentry:external, target:internal. Give dentry's
28891da177e4SLinus Torvalds * storage to target and make dentry internal
28901da177e4SLinus Torvalds */
28911da177e4SLinus Torvalds memcpy(dentry->d_iname, target->d_name.name,
28921da177e4SLinus Torvalds target->d_name.len + 1);
28931da177e4SLinus Torvalds target->d_name.name = dentry->d_name.name;
28941da177e4SLinus Torvalds dentry->d_name.name = dentry->d_iname;
28951da177e4SLinus Torvalds } else {
28961da177e4SLinus Torvalds /*
2897da1ce067SMiklos Szeredi * Both are internal.
28981da177e4SLinus Torvalds */
2899da1ce067SMiklos Szeredi unsigned int i;
2900da1ce067SMiklos Szeredi BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long)));
2901da1ce067SMiklos Szeredi for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) {
2902da1ce067SMiklos Szeredi swap(((long *) &dentry->d_iname)[i],
2903da1ce067SMiklos Szeredi ((long *) &target->d_iname)[i]);
2904da1ce067SMiklos Szeredi }
29051da177e4SLinus Torvalds }
29061da177e4SLinus Torvalds }
2907a28ddb87SLinus Torvalds swap(dentry->d_name.hash_len, target->d_name.hash_len);
29081da177e4SLinus Torvalds }
29091da177e4SLinus Torvalds
copy_name(struct dentry * dentry,struct dentry * target)29108d85b484SAl Viro static void copy_name(struct dentry *dentry, struct dentry *target)
29118d85b484SAl Viro {
29128d85b484SAl Viro struct external_name *old_name = NULL;
29138d85b484SAl Viro if (unlikely(dname_external(dentry)))
29148d85b484SAl Viro old_name = external_name(dentry);
29158d85b484SAl Viro if (unlikely(dname_external(target))) {
29168d85b484SAl Viro atomic_inc(&external_name(target)->u.count);
29178d85b484SAl Viro dentry->d_name = target->d_name;
29188d85b484SAl Viro } else {
29198d85b484SAl Viro memcpy(dentry->d_iname, target->d_name.name,
29208d85b484SAl Viro target->d_name.len + 1);
29218d85b484SAl Viro dentry->d_name.name = dentry->d_iname;
29228d85b484SAl Viro dentry->d_name.hash_len = target->d_name.hash_len;
29238d85b484SAl Viro }
29248d85b484SAl Viro if (old_name && likely(atomic_dec_and_test(&old_name->u.count)))
29252e03b4bcSVlastimil Babka kfree_rcu(old_name, u.head);
29268d85b484SAl Viro }
29278d85b484SAl Viro
29289eaef27bSTrond Myklebust /*
292918367501SAl Viro * __d_move - move a dentry
29301da177e4SLinus Torvalds * @dentry: entry to move
29311da177e4SLinus Torvalds * @target: new dentry
2932da1ce067SMiklos Szeredi * @exchange: exchange the two dentries
29331da177e4SLinus Torvalds *
29341da177e4SLinus Torvalds * Update the dcache to reflect the move of a file name. Negative
2935c46c8877SJeff Layton * dcache entries should not be moved in this way. Caller must hold
2936c46c8877SJeff Layton * rename_lock, the i_mutex of the source and target directories,
2937c46c8877SJeff Layton * and the sb->s_vfs_rename_mutex if they differ. See lock_rename().
29381da177e4SLinus Torvalds */
__d_move(struct dentry * dentry,struct dentry * target,bool exchange)2939da1ce067SMiklos Szeredi static void __d_move(struct dentry *dentry, struct dentry *target,
2940da1ce067SMiklos Szeredi bool exchange)
29411da177e4SLinus Torvalds {
294242177007SAl Viro struct dentry *old_parent, *p;
294345f78b0aSSebastian Andrzej Siewior wait_queue_head_t *d_wait;
294484e710daSAl Viro struct inode *dir = NULL;
294584e710daSAl Viro unsigned n;
29461da177e4SLinus Torvalds
294742177007SAl Viro WARN_ON(!dentry->d_inode);
294842177007SAl Viro if (WARN_ON(dentry == target))
294942177007SAl Viro return;
295042177007SAl Viro
29512fd6b7f5SNick Piggin BUG_ON(d_ancestor(target, dentry));
295242177007SAl Viro old_parent = dentry->d_parent;
295342177007SAl Viro p = d_ancestor(old_parent, target);
295442177007SAl Viro if (IS_ROOT(dentry)) {
295542177007SAl Viro BUG_ON(p);
295642177007SAl Viro spin_lock(&target->d_parent->d_lock);
295742177007SAl Viro } else if (!p) {
295842177007SAl Viro /* target is not a descendent of dentry->d_parent */
295942177007SAl Viro spin_lock(&target->d_parent->d_lock);
296042177007SAl Viro spin_lock_nested(&old_parent->d_lock, DENTRY_D_LOCK_NESTED);
296142177007SAl Viro } else {
296242177007SAl Viro BUG_ON(p == dentry);
296342177007SAl Viro spin_lock(&old_parent->d_lock);
296442177007SAl Viro if (p != target)
296542177007SAl Viro spin_lock_nested(&target->d_parent->d_lock,
296642177007SAl Viro DENTRY_D_LOCK_NESTED);
296742177007SAl Viro }
296842177007SAl Viro spin_lock_nested(&dentry->d_lock, 2);
296942177007SAl Viro spin_lock_nested(&target->d_lock, 3);
29702fd6b7f5SNick Piggin
297184e710daSAl Viro if (unlikely(d_in_lookup(target))) {
297284e710daSAl Viro dir = target->d_parent->d_inode;
297384e710daSAl Viro n = start_dir_add(dir);
297445f78b0aSSebastian Andrzej Siewior d_wait = __d_lookup_unhash(target);
297584e710daSAl Viro }
29761da177e4SLinus Torvalds
297731e6b01fSNick Piggin write_seqcount_begin(&dentry->d_seq);
29781ca7d67cSJohn Stultz write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED);
297931e6b01fSNick Piggin
298015d3c589SAl Viro /* unhash both */
29810632a9acSAl Viro if (!d_unhashed(dentry))
298261647823SNeilBrown ___d_drop(dentry);
29830632a9acSAl Viro if (!d_unhashed(target))
298461647823SNeilBrown ___d_drop(target);
29851da177e4SLinus Torvalds
298663cf427aSAl Viro /* ... and switch them in the tree */
29871da177e4SLinus Torvalds dentry->d_parent = target->d_parent;
2988076515fcSAl Viro if (!exchange) {
2989076515fcSAl Viro copy_name(dentry, target);
2990076515fcSAl Viro target->d_hash.pprev = NULL;
2991076515fcSAl Viro dentry->d_parent->d_lockref.count++;
29925467a68cSAl Viro if (dentry != old_parent) /* wasn't IS_ROOT */
2993076515fcSAl Viro WARN_ON(!--old_parent->d_lockref.count);
299463cf427aSAl Viro } else {
2995076515fcSAl Viro target->d_parent = old_parent;
2996076515fcSAl Viro swap_names(dentry, target);
2997946e51f2SAl Viro list_move(&target->d_child, &target->d_parent->d_subdirs);
2998076515fcSAl Viro __d_rehash(target);
2999affda484SAl Viro fsnotify_update_flags(target);
300063cf427aSAl Viro }
3001076515fcSAl Viro list_move(&dentry->d_child, &dentry->d_parent->d_subdirs);
3002076515fcSAl Viro __d_rehash(dentry);
3003076515fcSAl Viro fsnotify_update_flags(dentry);
30040bf3d5c1SEric Biggers fscrypt_handle_d_move(dentry);
30052fd6b7f5SNick Piggin
300631e6b01fSNick Piggin write_seqcount_end(&target->d_seq);
300731e6b01fSNick Piggin write_seqcount_end(&dentry->d_seq);
300831e6b01fSNick Piggin
300984e710daSAl Viro if (dir)
301050417d22SSebastian Andrzej Siewior end_dir_add(dir, n, d_wait);
3011076515fcSAl Viro
3012076515fcSAl Viro if (dentry->d_parent != old_parent)
3013076515fcSAl Viro spin_unlock(&dentry->d_parent->d_lock);
3014076515fcSAl Viro if (dentry != old_parent)
3015076515fcSAl Viro spin_unlock(&old_parent->d_lock);
3016076515fcSAl Viro spin_unlock(&target->d_lock);
3017076515fcSAl Viro spin_unlock(&dentry->d_lock);
301818367501SAl Viro }
301918367501SAl Viro
302018367501SAl Viro /*
302118367501SAl Viro * d_move - move a dentry
302218367501SAl Viro * @dentry: entry to move
302318367501SAl Viro * @target: new dentry
302418367501SAl Viro *
302518367501SAl Viro * Update the dcache to reflect the move of a file name. Negative
3026c46c8877SJeff Layton * dcache entries should not be moved in this way. See the locking
3027c46c8877SJeff Layton * requirements for __d_move.
302818367501SAl Viro */
d_move(struct dentry * dentry,struct dentry * target)302918367501SAl Viro void d_move(struct dentry *dentry, struct dentry *target)
303018367501SAl Viro {
303118367501SAl Viro write_seqlock(&rename_lock);
3032da1ce067SMiklos Szeredi __d_move(dentry, target, false);
30331da177e4SLinus Torvalds write_sequnlock(&rename_lock);
30349eaef27bSTrond Myklebust }
3035ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(d_move);
30361da177e4SLinus Torvalds
3037da1ce067SMiklos Szeredi /*
3038da1ce067SMiklos Szeredi * d_exchange - exchange two dentries
3039da1ce067SMiklos Szeredi * @dentry1: first dentry
3040da1ce067SMiklos Szeredi * @dentry2: second dentry
3041da1ce067SMiklos Szeredi */
d_exchange(struct dentry * dentry1,struct dentry * dentry2)3042da1ce067SMiklos Szeredi void d_exchange(struct dentry *dentry1, struct dentry *dentry2)
3043da1ce067SMiklos Szeredi {
3044da1ce067SMiklos Szeredi write_seqlock(&rename_lock);
3045da1ce067SMiklos Szeredi
3046da1ce067SMiklos Szeredi WARN_ON(!dentry1->d_inode);
3047da1ce067SMiklos Szeredi WARN_ON(!dentry2->d_inode);
3048da1ce067SMiklos Szeredi WARN_ON(IS_ROOT(dentry1));
3049da1ce067SMiklos Szeredi WARN_ON(IS_ROOT(dentry2));
3050da1ce067SMiklos Szeredi
3051da1ce067SMiklos Szeredi __d_move(dentry1, dentry2, true);
3052da1ce067SMiklos Szeredi
3053da1ce067SMiklos Szeredi write_sequnlock(&rename_lock);
3054da1ce067SMiklos Szeredi }
3055da1ce067SMiklos Szeredi
3056e2761a11SOGAWA Hirofumi /**
3057e2761a11SOGAWA Hirofumi * d_ancestor - search for an ancestor
3058e2761a11SOGAWA Hirofumi * @p1: ancestor dentry
3059e2761a11SOGAWA Hirofumi * @p2: child dentry
3060e2761a11SOGAWA Hirofumi *
3061e2761a11SOGAWA Hirofumi * Returns the ancestor dentry of p2 which is a child of p1, if p1 is
3062e2761a11SOGAWA Hirofumi * an ancestor of p2, else NULL.
30639eaef27bSTrond Myklebust */
d_ancestor(struct dentry * p1,struct dentry * p2)3064e2761a11SOGAWA Hirofumi struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2)
30659eaef27bSTrond Myklebust {
30669eaef27bSTrond Myklebust struct dentry *p;
30679eaef27bSTrond Myklebust
3068871c0067SOGAWA Hirofumi for (p = p2; !IS_ROOT(p); p = p->d_parent) {
30699eaef27bSTrond Myklebust if (p->d_parent == p1)
3070e2761a11SOGAWA Hirofumi return p;
30719eaef27bSTrond Myklebust }
3072e2761a11SOGAWA Hirofumi return NULL;
30739eaef27bSTrond Myklebust }
30749eaef27bSTrond Myklebust
30759eaef27bSTrond Myklebust /*
30769eaef27bSTrond Myklebust * This helper attempts to cope with remotely renamed directories
30779eaef27bSTrond Myklebust *
30789eaef27bSTrond Myklebust * It assumes that the caller is already holding
3079a03e283bSEric W. Biederman * dentry->d_parent->d_inode->i_mutex, and rename_lock
30809eaef27bSTrond Myklebust *
30819eaef27bSTrond Myklebust * Note: If ever the locking in lock_rename() changes, then please
30829eaef27bSTrond Myklebust * remember to update this too...
30839eaef27bSTrond Myklebust */
__d_unalias(struct inode * inode,struct dentry * dentry,struct dentry * alias)3084b5ae6b15SAl Viro static int __d_unalias(struct inode *inode,
3085873feea0SNick Piggin struct dentry *dentry, struct dentry *alias)
30869eaef27bSTrond Myklebust {
30879902af79SAl Viro struct mutex *m1 = NULL;
30889902af79SAl Viro struct rw_semaphore *m2 = NULL;
30893d330dc1SJ. Bruce Fields int ret = -ESTALE;
30909eaef27bSTrond Myklebust
30919eaef27bSTrond Myklebust /* If alias and dentry share a parent, then no extra locks required */
30929eaef27bSTrond Myklebust if (alias->d_parent == dentry->d_parent)
30939eaef27bSTrond Myklebust goto out_unalias;
30949eaef27bSTrond Myklebust
30959eaef27bSTrond Myklebust /* See lock_rename() */
30969eaef27bSTrond Myklebust if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex))
30979eaef27bSTrond Myklebust goto out_err;
30989eaef27bSTrond Myklebust m1 = &dentry->d_sb->s_vfs_rename_mutex;
30999902af79SAl Viro if (!inode_trylock_shared(alias->d_parent->d_inode))
31009eaef27bSTrond Myklebust goto out_err;
31019902af79SAl Viro m2 = &alias->d_parent->d_inode->i_rwsem;
31029eaef27bSTrond Myklebust out_unalias:
3103da1ce067SMiklos Szeredi __d_move(alias, dentry, false);
3104b5ae6b15SAl Viro ret = 0;
31059eaef27bSTrond Myklebust out_err:
31069eaef27bSTrond Myklebust if (m2)
31079902af79SAl Viro up_read(m2);
31089eaef27bSTrond Myklebust if (m1)
31099eaef27bSTrond Myklebust mutex_unlock(m1);
31109eaef27bSTrond Myklebust return ret;
31119eaef27bSTrond Myklebust }
31129eaef27bSTrond Myklebust
3113770bfad8SDavid Howells /**
31143f70bd51SJ. Bruce Fields * d_splice_alias - splice a disconnected dentry into the tree if one exists
31153f70bd51SJ. Bruce Fields * @inode: the inode which may have a disconnected dentry
31163f70bd51SJ. Bruce Fields * @dentry: a negative dentry which we want to point to the inode.
31173f70bd51SJ. Bruce Fields *
3118da093a9bSJ. Bruce Fields * If inode is a directory and has an IS_ROOT alias, then d_move that in
3119da093a9bSJ. Bruce Fields * place of the given dentry and return it, else simply d_add the inode
3120da093a9bSJ. Bruce Fields * to the dentry and return NULL.
31213f70bd51SJ. Bruce Fields *
3122908790faSJ. Bruce Fields * If a non-IS_ROOT directory is found, the filesystem is corrupt, and
3123908790faSJ. Bruce Fields * we should error out: directories can't have multiple aliases.
3124908790faSJ. Bruce Fields *
31253f70bd51SJ. Bruce Fields * This is needed in the lookup routine of any filesystem that is exportable
31263f70bd51SJ. Bruce Fields * (via knfsd) so that we can build dcache paths to directories effectively.
31273f70bd51SJ. Bruce Fields *
31283f70bd51SJ. Bruce Fields * If a dentry was found and moved, then it is returned. Otherwise NULL
31293f70bd51SJ. Bruce Fields * is returned. This matches the expected return value of ->lookup.
31303f70bd51SJ. Bruce Fields *
31313f70bd51SJ. Bruce Fields * Cluster filesystems may call this function with a negative, hashed dentry.
31323f70bd51SJ. Bruce Fields * In that case, we know that the inode will be a regular file, and also this
31333f70bd51SJ. Bruce Fields * will only occur during atomic_open. So we need to check for the dentry
31343f70bd51SJ. Bruce Fields * being already hashed only in the final case.
31353f70bd51SJ. Bruce Fields */
d_splice_alias(struct inode * inode,struct dentry * dentry)31363f70bd51SJ. Bruce Fields struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry)
31373f70bd51SJ. Bruce Fields {
31383f70bd51SJ. Bruce Fields if (IS_ERR(inode))
31393f70bd51SJ. Bruce Fields return ERR_CAST(inode);
31403f70bd51SJ. Bruce Fields
3141770bfad8SDavid Howells BUG_ON(!d_unhashed(dentry));
3142770bfad8SDavid Howells
3143de689f5eSAl Viro if (!inode)
3144b5ae6b15SAl Viro goto out;
3145de689f5eSAl Viro
3146b9680917SAl Viro security_d_instantiate(dentry, inode);
3147873feea0SNick Piggin spin_lock(&inode->i_lock);
31489eaef27bSTrond Myklebust if (S_ISDIR(inode->i_mode)) {
3149b5ae6b15SAl Viro struct dentry *new = __d_find_any_alias(inode);
3150b5ae6b15SAl Viro if (unlikely(new)) {
3151a03e283bSEric W. Biederman /* The reference to new ensures it remains an alias */
3152a03e283bSEric W. Biederman spin_unlock(&inode->i_lock);
315318367501SAl Viro write_seqlock(&rename_lock);
3154b5ae6b15SAl Viro if (unlikely(d_ancestor(new, dentry))) {
3155b5ae6b15SAl Viro write_sequnlock(&rename_lock);
3156b5ae6b15SAl Viro dput(new);
3157b5ae6b15SAl Viro new = ERR_PTR(-ELOOP);
3158dd179946SDavid Howells pr_warn_ratelimited(
3159dd179946SDavid Howells "VFS: Lookup of '%s' in %s %s"
3160dd179946SDavid Howells " would have caused loop\n",
3161dd179946SDavid Howells dentry->d_name.name,
3162dd179946SDavid Howells inode->i_sb->s_type->name,
3163dd179946SDavid Howells inode->i_sb->s_id);
3164b5ae6b15SAl Viro } else if (!IS_ROOT(new)) {
3165076515fcSAl Viro struct dentry *old_parent = dget(new->d_parent);
3166b5ae6b15SAl Viro int err = __d_unalias(inode, dentry, new);
3167b5ae6b15SAl Viro write_sequnlock(&rename_lock);
3168b5ae6b15SAl Viro if (err) {
3169b5ae6b15SAl Viro dput(new);
3170b5ae6b15SAl Viro new = ERR_PTR(err);
3171dd179946SDavid Howells }
3172076515fcSAl Viro dput(old_parent);
3173b5ae6b15SAl Viro } else {
3174b5ae6b15SAl Viro __d_move(new, dentry, false);
3175b5ae6b15SAl Viro write_sequnlock(&rename_lock);
3176b5ae6b15SAl Viro }
3177b5ae6b15SAl Viro iput(inode);
3178b5ae6b15SAl Viro return new;
3179b5ae6b15SAl Viro }
3180b5ae6b15SAl Viro }
3181b5ae6b15SAl Viro out:
3182ed782b5aSAl Viro __d_add(dentry, inode);
3183770bfad8SDavid Howells return NULL;
3184770bfad8SDavid Howells }
3185b5ae6b15SAl Viro EXPORT_SYMBOL(d_splice_alias);
3186770bfad8SDavid Howells
31871da177e4SLinus Torvalds /*
31881da177e4SLinus Torvalds * Test whether new_dentry is a subdirectory of old_dentry.
31891da177e4SLinus Torvalds *
31901da177e4SLinus Torvalds * Trivially implemented using the dcache structure
31911da177e4SLinus Torvalds */
31921da177e4SLinus Torvalds
31931da177e4SLinus Torvalds /**
31941da177e4SLinus Torvalds * is_subdir - is new dentry a subdirectory of old_dentry
31951da177e4SLinus Torvalds * @new_dentry: new dentry
31961da177e4SLinus Torvalds * @old_dentry: old dentry
31971da177e4SLinus Torvalds *
3198a6e5787fSYaowei Bai * Returns true if new_dentry is a subdirectory of the parent (at any depth).
3199a6e5787fSYaowei Bai * Returns false otherwise.
32001da177e4SLinus Torvalds * Caller must ensure that "new_dentry" is pinned before calling is_subdir()
32011da177e4SLinus Torvalds */
32021da177e4SLinus Torvalds
is_subdir(struct dentry * new_dentry,struct dentry * old_dentry)3203a6e5787fSYaowei Bai bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry)
32041da177e4SLinus Torvalds {
3205a6e5787fSYaowei Bai bool result;
3206949854d0SNick Piggin unsigned seq;
32071da177e4SLinus Torvalds
3208e2761a11SOGAWA Hirofumi if (new_dentry == old_dentry)
3209a6e5787fSYaowei Bai return true;
3210e2761a11SOGAWA Hirofumi
3211949854d0SNick Piggin do {
3212949854d0SNick Piggin /* for restarting inner loop in case of seq retry */
3213949854d0SNick Piggin seq = read_seqbegin(&rename_lock);
3214e2761a11SOGAWA Hirofumi /*
3215e2761a11SOGAWA Hirofumi * Need rcu_readlock to protect against the d_parent trashing
3216e2761a11SOGAWA Hirofumi * due to d_move
32171da177e4SLinus Torvalds */
32181da177e4SLinus Torvalds rcu_read_lock();
3219e2761a11SOGAWA Hirofumi if (d_ancestor(old_dentry, new_dentry))
3220a6e5787fSYaowei Bai result = true;
3221e2761a11SOGAWA Hirofumi else
3222a6e5787fSYaowei Bai result = false;
32231da177e4SLinus Torvalds rcu_read_unlock();
3224949854d0SNick Piggin } while (read_seqretry(&rename_lock, seq));
32251da177e4SLinus Torvalds
32261da177e4SLinus Torvalds return result;
32271da177e4SLinus Torvalds }
3228e8f9e5b7SAmir Goldstein EXPORT_SYMBOL(is_subdir);
32291da177e4SLinus Torvalds
d_genocide_kill(void * data,struct dentry * dentry)3230db14fc3aSMiklos Szeredi static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry)
32311da177e4SLinus Torvalds {
3232db14fc3aSMiklos Szeredi struct dentry *root = data;
3233db14fc3aSMiklos Szeredi if (dentry != root) {
3234db14fc3aSMiklos Szeredi if (d_unhashed(dentry) || !dentry->d_inode)
3235db14fc3aSMiklos Szeredi return D_WALK_SKIP;
32361da177e4SLinus Torvalds
323701ddc4edSMiklos Szeredi if (!(dentry->d_flags & DCACHE_GENOCIDE)) {
323801ddc4edSMiklos Szeredi dentry->d_flags |= DCACHE_GENOCIDE;
323901ddc4edSMiklos Szeredi dentry->d_lockref.count--;
324001ddc4edSMiklos Szeredi }
32411da177e4SLinus Torvalds }
3242db14fc3aSMiklos Szeredi return D_WALK_CONTINUE;
32431da177e4SLinus Torvalds }
324458db63d0SNick Piggin
d_genocide(struct dentry * parent)3245db14fc3aSMiklos Szeredi void d_genocide(struct dentry *parent)
3246db14fc3aSMiklos Szeredi {
32473a8e3611SAl Viro d_walk(parent, parent, d_genocide_kill);
32481da177e4SLinus Torvalds }
32491da177e4SLinus Torvalds
d_tmpfile(struct file * file,struct inode * inode)3250863f144fSMiklos Szeredi void d_tmpfile(struct file *file, struct inode *inode)
32511da177e4SLinus Torvalds {
3252863f144fSMiklos Szeredi struct dentry *dentry = file->f_path.dentry;
3253863f144fSMiklos Szeredi
325460545d0dSAl Viro inode_dec_link_count(inode);
325560545d0dSAl Viro BUG_ON(dentry->d_name.name != dentry->d_iname ||
3256946e51f2SAl Viro !hlist_unhashed(&dentry->d_u.d_alias) ||
325760545d0dSAl Viro !d_unlinked(dentry));
325860545d0dSAl Viro spin_lock(&dentry->d_parent->d_lock);
325960545d0dSAl Viro spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED);
326060545d0dSAl Viro dentry->d_name.len = sprintf(dentry->d_iname, "#%llu",
326160545d0dSAl Viro (unsigned long long)inode->i_ino);
326260545d0dSAl Viro spin_unlock(&dentry->d_lock);
326360545d0dSAl Viro spin_unlock(&dentry->d_parent->d_lock);
326460545d0dSAl Viro d_instantiate(dentry, inode);
32651da177e4SLinus Torvalds }
326660545d0dSAl Viro EXPORT_SYMBOL(d_tmpfile);
32671da177e4SLinus Torvalds
32681da177e4SLinus Torvalds static __initdata unsigned long dhash_entries;
set_dhash_entries(char * str)32691da177e4SLinus Torvalds static int __init set_dhash_entries(char *str)
32701da177e4SLinus Torvalds {
32711da177e4SLinus Torvalds if (!str)
32721da177e4SLinus Torvalds return 0;
32731da177e4SLinus Torvalds dhash_entries = simple_strtoul(str, &str, 0);
32741da177e4SLinus Torvalds return 1;
32751da177e4SLinus Torvalds }
32761da177e4SLinus Torvalds __setup("dhash_entries=", set_dhash_entries);
32771da177e4SLinus Torvalds
dcache_init_early(void)32781da177e4SLinus Torvalds static void __init dcache_init_early(void)
32791da177e4SLinus Torvalds {
32801da177e4SLinus Torvalds /* If hashes are distributed across NUMA nodes, defer
32811da177e4SLinus Torvalds * hash allocation until vmalloc space is available.
32821da177e4SLinus Torvalds */
32831da177e4SLinus Torvalds if (hashdist)
32841da177e4SLinus Torvalds return;
32851da177e4SLinus Torvalds
32861da177e4SLinus Torvalds dentry_hashtable =
32871da177e4SLinus Torvalds alloc_large_system_hash("Dentry cache",
3288b07ad996SLinus Torvalds sizeof(struct hlist_bl_head),
32891da177e4SLinus Torvalds dhash_entries,
32901da177e4SLinus Torvalds 13,
32913d375d78SPavel Tatashin HASH_EARLY | HASH_ZERO,
32921da177e4SLinus Torvalds &d_hash_shift,
3293b35d786bSAlexey Dobriyan NULL,
329431fe62b9STim Bird 0,
32951da177e4SLinus Torvalds 0);
3296854d3e63SAlexey Dobriyan d_hash_shift = 32 - d_hash_shift;
32971da177e4SLinus Torvalds }
32981da177e4SLinus Torvalds
dcache_init(void)329974bf17cfSDenis Cheng static void __init dcache_init(void)
33001da177e4SLinus Torvalds {
33011da177e4SLinus Torvalds /*
33021da177e4SLinus Torvalds * A constructor could be added for stable state like the lists,
33031da177e4SLinus Torvalds * but it is probably not worth it because of the cache nature
33041da177e4SLinus Torvalds * of the dcache.
33051da177e4SLinus Torvalds */
330680344266SDavid Windsor dentry_cache = KMEM_CACHE_USERCOPY(dentry,
330780344266SDavid Windsor SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT,
330880344266SDavid Windsor d_iname);
33091da177e4SLinus Torvalds
33101da177e4SLinus Torvalds /* Hash may have been set up in dcache_init_early */
33111da177e4SLinus Torvalds if (!hashdist)
33121da177e4SLinus Torvalds return;
33131da177e4SLinus Torvalds
33141da177e4SLinus Torvalds dentry_hashtable =
33151da177e4SLinus Torvalds alloc_large_system_hash("Dentry cache",
3316b07ad996SLinus Torvalds sizeof(struct hlist_bl_head),
33171da177e4SLinus Torvalds dhash_entries,
33181da177e4SLinus Torvalds 13,
33193d375d78SPavel Tatashin HASH_ZERO,
33201da177e4SLinus Torvalds &d_hash_shift,
3321b35d786bSAlexey Dobriyan NULL,
332231fe62b9STim Bird 0,
33231da177e4SLinus Torvalds 0);
3324854d3e63SAlexey Dobriyan d_hash_shift = 32 - d_hash_shift;
33251da177e4SLinus Torvalds }
33261da177e4SLinus Torvalds
33271da177e4SLinus Torvalds /* SLAB cache for __getname() consumers */
3328e18b890bSChristoph Lameter struct kmem_cache *names_cachep __read_mostly;
3329ec4f8605SH Hartley Sweeten EXPORT_SYMBOL(names_cachep);
33301da177e4SLinus Torvalds
vfs_caches_init_early(void)33311da177e4SLinus Torvalds void __init vfs_caches_init_early(void)
33321da177e4SLinus Torvalds {
33336916363fSSebastian Andrzej Siewior int i;
33346916363fSSebastian Andrzej Siewior
33356916363fSSebastian Andrzej Siewior for (i = 0; i < ARRAY_SIZE(in_lookup_hashtable); i++)
33366916363fSSebastian Andrzej Siewior INIT_HLIST_BL_HEAD(&in_lookup_hashtable[i]);
33376916363fSSebastian Andrzej Siewior
33381da177e4SLinus Torvalds dcache_init_early();
33391da177e4SLinus Torvalds inode_init_early();
33401da177e4SLinus Torvalds }
33411da177e4SLinus Torvalds
vfs_caches_init(void)33424248b0daSMel Gorman void __init vfs_caches_init(void)
33431da177e4SLinus Torvalds {
33446a9b8820SDavid Windsor names_cachep = kmem_cache_create_usercopy("names_cache", PATH_MAX, 0,
33456a9b8820SDavid Windsor SLAB_HWCACHE_ALIGN|SLAB_PANIC, 0, PATH_MAX, NULL);
33461da177e4SLinus Torvalds
334774bf17cfSDenis Cheng dcache_init();
334874bf17cfSDenis Cheng inode_init();
33494248b0daSMel Gorman files_init();
33504248b0daSMel Gorman files_maxfiles_init();
335174bf17cfSDenis Cheng mnt_init();
33521da177e4SLinus Torvalds bdev_cache_init();
33531da177e4SLinus Torvalds chrdev_init();
33541da177e4SLinus Torvalds }
3355