1 /* 2 * fs/dcache.c 3 * 4 * Complete reimplementation 5 * (C) 1997 Thomas Schoebel-Theuer, 6 * with heavy changes by Linus Torvalds 7 */ 8 9 /* 10 * Notes on the allocation strategy: 11 * 12 * The dcache is a master of the icache - whenever a dcache entry 13 * exists, the inode will always exist. "iput()" is done either when 14 * the dcache entry is deleted or garbage collected. 15 */ 16 17 #include <linux/syscalls.h> 18 #include <linux/string.h> 19 #include <linux/mm.h> 20 #include <linux/fs.h> 21 #include <linux/fsnotify.h> 22 #include <linux/slab.h> 23 #include <linux/init.h> 24 #include <linux/hash.h> 25 #include <linux/cache.h> 26 #include <linux/export.h> 27 #include <linux/mount.h> 28 #include <linux/file.h> 29 #include <linux/uaccess.h> 30 #include <linux/security.h> 31 #include <linux/seqlock.h> 32 #include <linux/swap.h> 33 #include <linux/bootmem.h> 34 #include <linux/fs_struct.h> 35 #include <linux/hardirq.h> 36 #include <linux/bit_spinlock.h> 37 #include <linux/rculist_bl.h> 38 #include <linux/prefetch.h> 39 #include <linux/ratelimit.h> 40 #include <linux/list_lru.h> 41 #include <linux/kasan.h> 42 43 #include "internal.h" 44 #include "mount.h" 45 46 /* 47 * Usage: 48 * dcache->d_inode->i_lock protects: 49 * - i_dentry, d_u.d_alias, d_inode of aliases 50 * dcache_hash_bucket lock protects: 51 * - the dcache hash table 52 * s_anon bl list spinlock protects: 53 * - the s_anon list (see __d_drop) 54 * dentry->d_sb->s_dentry_lru_lock protects: 55 * - the dcache lru lists and counters 56 * d_lock protects: 57 * - d_flags 58 * - d_name 59 * - d_lru 60 * - d_count 61 * - d_unhashed() 62 * - d_parent and d_subdirs 63 * - childrens' d_child and d_parent 64 * - d_u.d_alias, d_inode 65 * 66 * Ordering: 67 * dentry->d_inode->i_lock 68 * dentry->d_lock 69 * dentry->d_sb->s_dentry_lru_lock 70 * dcache_hash_bucket lock 71 * s_anon lock 72 * 73 * If there is an ancestor relationship: 74 * dentry->d_parent->...->d_parent->d_lock 75 * ... 76 * dentry->d_parent->d_lock 77 * dentry->d_lock 78 * 79 * If no ancestor relationship: 80 * if (dentry1 < dentry2) 81 * dentry1->d_lock 82 * dentry2->d_lock 83 */ 84 int sysctl_vfs_cache_pressure __read_mostly = 100; 85 EXPORT_SYMBOL_GPL(sysctl_vfs_cache_pressure); 86 87 __cacheline_aligned_in_smp DEFINE_SEQLOCK(rename_lock); 88 89 EXPORT_SYMBOL(rename_lock); 90 91 static struct kmem_cache *dentry_cache __read_mostly; 92 93 /* 94 * This is the single most critical data structure when it comes 95 * to the dcache: the hashtable for lookups. Somebody should try 96 * to make this good - I've just made it work. 97 * 98 * This hash-function tries to avoid losing too many bits of hash 99 * information, yet avoid using a prime hash-size or similar. 100 */ 101 102 static unsigned int d_hash_mask __read_mostly; 103 static unsigned int d_hash_shift __read_mostly; 104 105 static struct hlist_bl_head *dentry_hashtable __read_mostly; 106 107 static inline struct hlist_bl_head *d_hash(unsigned int hash) 108 { 109 return dentry_hashtable + (hash >> (32 - d_hash_shift)); 110 } 111 112 #define IN_LOOKUP_SHIFT 10 113 static struct hlist_bl_head in_lookup_hashtable[1 << IN_LOOKUP_SHIFT]; 114 115 static inline struct hlist_bl_head *in_lookup_hash(const struct dentry *parent, 116 unsigned int hash) 117 { 118 hash += (unsigned long) parent / L1_CACHE_BYTES; 119 return in_lookup_hashtable + hash_32(hash, IN_LOOKUP_SHIFT); 120 } 121 122 123 /* Statistics gathering. */ 124 struct dentry_stat_t dentry_stat = { 125 .age_limit = 45, 126 }; 127 128 static DEFINE_PER_CPU(long, nr_dentry); 129 static DEFINE_PER_CPU(long, nr_dentry_unused); 130 131 #if defined(CONFIG_SYSCTL) && defined(CONFIG_PROC_FS) 132 133 /* 134 * Here we resort to our own counters instead of using generic per-cpu counters 135 * for consistency with what the vfs inode code does. We are expected to harvest 136 * better code and performance by having our own specialized counters. 137 * 138 * Please note that the loop is done over all possible CPUs, not over all online 139 * CPUs. The reason for this is that we don't want to play games with CPUs going 140 * on and off. If one of them goes off, we will just keep their counters. 141 * 142 * glommer: See cffbc8a for details, and if you ever intend to change this, 143 * please update all vfs counters to match. 144 */ 145 static long get_nr_dentry(void) 146 { 147 int i; 148 long sum = 0; 149 for_each_possible_cpu(i) 150 sum += per_cpu(nr_dentry, i); 151 return sum < 0 ? 0 : sum; 152 } 153 154 static long get_nr_dentry_unused(void) 155 { 156 int i; 157 long sum = 0; 158 for_each_possible_cpu(i) 159 sum += per_cpu(nr_dentry_unused, i); 160 return sum < 0 ? 0 : sum; 161 } 162 163 int proc_nr_dentry(struct ctl_table *table, int write, void __user *buffer, 164 size_t *lenp, loff_t *ppos) 165 { 166 dentry_stat.nr_dentry = get_nr_dentry(); 167 dentry_stat.nr_unused = get_nr_dentry_unused(); 168 return proc_doulongvec_minmax(table, write, buffer, lenp, ppos); 169 } 170 #endif 171 172 /* 173 * Compare 2 name strings, return 0 if they match, otherwise non-zero. 174 * The strings are both count bytes long, and count is non-zero. 175 */ 176 #ifdef CONFIG_DCACHE_WORD_ACCESS 177 178 #include <asm/word-at-a-time.h> 179 /* 180 * NOTE! 'cs' and 'scount' come from a dentry, so it has a 181 * aligned allocation for this particular component. We don't 182 * strictly need the load_unaligned_zeropad() safety, but it 183 * doesn't hurt either. 184 * 185 * In contrast, 'ct' and 'tcount' can be from a pathname, and do 186 * need the careful unaligned handling. 187 */ 188 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) 189 { 190 unsigned long a,b,mask; 191 192 for (;;) { 193 a = *(unsigned long *)cs; 194 b = load_unaligned_zeropad(ct); 195 if (tcount < sizeof(unsigned long)) 196 break; 197 if (unlikely(a != b)) 198 return 1; 199 cs += sizeof(unsigned long); 200 ct += sizeof(unsigned long); 201 tcount -= sizeof(unsigned long); 202 if (!tcount) 203 return 0; 204 } 205 mask = bytemask_from_count(tcount); 206 return unlikely(!!((a ^ b) & mask)); 207 } 208 209 #else 210 211 static inline int dentry_string_cmp(const unsigned char *cs, const unsigned char *ct, unsigned tcount) 212 { 213 do { 214 if (*cs != *ct) 215 return 1; 216 cs++; 217 ct++; 218 tcount--; 219 } while (tcount); 220 return 0; 221 } 222 223 #endif 224 225 static inline int dentry_cmp(const struct dentry *dentry, const unsigned char *ct, unsigned tcount) 226 { 227 /* 228 * Be careful about RCU walk racing with rename: 229 * use 'lockless_dereference' to fetch the name pointer. 230 * 231 * NOTE! Even if a rename will mean that the length 232 * was not loaded atomically, we don't care. The 233 * RCU walk will check the sequence count eventually, 234 * and catch it. And we won't overrun the buffer, 235 * because we're reading the name pointer atomically, 236 * and a dentry name is guaranteed to be properly 237 * terminated with a NUL byte. 238 * 239 * End result: even if 'len' is wrong, we'll exit 240 * early because the data cannot match (there can 241 * be no NUL in the ct/tcount data) 242 */ 243 const unsigned char *cs = lockless_dereference(dentry->d_name.name); 244 245 return dentry_string_cmp(cs, ct, tcount); 246 } 247 248 struct external_name { 249 union { 250 atomic_t count; 251 struct rcu_head head; 252 } u; 253 unsigned char name[]; 254 }; 255 256 static inline struct external_name *external_name(struct dentry *dentry) 257 { 258 return container_of(dentry->d_name.name, struct external_name, name[0]); 259 } 260 261 static void __d_free(struct rcu_head *head) 262 { 263 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); 264 265 kmem_cache_free(dentry_cache, dentry); 266 } 267 268 static void __d_free_external(struct rcu_head *head) 269 { 270 struct dentry *dentry = container_of(head, struct dentry, d_u.d_rcu); 271 kfree(external_name(dentry)); 272 kmem_cache_free(dentry_cache, dentry); 273 } 274 275 static inline int dname_external(const struct dentry *dentry) 276 { 277 return dentry->d_name.name != dentry->d_iname; 278 } 279 280 static inline void __d_set_inode_and_type(struct dentry *dentry, 281 struct inode *inode, 282 unsigned type_flags) 283 { 284 unsigned flags; 285 286 dentry->d_inode = inode; 287 flags = READ_ONCE(dentry->d_flags); 288 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU); 289 flags |= type_flags; 290 WRITE_ONCE(dentry->d_flags, flags); 291 } 292 293 static inline void __d_clear_type_and_inode(struct dentry *dentry) 294 { 295 unsigned flags = READ_ONCE(dentry->d_flags); 296 297 flags &= ~(DCACHE_ENTRY_TYPE | DCACHE_FALLTHRU); 298 WRITE_ONCE(dentry->d_flags, flags); 299 dentry->d_inode = NULL; 300 } 301 302 static void dentry_free(struct dentry *dentry) 303 { 304 WARN_ON(!hlist_unhashed(&dentry->d_u.d_alias)); 305 if (unlikely(dname_external(dentry))) { 306 struct external_name *p = external_name(dentry); 307 if (likely(atomic_dec_and_test(&p->u.count))) { 308 call_rcu(&dentry->d_u.d_rcu, __d_free_external); 309 return; 310 } 311 } 312 /* if dentry was never visible to RCU, immediate free is OK */ 313 if (!(dentry->d_flags & DCACHE_RCUACCESS)) 314 __d_free(&dentry->d_u.d_rcu); 315 else 316 call_rcu(&dentry->d_u.d_rcu, __d_free); 317 } 318 319 /* 320 * Release the dentry's inode, using the filesystem 321 * d_iput() operation if defined. 322 */ 323 static void dentry_unlink_inode(struct dentry * dentry) 324 __releases(dentry->d_lock) 325 __releases(dentry->d_inode->i_lock) 326 { 327 struct inode *inode = dentry->d_inode; 328 bool hashed = !d_unhashed(dentry); 329 330 if (hashed) 331 raw_write_seqcount_begin(&dentry->d_seq); 332 __d_clear_type_and_inode(dentry); 333 hlist_del_init(&dentry->d_u.d_alias); 334 if (hashed) 335 raw_write_seqcount_end(&dentry->d_seq); 336 spin_unlock(&dentry->d_lock); 337 spin_unlock(&inode->i_lock); 338 if (!inode->i_nlink) 339 fsnotify_inoderemove(inode); 340 if (dentry->d_op && dentry->d_op->d_iput) 341 dentry->d_op->d_iput(dentry, inode); 342 else 343 iput(inode); 344 } 345 346 /* 347 * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry 348 * is in use - which includes both the "real" per-superblock 349 * LRU list _and_ the DCACHE_SHRINK_LIST use. 350 * 351 * The DCACHE_SHRINK_LIST bit is set whenever the dentry is 352 * on the shrink list (ie not on the superblock LRU list). 353 * 354 * The per-cpu "nr_dentry_unused" counters are updated with 355 * the DCACHE_LRU_LIST bit. 356 * 357 * These helper functions make sure we always follow the 358 * rules. d_lock must be held by the caller. 359 */ 360 #define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x)) 361 static void d_lru_add(struct dentry *dentry) 362 { 363 D_FLAG_VERIFY(dentry, 0); 364 dentry->d_flags |= DCACHE_LRU_LIST; 365 this_cpu_inc(nr_dentry_unused); 366 WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); 367 } 368 369 static void d_lru_del(struct dentry *dentry) 370 { 371 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); 372 dentry->d_flags &= ~DCACHE_LRU_LIST; 373 this_cpu_dec(nr_dentry_unused); 374 WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); 375 } 376 377 static void d_shrink_del(struct dentry *dentry) 378 { 379 D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); 380 list_del_init(&dentry->d_lru); 381 dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); 382 this_cpu_dec(nr_dentry_unused); 383 } 384 385 static void d_shrink_add(struct dentry *dentry, struct list_head *list) 386 { 387 D_FLAG_VERIFY(dentry, 0); 388 list_add(&dentry->d_lru, list); 389 dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; 390 this_cpu_inc(nr_dentry_unused); 391 } 392 393 /* 394 * These can only be called under the global LRU lock, ie during the 395 * callback for freeing the LRU list. "isolate" removes it from the 396 * LRU lists entirely, while shrink_move moves it to the indicated 397 * private list. 398 */ 399 static void d_lru_isolate(struct list_lru_one *lru, struct dentry *dentry) 400 { 401 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); 402 dentry->d_flags &= ~DCACHE_LRU_LIST; 403 this_cpu_dec(nr_dentry_unused); 404 list_lru_isolate(lru, &dentry->d_lru); 405 } 406 407 static void d_lru_shrink_move(struct list_lru_one *lru, struct dentry *dentry, 408 struct list_head *list) 409 { 410 D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); 411 dentry->d_flags |= DCACHE_SHRINK_LIST; 412 list_lru_isolate_move(lru, &dentry->d_lru, list); 413 } 414 415 /* 416 * dentry_lru_(add|del)_list) must be called with d_lock held. 417 */ 418 static void dentry_lru_add(struct dentry *dentry) 419 { 420 if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) 421 d_lru_add(dentry); 422 } 423 424 /** 425 * d_drop - drop a dentry 426 * @dentry: dentry to drop 427 * 428 * d_drop() unhashes the entry from the parent dentry hashes, so that it won't 429 * be found through a VFS lookup any more. Note that this is different from 430 * deleting the dentry - d_delete will try to mark the dentry negative if 431 * possible, giving a successful _negative_ lookup, while d_drop will 432 * just make the cache lookup fail. 433 * 434 * d_drop() is used mainly for stuff that wants to invalidate a dentry for some 435 * reason (NFS timeouts or autofs deletes). 436 * 437 * __d_drop requires dentry->d_lock. 438 */ 439 void __d_drop(struct dentry *dentry) 440 { 441 if (!d_unhashed(dentry)) { 442 struct hlist_bl_head *b; 443 /* 444 * Hashed dentries are normally on the dentry hashtable, 445 * with the exception of those newly allocated by 446 * d_obtain_alias, which are always IS_ROOT: 447 */ 448 if (unlikely(IS_ROOT(dentry))) 449 b = &dentry->d_sb->s_anon; 450 else 451 b = d_hash(dentry->d_name.hash); 452 453 hlist_bl_lock(b); 454 __hlist_bl_del(&dentry->d_hash); 455 dentry->d_hash.pprev = NULL; 456 hlist_bl_unlock(b); 457 /* After this call, in-progress rcu-walk path lookup will fail. */ 458 write_seqcount_invalidate(&dentry->d_seq); 459 } 460 } 461 EXPORT_SYMBOL(__d_drop); 462 463 void d_drop(struct dentry *dentry) 464 { 465 spin_lock(&dentry->d_lock); 466 __d_drop(dentry); 467 spin_unlock(&dentry->d_lock); 468 } 469 EXPORT_SYMBOL(d_drop); 470 471 static inline void dentry_unlist(struct dentry *dentry, struct dentry *parent) 472 { 473 struct dentry *next; 474 /* 475 * Inform d_walk() and shrink_dentry_list() that we are no longer 476 * attached to the dentry tree 477 */ 478 dentry->d_flags |= DCACHE_DENTRY_KILLED; 479 if (unlikely(list_empty(&dentry->d_child))) 480 return; 481 __list_del_entry(&dentry->d_child); 482 /* 483 * Cursors can move around the list of children. While we'd been 484 * a normal list member, it didn't matter - ->d_child.next would've 485 * been updated. However, from now on it won't be and for the 486 * things like d_walk() it might end up with a nasty surprise. 487 * Normally d_walk() doesn't care about cursors moving around - 488 * ->d_lock on parent prevents that and since a cursor has no children 489 * of its own, we get through it without ever unlocking the parent. 490 * There is one exception, though - if we ascend from a child that 491 * gets killed as soon as we unlock it, the next sibling is found 492 * using the value left in its ->d_child.next. And if _that_ 493 * pointed to a cursor, and cursor got moved (e.g. by lseek()) 494 * before d_walk() regains parent->d_lock, we'll end up skipping 495 * everything the cursor had been moved past. 496 * 497 * Solution: make sure that the pointer left behind in ->d_child.next 498 * points to something that won't be moving around. I.e. skip the 499 * cursors. 500 */ 501 while (dentry->d_child.next != &parent->d_subdirs) { 502 next = list_entry(dentry->d_child.next, struct dentry, d_child); 503 if (likely(!(next->d_flags & DCACHE_DENTRY_CURSOR))) 504 break; 505 dentry->d_child.next = next->d_child.next; 506 } 507 } 508 509 static void __dentry_kill(struct dentry *dentry) 510 { 511 struct dentry *parent = NULL; 512 bool can_free = true; 513 if (!IS_ROOT(dentry)) 514 parent = dentry->d_parent; 515 516 /* 517 * The dentry is now unrecoverably dead to the world. 518 */ 519 lockref_mark_dead(&dentry->d_lockref); 520 521 /* 522 * inform the fs via d_prune that this dentry is about to be 523 * unhashed and destroyed. 524 */ 525 if (dentry->d_flags & DCACHE_OP_PRUNE) 526 dentry->d_op->d_prune(dentry); 527 528 if (dentry->d_flags & DCACHE_LRU_LIST) { 529 if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) 530 d_lru_del(dentry); 531 } 532 /* if it was on the hash then remove it */ 533 __d_drop(dentry); 534 dentry_unlist(dentry, parent); 535 if (parent) 536 spin_unlock(&parent->d_lock); 537 if (dentry->d_inode) 538 dentry_unlink_inode(dentry); 539 else 540 spin_unlock(&dentry->d_lock); 541 this_cpu_dec(nr_dentry); 542 if (dentry->d_op && dentry->d_op->d_release) 543 dentry->d_op->d_release(dentry); 544 545 spin_lock(&dentry->d_lock); 546 if (dentry->d_flags & DCACHE_SHRINK_LIST) { 547 dentry->d_flags |= DCACHE_MAY_FREE; 548 can_free = false; 549 } 550 spin_unlock(&dentry->d_lock); 551 if (likely(can_free)) 552 dentry_free(dentry); 553 } 554 555 /* 556 * Finish off a dentry we've decided to kill. 557 * dentry->d_lock must be held, returns with it unlocked. 558 * If ref is non-zero, then decrement the refcount too. 559 * Returns dentry requiring refcount drop, or NULL if we're done. 560 */ 561 static struct dentry *dentry_kill(struct dentry *dentry) 562 __releases(dentry->d_lock) 563 { 564 struct inode *inode = dentry->d_inode; 565 struct dentry *parent = NULL; 566 567 if (inode && unlikely(!spin_trylock(&inode->i_lock))) 568 goto failed; 569 570 if (!IS_ROOT(dentry)) { 571 parent = dentry->d_parent; 572 if (unlikely(!spin_trylock(&parent->d_lock))) { 573 if (inode) 574 spin_unlock(&inode->i_lock); 575 goto failed; 576 } 577 } 578 579 __dentry_kill(dentry); 580 return parent; 581 582 failed: 583 spin_unlock(&dentry->d_lock); 584 return dentry; /* try again with same dentry */ 585 } 586 587 static inline struct dentry *lock_parent(struct dentry *dentry) 588 { 589 struct dentry *parent = dentry->d_parent; 590 if (IS_ROOT(dentry)) 591 return NULL; 592 if (unlikely(dentry->d_lockref.count < 0)) 593 return NULL; 594 if (likely(spin_trylock(&parent->d_lock))) 595 return parent; 596 rcu_read_lock(); 597 spin_unlock(&dentry->d_lock); 598 again: 599 parent = ACCESS_ONCE(dentry->d_parent); 600 spin_lock(&parent->d_lock); 601 /* 602 * We can't blindly lock dentry until we are sure 603 * that we won't violate the locking order. 604 * Any changes of dentry->d_parent must have 605 * been done with parent->d_lock held, so 606 * spin_lock() above is enough of a barrier 607 * for checking if it's still our child. 608 */ 609 if (unlikely(parent != dentry->d_parent)) { 610 spin_unlock(&parent->d_lock); 611 goto again; 612 } 613 rcu_read_unlock(); 614 if (parent != dentry) 615 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); 616 else 617 parent = NULL; 618 return parent; 619 } 620 621 /* 622 * Try to do a lockless dput(), and return whether that was successful. 623 * 624 * If unsuccessful, we return false, having already taken the dentry lock. 625 * 626 * The caller needs to hold the RCU read lock, so that the dentry is 627 * guaranteed to stay around even if the refcount goes down to zero! 628 */ 629 static inline bool fast_dput(struct dentry *dentry) 630 { 631 int ret; 632 unsigned int d_flags; 633 634 /* 635 * If we have a d_op->d_delete() operation, we sould not 636 * let the dentry count go to zero, so use "put_or_lock". 637 */ 638 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) 639 return lockref_put_or_lock(&dentry->d_lockref); 640 641 /* 642 * .. otherwise, we can try to just decrement the 643 * lockref optimistically. 644 */ 645 ret = lockref_put_return(&dentry->d_lockref); 646 647 /* 648 * If the lockref_put_return() failed due to the lock being held 649 * by somebody else, the fast path has failed. We will need to 650 * get the lock, and then check the count again. 651 */ 652 if (unlikely(ret < 0)) { 653 spin_lock(&dentry->d_lock); 654 if (dentry->d_lockref.count > 1) { 655 dentry->d_lockref.count--; 656 spin_unlock(&dentry->d_lock); 657 return 1; 658 } 659 return 0; 660 } 661 662 /* 663 * If we weren't the last ref, we're done. 664 */ 665 if (ret) 666 return 1; 667 668 /* 669 * Careful, careful. The reference count went down 670 * to zero, but we don't hold the dentry lock, so 671 * somebody else could get it again, and do another 672 * dput(), and we need to not race with that. 673 * 674 * However, there is a very special and common case 675 * where we don't care, because there is nothing to 676 * do: the dentry is still hashed, it does not have 677 * a 'delete' op, and it's referenced and already on 678 * the LRU list. 679 * 680 * NOTE! Since we aren't locked, these values are 681 * not "stable". However, it is sufficient that at 682 * some point after we dropped the reference the 683 * dentry was hashed and the flags had the proper 684 * value. Other dentry users may have re-gotten 685 * a reference to the dentry and change that, but 686 * our work is done - we can leave the dentry 687 * around with a zero refcount. 688 */ 689 smp_rmb(); 690 d_flags = ACCESS_ONCE(dentry->d_flags); 691 d_flags &= DCACHE_REFERENCED | DCACHE_LRU_LIST | DCACHE_DISCONNECTED; 692 693 /* Nothing to do? Dropping the reference was all we needed? */ 694 if (d_flags == (DCACHE_REFERENCED | DCACHE_LRU_LIST) && !d_unhashed(dentry)) 695 return 1; 696 697 /* 698 * Not the fast normal case? Get the lock. We've already decremented 699 * the refcount, but we'll need to re-check the situation after 700 * getting the lock. 701 */ 702 spin_lock(&dentry->d_lock); 703 704 /* 705 * Did somebody else grab a reference to it in the meantime, and 706 * we're no longer the last user after all? Alternatively, somebody 707 * else could have killed it and marked it dead. Either way, we 708 * don't need to do anything else. 709 */ 710 if (dentry->d_lockref.count) { 711 spin_unlock(&dentry->d_lock); 712 return 1; 713 } 714 715 /* 716 * Re-get the reference we optimistically dropped. We hold the 717 * lock, and we just tested that it was zero, so we can just 718 * set it to 1. 719 */ 720 dentry->d_lockref.count = 1; 721 return 0; 722 } 723 724 725 /* 726 * This is dput 727 * 728 * This is complicated by the fact that we do not want to put 729 * dentries that are no longer on any hash chain on the unused 730 * list: we'd much rather just get rid of them immediately. 731 * 732 * However, that implies that we have to traverse the dentry 733 * tree upwards to the parents which might _also_ now be 734 * scheduled for deletion (it may have been only waiting for 735 * its last child to go away). 736 * 737 * This tail recursion is done by hand as we don't want to depend 738 * on the compiler to always get this right (gcc generally doesn't). 739 * Real recursion would eat up our stack space. 740 */ 741 742 /* 743 * dput - release a dentry 744 * @dentry: dentry to release 745 * 746 * Release a dentry. This will drop the usage count and if appropriate 747 * call the dentry unlink method as well as removing it from the queues and 748 * releasing its resources. If the parent dentries were scheduled for release 749 * they too may now get deleted. 750 */ 751 void dput(struct dentry *dentry) 752 { 753 if (unlikely(!dentry)) 754 return; 755 756 repeat: 757 might_sleep(); 758 759 rcu_read_lock(); 760 if (likely(fast_dput(dentry))) { 761 rcu_read_unlock(); 762 return; 763 } 764 765 /* Slow case: now with the dentry lock held */ 766 rcu_read_unlock(); 767 768 WARN_ON(d_in_lookup(dentry)); 769 770 /* Unreachable? Get rid of it */ 771 if (unlikely(d_unhashed(dentry))) 772 goto kill_it; 773 774 if (unlikely(dentry->d_flags & DCACHE_DISCONNECTED)) 775 goto kill_it; 776 777 if (unlikely(dentry->d_flags & DCACHE_OP_DELETE)) { 778 if (dentry->d_op->d_delete(dentry)) 779 goto kill_it; 780 } 781 782 if (!(dentry->d_flags & DCACHE_REFERENCED)) 783 dentry->d_flags |= DCACHE_REFERENCED; 784 dentry_lru_add(dentry); 785 786 dentry->d_lockref.count--; 787 spin_unlock(&dentry->d_lock); 788 return; 789 790 kill_it: 791 dentry = dentry_kill(dentry); 792 if (dentry) { 793 cond_resched(); 794 goto repeat; 795 } 796 } 797 EXPORT_SYMBOL(dput); 798 799 800 /* This must be called with d_lock held */ 801 static inline void __dget_dlock(struct dentry *dentry) 802 { 803 dentry->d_lockref.count++; 804 } 805 806 static inline void __dget(struct dentry *dentry) 807 { 808 lockref_get(&dentry->d_lockref); 809 } 810 811 struct dentry *dget_parent(struct dentry *dentry) 812 { 813 int gotref; 814 struct dentry *ret; 815 816 /* 817 * Do optimistic parent lookup without any 818 * locking. 819 */ 820 rcu_read_lock(); 821 ret = ACCESS_ONCE(dentry->d_parent); 822 gotref = lockref_get_not_zero(&ret->d_lockref); 823 rcu_read_unlock(); 824 if (likely(gotref)) { 825 if (likely(ret == ACCESS_ONCE(dentry->d_parent))) 826 return ret; 827 dput(ret); 828 } 829 830 repeat: 831 /* 832 * Don't need rcu_dereference because we re-check it was correct under 833 * the lock. 834 */ 835 rcu_read_lock(); 836 ret = dentry->d_parent; 837 spin_lock(&ret->d_lock); 838 if (unlikely(ret != dentry->d_parent)) { 839 spin_unlock(&ret->d_lock); 840 rcu_read_unlock(); 841 goto repeat; 842 } 843 rcu_read_unlock(); 844 BUG_ON(!ret->d_lockref.count); 845 ret->d_lockref.count++; 846 spin_unlock(&ret->d_lock); 847 return ret; 848 } 849 EXPORT_SYMBOL(dget_parent); 850 851 /** 852 * d_find_alias - grab a hashed alias of inode 853 * @inode: inode in question 854 * 855 * If inode has a hashed alias, or is a directory and has any alias, 856 * acquire the reference to alias and return it. Otherwise return NULL. 857 * Notice that if inode is a directory there can be only one alias and 858 * it can be unhashed only if it has no children, or if it is the root 859 * of a filesystem, or if the directory was renamed and d_revalidate 860 * was the first vfs operation to notice. 861 * 862 * If the inode has an IS_ROOT, DCACHE_DISCONNECTED alias, then prefer 863 * any other hashed alias over that one. 864 */ 865 static struct dentry *__d_find_alias(struct inode *inode) 866 { 867 struct dentry *alias, *discon_alias; 868 869 again: 870 discon_alias = NULL; 871 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) { 872 spin_lock(&alias->d_lock); 873 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { 874 if (IS_ROOT(alias) && 875 (alias->d_flags & DCACHE_DISCONNECTED)) { 876 discon_alias = alias; 877 } else { 878 __dget_dlock(alias); 879 spin_unlock(&alias->d_lock); 880 return alias; 881 } 882 } 883 spin_unlock(&alias->d_lock); 884 } 885 if (discon_alias) { 886 alias = discon_alias; 887 spin_lock(&alias->d_lock); 888 if (S_ISDIR(inode->i_mode) || !d_unhashed(alias)) { 889 __dget_dlock(alias); 890 spin_unlock(&alias->d_lock); 891 return alias; 892 } 893 spin_unlock(&alias->d_lock); 894 goto again; 895 } 896 return NULL; 897 } 898 899 struct dentry *d_find_alias(struct inode *inode) 900 { 901 struct dentry *de = NULL; 902 903 if (!hlist_empty(&inode->i_dentry)) { 904 spin_lock(&inode->i_lock); 905 de = __d_find_alias(inode); 906 spin_unlock(&inode->i_lock); 907 } 908 return de; 909 } 910 EXPORT_SYMBOL(d_find_alias); 911 912 /* 913 * Try to kill dentries associated with this inode. 914 * WARNING: you must own a reference to inode. 915 */ 916 void d_prune_aliases(struct inode *inode) 917 { 918 struct dentry *dentry; 919 restart: 920 spin_lock(&inode->i_lock); 921 hlist_for_each_entry(dentry, &inode->i_dentry, d_u.d_alias) { 922 spin_lock(&dentry->d_lock); 923 if (!dentry->d_lockref.count) { 924 struct dentry *parent = lock_parent(dentry); 925 if (likely(!dentry->d_lockref.count)) { 926 __dentry_kill(dentry); 927 dput(parent); 928 goto restart; 929 } 930 if (parent) 931 spin_unlock(&parent->d_lock); 932 } 933 spin_unlock(&dentry->d_lock); 934 } 935 spin_unlock(&inode->i_lock); 936 } 937 EXPORT_SYMBOL(d_prune_aliases); 938 939 static void shrink_dentry_list(struct list_head *list) 940 { 941 struct dentry *dentry, *parent; 942 943 while (!list_empty(list)) { 944 struct inode *inode; 945 dentry = list_entry(list->prev, struct dentry, d_lru); 946 spin_lock(&dentry->d_lock); 947 parent = lock_parent(dentry); 948 949 /* 950 * The dispose list is isolated and dentries are not accounted 951 * to the LRU here, so we can simply remove it from the list 952 * here regardless of whether it is referenced or not. 953 */ 954 d_shrink_del(dentry); 955 956 /* 957 * We found an inuse dentry which was not removed from 958 * the LRU because of laziness during lookup. Do not free it. 959 */ 960 if (dentry->d_lockref.count > 0) { 961 spin_unlock(&dentry->d_lock); 962 if (parent) 963 spin_unlock(&parent->d_lock); 964 continue; 965 } 966 967 968 if (unlikely(dentry->d_flags & DCACHE_DENTRY_KILLED)) { 969 bool can_free = dentry->d_flags & DCACHE_MAY_FREE; 970 spin_unlock(&dentry->d_lock); 971 if (parent) 972 spin_unlock(&parent->d_lock); 973 if (can_free) 974 dentry_free(dentry); 975 continue; 976 } 977 978 inode = dentry->d_inode; 979 if (inode && unlikely(!spin_trylock(&inode->i_lock))) { 980 d_shrink_add(dentry, list); 981 spin_unlock(&dentry->d_lock); 982 if (parent) 983 spin_unlock(&parent->d_lock); 984 continue; 985 } 986 987 __dentry_kill(dentry); 988 989 /* 990 * We need to prune ancestors too. This is necessary to prevent 991 * quadratic behavior of shrink_dcache_parent(), but is also 992 * expected to be beneficial in reducing dentry cache 993 * fragmentation. 994 */ 995 dentry = parent; 996 while (dentry && !lockref_put_or_lock(&dentry->d_lockref)) { 997 parent = lock_parent(dentry); 998 if (dentry->d_lockref.count != 1) { 999 dentry->d_lockref.count--; 1000 spin_unlock(&dentry->d_lock); 1001 if (parent) 1002 spin_unlock(&parent->d_lock); 1003 break; 1004 } 1005 inode = dentry->d_inode; /* can't be NULL */ 1006 if (unlikely(!spin_trylock(&inode->i_lock))) { 1007 spin_unlock(&dentry->d_lock); 1008 if (parent) 1009 spin_unlock(&parent->d_lock); 1010 cpu_relax(); 1011 continue; 1012 } 1013 __dentry_kill(dentry); 1014 dentry = parent; 1015 } 1016 } 1017 } 1018 1019 static enum lru_status dentry_lru_isolate(struct list_head *item, 1020 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg) 1021 { 1022 struct list_head *freeable = arg; 1023 struct dentry *dentry = container_of(item, struct dentry, d_lru); 1024 1025 1026 /* 1027 * we are inverting the lru lock/dentry->d_lock here, 1028 * so use a trylock. If we fail to get the lock, just skip 1029 * it 1030 */ 1031 if (!spin_trylock(&dentry->d_lock)) 1032 return LRU_SKIP; 1033 1034 /* 1035 * Referenced dentries are still in use. If they have active 1036 * counts, just remove them from the LRU. Otherwise give them 1037 * another pass through the LRU. 1038 */ 1039 if (dentry->d_lockref.count) { 1040 d_lru_isolate(lru, dentry); 1041 spin_unlock(&dentry->d_lock); 1042 return LRU_REMOVED; 1043 } 1044 1045 if (dentry->d_flags & DCACHE_REFERENCED) { 1046 dentry->d_flags &= ~DCACHE_REFERENCED; 1047 spin_unlock(&dentry->d_lock); 1048 1049 /* 1050 * The list move itself will be made by the common LRU code. At 1051 * this point, we've dropped the dentry->d_lock but keep the 1052 * lru lock. This is safe to do, since every list movement is 1053 * protected by the lru lock even if both locks are held. 1054 * 1055 * This is guaranteed by the fact that all LRU management 1056 * functions are intermediated by the LRU API calls like 1057 * list_lru_add and list_lru_del. List movement in this file 1058 * only ever occur through this functions or through callbacks 1059 * like this one, that are called from the LRU API. 1060 * 1061 * The only exceptions to this are functions like 1062 * shrink_dentry_list, and code that first checks for the 1063 * DCACHE_SHRINK_LIST flag. Those are guaranteed to be 1064 * operating only with stack provided lists after they are 1065 * properly isolated from the main list. It is thus, always a 1066 * local access. 1067 */ 1068 return LRU_ROTATE; 1069 } 1070 1071 d_lru_shrink_move(lru, dentry, freeable); 1072 spin_unlock(&dentry->d_lock); 1073 1074 return LRU_REMOVED; 1075 } 1076 1077 /** 1078 * prune_dcache_sb - shrink the dcache 1079 * @sb: superblock 1080 * @sc: shrink control, passed to list_lru_shrink_walk() 1081 * 1082 * Attempt to shrink the superblock dcache LRU by @sc->nr_to_scan entries. This 1083 * is done when we need more memory and called from the superblock shrinker 1084 * function. 1085 * 1086 * This function may fail to free any resources if all the dentries are in 1087 * use. 1088 */ 1089 long prune_dcache_sb(struct super_block *sb, struct shrink_control *sc) 1090 { 1091 LIST_HEAD(dispose); 1092 long freed; 1093 1094 freed = list_lru_shrink_walk(&sb->s_dentry_lru, sc, 1095 dentry_lru_isolate, &dispose); 1096 shrink_dentry_list(&dispose); 1097 return freed; 1098 } 1099 1100 static enum lru_status dentry_lru_isolate_shrink(struct list_head *item, 1101 struct list_lru_one *lru, spinlock_t *lru_lock, void *arg) 1102 { 1103 struct list_head *freeable = arg; 1104 struct dentry *dentry = container_of(item, struct dentry, d_lru); 1105 1106 /* 1107 * we are inverting the lru lock/dentry->d_lock here, 1108 * so use a trylock. If we fail to get the lock, just skip 1109 * it 1110 */ 1111 if (!spin_trylock(&dentry->d_lock)) 1112 return LRU_SKIP; 1113 1114 d_lru_shrink_move(lru, dentry, freeable); 1115 spin_unlock(&dentry->d_lock); 1116 1117 return LRU_REMOVED; 1118 } 1119 1120 1121 /** 1122 * shrink_dcache_sb - shrink dcache for a superblock 1123 * @sb: superblock 1124 * 1125 * Shrink the dcache for the specified super block. This is used to free 1126 * the dcache before unmounting a file system. 1127 */ 1128 void shrink_dcache_sb(struct super_block *sb) 1129 { 1130 long freed; 1131 1132 do { 1133 LIST_HEAD(dispose); 1134 1135 freed = list_lru_walk(&sb->s_dentry_lru, 1136 dentry_lru_isolate_shrink, &dispose, UINT_MAX); 1137 1138 this_cpu_sub(nr_dentry_unused, freed); 1139 shrink_dentry_list(&dispose); 1140 } while (freed > 0); 1141 } 1142 EXPORT_SYMBOL(shrink_dcache_sb); 1143 1144 /** 1145 * enum d_walk_ret - action to talke during tree walk 1146 * @D_WALK_CONTINUE: contrinue walk 1147 * @D_WALK_QUIT: quit walk 1148 * @D_WALK_NORETRY: quit when retry is needed 1149 * @D_WALK_SKIP: skip this dentry and its children 1150 */ 1151 enum d_walk_ret { 1152 D_WALK_CONTINUE, 1153 D_WALK_QUIT, 1154 D_WALK_NORETRY, 1155 D_WALK_SKIP, 1156 }; 1157 1158 /** 1159 * d_walk - walk the dentry tree 1160 * @parent: start of walk 1161 * @data: data passed to @enter() and @finish() 1162 * @enter: callback when first entering the dentry 1163 * @finish: callback when successfully finished the walk 1164 * 1165 * The @enter() and @finish() callbacks are called with d_lock held. 1166 */ 1167 static void d_walk(struct dentry *parent, void *data, 1168 enum d_walk_ret (*enter)(void *, struct dentry *), 1169 void (*finish)(void *)) 1170 { 1171 struct dentry *this_parent; 1172 struct list_head *next; 1173 unsigned seq = 0; 1174 enum d_walk_ret ret; 1175 bool retry = true; 1176 1177 again: 1178 read_seqbegin_or_lock(&rename_lock, &seq); 1179 this_parent = parent; 1180 spin_lock(&this_parent->d_lock); 1181 1182 ret = enter(data, this_parent); 1183 switch (ret) { 1184 case D_WALK_CONTINUE: 1185 break; 1186 case D_WALK_QUIT: 1187 case D_WALK_SKIP: 1188 goto out_unlock; 1189 case D_WALK_NORETRY: 1190 retry = false; 1191 break; 1192 } 1193 repeat: 1194 next = this_parent->d_subdirs.next; 1195 resume: 1196 while (next != &this_parent->d_subdirs) { 1197 struct list_head *tmp = next; 1198 struct dentry *dentry = list_entry(tmp, struct dentry, d_child); 1199 next = tmp->next; 1200 1201 if (unlikely(dentry->d_flags & DCACHE_DENTRY_CURSOR)) 1202 continue; 1203 1204 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); 1205 1206 ret = enter(data, dentry); 1207 switch (ret) { 1208 case D_WALK_CONTINUE: 1209 break; 1210 case D_WALK_QUIT: 1211 spin_unlock(&dentry->d_lock); 1212 goto out_unlock; 1213 case D_WALK_NORETRY: 1214 retry = false; 1215 break; 1216 case D_WALK_SKIP: 1217 spin_unlock(&dentry->d_lock); 1218 continue; 1219 } 1220 1221 if (!list_empty(&dentry->d_subdirs)) { 1222 spin_unlock(&this_parent->d_lock); 1223 spin_release(&dentry->d_lock.dep_map, 1, _RET_IP_); 1224 this_parent = dentry; 1225 spin_acquire(&this_parent->d_lock.dep_map, 0, 1, _RET_IP_); 1226 goto repeat; 1227 } 1228 spin_unlock(&dentry->d_lock); 1229 } 1230 /* 1231 * All done at this level ... ascend and resume the search. 1232 */ 1233 rcu_read_lock(); 1234 ascend: 1235 if (this_parent != parent) { 1236 struct dentry *child = this_parent; 1237 this_parent = child->d_parent; 1238 1239 spin_unlock(&child->d_lock); 1240 spin_lock(&this_parent->d_lock); 1241 1242 /* might go back up the wrong parent if we have had a rename. */ 1243 if (need_seqretry(&rename_lock, seq)) 1244 goto rename_retry; 1245 /* go into the first sibling still alive */ 1246 do { 1247 next = child->d_child.next; 1248 if (next == &this_parent->d_subdirs) 1249 goto ascend; 1250 child = list_entry(next, struct dentry, d_child); 1251 } while (unlikely(child->d_flags & DCACHE_DENTRY_KILLED)); 1252 rcu_read_unlock(); 1253 goto resume; 1254 } 1255 if (need_seqretry(&rename_lock, seq)) 1256 goto rename_retry; 1257 rcu_read_unlock(); 1258 if (finish) 1259 finish(data); 1260 1261 out_unlock: 1262 spin_unlock(&this_parent->d_lock); 1263 done_seqretry(&rename_lock, seq); 1264 return; 1265 1266 rename_retry: 1267 spin_unlock(&this_parent->d_lock); 1268 rcu_read_unlock(); 1269 BUG_ON(seq & 1); 1270 if (!retry) 1271 return; 1272 seq = 1; 1273 goto again; 1274 } 1275 1276 struct check_mount { 1277 struct vfsmount *mnt; 1278 unsigned int mounted; 1279 }; 1280 1281 static enum d_walk_ret path_check_mount(void *data, struct dentry *dentry) 1282 { 1283 struct check_mount *info = data; 1284 struct path path = { .mnt = info->mnt, .dentry = dentry }; 1285 1286 if (likely(!d_mountpoint(dentry))) 1287 return D_WALK_CONTINUE; 1288 if (__path_is_mountpoint(&path)) { 1289 info->mounted = 1; 1290 return D_WALK_QUIT; 1291 } 1292 return D_WALK_CONTINUE; 1293 } 1294 1295 /** 1296 * path_has_submounts - check for mounts over a dentry in the 1297 * current namespace. 1298 * @parent: path to check. 1299 * 1300 * Return true if the parent or its subdirectories contain 1301 * a mount point in the current namespace. 1302 */ 1303 int path_has_submounts(const struct path *parent) 1304 { 1305 struct check_mount data = { .mnt = parent->mnt, .mounted = 0 }; 1306 1307 read_seqlock_excl(&mount_lock); 1308 d_walk(parent->dentry, &data, path_check_mount, NULL); 1309 read_sequnlock_excl(&mount_lock); 1310 1311 return data.mounted; 1312 } 1313 EXPORT_SYMBOL(path_has_submounts); 1314 1315 /* 1316 * Called by mount code to set a mountpoint and check if the mountpoint is 1317 * reachable (e.g. NFS can unhash a directory dentry and then the complete 1318 * subtree can become unreachable). 1319 * 1320 * Only one of d_invalidate() and d_set_mounted() must succeed. For 1321 * this reason take rename_lock and d_lock on dentry and ancestors. 1322 */ 1323 int d_set_mounted(struct dentry *dentry) 1324 { 1325 struct dentry *p; 1326 int ret = -ENOENT; 1327 write_seqlock(&rename_lock); 1328 for (p = dentry->d_parent; !IS_ROOT(p); p = p->d_parent) { 1329 /* Need exclusion wrt. d_invalidate() */ 1330 spin_lock(&p->d_lock); 1331 if (unlikely(d_unhashed(p))) { 1332 spin_unlock(&p->d_lock); 1333 goto out; 1334 } 1335 spin_unlock(&p->d_lock); 1336 } 1337 spin_lock(&dentry->d_lock); 1338 if (!d_unlinked(dentry)) { 1339 dentry->d_flags |= DCACHE_MOUNTED; 1340 ret = 0; 1341 } 1342 spin_unlock(&dentry->d_lock); 1343 out: 1344 write_sequnlock(&rename_lock); 1345 return ret; 1346 } 1347 1348 /* 1349 * Search the dentry child list of the specified parent, 1350 * and move any unused dentries to the end of the unused 1351 * list for prune_dcache(). We descend to the next level 1352 * whenever the d_subdirs list is non-empty and continue 1353 * searching. 1354 * 1355 * It returns zero iff there are no unused children, 1356 * otherwise it returns the number of children moved to 1357 * the end of the unused list. This may not be the total 1358 * number of unused children, because select_parent can 1359 * drop the lock and return early due to latency 1360 * constraints. 1361 */ 1362 1363 struct select_data { 1364 struct dentry *start; 1365 struct list_head dispose; 1366 int found; 1367 }; 1368 1369 static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) 1370 { 1371 struct select_data *data = _data; 1372 enum d_walk_ret ret = D_WALK_CONTINUE; 1373 1374 if (data->start == dentry) 1375 goto out; 1376 1377 if (dentry->d_flags & DCACHE_SHRINK_LIST) { 1378 data->found++; 1379 } else { 1380 if (dentry->d_flags & DCACHE_LRU_LIST) 1381 d_lru_del(dentry); 1382 if (!dentry->d_lockref.count) { 1383 d_shrink_add(dentry, &data->dispose); 1384 data->found++; 1385 } 1386 } 1387 /* 1388 * We can return to the caller if we have found some (this 1389 * ensures forward progress). We'll be coming back to find 1390 * the rest. 1391 */ 1392 if (!list_empty(&data->dispose)) 1393 ret = need_resched() ? D_WALK_QUIT : D_WALK_NORETRY; 1394 out: 1395 return ret; 1396 } 1397 1398 /** 1399 * shrink_dcache_parent - prune dcache 1400 * @parent: parent of entries to prune 1401 * 1402 * Prune the dcache to remove unused children of the parent dentry. 1403 */ 1404 void shrink_dcache_parent(struct dentry *parent) 1405 { 1406 for (;;) { 1407 struct select_data data; 1408 1409 INIT_LIST_HEAD(&data.dispose); 1410 data.start = parent; 1411 data.found = 0; 1412 1413 d_walk(parent, &data, select_collect, NULL); 1414 if (!data.found) 1415 break; 1416 1417 shrink_dentry_list(&data.dispose); 1418 cond_resched(); 1419 } 1420 } 1421 EXPORT_SYMBOL(shrink_dcache_parent); 1422 1423 static enum d_walk_ret umount_check(void *_data, struct dentry *dentry) 1424 { 1425 /* it has busy descendents; complain about those instead */ 1426 if (!list_empty(&dentry->d_subdirs)) 1427 return D_WALK_CONTINUE; 1428 1429 /* root with refcount 1 is fine */ 1430 if (dentry == _data && dentry->d_lockref.count == 1) 1431 return D_WALK_CONTINUE; 1432 1433 printk(KERN_ERR "BUG: Dentry %p{i=%lx,n=%pd} " 1434 " still in use (%d) [unmount of %s %s]\n", 1435 dentry, 1436 dentry->d_inode ? 1437 dentry->d_inode->i_ino : 0UL, 1438 dentry, 1439 dentry->d_lockref.count, 1440 dentry->d_sb->s_type->name, 1441 dentry->d_sb->s_id); 1442 WARN_ON(1); 1443 return D_WALK_CONTINUE; 1444 } 1445 1446 static void do_one_tree(struct dentry *dentry) 1447 { 1448 shrink_dcache_parent(dentry); 1449 d_walk(dentry, dentry, umount_check, NULL); 1450 d_drop(dentry); 1451 dput(dentry); 1452 } 1453 1454 /* 1455 * destroy the dentries attached to a superblock on unmounting 1456 */ 1457 void shrink_dcache_for_umount(struct super_block *sb) 1458 { 1459 struct dentry *dentry; 1460 1461 WARN(down_read_trylock(&sb->s_umount), "s_umount should've been locked"); 1462 1463 dentry = sb->s_root; 1464 sb->s_root = NULL; 1465 do_one_tree(dentry); 1466 1467 while (!hlist_bl_empty(&sb->s_anon)) { 1468 dentry = dget(hlist_bl_entry(hlist_bl_first(&sb->s_anon), struct dentry, d_hash)); 1469 do_one_tree(dentry); 1470 } 1471 } 1472 1473 struct detach_data { 1474 struct select_data select; 1475 struct dentry *mountpoint; 1476 }; 1477 static enum d_walk_ret detach_and_collect(void *_data, struct dentry *dentry) 1478 { 1479 struct detach_data *data = _data; 1480 1481 if (d_mountpoint(dentry)) { 1482 __dget_dlock(dentry); 1483 data->mountpoint = dentry; 1484 return D_WALK_QUIT; 1485 } 1486 1487 return select_collect(&data->select, dentry); 1488 } 1489 1490 static void check_and_drop(void *_data) 1491 { 1492 struct detach_data *data = _data; 1493 1494 if (!data->mountpoint && !data->select.found) 1495 __d_drop(data->select.start); 1496 } 1497 1498 /** 1499 * d_invalidate - detach submounts, prune dcache, and drop 1500 * @dentry: dentry to invalidate (aka detach, prune and drop) 1501 * 1502 * no dcache lock. 1503 * 1504 * The final d_drop is done as an atomic operation relative to 1505 * rename_lock ensuring there are no races with d_set_mounted. This 1506 * ensures there are no unhashed dentries on the path to a mountpoint. 1507 */ 1508 void d_invalidate(struct dentry *dentry) 1509 { 1510 /* 1511 * If it's already been dropped, return OK. 1512 */ 1513 spin_lock(&dentry->d_lock); 1514 if (d_unhashed(dentry)) { 1515 spin_unlock(&dentry->d_lock); 1516 return; 1517 } 1518 spin_unlock(&dentry->d_lock); 1519 1520 /* Negative dentries can be dropped without further checks */ 1521 if (!dentry->d_inode) { 1522 d_drop(dentry); 1523 return; 1524 } 1525 1526 for (;;) { 1527 struct detach_data data; 1528 1529 data.mountpoint = NULL; 1530 INIT_LIST_HEAD(&data.select.dispose); 1531 data.select.start = dentry; 1532 data.select.found = 0; 1533 1534 d_walk(dentry, &data, detach_and_collect, check_and_drop); 1535 1536 if (data.select.found) 1537 shrink_dentry_list(&data.select.dispose); 1538 1539 if (data.mountpoint) { 1540 detach_mounts(data.mountpoint); 1541 dput(data.mountpoint); 1542 } 1543 1544 if (!data.mountpoint && !data.select.found) 1545 break; 1546 1547 cond_resched(); 1548 } 1549 } 1550 EXPORT_SYMBOL(d_invalidate); 1551 1552 /** 1553 * __d_alloc - allocate a dcache entry 1554 * @sb: filesystem it will belong to 1555 * @name: qstr of the name 1556 * 1557 * Allocates a dentry. It returns %NULL if there is insufficient memory 1558 * available. On a success the dentry is returned. The name passed in is 1559 * copied and the copy passed in may be reused after this call. 1560 */ 1561 1562 struct dentry *__d_alloc(struct super_block *sb, const struct qstr *name) 1563 { 1564 struct dentry *dentry; 1565 char *dname; 1566 int err; 1567 1568 dentry = kmem_cache_alloc(dentry_cache, GFP_KERNEL); 1569 if (!dentry) 1570 return NULL; 1571 1572 /* 1573 * We guarantee that the inline name is always NUL-terminated. 1574 * This way the memcpy() done by the name switching in rename 1575 * will still always have a NUL at the end, even if we might 1576 * be overwriting an internal NUL character 1577 */ 1578 dentry->d_iname[DNAME_INLINE_LEN-1] = 0; 1579 if (unlikely(!name)) { 1580 static const struct qstr anon = QSTR_INIT("/", 1); 1581 name = &anon; 1582 dname = dentry->d_iname; 1583 } else if (name->len > DNAME_INLINE_LEN-1) { 1584 size_t size = offsetof(struct external_name, name[1]); 1585 struct external_name *p = kmalloc(size + name->len, 1586 GFP_KERNEL_ACCOUNT); 1587 if (!p) { 1588 kmem_cache_free(dentry_cache, dentry); 1589 return NULL; 1590 } 1591 atomic_set(&p->u.count, 1); 1592 dname = p->name; 1593 if (IS_ENABLED(CONFIG_DCACHE_WORD_ACCESS)) 1594 kasan_unpoison_shadow(dname, 1595 round_up(name->len + 1, sizeof(unsigned long))); 1596 } else { 1597 dname = dentry->d_iname; 1598 } 1599 1600 dentry->d_name.len = name->len; 1601 dentry->d_name.hash = name->hash; 1602 memcpy(dname, name->name, name->len); 1603 dname[name->len] = 0; 1604 1605 /* Make sure we always see the terminating NUL character */ 1606 smp_wmb(); 1607 dentry->d_name.name = dname; 1608 1609 dentry->d_lockref.count = 1; 1610 dentry->d_flags = 0; 1611 spin_lock_init(&dentry->d_lock); 1612 seqcount_init(&dentry->d_seq); 1613 dentry->d_inode = NULL; 1614 dentry->d_parent = dentry; 1615 dentry->d_sb = sb; 1616 dentry->d_op = NULL; 1617 dentry->d_fsdata = NULL; 1618 INIT_HLIST_BL_NODE(&dentry->d_hash); 1619 INIT_LIST_HEAD(&dentry->d_lru); 1620 INIT_LIST_HEAD(&dentry->d_subdirs); 1621 INIT_HLIST_NODE(&dentry->d_u.d_alias); 1622 INIT_LIST_HEAD(&dentry->d_child); 1623 d_set_d_op(dentry, dentry->d_sb->s_d_op); 1624 1625 if (dentry->d_op && dentry->d_op->d_init) { 1626 err = dentry->d_op->d_init(dentry); 1627 if (err) { 1628 if (dname_external(dentry)) 1629 kfree(external_name(dentry)); 1630 kmem_cache_free(dentry_cache, dentry); 1631 return NULL; 1632 } 1633 } 1634 1635 this_cpu_inc(nr_dentry); 1636 1637 return dentry; 1638 } 1639 1640 /** 1641 * d_alloc - allocate a dcache entry 1642 * @parent: parent of entry to allocate 1643 * @name: qstr of the name 1644 * 1645 * Allocates a dentry. It returns %NULL if there is insufficient memory 1646 * available. On a success the dentry is returned. The name passed in is 1647 * copied and the copy passed in may be reused after this call. 1648 */ 1649 struct dentry *d_alloc(struct dentry * parent, const struct qstr *name) 1650 { 1651 struct dentry *dentry = __d_alloc(parent->d_sb, name); 1652 if (!dentry) 1653 return NULL; 1654 dentry->d_flags |= DCACHE_RCUACCESS; 1655 spin_lock(&parent->d_lock); 1656 /* 1657 * don't need child lock because it is not subject 1658 * to concurrency here 1659 */ 1660 __dget_dlock(parent); 1661 dentry->d_parent = parent; 1662 list_add(&dentry->d_child, &parent->d_subdirs); 1663 spin_unlock(&parent->d_lock); 1664 1665 return dentry; 1666 } 1667 EXPORT_SYMBOL(d_alloc); 1668 1669 struct dentry *d_alloc_cursor(struct dentry * parent) 1670 { 1671 struct dentry *dentry = __d_alloc(parent->d_sb, NULL); 1672 if (dentry) { 1673 dentry->d_flags |= DCACHE_RCUACCESS | DCACHE_DENTRY_CURSOR; 1674 dentry->d_parent = dget(parent); 1675 } 1676 return dentry; 1677 } 1678 1679 /** 1680 * d_alloc_pseudo - allocate a dentry (for lookup-less filesystems) 1681 * @sb: the superblock 1682 * @name: qstr of the name 1683 * 1684 * For a filesystem that just pins its dentries in memory and never 1685 * performs lookups at all, return an unhashed IS_ROOT dentry. 1686 */ 1687 struct dentry *d_alloc_pseudo(struct super_block *sb, const struct qstr *name) 1688 { 1689 return __d_alloc(sb, name); 1690 } 1691 EXPORT_SYMBOL(d_alloc_pseudo); 1692 1693 struct dentry *d_alloc_name(struct dentry *parent, const char *name) 1694 { 1695 struct qstr q; 1696 1697 q.name = name; 1698 q.hash_len = hashlen_string(parent, name); 1699 return d_alloc(parent, &q); 1700 } 1701 EXPORT_SYMBOL(d_alloc_name); 1702 1703 void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op) 1704 { 1705 WARN_ON_ONCE(dentry->d_op); 1706 WARN_ON_ONCE(dentry->d_flags & (DCACHE_OP_HASH | 1707 DCACHE_OP_COMPARE | 1708 DCACHE_OP_REVALIDATE | 1709 DCACHE_OP_WEAK_REVALIDATE | 1710 DCACHE_OP_DELETE | 1711 DCACHE_OP_REAL)); 1712 dentry->d_op = op; 1713 if (!op) 1714 return; 1715 if (op->d_hash) 1716 dentry->d_flags |= DCACHE_OP_HASH; 1717 if (op->d_compare) 1718 dentry->d_flags |= DCACHE_OP_COMPARE; 1719 if (op->d_revalidate) 1720 dentry->d_flags |= DCACHE_OP_REVALIDATE; 1721 if (op->d_weak_revalidate) 1722 dentry->d_flags |= DCACHE_OP_WEAK_REVALIDATE; 1723 if (op->d_delete) 1724 dentry->d_flags |= DCACHE_OP_DELETE; 1725 if (op->d_prune) 1726 dentry->d_flags |= DCACHE_OP_PRUNE; 1727 if (op->d_real) 1728 dentry->d_flags |= DCACHE_OP_REAL; 1729 1730 } 1731 EXPORT_SYMBOL(d_set_d_op); 1732 1733 1734 /* 1735 * d_set_fallthru - Mark a dentry as falling through to a lower layer 1736 * @dentry - The dentry to mark 1737 * 1738 * Mark a dentry as falling through to the lower layer (as set with 1739 * d_pin_lower()). This flag may be recorded on the medium. 1740 */ 1741 void d_set_fallthru(struct dentry *dentry) 1742 { 1743 spin_lock(&dentry->d_lock); 1744 dentry->d_flags |= DCACHE_FALLTHRU; 1745 spin_unlock(&dentry->d_lock); 1746 } 1747 EXPORT_SYMBOL(d_set_fallthru); 1748 1749 static unsigned d_flags_for_inode(struct inode *inode) 1750 { 1751 unsigned add_flags = DCACHE_REGULAR_TYPE; 1752 1753 if (!inode) 1754 return DCACHE_MISS_TYPE; 1755 1756 if (S_ISDIR(inode->i_mode)) { 1757 add_flags = DCACHE_DIRECTORY_TYPE; 1758 if (unlikely(!(inode->i_opflags & IOP_LOOKUP))) { 1759 if (unlikely(!inode->i_op->lookup)) 1760 add_flags = DCACHE_AUTODIR_TYPE; 1761 else 1762 inode->i_opflags |= IOP_LOOKUP; 1763 } 1764 goto type_determined; 1765 } 1766 1767 if (unlikely(!(inode->i_opflags & IOP_NOFOLLOW))) { 1768 if (unlikely(inode->i_op->get_link)) { 1769 add_flags = DCACHE_SYMLINK_TYPE; 1770 goto type_determined; 1771 } 1772 inode->i_opflags |= IOP_NOFOLLOW; 1773 } 1774 1775 if (unlikely(!S_ISREG(inode->i_mode))) 1776 add_flags = DCACHE_SPECIAL_TYPE; 1777 1778 type_determined: 1779 if (unlikely(IS_AUTOMOUNT(inode))) 1780 add_flags |= DCACHE_NEED_AUTOMOUNT; 1781 return add_flags; 1782 } 1783 1784 static void __d_instantiate(struct dentry *dentry, struct inode *inode) 1785 { 1786 unsigned add_flags = d_flags_for_inode(inode); 1787 WARN_ON(d_in_lookup(dentry)); 1788 1789 spin_lock(&dentry->d_lock); 1790 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); 1791 raw_write_seqcount_begin(&dentry->d_seq); 1792 __d_set_inode_and_type(dentry, inode, add_flags); 1793 raw_write_seqcount_end(&dentry->d_seq); 1794 fsnotify_update_flags(dentry); 1795 spin_unlock(&dentry->d_lock); 1796 } 1797 1798 /** 1799 * d_instantiate - fill in inode information for a dentry 1800 * @entry: dentry to complete 1801 * @inode: inode to attach to this dentry 1802 * 1803 * Fill in inode information in the entry. 1804 * 1805 * This turns negative dentries into productive full members 1806 * of society. 1807 * 1808 * NOTE! This assumes that the inode count has been incremented 1809 * (or otherwise set) by the caller to indicate that it is now 1810 * in use by the dcache. 1811 */ 1812 1813 void d_instantiate(struct dentry *entry, struct inode * inode) 1814 { 1815 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias)); 1816 if (inode) { 1817 security_d_instantiate(entry, inode); 1818 spin_lock(&inode->i_lock); 1819 __d_instantiate(entry, inode); 1820 spin_unlock(&inode->i_lock); 1821 } 1822 } 1823 EXPORT_SYMBOL(d_instantiate); 1824 1825 /** 1826 * d_instantiate_no_diralias - instantiate a non-aliased dentry 1827 * @entry: dentry to complete 1828 * @inode: inode to attach to this dentry 1829 * 1830 * Fill in inode information in the entry. If a directory alias is found, then 1831 * return an error (and drop inode). Together with d_materialise_unique() this 1832 * guarantees that a directory inode may never have more than one alias. 1833 */ 1834 int d_instantiate_no_diralias(struct dentry *entry, struct inode *inode) 1835 { 1836 BUG_ON(!hlist_unhashed(&entry->d_u.d_alias)); 1837 1838 security_d_instantiate(entry, inode); 1839 spin_lock(&inode->i_lock); 1840 if (S_ISDIR(inode->i_mode) && !hlist_empty(&inode->i_dentry)) { 1841 spin_unlock(&inode->i_lock); 1842 iput(inode); 1843 return -EBUSY; 1844 } 1845 __d_instantiate(entry, inode); 1846 spin_unlock(&inode->i_lock); 1847 1848 return 0; 1849 } 1850 EXPORT_SYMBOL(d_instantiate_no_diralias); 1851 1852 struct dentry *d_make_root(struct inode *root_inode) 1853 { 1854 struct dentry *res = NULL; 1855 1856 if (root_inode) { 1857 res = __d_alloc(root_inode->i_sb, NULL); 1858 if (res) 1859 d_instantiate(res, root_inode); 1860 else 1861 iput(root_inode); 1862 } 1863 return res; 1864 } 1865 EXPORT_SYMBOL(d_make_root); 1866 1867 static struct dentry * __d_find_any_alias(struct inode *inode) 1868 { 1869 struct dentry *alias; 1870 1871 if (hlist_empty(&inode->i_dentry)) 1872 return NULL; 1873 alias = hlist_entry(inode->i_dentry.first, struct dentry, d_u.d_alias); 1874 __dget(alias); 1875 return alias; 1876 } 1877 1878 /** 1879 * d_find_any_alias - find any alias for a given inode 1880 * @inode: inode to find an alias for 1881 * 1882 * If any aliases exist for the given inode, take and return a 1883 * reference for one of them. If no aliases exist, return %NULL. 1884 */ 1885 struct dentry *d_find_any_alias(struct inode *inode) 1886 { 1887 struct dentry *de; 1888 1889 spin_lock(&inode->i_lock); 1890 de = __d_find_any_alias(inode); 1891 spin_unlock(&inode->i_lock); 1892 return de; 1893 } 1894 EXPORT_SYMBOL(d_find_any_alias); 1895 1896 static struct dentry *__d_obtain_alias(struct inode *inode, int disconnected) 1897 { 1898 struct dentry *tmp; 1899 struct dentry *res; 1900 unsigned add_flags; 1901 1902 if (!inode) 1903 return ERR_PTR(-ESTALE); 1904 if (IS_ERR(inode)) 1905 return ERR_CAST(inode); 1906 1907 res = d_find_any_alias(inode); 1908 if (res) 1909 goto out_iput; 1910 1911 tmp = __d_alloc(inode->i_sb, NULL); 1912 if (!tmp) { 1913 res = ERR_PTR(-ENOMEM); 1914 goto out_iput; 1915 } 1916 1917 security_d_instantiate(tmp, inode); 1918 spin_lock(&inode->i_lock); 1919 res = __d_find_any_alias(inode); 1920 if (res) { 1921 spin_unlock(&inode->i_lock); 1922 dput(tmp); 1923 goto out_iput; 1924 } 1925 1926 /* attach a disconnected dentry */ 1927 add_flags = d_flags_for_inode(inode); 1928 1929 if (disconnected) 1930 add_flags |= DCACHE_DISCONNECTED; 1931 1932 spin_lock(&tmp->d_lock); 1933 __d_set_inode_and_type(tmp, inode, add_flags); 1934 hlist_add_head(&tmp->d_u.d_alias, &inode->i_dentry); 1935 hlist_bl_lock(&tmp->d_sb->s_anon); 1936 hlist_bl_add_head(&tmp->d_hash, &tmp->d_sb->s_anon); 1937 hlist_bl_unlock(&tmp->d_sb->s_anon); 1938 spin_unlock(&tmp->d_lock); 1939 spin_unlock(&inode->i_lock); 1940 1941 return tmp; 1942 1943 out_iput: 1944 iput(inode); 1945 return res; 1946 } 1947 1948 /** 1949 * d_obtain_alias - find or allocate a DISCONNECTED dentry for a given inode 1950 * @inode: inode to allocate the dentry for 1951 * 1952 * Obtain a dentry for an inode resulting from NFS filehandle conversion or 1953 * similar open by handle operations. The returned dentry may be anonymous, 1954 * or may have a full name (if the inode was already in the cache). 1955 * 1956 * When called on a directory inode, we must ensure that the inode only ever 1957 * has one dentry. If a dentry is found, that is returned instead of 1958 * allocating a new one. 1959 * 1960 * On successful return, the reference to the inode has been transferred 1961 * to the dentry. In case of an error the reference on the inode is released. 1962 * To make it easier to use in export operations a %NULL or IS_ERR inode may 1963 * be passed in and the error will be propagated to the return value, 1964 * with a %NULL @inode replaced by ERR_PTR(-ESTALE). 1965 */ 1966 struct dentry *d_obtain_alias(struct inode *inode) 1967 { 1968 return __d_obtain_alias(inode, 1); 1969 } 1970 EXPORT_SYMBOL(d_obtain_alias); 1971 1972 /** 1973 * d_obtain_root - find or allocate a dentry for a given inode 1974 * @inode: inode to allocate the dentry for 1975 * 1976 * Obtain an IS_ROOT dentry for the root of a filesystem. 1977 * 1978 * We must ensure that directory inodes only ever have one dentry. If a 1979 * dentry is found, that is returned instead of allocating a new one. 1980 * 1981 * On successful return, the reference to the inode has been transferred 1982 * to the dentry. In case of an error the reference on the inode is 1983 * released. A %NULL or IS_ERR inode may be passed in and will be the 1984 * error will be propagate to the return value, with a %NULL @inode 1985 * replaced by ERR_PTR(-ESTALE). 1986 */ 1987 struct dentry *d_obtain_root(struct inode *inode) 1988 { 1989 return __d_obtain_alias(inode, 0); 1990 } 1991 EXPORT_SYMBOL(d_obtain_root); 1992 1993 /** 1994 * d_add_ci - lookup or allocate new dentry with case-exact name 1995 * @inode: the inode case-insensitive lookup has found 1996 * @dentry: the negative dentry that was passed to the parent's lookup func 1997 * @name: the case-exact name to be associated with the returned dentry 1998 * 1999 * This is to avoid filling the dcache with case-insensitive names to the 2000 * same inode, only the actual correct case is stored in the dcache for 2001 * case-insensitive filesystems. 2002 * 2003 * For a case-insensitive lookup match and if the the case-exact dentry 2004 * already exists in in the dcache, use it and return it. 2005 * 2006 * If no entry exists with the exact case name, allocate new dentry with 2007 * the exact case, and return the spliced entry. 2008 */ 2009 struct dentry *d_add_ci(struct dentry *dentry, struct inode *inode, 2010 struct qstr *name) 2011 { 2012 struct dentry *found, *res; 2013 2014 /* 2015 * First check if a dentry matching the name already exists, 2016 * if not go ahead and create it now. 2017 */ 2018 found = d_hash_and_lookup(dentry->d_parent, name); 2019 if (found) { 2020 iput(inode); 2021 return found; 2022 } 2023 if (d_in_lookup(dentry)) { 2024 found = d_alloc_parallel(dentry->d_parent, name, 2025 dentry->d_wait); 2026 if (IS_ERR(found) || !d_in_lookup(found)) { 2027 iput(inode); 2028 return found; 2029 } 2030 } else { 2031 found = d_alloc(dentry->d_parent, name); 2032 if (!found) { 2033 iput(inode); 2034 return ERR_PTR(-ENOMEM); 2035 } 2036 } 2037 res = d_splice_alias(inode, found); 2038 if (res) { 2039 dput(found); 2040 return res; 2041 } 2042 return found; 2043 } 2044 EXPORT_SYMBOL(d_add_ci); 2045 2046 2047 static inline bool d_same_name(const struct dentry *dentry, 2048 const struct dentry *parent, 2049 const struct qstr *name) 2050 { 2051 if (likely(!(parent->d_flags & DCACHE_OP_COMPARE))) { 2052 if (dentry->d_name.len != name->len) 2053 return false; 2054 return dentry_cmp(dentry, name->name, name->len) == 0; 2055 } 2056 return parent->d_op->d_compare(dentry, 2057 dentry->d_name.len, dentry->d_name.name, 2058 name) == 0; 2059 } 2060 2061 /** 2062 * __d_lookup_rcu - search for a dentry (racy, store-free) 2063 * @parent: parent dentry 2064 * @name: qstr of name we wish to find 2065 * @seqp: returns d_seq value at the point where the dentry was found 2066 * Returns: dentry, or NULL 2067 * 2068 * __d_lookup_rcu is the dcache lookup function for rcu-walk name 2069 * resolution (store-free path walking) design described in 2070 * Documentation/filesystems/path-lookup.txt. 2071 * 2072 * This is not to be used outside core vfs. 2073 * 2074 * __d_lookup_rcu must only be used in rcu-walk mode, ie. with vfsmount lock 2075 * held, and rcu_read_lock held. The returned dentry must not be stored into 2076 * without taking d_lock and checking d_seq sequence count against @seq 2077 * returned here. 2078 * 2079 * A refcount may be taken on the found dentry with the d_rcu_to_refcount 2080 * function. 2081 * 2082 * Alternatively, __d_lookup_rcu may be called again to look up the child of 2083 * the returned dentry, so long as its parent's seqlock is checked after the 2084 * child is looked up. Thus, an interlocking stepping of sequence lock checks 2085 * is formed, giving integrity down the path walk. 2086 * 2087 * NOTE! The caller *has* to check the resulting dentry against the sequence 2088 * number we've returned before using any of the resulting dentry state! 2089 */ 2090 struct dentry *__d_lookup_rcu(const struct dentry *parent, 2091 const struct qstr *name, 2092 unsigned *seqp) 2093 { 2094 u64 hashlen = name->hash_len; 2095 const unsigned char *str = name->name; 2096 struct hlist_bl_head *b = d_hash(hashlen_hash(hashlen)); 2097 struct hlist_bl_node *node; 2098 struct dentry *dentry; 2099 2100 /* 2101 * Note: There is significant duplication with __d_lookup_rcu which is 2102 * required to prevent single threaded performance regressions 2103 * especially on architectures where smp_rmb (in seqcounts) are costly. 2104 * Keep the two functions in sync. 2105 */ 2106 2107 /* 2108 * The hash list is protected using RCU. 2109 * 2110 * Carefully use d_seq when comparing a candidate dentry, to avoid 2111 * races with d_move(). 2112 * 2113 * It is possible that concurrent renames can mess up our list 2114 * walk here and result in missing our dentry, resulting in the 2115 * false-negative result. d_lookup() protects against concurrent 2116 * renames using rename_lock seqlock. 2117 * 2118 * See Documentation/filesystems/path-lookup.txt for more details. 2119 */ 2120 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2121 unsigned seq; 2122 2123 seqretry: 2124 /* 2125 * The dentry sequence count protects us from concurrent 2126 * renames, and thus protects parent and name fields. 2127 * 2128 * The caller must perform a seqcount check in order 2129 * to do anything useful with the returned dentry. 2130 * 2131 * NOTE! We do a "raw" seqcount_begin here. That means that 2132 * we don't wait for the sequence count to stabilize if it 2133 * is in the middle of a sequence change. If we do the slow 2134 * dentry compare, we will do seqretries until it is stable, 2135 * and if we end up with a successful lookup, we actually 2136 * want to exit RCU lookup anyway. 2137 * 2138 * Note that raw_seqcount_begin still *does* smp_rmb(), so 2139 * we are still guaranteed NUL-termination of ->d_name.name. 2140 */ 2141 seq = raw_seqcount_begin(&dentry->d_seq); 2142 if (dentry->d_parent != parent) 2143 continue; 2144 if (d_unhashed(dentry)) 2145 continue; 2146 2147 if (unlikely(parent->d_flags & DCACHE_OP_COMPARE)) { 2148 int tlen; 2149 const char *tname; 2150 if (dentry->d_name.hash != hashlen_hash(hashlen)) 2151 continue; 2152 tlen = dentry->d_name.len; 2153 tname = dentry->d_name.name; 2154 /* we want a consistent (name,len) pair */ 2155 if (read_seqcount_retry(&dentry->d_seq, seq)) { 2156 cpu_relax(); 2157 goto seqretry; 2158 } 2159 if (parent->d_op->d_compare(dentry, 2160 tlen, tname, name) != 0) 2161 continue; 2162 } else { 2163 if (dentry->d_name.hash_len != hashlen) 2164 continue; 2165 if (dentry_cmp(dentry, str, hashlen_len(hashlen)) != 0) 2166 continue; 2167 } 2168 *seqp = seq; 2169 return dentry; 2170 } 2171 return NULL; 2172 } 2173 2174 /** 2175 * d_lookup - search for a dentry 2176 * @parent: parent dentry 2177 * @name: qstr of name we wish to find 2178 * Returns: dentry, or NULL 2179 * 2180 * d_lookup searches the children of the parent dentry for the name in 2181 * question. If the dentry is found its reference count is incremented and the 2182 * dentry is returned. The caller must use dput to free the entry when it has 2183 * finished using it. %NULL is returned if the dentry does not exist. 2184 */ 2185 struct dentry *d_lookup(const struct dentry *parent, const struct qstr *name) 2186 { 2187 struct dentry *dentry; 2188 unsigned seq; 2189 2190 do { 2191 seq = read_seqbegin(&rename_lock); 2192 dentry = __d_lookup(parent, name); 2193 if (dentry) 2194 break; 2195 } while (read_seqretry(&rename_lock, seq)); 2196 return dentry; 2197 } 2198 EXPORT_SYMBOL(d_lookup); 2199 2200 /** 2201 * __d_lookup - search for a dentry (racy) 2202 * @parent: parent dentry 2203 * @name: qstr of name we wish to find 2204 * Returns: dentry, or NULL 2205 * 2206 * __d_lookup is like d_lookup, however it may (rarely) return a 2207 * false-negative result due to unrelated rename activity. 2208 * 2209 * __d_lookup is slightly faster by avoiding rename_lock read seqlock, 2210 * however it must be used carefully, eg. with a following d_lookup in 2211 * the case of failure. 2212 * 2213 * __d_lookup callers must be commented. 2214 */ 2215 struct dentry *__d_lookup(const struct dentry *parent, const struct qstr *name) 2216 { 2217 unsigned int hash = name->hash; 2218 struct hlist_bl_head *b = d_hash(hash); 2219 struct hlist_bl_node *node; 2220 struct dentry *found = NULL; 2221 struct dentry *dentry; 2222 2223 /* 2224 * Note: There is significant duplication with __d_lookup_rcu which is 2225 * required to prevent single threaded performance regressions 2226 * especially on architectures where smp_rmb (in seqcounts) are costly. 2227 * Keep the two functions in sync. 2228 */ 2229 2230 /* 2231 * The hash list is protected using RCU. 2232 * 2233 * Take d_lock when comparing a candidate dentry, to avoid races 2234 * with d_move(). 2235 * 2236 * It is possible that concurrent renames can mess up our list 2237 * walk here and result in missing our dentry, resulting in the 2238 * false-negative result. d_lookup() protects against concurrent 2239 * renames using rename_lock seqlock. 2240 * 2241 * See Documentation/filesystems/path-lookup.txt for more details. 2242 */ 2243 rcu_read_lock(); 2244 2245 hlist_bl_for_each_entry_rcu(dentry, node, b, d_hash) { 2246 2247 if (dentry->d_name.hash != hash) 2248 continue; 2249 2250 spin_lock(&dentry->d_lock); 2251 if (dentry->d_parent != parent) 2252 goto next; 2253 if (d_unhashed(dentry)) 2254 goto next; 2255 2256 if (!d_same_name(dentry, parent, name)) 2257 goto next; 2258 2259 dentry->d_lockref.count++; 2260 found = dentry; 2261 spin_unlock(&dentry->d_lock); 2262 break; 2263 next: 2264 spin_unlock(&dentry->d_lock); 2265 } 2266 rcu_read_unlock(); 2267 2268 return found; 2269 } 2270 2271 /** 2272 * d_hash_and_lookup - hash the qstr then search for a dentry 2273 * @dir: Directory to search in 2274 * @name: qstr of name we wish to find 2275 * 2276 * On lookup failure NULL is returned; on bad name - ERR_PTR(-error) 2277 */ 2278 struct dentry *d_hash_and_lookup(struct dentry *dir, struct qstr *name) 2279 { 2280 /* 2281 * Check for a fs-specific hash function. Note that we must 2282 * calculate the standard hash first, as the d_op->d_hash() 2283 * routine may choose to leave the hash value unchanged. 2284 */ 2285 name->hash = full_name_hash(dir, name->name, name->len); 2286 if (dir->d_flags & DCACHE_OP_HASH) { 2287 int err = dir->d_op->d_hash(dir, name); 2288 if (unlikely(err < 0)) 2289 return ERR_PTR(err); 2290 } 2291 return d_lookup(dir, name); 2292 } 2293 EXPORT_SYMBOL(d_hash_and_lookup); 2294 2295 /* 2296 * When a file is deleted, we have two options: 2297 * - turn this dentry into a negative dentry 2298 * - unhash this dentry and free it. 2299 * 2300 * Usually, we want to just turn this into 2301 * a negative dentry, but if anybody else is 2302 * currently using the dentry or the inode 2303 * we can't do that and we fall back on removing 2304 * it from the hash queues and waiting for 2305 * it to be deleted later when it has no users 2306 */ 2307 2308 /** 2309 * d_delete - delete a dentry 2310 * @dentry: The dentry to delete 2311 * 2312 * Turn the dentry into a negative dentry if possible, otherwise 2313 * remove it from the hash queues so it can be deleted later 2314 */ 2315 2316 void d_delete(struct dentry * dentry) 2317 { 2318 struct inode *inode; 2319 int isdir = 0; 2320 /* 2321 * Are we the only user? 2322 */ 2323 again: 2324 spin_lock(&dentry->d_lock); 2325 inode = dentry->d_inode; 2326 isdir = S_ISDIR(inode->i_mode); 2327 if (dentry->d_lockref.count == 1) { 2328 if (!spin_trylock(&inode->i_lock)) { 2329 spin_unlock(&dentry->d_lock); 2330 cpu_relax(); 2331 goto again; 2332 } 2333 dentry->d_flags &= ~DCACHE_CANT_MOUNT; 2334 dentry_unlink_inode(dentry); 2335 fsnotify_nameremove(dentry, isdir); 2336 return; 2337 } 2338 2339 if (!d_unhashed(dentry)) 2340 __d_drop(dentry); 2341 2342 spin_unlock(&dentry->d_lock); 2343 2344 fsnotify_nameremove(dentry, isdir); 2345 } 2346 EXPORT_SYMBOL(d_delete); 2347 2348 static void __d_rehash(struct dentry *entry) 2349 { 2350 struct hlist_bl_head *b = d_hash(entry->d_name.hash); 2351 BUG_ON(!d_unhashed(entry)); 2352 hlist_bl_lock(b); 2353 hlist_bl_add_head_rcu(&entry->d_hash, b); 2354 hlist_bl_unlock(b); 2355 } 2356 2357 /** 2358 * d_rehash - add an entry back to the hash 2359 * @entry: dentry to add to the hash 2360 * 2361 * Adds a dentry to the hash according to its name. 2362 */ 2363 2364 void d_rehash(struct dentry * entry) 2365 { 2366 spin_lock(&entry->d_lock); 2367 __d_rehash(entry); 2368 spin_unlock(&entry->d_lock); 2369 } 2370 EXPORT_SYMBOL(d_rehash); 2371 2372 static inline unsigned start_dir_add(struct inode *dir) 2373 { 2374 2375 for (;;) { 2376 unsigned n = dir->i_dir_seq; 2377 if (!(n & 1) && cmpxchg(&dir->i_dir_seq, n, n + 1) == n) 2378 return n; 2379 cpu_relax(); 2380 } 2381 } 2382 2383 static inline void end_dir_add(struct inode *dir, unsigned n) 2384 { 2385 smp_store_release(&dir->i_dir_seq, n + 2); 2386 } 2387 2388 static void d_wait_lookup(struct dentry *dentry) 2389 { 2390 if (d_in_lookup(dentry)) { 2391 DECLARE_WAITQUEUE(wait, current); 2392 add_wait_queue(dentry->d_wait, &wait); 2393 do { 2394 set_current_state(TASK_UNINTERRUPTIBLE); 2395 spin_unlock(&dentry->d_lock); 2396 schedule(); 2397 spin_lock(&dentry->d_lock); 2398 } while (d_in_lookup(dentry)); 2399 } 2400 } 2401 2402 struct dentry *d_alloc_parallel(struct dentry *parent, 2403 const struct qstr *name, 2404 wait_queue_head_t *wq) 2405 { 2406 unsigned int hash = name->hash; 2407 struct hlist_bl_head *b = in_lookup_hash(parent, hash); 2408 struct hlist_bl_node *node; 2409 struct dentry *new = d_alloc(parent, name); 2410 struct dentry *dentry; 2411 unsigned seq, r_seq, d_seq; 2412 2413 if (unlikely(!new)) 2414 return ERR_PTR(-ENOMEM); 2415 2416 retry: 2417 rcu_read_lock(); 2418 seq = smp_load_acquire(&parent->d_inode->i_dir_seq) & ~1; 2419 r_seq = read_seqbegin(&rename_lock); 2420 dentry = __d_lookup_rcu(parent, name, &d_seq); 2421 if (unlikely(dentry)) { 2422 if (!lockref_get_not_dead(&dentry->d_lockref)) { 2423 rcu_read_unlock(); 2424 goto retry; 2425 } 2426 if (read_seqcount_retry(&dentry->d_seq, d_seq)) { 2427 rcu_read_unlock(); 2428 dput(dentry); 2429 goto retry; 2430 } 2431 rcu_read_unlock(); 2432 dput(new); 2433 return dentry; 2434 } 2435 if (unlikely(read_seqretry(&rename_lock, r_seq))) { 2436 rcu_read_unlock(); 2437 goto retry; 2438 } 2439 hlist_bl_lock(b); 2440 if (unlikely(parent->d_inode->i_dir_seq != seq)) { 2441 hlist_bl_unlock(b); 2442 rcu_read_unlock(); 2443 goto retry; 2444 } 2445 /* 2446 * No changes for the parent since the beginning of d_lookup(). 2447 * Since all removals from the chain happen with hlist_bl_lock(), 2448 * any potential in-lookup matches are going to stay here until 2449 * we unlock the chain. All fields are stable in everything 2450 * we encounter. 2451 */ 2452 hlist_bl_for_each_entry(dentry, node, b, d_u.d_in_lookup_hash) { 2453 if (dentry->d_name.hash != hash) 2454 continue; 2455 if (dentry->d_parent != parent) 2456 continue; 2457 if (!d_same_name(dentry, parent, name)) 2458 continue; 2459 hlist_bl_unlock(b); 2460 /* now we can try to grab a reference */ 2461 if (!lockref_get_not_dead(&dentry->d_lockref)) { 2462 rcu_read_unlock(); 2463 goto retry; 2464 } 2465 2466 rcu_read_unlock(); 2467 /* 2468 * somebody is likely to be still doing lookup for it; 2469 * wait for them to finish 2470 */ 2471 spin_lock(&dentry->d_lock); 2472 d_wait_lookup(dentry); 2473 /* 2474 * it's not in-lookup anymore; in principle we should repeat 2475 * everything from dcache lookup, but it's likely to be what 2476 * d_lookup() would've found anyway. If it is, just return it; 2477 * otherwise we really have to repeat the whole thing. 2478 */ 2479 if (unlikely(dentry->d_name.hash != hash)) 2480 goto mismatch; 2481 if (unlikely(dentry->d_parent != parent)) 2482 goto mismatch; 2483 if (unlikely(d_unhashed(dentry))) 2484 goto mismatch; 2485 if (unlikely(!d_same_name(dentry, parent, name))) 2486 goto mismatch; 2487 /* OK, it *is* a hashed match; return it */ 2488 spin_unlock(&dentry->d_lock); 2489 dput(new); 2490 return dentry; 2491 } 2492 rcu_read_unlock(); 2493 /* we can't take ->d_lock here; it's OK, though. */ 2494 new->d_flags |= DCACHE_PAR_LOOKUP; 2495 new->d_wait = wq; 2496 hlist_bl_add_head_rcu(&new->d_u.d_in_lookup_hash, b); 2497 hlist_bl_unlock(b); 2498 return new; 2499 mismatch: 2500 spin_unlock(&dentry->d_lock); 2501 dput(dentry); 2502 goto retry; 2503 } 2504 EXPORT_SYMBOL(d_alloc_parallel); 2505 2506 void __d_lookup_done(struct dentry *dentry) 2507 { 2508 struct hlist_bl_head *b = in_lookup_hash(dentry->d_parent, 2509 dentry->d_name.hash); 2510 hlist_bl_lock(b); 2511 dentry->d_flags &= ~DCACHE_PAR_LOOKUP; 2512 __hlist_bl_del(&dentry->d_u.d_in_lookup_hash); 2513 wake_up_all(dentry->d_wait); 2514 dentry->d_wait = NULL; 2515 hlist_bl_unlock(b); 2516 INIT_HLIST_NODE(&dentry->d_u.d_alias); 2517 INIT_LIST_HEAD(&dentry->d_lru); 2518 } 2519 EXPORT_SYMBOL(__d_lookup_done); 2520 2521 /* inode->i_lock held if inode is non-NULL */ 2522 2523 static inline void __d_add(struct dentry *dentry, struct inode *inode) 2524 { 2525 struct inode *dir = NULL; 2526 unsigned n; 2527 spin_lock(&dentry->d_lock); 2528 if (unlikely(d_in_lookup(dentry))) { 2529 dir = dentry->d_parent->d_inode; 2530 n = start_dir_add(dir); 2531 __d_lookup_done(dentry); 2532 } 2533 if (inode) { 2534 unsigned add_flags = d_flags_for_inode(inode); 2535 hlist_add_head(&dentry->d_u.d_alias, &inode->i_dentry); 2536 raw_write_seqcount_begin(&dentry->d_seq); 2537 __d_set_inode_and_type(dentry, inode, add_flags); 2538 raw_write_seqcount_end(&dentry->d_seq); 2539 fsnotify_update_flags(dentry); 2540 } 2541 __d_rehash(dentry); 2542 if (dir) 2543 end_dir_add(dir, n); 2544 spin_unlock(&dentry->d_lock); 2545 if (inode) 2546 spin_unlock(&inode->i_lock); 2547 } 2548 2549 /** 2550 * d_add - add dentry to hash queues 2551 * @entry: dentry to add 2552 * @inode: The inode to attach to this dentry 2553 * 2554 * This adds the entry to the hash queues and initializes @inode. 2555 * The entry was actually filled in earlier during d_alloc(). 2556 */ 2557 2558 void d_add(struct dentry *entry, struct inode *inode) 2559 { 2560 if (inode) { 2561 security_d_instantiate(entry, inode); 2562 spin_lock(&inode->i_lock); 2563 } 2564 __d_add(entry, inode); 2565 } 2566 EXPORT_SYMBOL(d_add); 2567 2568 /** 2569 * d_exact_alias - find and hash an exact unhashed alias 2570 * @entry: dentry to add 2571 * @inode: The inode to go with this dentry 2572 * 2573 * If an unhashed dentry with the same name/parent and desired 2574 * inode already exists, hash and return it. Otherwise, return 2575 * NULL. 2576 * 2577 * Parent directory should be locked. 2578 */ 2579 struct dentry *d_exact_alias(struct dentry *entry, struct inode *inode) 2580 { 2581 struct dentry *alias; 2582 unsigned int hash = entry->d_name.hash; 2583 2584 spin_lock(&inode->i_lock); 2585 hlist_for_each_entry(alias, &inode->i_dentry, d_u.d_alias) { 2586 /* 2587 * Don't need alias->d_lock here, because aliases with 2588 * d_parent == entry->d_parent are not subject to name or 2589 * parent changes, because the parent inode i_mutex is held. 2590 */ 2591 if (alias->d_name.hash != hash) 2592 continue; 2593 if (alias->d_parent != entry->d_parent) 2594 continue; 2595 if (!d_same_name(alias, entry->d_parent, &entry->d_name)) 2596 continue; 2597 spin_lock(&alias->d_lock); 2598 if (!d_unhashed(alias)) { 2599 spin_unlock(&alias->d_lock); 2600 alias = NULL; 2601 } else { 2602 __dget_dlock(alias); 2603 __d_rehash(alias); 2604 spin_unlock(&alias->d_lock); 2605 } 2606 spin_unlock(&inode->i_lock); 2607 return alias; 2608 } 2609 spin_unlock(&inode->i_lock); 2610 return NULL; 2611 } 2612 EXPORT_SYMBOL(d_exact_alias); 2613 2614 /** 2615 * dentry_update_name_case - update case insensitive dentry with a new name 2616 * @dentry: dentry to be updated 2617 * @name: new name 2618 * 2619 * Update a case insensitive dentry with new case of name. 2620 * 2621 * dentry must have been returned by d_lookup with name @name. Old and new 2622 * name lengths must match (ie. no d_compare which allows mismatched name 2623 * lengths). 2624 * 2625 * Parent inode i_mutex must be held over d_lookup and into this call (to 2626 * keep renames and concurrent inserts, and readdir(2) away). 2627 */ 2628 void dentry_update_name_case(struct dentry *dentry, const struct qstr *name) 2629 { 2630 BUG_ON(!inode_is_locked(dentry->d_parent->d_inode)); 2631 BUG_ON(dentry->d_name.len != name->len); /* d_lookup gives this */ 2632 2633 spin_lock(&dentry->d_lock); 2634 write_seqcount_begin(&dentry->d_seq); 2635 memcpy((unsigned char *)dentry->d_name.name, name->name, name->len); 2636 write_seqcount_end(&dentry->d_seq); 2637 spin_unlock(&dentry->d_lock); 2638 } 2639 EXPORT_SYMBOL(dentry_update_name_case); 2640 2641 static void swap_names(struct dentry *dentry, struct dentry *target) 2642 { 2643 if (unlikely(dname_external(target))) { 2644 if (unlikely(dname_external(dentry))) { 2645 /* 2646 * Both external: swap the pointers 2647 */ 2648 swap(target->d_name.name, dentry->d_name.name); 2649 } else { 2650 /* 2651 * dentry:internal, target:external. Steal target's 2652 * storage and make target internal. 2653 */ 2654 memcpy(target->d_iname, dentry->d_name.name, 2655 dentry->d_name.len + 1); 2656 dentry->d_name.name = target->d_name.name; 2657 target->d_name.name = target->d_iname; 2658 } 2659 } else { 2660 if (unlikely(dname_external(dentry))) { 2661 /* 2662 * dentry:external, target:internal. Give dentry's 2663 * storage to target and make dentry internal 2664 */ 2665 memcpy(dentry->d_iname, target->d_name.name, 2666 target->d_name.len + 1); 2667 target->d_name.name = dentry->d_name.name; 2668 dentry->d_name.name = dentry->d_iname; 2669 } else { 2670 /* 2671 * Both are internal. 2672 */ 2673 unsigned int i; 2674 BUILD_BUG_ON(!IS_ALIGNED(DNAME_INLINE_LEN, sizeof(long))); 2675 kmemcheck_mark_initialized(dentry->d_iname, DNAME_INLINE_LEN); 2676 kmemcheck_mark_initialized(target->d_iname, DNAME_INLINE_LEN); 2677 for (i = 0; i < DNAME_INLINE_LEN / sizeof(long); i++) { 2678 swap(((long *) &dentry->d_iname)[i], 2679 ((long *) &target->d_iname)[i]); 2680 } 2681 } 2682 } 2683 swap(dentry->d_name.hash_len, target->d_name.hash_len); 2684 } 2685 2686 static void copy_name(struct dentry *dentry, struct dentry *target) 2687 { 2688 struct external_name *old_name = NULL; 2689 if (unlikely(dname_external(dentry))) 2690 old_name = external_name(dentry); 2691 if (unlikely(dname_external(target))) { 2692 atomic_inc(&external_name(target)->u.count); 2693 dentry->d_name = target->d_name; 2694 } else { 2695 memcpy(dentry->d_iname, target->d_name.name, 2696 target->d_name.len + 1); 2697 dentry->d_name.name = dentry->d_iname; 2698 dentry->d_name.hash_len = target->d_name.hash_len; 2699 } 2700 if (old_name && likely(atomic_dec_and_test(&old_name->u.count))) 2701 kfree_rcu(old_name, u.head); 2702 } 2703 2704 static void dentry_lock_for_move(struct dentry *dentry, struct dentry *target) 2705 { 2706 /* 2707 * XXXX: do we really need to take target->d_lock? 2708 */ 2709 if (IS_ROOT(dentry) || dentry->d_parent == target->d_parent) 2710 spin_lock(&target->d_parent->d_lock); 2711 else { 2712 if (d_ancestor(dentry->d_parent, target->d_parent)) { 2713 spin_lock(&dentry->d_parent->d_lock); 2714 spin_lock_nested(&target->d_parent->d_lock, 2715 DENTRY_D_LOCK_NESTED); 2716 } else { 2717 spin_lock(&target->d_parent->d_lock); 2718 spin_lock_nested(&dentry->d_parent->d_lock, 2719 DENTRY_D_LOCK_NESTED); 2720 } 2721 } 2722 if (target < dentry) { 2723 spin_lock_nested(&target->d_lock, 2); 2724 spin_lock_nested(&dentry->d_lock, 3); 2725 } else { 2726 spin_lock_nested(&dentry->d_lock, 2); 2727 spin_lock_nested(&target->d_lock, 3); 2728 } 2729 } 2730 2731 static void dentry_unlock_for_move(struct dentry *dentry, struct dentry *target) 2732 { 2733 if (target->d_parent != dentry->d_parent) 2734 spin_unlock(&dentry->d_parent->d_lock); 2735 if (target->d_parent != target) 2736 spin_unlock(&target->d_parent->d_lock); 2737 spin_unlock(&target->d_lock); 2738 spin_unlock(&dentry->d_lock); 2739 } 2740 2741 /* 2742 * When switching names, the actual string doesn't strictly have to 2743 * be preserved in the target - because we're dropping the target 2744 * anyway. As such, we can just do a simple memcpy() to copy over 2745 * the new name before we switch, unless we are going to rehash 2746 * it. Note that if we *do* unhash the target, we are not allowed 2747 * to rehash it without giving it a new name/hash key - whether 2748 * we swap or overwrite the names here, resulting name won't match 2749 * the reality in filesystem; it's only there for d_path() purposes. 2750 * Note that all of this is happening under rename_lock, so the 2751 * any hash lookup seeing it in the middle of manipulations will 2752 * be discarded anyway. So we do not care what happens to the hash 2753 * key in that case. 2754 */ 2755 /* 2756 * __d_move - move a dentry 2757 * @dentry: entry to move 2758 * @target: new dentry 2759 * @exchange: exchange the two dentries 2760 * 2761 * Update the dcache to reflect the move of a file name. Negative 2762 * dcache entries should not be moved in this way. Caller must hold 2763 * rename_lock, the i_mutex of the source and target directories, 2764 * and the sb->s_vfs_rename_mutex if they differ. See lock_rename(). 2765 */ 2766 static void __d_move(struct dentry *dentry, struct dentry *target, 2767 bool exchange) 2768 { 2769 struct inode *dir = NULL; 2770 unsigned n; 2771 if (!dentry->d_inode) 2772 printk(KERN_WARNING "VFS: moving negative dcache entry\n"); 2773 2774 BUG_ON(d_ancestor(dentry, target)); 2775 BUG_ON(d_ancestor(target, dentry)); 2776 2777 dentry_lock_for_move(dentry, target); 2778 if (unlikely(d_in_lookup(target))) { 2779 dir = target->d_parent->d_inode; 2780 n = start_dir_add(dir); 2781 __d_lookup_done(target); 2782 } 2783 2784 write_seqcount_begin(&dentry->d_seq); 2785 write_seqcount_begin_nested(&target->d_seq, DENTRY_D_LOCK_NESTED); 2786 2787 /* unhash both */ 2788 /* __d_drop does write_seqcount_barrier, but they're OK to nest. */ 2789 __d_drop(dentry); 2790 __d_drop(target); 2791 2792 /* Switch the names.. */ 2793 if (exchange) 2794 swap_names(dentry, target); 2795 else 2796 copy_name(dentry, target); 2797 2798 /* rehash in new place(s) */ 2799 __d_rehash(dentry); 2800 if (exchange) 2801 __d_rehash(target); 2802 2803 /* ... and switch them in the tree */ 2804 if (IS_ROOT(dentry)) { 2805 /* splicing a tree */ 2806 dentry->d_flags |= DCACHE_RCUACCESS; 2807 dentry->d_parent = target->d_parent; 2808 target->d_parent = target; 2809 list_del_init(&target->d_child); 2810 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs); 2811 } else { 2812 /* swapping two dentries */ 2813 swap(dentry->d_parent, target->d_parent); 2814 list_move(&target->d_child, &target->d_parent->d_subdirs); 2815 list_move(&dentry->d_child, &dentry->d_parent->d_subdirs); 2816 if (exchange) 2817 fsnotify_update_flags(target); 2818 fsnotify_update_flags(dentry); 2819 } 2820 2821 write_seqcount_end(&target->d_seq); 2822 write_seqcount_end(&dentry->d_seq); 2823 2824 if (dir) 2825 end_dir_add(dir, n); 2826 dentry_unlock_for_move(dentry, target); 2827 } 2828 2829 /* 2830 * d_move - move a dentry 2831 * @dentry: entry to move 2832 * @target: new dentry 2833 * 2834 * Update the dcache to reflect the move of a file name. Negative 2835 * dcache entries should not be moved in this way. See the locking 2836 * requirements for __d_move. 2837 */ 2838 void d_move(struct dentry *dentry, struct dentry *target) 2839 { 2840 write_seqlock(&rename_lock); 2841 __d_move(dentry, target, false); 2842 write_sequnlock(&rename_lock); 2843 } 2844 EXPORT_SYMBOL(d_move); 2845 2846 /* 2847 * d_exchange - exchange two dentries 2848 * @dentry1: first dentry 2849 * @dentry2: second dentry 2850 */ 2851 void d_exchange(struct dentry *dentry1, struct dentry *dentry2) 2852 { 2853 write_seqlock(&rename_lock); 2854 2855 WARN_ON(!dentry1->d_inode); 2856 WARN_ON(!dentry2->d_inode); 2857 WARN_ON(IS_ROOT(dentry1)); 2858 WARN_ON(IS_ROOT(dentry2)); 2859 2860 __d_move(dentry1, dentry2, true); 2861 2862 write_sequnlock(&rename_lock); 2863 } 2864 2865 /** 2866 * d_ancestor - search for an ancestor 2867 * @p1: ancestor dentry 2868 * @p2: child dentry 2869 * 2870 * Returns the ancestor dentry of p2 which is a child of p1, if p1 is 2871 * an ancestor of p2, else NULL. 2872 */ 2873 struct dentry *d_ancestor(struct dentry *p1, struct dentry *p2) 2874 { 2875 struct dentry *p; 2876 2877 for (p = p2; !IS_ROOT(p); p = p->d_parent) { 2878 if (p->d_parent == p1) 2879 return p; 2880 } 2881 return NULL; 2882 } 2883 2884 /* 2885 * This helper attempts to cope with remotely renamed directories 2886 * 2887 * It assumes that the caller is already holding 2888 * dentry->d_parent->d_inode->i_mutex, and rename_lock 2889 * 2890 * Note: If ever the locking in lock_rename() changes, then please 2891 * remember to update this too... 2892 */ 2893 static int __d_unalias(struct inode *inode, 2894 struct dentry *dentry, struct dentry *alias) 2895 { 2896 struct mutex *m1 = NULL; 2897 struct rw_semaphore *m2 = NULL; 2898 int ret = -ESTALE; 2899 2900 /* If alias and dentry share a parent, then no extra locks required */ 2901 if (alias->d_parent == dentry->d_parent) 2902 goto out_unalias; 2903 2904 /* See lock_rename() */ 2905 if (!mutex_trylock(&dentry->d_sb->s_vfs_rename_mutex)) 2906 goto out_err; 2907 m1 = &dentry->d_sb->s_vfs_rename_mutex; 2908 if (!inode_trylock_shared(alias->d_parent->d_inode)) 2909 goto out_err; 2910 m2 = &alias->d_parent->d_inode->i_rwsem; 2911 out_unalias: 2912 __d_move(alias, dentry, false); 2913 ret = 0; 2914 out_err: 2915 if (m2) 2916 up_read(m2); 2917 if (m1) 2918 mutex_unlock(m1); 2919 return ret; 2920 } 2921 2922 /** 2923 * d_splice_alias - splice a disconnected dentry into the tree if one exists 2924 * @inode: the inode which may have a disconnected dentry 2925 * @dentry: a negative dentry which we want to point to the inode. 2926 * 2927 * If inode is a directory and has an IS_ROOT alias, then d_move that in 2928 * place of the given dentry and return it, else simply d_add the inode 2929 * to the dentry and return NULL. 2930 * 2931 * If a non-IS_ROOT directory is found, the filesystem is corrupt, and 2932 * we should error out: directories can't have multiple aliases. 2933 * 2934 * This is needed in the lookup routine of any filesystem that is exportable 2935 * (via knfsd) so that we can build dcache paths to directories effectively. 2936 * 2937 * If a dentry was found and moved, then it is returned. Otherwise NULL 2938 * is returned. This matches the expected return value of ->lookup. 2939 * 2940 * Cluster filesystems may call this function with a negative, hashed dentry. 2941 * In that case, we know that the inode will be a regular file, and also this 2942 * will only occur during atomic_open. So we need to check for the dentry 2943 * being already hashed only in the final case. 2944 */ 2945 struct dentry *d_splice_alias(struct inode *inode, struct dentry *dentry) 2946 { 2947 if (IS_ERR(inode)) 2948 return ERR_CAST(inode); 2949 2950 BUG_ON(!d_unhashed(dentry)); 2951 2952 if (!inode) 2953 goto out; 2954 2955 security_d_instantiate(dentry, inode); 2956 spin_lock(&inode->i_lock); 2957 if (S_ISDIR(inode->i_mode)) { 2958 struct dentry *new = __d_find_any_alias(inode); 2959 if (unlikely(new)) { 2960 /* The reference to new ensures it remains an alias */ 2961 spin_unlock(&inode->i_lock); 2962 write_seqlock(&rename_lock); 2963 if (unlikely(d_ancestor(new, dentry))) { 2964 write_sequnlock(&rename_lock); 2965 dput(new); 2966 new = ERR_PTR(-ELOOP); 2967 pr_warn_ratelimited( 2968 "VFS: Lookup of '%s' in %s %s" 2969 " would have caused loop\n", 2970 dentry->d_name.name, 2971 inode->i_sb->s_type->name, 2972 inode->i_sb->s_id); 2973 } else if (!IS_ROOT(new)) { 2974 int err = __d_unalias(inode, dentry, new); 2975 write_sequnlock(&rename_lock); 2976 if (err) { 2977 dput(new); 2978 new = ERR_PTR(err); 2979 } 2980 } else { 2981 __d_move(new, dentry, false); 2982 write_sequnlock(&rename_lock); 2983 } 2984 iput(inode); 2985 return new; 2986 } 2987 } 2988 out: 2989 __d_add(dentry, inode); 2990 return NULL; 2991 } 2992 EXPORT_SYMBOL(d_splice_alias); 2993 2994 static int prepend(char **buffer, int *buflen, const char *str, int namelen) 2995 { 2996 *buflen -= namelen; 2997 if (*buflen < 0) 2998 return -ENAMETOOLONG; 2999 *buffer -= namelen; 3000 memcpy(*buffer, str, namelen); 3001 return 0; 3002 } 3003 3004 /** 3005 * prepend_name - prepend a pathname in front of current buffer pointer 3006 * @buffer: buffer pointer 3007 * @buflen: allocated length of the buffer 3008 * @name: name string and length qstr structure 3009 * 3010 * With RCU path tracing, it may race with d_move(). Use ACCESS_ONCE() to 3011 * make sure that either the old or the new name pointer and length are 3012 * fetched. However, there may be mismatch between length and pointer. 3013 * The length cannot be trusted, we need to copy it byte-by-byte until 3014 * the length is reached or a null byte is found. It also prepends "/" at 3015 * the beginning of the name. The sequence number check at the caller will 3016 * retry it again when a d_move() does happen. So any garbage in the buffer 3017 * due to mismatched pointer and length will be discarded. 3018 * 3019 * Data dependency barrier is needed to make sure that we see that terminating 3020 * NUL. Alpha strikes again, film at 11... 3021 */ 3022 static int prepend_name(char **buffer, int *buflen, const struct qstr *name) 3023 { 3024 const char *dname = ACCESS_ONCE(name->name); 3025 u32 dlen = ACCESS_ONCE(name->len); 3026 char *p; 3027 3028 smp_read_barrier_depends(); 3029 3030 *buflen -= dlen + 1; 3031 if (*buflen < 0) 3032 return -ENAMETOOLONG; 3033 p = *buffer -= dlen + 1; 3034 *p++ = '/'; 3035 while (dlen--) { 3036 char c = *dname++; 3037 if (!c) 3038 break; 3039 *p++ = c; 3040 } 3041 return 0; 3042 } 3043 3044 /** 3045 * prepend_path - Prepend path string to a buffer 3046 * @path: the dentry/vfsmount to report 3047 * @root: root vfsmnt/dentry 3048 * @buffer: pointer to the end of the buffer 3049 * @buflen: pointer to buffer length 3050 * 3051 * The function will first try to write out the pathname without taking any 3052 * lock other than the RCU read lock to make sure that dentries won't go away. 3053 * It only checks the sequence number of the global rename_lock as any change 3054 * in the dentry's d_seq will be preceded by changes in the rename_lock 3055 * sequence number. If the sequence number had been changed, it will restart 3056 * the whole pathname back-tracing sequence again by taking the rename_lock. 3057 * In this case, there is no need to take the RCU read lock as the recursive 3058 * parent pointer references will keep the dentry chain alive as long as no 3059 * rename operation is performed. 3060 */ 3061 static int prepend_path(const struct path *path, 3062 const struct path *root, 3063 char **buffer, int *buflen) 3064 { 3065 struct dentry *dentry; 3066 struct vfsmount *vfsmnt; 3067 struct mount *mnt; 3068 int error = 0; 3069 unsigned seq, m_seq = 0; 3070 char *bptr; 3071 int blen; 3072 3073 rcu_read_lock(); 3074 restart_mnt: 3075 read_seqbegin_or_lock(&mount_lock, &m_seq); 3076 seq = 0; 3077 rcu_read_lock(); 3078 restart: 3079 bptr = *buffer; 3080 blen = *buflen; 3081 error = 0; 3082 dentry = path->dentry; 3083 vfsmnt = path->mnt; 3084 mnt = real_mount(vfsmnt); 3085 read_seqbegin_or_lock(&rename_lock, &seq); 3086 while (dentry != root->dentry || vfsmnt != root->mnt) { 3087 struct dentry * parent; 3088 3089 if (dentry == vfsmnt->mnt_root || IS_ROOT(dentry)) { 3090 struct mount *parent = ACCESS_ONCE(mnt->mnt_parent); 3091 /* Escaped? */ 3092 if (dentry != vfsmnt->mnt_root) { 3093 bptr = *buffer; 3094 blen = *buflen; 3095 error = 3; 3096 break; 3097 } 3098 /* Global root? */ 3099 if (mnt != parent) { 3100 dentry = ACCESS_ONCE(mnt->mnt_mountpoint); 3101 mnt = parent; 3102 vfsmnt = &mnt->mnt; 3103 continue; 3104 } 3105 if (!error) 3106 error = is_mounted(vfsmnt) ? 1 : 2; 3107 break; 3108 } 3109 parent = dentry->d_parent; 3110 prefetch(parent); 3111 error = prepend_name(&bptr, &blen, &dentry->d_name); 3112 if (error) 3113 break; 3114 3115 dentry = parent; 3116 } 3117 if (!(seq & 1)) 3118 rcu_read_unlock(); 3119 if (need_seqretry(&rename_lock, seq)) { 3120 seq = 1; 3121 goto restart; 3122 } 3123 done_seqretry(&rename_lock, seq); 3124 3125 if (!(m_seq & 1)) 3126 rcu_read_unlock(); 3127 if (need_seqretry(&mount_lock, m_seq)) { 3128 m_seq = 1; 3129 goto restart_mnt; 3130 } 3131 done_seqretry(&mount_lock, m_seq); 3132 3133 if (error >= 0 && bptr == *buffer) { 3134 if (--blen < 0) 3135 error = -ENAMETOOLONG; 3136 else 3137 *--bptr = '/'; 3138 } 3139 *buffer = bptr; 3140 *buflen = blen; 3141 return error; 3142 } 3143 3144 /** 3145 * __d_path - return the path of a dentry 3146 * @path: the dentry/vfsmount to report 3147 * @root: root vfsmnt/dentry 3148 * @buf: buffer to return value in 3149 * @buflen: buffer length 3150 * 3151 * Convert a dentry into an ASCII path name. 3152 * 3153 * Returns a pointer into the buffer or an error code if the 3154 * path was too long. 3155 * 3156 * "buflen" should be positive. 3157 * 3158 * If the path is not reachable from the supplied root, return %NULL. 3159 */ 3160 char *__d_path(const struct path *path, 3161 const struct path *root, 3162 char *buf, int buflen) 3163 { 3164 char *res = buf + buflen; 3165 int error; 3166 3167 prepend(&res, &buflen, "\0", 1); 3168 error = prepend_path(path, root, &res, &buflen); 3169 3170 if (error < 0) 3171 return ERR_PTR(error); 3172 if (error > 0) 3173 return NULL; 3174 return res; 3175 } 3176 3177 char *d_absolute_path(const struct path *path, 3178 char *buf, int buflen) 3179 { 3180 struct path root = {}; 3181 char *res = buf + buflen; 3182 int error; 3183 3184 prepend(&res, &buflen, "\0", 1); 3185 error = prepend_path(path, &root, &res, &buflen); 3186 3187 if (error > 1) 3188 error = -EINVAL; 3189 if (error < 0) 3190 return ERR_PTR(error); 3191 return res; 3192 } 3193 3194 /* 3195 * same as __d_path but appends "(deleted)" for unlinked files. 3196 */ 3197 static int path_with_deleted(const struct path *path, 3198 const struct path *root, 3199 char **buf, int *buflen) 3200 { 3201 prepend(buf, buflen, "\0", 1); 3202 if (d_unlinked(path->dentry)) { 3203 int error = prepend(buf, buflen, " (deleted)", 10); 3204 if (error) 3205 return error; 3206 } 3207 3208 return prepend_path(path, root, buf, buflen); 3209 } 3210 3211 static int prepend_unreachable(char **buffer, int *buflen) 3212 { 3213 return prepend(buffer, buflen, "(unreachable)", 13); 3214 } 3215 3216 static void get_fs_root_rcu(struct fs_struct *fs, struct path *root) 3217 { 3218 unsigned seq; 3219 3220 do { 3221 seq = read_seqcount_begin(&fs->seq); 3222 *root = fs->root; 3223 } while (read_seqcount_retry(&fs->seq, seq)); 3224 } 3225 3226 /** 3227 * d_path - return the path of a dentry 3228 * @path: path to report 3229 * @buf: buffer to return value in 3230 * @buflen: buffer length 3231 * 3232 * Convert a dentry into an ASCII path name. If the entry has been deleted 3233 * the string " (deleted)" is appended. Note that this is ambiguous. 3234 * 3235 * Returns a pointer into the buffer or an error code if the path was 3236 * too long. Note: Callers should use the returned pointer, not the passed 3237 * in buffer, to use the name! The implementation often starts at an offset 3238 * into the buffer, and may leave 0 bytes at the start. 3239 * 3240 * "buflen" should be positive. 3241 */ 3242 char *d_path(const struct path *path, char *buf, int buflen) 3243 { 3244 char *res = buf + buflen; 3245 struct path root; 3246 int error; 3247 3248 /* 3249 * We have various synthetic filesystems that never get mounted. On 3250 * these filesystems dentries are never used for lookup purposes, and 3251 * thus don't need to be hashed. They also don't need a name until a 3252 * user wants to identify the object in /proc/pid/fd/. The little hack 3253 * below allows us to generate a name for these objects on demand: 3254 * 3255 * Some pseudo inodes are mountable. When they are mounted 3256 * path->dentry == path->mnt->mnt_root. In that case don't call d_dname 3257 * and instead have d_path return the mounted path. 3258 */ 3259 if (path->dentry->d_op && path->dentry->d_op->d_dname && 3260 (!IS_ROOT(path->dentry) || path->dentry != path->mnt->mnt_root)) 3261 return path->dentry->d_op->d_dname(path->dentry, buf, buflen); 3262 3263 rcu_read_lock(); 3264 get_fs_root_rcu(current->fs, &root); 3265 error = path_with_deleted(path, &root, &res, &buflen); 3266 rcu_read_unlock(); 3267 3268 if (error < 0) 3269 res = ERR_PTR(error); 3270 return res; 3271 } 3272 EXPORT_SYMBOL(d_path); 3273 3274 /* 3275 * Helper function for dentry_operations.d_dname() members 3276 */ 3277 char *dynamic_dname(struct dentry *dentry, char *buffer, int buflen, 3278 const char *fmt, ...) 3279 { 3280 va_list args; 3281 char temp[64]; 3282 int sz; 3283 3284 va_start(args, fmt); 3285 sz = vsnprintf(temp, sizeof(temp), fmt, args) + 1; 3286 va_end(args); 3287 3288 if (sz > sizeof(temp) || sz > buflen) 3289 return ERR_PTR(-ENAMETOOLONG); 3290 3291 buffer += buflen - sz; 3292 return memcpy(buffer, temp, sz); 3293 } 3294 3295 char *simple_dname(struct dentry *dentry, char *buffer, int buflen) 3296 { 3297 char *end = buffer + buflen; 3298 /* these dentries are never renamed, so d_lock is not needed */ 3299 if (prepend(&end, &buflen, " (deleted)", 11) || 3300 prepend(&end, &buflen, dentry->d_name.name, dentry->d_name.len) || 3301 prepend(&end, &buflen, "/", 1)) 3302 end = ERR_PTR(-ENAMETOOLONG); 3303 return end; 3304 } 3305 EXPORT_SYMBOL(simple_dname); 3306 3307 /* 3308 * Write full pathname from the root of the filesystem into the buffer. 3309 */ 3310 static char *__dentry_path(struct dentry *d, char *buf, int buflen) 3311 { 3312 struct dentry *dentry; 3313 char *end, *retval; 3314 int len, seq = 0; 3315 int error = 0; 3316 3317 if (buflen < 2) 3318 goto Elong; 3319 3320 rcu_read_lock(); 3321 restart: 3322 dentry = d; 3323 end = buf + buflen; 3324 len = buflen; 3325 prepend(&end, &len, "\0", 1); 3326 /* Get '/' right */ 3327 retval = end-1; 3328 *retval = '/'; 3329 read_seqbegin_or_lock(&rename_lock, &seq); 3330 while (!IS_ROOT(dentry)) { 3331 struct dentry *parent = dentry->d_parent; 3332 3333 prefetch(parent); 3334 error = prepend_name(&end, &len, &dentry->d_name); 3335 if (error) 3336 break; 3337 3338 retval = end; 3339 dentry = parent; 3340 } 3341 if (!(seq & 1)) 3342 rcu_read_unlock(); 3343 if (need_seqretry(&rename_lock, seq)) { 3344 seq = 1; 3345 goto restart; 3346 } 3347 done_seqretry(&rename_lock, seq); 3348 if (error) 3349 goto Elong; 3350 return retval; 3351 Elong: 3352 return ERR_PTR(-ENAMETOOLONG); 3353 } 3354 3355 char *dentry_path_raw(struct dentry *dentry, char *buf, int buflen) 3356 { 3357 return __dentry_path(dentry, buf, buflen); 3358 } 3359 EXPORT_SYMBOL(dentry_path_raw); 3360 3361 char *dentry_path(struct dentry *dentry, char *buf, int buflen) 3362 { 3363 char *p = NULL; 3364 char *retval; 3365 3366 if (d_unlinked(dentry)) { 3367 p = buf + buflen; 3368 if (prepend(&p, &buflen, "//deleted", 10) != 0) 3369 goto Elong; 3370 buflen++; 3371 } 3372 retval = __dentry_path(dentry, buf, buflen); 3373 if (!IS_ERR(retval) && p) 3374 *p = '/'; /* restore '/' overriden with '\0' */ 3375 return retval; 3376 Elong: 3377 return ERR_PTR(-ENAMETOOLONG); 3378 } 3379 3380 static void get_fs_root_and_pwd_rcu(struct fs_struct *fs, struct path *root, 3381 struct path *pwd) 3382 { 3383 unsigned seq; 3384 3385 do { 3386 seq = read_seqcount_begin(&fs->seq); 3387 *root = fs->root; 3388 *pwd = fs->pwd; 3389 } while (read_seqcount_retry(&fs->seq, seq)); 3390 } 3391 3392 /* 3393 * NOTE! The user-level library version returns a 3394 * character pointer. The kernel system call just 3395 * returns the length of the buffer filled (which 3396 * includes the ending '\0' character), or a negative 3397 * error value. So libc would do something like 3398 * 3399 * char *getcwd(char * buf, size_t size) 3400 * { 3401 * int retval; 3402 * 3403 * retval = sys_getcwd(buf, size); 3404 * if (retval >= 0) 3405 * return buf; 3406 * errno = -retval; 3407 * return NULL; 3408 * } 3409 */ 3410 SYSCALL_DEFINE2(getcwd, char __user *, buf, unsigned long, size) 3411 { 3412 int error; 3413 struct path pwd, root; 3414 char *page = __getname(); 3415 3416 if (!page) 3417 return -ENOMEM; 3418 3419 rcu_read_lock(); 3420 get_fs_root_and_pwd_rcu(current->fs, &root, &pwd); 3421 3422 error = -ENOENT; 3423 if (!d_unlinked(pwd.dentry)) { 3424 unsigned long len; 3425 char *cwd = page + PATH_MAX; 3426 int buflen = PATH_MAX; 3427 3428 prepend(&cwd, &buflen, "\0", 1); 3429 error = prepend_path(&pwd, &root, &cwd, &buflen); 3430 rcu_read_unlock(); 3431 3432 if (error < 0) 3433 goto out; 3434 3435 /* Unreachable from current root */ 3436 if (error > 0) { 3437 error = prepend_unreachable(&cwd, &buflen); 3438 if (error) 3439 goto out; 3440 } 3441 3442 error = -ERANGE; 3443 len = PATH_MAX + page - cwd; 3444 if (len <= size) { 3445 error = len; 3446 if (copy_to_user(buf, cwd, len)) 3447 error = -EFAULT; 3448 } 3449 } else { 3450 rcu_read_unlock(); 3451 } 3452 3453 out: 3454 __putname(page); 3455 return error; 3456 } 3457 3458 /* 3459 * Test whether new_dentry is a subdirectory of old_dentry. 3460 * 3461 * Trivially implemented using the dcache structure 3462 */ 3463 3464 /** 3465 * is_subdir - is new dentry a subdirectory of old_dentry 3466 * @new_dentry: new dentry 3467 * @old_dentry: old dentry 3468 * 3469 * Returns true if new_dentry is a subdirectory of the parent (at any depth). 3470 * Returns false otherwise. 3471 * Caller must ensure that "new_dentry" is pinned before calling is_subdir() 3472 */ 3473 3474 bool is_subdir(struct dentry *new_dentry, struct dentry *old_dentry) 3475 { 3476 bool result; 3477 unsigned seq; 3478 3479 if (new_dentry == old_dentry) 3480 return true; 3481 3482 do { 3483 /* for restarting inner loop in case of seq retry */ 3484 seq = read_seqbegin(&rename_lock); 3485 /* 3486 * Need rcu_readlock to protect against the d_parent trashing 3487 * due to d_move 3488 */ 3489 rcu_read_lock(); 3490 if (d_ancestor(old_dentry, new_dentry)) 3491 result = true; 3492 else 3493 result = false; 3494 rcu_read_unlock(); 3495 } while (read_seqretry(&rename_lock, seq)); 3496 3497 return result; 3498 } 3499 3500 static enum d_walk_ret d_genocide_kill(void *data, struct dentry *dentry) 3501 { 3502 struct dentry *root = data; 3503 if (dentry != root) { 3504 if (d_unhashed(dentry) || !dentry->d_inode) 3505 return D_WALK_SKIP; 3506 3507 if (!(dentry->d_flags & DCACHE_GENOCIDE)) { 3508 dentry->d_flags |= DCACHE_GENOCIDE; 3509 dentry->d_lockref.count--; 3510 } 3511 } 3512 return D_WALK_CONTINUE; 3513 } 3514 3515 void d_genocide(struct dentry *parent) 3516 { 3517 d_walk(parent, parent, d_genocide_kill, NULL); 3518 } 3519 3520 void d_tmpfile(struct dentry *dentry, struct inode *inode) 3521 { 3522 inode_dec_link_count(inode); 3523 BUG_ON(dentry->d_name.name != dentry->d_iname || 3524 !hlist_unhashed(&dentry->d_u.d_alias) || 3525 !d_unlinked(dentry)); 3526 spin_lock(&dentry->d_parent->d_lock); 3527 spin_lock_nested(&dentry->d_lock, DENTRY_D_LOCK_NESTED); 3528 dentry->d_name.len = sprintf(dentry->d_iname, "#%llu", 3529 (unsigned long long)inode->i_ino); 3530 spin_unlock(&dentry->d_lock); 3531 spin_unlock(&dentry->d_parent->d_lock); 3532 d_instantiate(dentry, inode); 3533 } 3534 EXPORT_SYMBOL(d_tmpfile); 3535 3536 static __initdata unsigned long dhash_entries; 3537 static int __init set_dhash_entries(char *str) 3538 { 3539 if (!str) 3540 return 0; 3541 dhash_entries = simple_strtoul(str, &str, 0); 3542 return 1; 3543 } 3544 __setup("dhash_entries=", set_dhash_entries); 3545 3546 static void __init dcache_init_early(void) 3547 { 3548 unsigned int loop; 3549 3550 /* If hashes are distributed across NUMA nodes, defer 3551 * hash allocation until vmalloc space is available. 3552 */ 3553 if (hashdist) 3554 return; 3555 3556 dentry_hashtable = 3557 alloc_large_system_hash("Dentry cache", 3558 sizeof(struct hlist_bl_head), 3559 dhash_entries, 3560 13, 3561 HASH_EARLY, 3562 &d_hash_shift, 3563 &d_hash_mask, 3564 0, 3565 0); 3566 3567 for (loop = 0; loop < (1U << d_hash_shift); loop++) 3568 INIT_HLIST_BL_HEAD(dentry_hashtable + loop); 3569 } 3570 3571 static void __init dcache_init(void) 3572 { 3573 unsigned int loop; 3574 3575 /* 3576 * A constructor could be added for stable state like the lists, 3577 * but it is probably not worth it because of the cache nature 3578 * of the dcache. 3579 */ 3580 dentry_cache = KMEM_CACHE(dentry, 3581 SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|SLAB_MEM_SPREAD|SLAB_ACCOUNT); 3582 3583 /* Hash may have been set up in dcache_init_early */ 3584 if (!hashdist) 3585 return; 3586 3587 dentry_hashtable = 3588 alloc_large_system_hash("Dentry cache", 3589 sizeof(struct hlist_bl_head), 3590 dhash_entries, 3591 13, 3592 0, 3593 &d_hash_shift, 3594 &d_hash_mask, 3595 0, 3596 0); 3597 3598 for (loop = 0; loop < (1U << d_hash_shift); loop++) 3599 INIT_HLIST_BL_HEAD(dentry_hashtable + loop); 3600 } 3601 3602 /* SLAB cache for __getname() consumers */ 3603 struct kmem_cache *names_cachep __read_mostly; 3604 EXPORT_SYMBOL(names_cachep); 3605 3606 EXPORT_SYMBOL(d_genocide); 3607 3608 void __init vfs_caches_init_early(void) 3609 { 3610 dcache_init_early(); 3611 inode_init_early(); 3612 } 3613 3614 void __init vfs_caches_init(void) 3615 { 3616 names_cachep = kmem_cache_create("names_cache", PATH_MAX, 0, 3617 SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); 3618 3619 dcache_init(); 3620 inode_init(); 3621 files_init(); 3622 files_maxfiles_init(); 3623 mnt_init(); 3624 bdev_cache_init(); 3625 chrdev_init(); 3626 } 3627