1 /* 2 * linux/fs/inode.c 3 * 4 * (C) 1997 Linus Torvalds 5 */ 6 7 #include <linux/fs.h> 8 #include <linux/mm.h> 9 #include <linux/dcache.h> 10 #include <linux/init.h> 11 #include <linux/quotaops.h> 12 #include <linux/slab.h> 13 #include <linux/writeback.h> 14 #include <linux/module.h> 15 #include <linux/backing-dev.h> 16 #include <linux/wait.h> 17 #include <linux/hash.h> 18 #include <linux/swap.h> 19 #include <linux/security.h> 20 #include <linux/pagemap.h> 21 #include <linux/cdev.h> 22 #include <linux/bootmem.h> 23 #include <linux/inotify.h> 24 #include <linux/mount.h> 25 26 /* 27 * This is needed for the following functions: 28 * - inode_has_buffers 29 * - invalidate_inode_buffers 30 * - invalidate_bdev 31 * 32 * FIXME: remove all knowledge of the buffer layer from this file 33 */ 34 #include <linux/buffer_head.h> 35 36 /* 37 * New inode.c implementation. 38 * 39 * This implementation has the basic premise of trying 40 * to be extremely low-overhead and SMP-safe, yet be 41 * simple enough to be "obviously correct". 42 * 43 * Famous last words. 44 */ 45 46 /* inode dynamic allocation 1999, Andrea Arcangeli <andrea@suse.de> */ 47 48 /* #define INODE_PARANOIA 1 */ 49 /* #define INODE_DEBUG 1 */ 50 51 /* 52 * Inode lookup is no longer as critical as it used to be: 53 * most of the lookups are going to be through the dcache. 54 */ 55 #define I_HASHBITS i_hash_shift 56 #define I_HASHMASK i_hash_mask 57 58 static unsigned int i_hash_mask __read_mostly; 59 static unsigned int i_hash_shift __read_mostly; 60 61 /* 62 * Each inode can be on two separate lists. One is 63 * the hash list of the inode, used for lookups. The 64 * other linked list is the "type" list: 65 * "in_use" - valid inode, i_count > 0, i_nlink > 0 66 * "dirty" - as "in_use" but also dirty 67 * "unused" - valid inode, i_count = 0 68 * 69 * A "dirty" list is maintained for each super block, 70 * allowing for low-overhead inode sync() operations. 71 */ 72 73 LIST_HEAD(inode_in_use); 74 LIST_HEAD(inode_unused); 75 static struct hlist_head *inode_hashtable __read_mostly; 76 77 /* 78 * A simple spinlock to protect the list manipulations. 79 * 80 * NOTE! You also have to own the lock if you change 81 * the i_state of an inode while it is in use.. 82 */ 83 DEFINE_SPINLOCK(inode_lock); 84 85 /* 86 * iprune_mutex provides exclusion between the kswapd or try_to_free_pages 87 * icache shrinking path, and the umount path. Without this exclusion, 88 * by the time prune_icache calls iput for the inode whose pages it has 89 * been invalidating, or by the time it calls clear_inode & destroy_inode 90 * from its final dispose_list, the struct super_block they refer to 91 * (for inode->i_sb->s_op) may already have been freed and reused. 92 */ 93 static DEFINE_MUTEX(iprune_mutex); 94 95 /* 96 * Statistics gathering.. 97 */ 98 struct inodes_stat_t inodes_stat; 99 100 static struct kmem_cache * inode_cachep __read_mostly; 101 102 static struct inode *alloc_inode(struct super_block *sb) 103 { 104 static const struct address_space_operations empty_aops; 105 static struct inode_operations empty_iops; 106 static const struct file_operations empty_fops; 107 struct inode *inode; 108 109 if (sb->s_op->alloc_inode) 110 inode = sb->s_op->alloc_inode(sb); 111 else 112 inode = (struct inode *) kmem_cache_alloc(inode_cachep, GFP_KERNEL); 113 114 if (inode) { 115 struct address_space * const mapping = &inode->i_data; 116 117 inode->i_sb = sb; 118 inode->i_blkbits = sb->s_blocksize_bits; 119 inode->i_flags = 0; 120 atomic_set(&inode->i_count, 1); 121 inode->i_op = &empty_iops; 122 inode->i_fop = &empty_fops; 123 inode->i_nlink = 1; 124 atomic_set(&inode->i_writecount, 0); 125 inode->i_size = 0; 126 inode->i_blocks = 0; 127 inode->i_bytes = 0; 128 inode->i_generation = 0; 129 #ifdef CONFIG_QUOTA 130 memset(&inode->i_dquot, 0, sizeof(inode->i_dquot)); 131 #endif 132 inode->i_pipe = NULL; 133 inode->i_bdev = NULL; 134 inode->i_cdev = NULL; 135 inode->i_rdev = 0; 136 inode->dirtied_when = 0; 137 if (security_inode_alloc(inode)) { 138 if (inode->i_sb->s_op->destroy_inode) 139 inode->i_sb->s_op->destroy_inode(inode); 140 else 141 kmem_cache_free(inode_cachep, (inode)); 142 return NULL; 143 } 144 145 mapping->a_ops = &empty_aops; 146 mapping->host = inode; 147 mapping->flags = 0; 148 mapping_set_gfp_mask(mapping, GFP_HIGHUSER); 149 mapping->assoc_mapping = NULL; 150 mapping->backing_dev_info = &default_backing_dev_info; 151 152 /* 153 * If the block_device provides a backing_dev_info for client 154 * inodes then use that. Otherwise the inode share the bdev's 155 * backing_dev_info. 156 */ 157 if (sb->s_bdev) { 158 struct backing_dev_info *bdi; 159 160 bdi = sb->s_bdev->bd_inode_backing_dev_info; 161 if (!bdi) 162 bdi = sb->s_bdev->bd_inode->i_mapping->backing_dev_info; 163 mapping->backing_dev_info = bdi; 164 } 165 inode->i_private = NULL; 166 inode->i_mapping = mapping; 167 } 168 return inode; 169 } 170 171 void destroy_inode(struct inode *inode) 172 { 173 BUG_ON(inode_has_buffers(inode)); 174 security_inode_free(inode); 175 if (inode->i_sb->s_op->destroy_inode) 176 inode->i_sb->s_op->destroy_inode(inode); 177 else 178 kmem_cache_free(inode_cachep, (inode)); 179 } 180 181 182 /* 183 * These are initializations that only need to be done 184 * once, because the fields are idempotent across use 185 * of the inode, so let the slab aware of that. 186 */ 187 void inode_init_once(struct inode *inode) 188 { 189 memset(inode, 0, sizeof(*inode)); 190 INIT_HLIST_NODE(&inode->i_hash); 191 INIT_LIST_HEAD(&inode->i_dentry); 192 INIT_LIST_HEAD(&inode->i_devices); 193 mutex_init(&inode->i_mutex); 194 init_rwsem(&inode->i_alloc_sem); 195 INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC); 196 rwlock_init(&inode->i_data.tree_lock); 197 spin_lock_init(&inode->i_data.i_mmap_lock); 198 INIT_LIST_HEAD(&inode->i_data.private_list); 199 spin_lock_init(&inode->i_data.private_lock); 200 INIT_RAW_PRIO_TREE_ROOT(&inode->i_data.i_mmap); 201 INIT_LIST_HEAD(&inode->i_data.i_mmap_nonlinear); 202 spin_lock_init(&inode->i_lock); 203 i_size_ordered_init(inode); 204 #ifdef CONFIG_INOTIFY 205 INIT_LIST_HEAD(&inode->inotify_watches); 206 mutex_init(&inode->inotify_mutex); 207 #endif 208 } 209 210 EXPORT_SYMBOL(inode_init_once); 211 212 static void init_once(void * foo, struct kmem_cache * cachep, unsigned long flags) 213 { 214 struct inode * inode = (struct inode *) foo; 215 216 if (flags & SLAB_CTOR_CONSTRUCTOR) 217 inode_init_once(inode); 218 } 219 220 /* 221 * inode_lock must be held 222 */ 223 void __iget(struct inode * inode) 224 { 225 if (atomic_read(&inode->i_count)) { 226 atomic_inc(&inode->i_count); 227 return; 228 } 229 atomic_inc(&inode->i_count); 230 if (!(inode->i_state & (I_DIRTY|I_LOCK))) 231 list_move(&inode->i_list, &inode_in_use); 232 inodes_stat.nr_unused--; 233 } 234 235 /** 236 * clear_inode - clear an inode 237 * @inode: inode to clear 238 * 239 * This is called by the filesystem to tell us 240 * that the inode is no longer useful. We just 241 * terminate it with extreme prejudice. 242 */ 243 void clear_inode(struct inode *inode) 244 { 245 might_sleep(); 246 invalidate_inode_buffers(inode); 247 248 BUG_ON(inode->i_data.nrpages); 249 BUG_ON(!(inode->i_state & I_FREEING)); 250 BUG_ON(inode->i_state & I_CLEAR); 251 wait_on_inode(inode); 252 DQUOT_DROP(inode); 253 if (inode->i_sb->s_op->clear_inode) 254 inode->i_sb->s_op->clear_inode(inode); 255 if (S_ISBLK(inode->i_mode) && inode->i_bdev) 256 bd_forget(inode); 257 if (S_ISCHR(inode->i_mode) && inode->i_cdev) 258 cd_forget(inode); 259 inode->i_state = I_CLEAR; 260 } 261 262 EXPORT_SYMBOL(clear_inode); 263 264 /* 265 * dispose_list - dispose of the contents of a local list 266 * @head: the head of the list to free 267 * 268 * Dispose-list gets a local list with local inodes in it, so it doesn't 269 * need to worry about list corruption and SMP locks. 270 */ 271 static void dispose_list(struct list_head *head) 272 { 273 int nr_disposed = 0; 274 275 while (!list_empty(head)) { 276 struct inode *inode; 277 278 inode = list_first_entry(head, struct inode, i_list); 279 list_del(&inode->i_list); 280 281 if (inode->i_data.nrpages) 282 truncate_inode_pages(&inode->i_data, 0); 283 clear_inode(inode); 284 285 spin_lock(&inode_lock); 286 hlist_del_init(&inode->i_hash); 287 list_del_init(&inode->i_sb_list); 288 spin_unlock(&inode_lock); 289 290 wake_up_inode(inode); 291 destroy_inode(inode); 292 nr_disposed++; 293 } 294 spin_lock(&inode_lock); 295 inodes_stat.nr_inodes -= nr_disposed; 296 spin_unlock(&inode_lock); 297 } 298 299 /* 300 * Invalidate all inodes for a device. 301 */ 302 static int invalidate_list(struct list_head *head, struct list_head *dispose) 303 { 304 struct list_head *next; 305 int busy = 0, count = 0; 306 307 next = head->next; 308 for (;;) { 309 struct list_head * tmp = next; 310 struct inode * inode; 311 312 /* 313 * We can reschedule here without worrying about the list's 314 * consistency because the per-sb list of inodes must not 315 * change during umount anymore, and because iprune_mutex keeps 316 * shrink_icache_memory() away. 317 */ 318 cond_resched_lock(&inode_lock); 319 320 next = next->next; 321 if (tmp == head) 322 break; 323 inode = list_entry(tmp, struct inode, i_sb_list); 324 invalidate_inode_buffers(inode); 325 if (!atomic_read(&inode->i_count)) { 326 list_move(&inode->i_list, dispose); 327 inode->i_state |= I_FREEING; 328 count++; 329 continue; 330 } 331 busy = 1; 332 } 333 /* only unused inodes may be cached with i_count zero */ 334 inodes_stat.nr_unused -= count; 335 return busy; 336 } 337 338 /** 339 * invalidate_inodes - discard the inodes on a device 340 * @sb: superblock 341 * 342 * Discard all of the inodes for a given superblock. If the discard 343 * fails because there are busy inodes then a non zero value is returned. 344 * If the discard is successful all the inodes have been discarded. 345 */ 346 int invalidate_inodes(struct super_block * sb) 347 { 348 int busy; 349 LIST_HEAD(throw_away); 350 351 mutex_lock(&iprune_mutex); 352 spin_lock(&inode_lock); 353 inotify_unmount_inodes(&sb->s_inodes); 354 busy = invalidate_list(&sb->s_inodes, &throw_away); 355 spin_unlock(&inode_lock); 356 357 dispose_list(&throw_away); 358 mutex_unlock(&iprune_mutex); 359 360 return busy; 361 } 362 363 EXPORT_SYMBOL(invalidate_inodes); 364 365 static int can_unuse(struct inode *inode) 366 { 367 if (inode->i_state) 368 return 0; 369 if (inode_has_buffers(inode)) 370 return 0; 371 if (atomic_read(&inode->i_count)) 372 return 0; 373 if (inode->i_data.nrpages) 374 return 0; 375 return 1; 376 } 377 378 /* 379 * Scan `goal' inodes on the unused list for freeable ones. They are moved to 380 * a temporary list and then are freed outside inode_lock by dispose_list(). 381 * 382 * Any inodes which are pinned purely because of attached pagecache have their 383 * pagecache removed. We expect the final iput() on that inode to add it to 384 * the front of the inode_unused list. So look for it there and if the 385 * inode is still freeable, proceed. The right inode is found 99.9% of the 386 * time in testing on a 4-way. 387 * 388 * If the inode has metadata buffers attached to mapping->private_list then 389 * try to remove them. 390 */ 391 static void prune_icache(int nr_to_scan) 392 { 393 LIST_HEAD(freeable); 394 int nr_pruned = 0; 395 int nr_scanned; 396 unsigned long reap = 0; 397 398 mutex_lock(&iprune_mutex); 399 spin_lock(&inode_lock); 400 for (nr_scanned = 0; nr_scanned < nr_to_scan; nr_scanned++) { 401 struct inode *inode; 402 403 if (list_empty(&inode_unused)) 404 break; 405 406 inode = list_entry(inode_unused.prev, struct inode, i_list); 407 408 if (inode->i_state || atomic_read(&inode->i_count)) { 409 list_move(&inode->i_list, &inode_unused); 410 continue; 411 } 412 if (inode_has_buffers(inode) || inode->i_data.nrpages) { 413 __iget(inode); 414 spin_unlock(&inode_lock); 415 if (remove_inode_buffers(inode)) 416 reap += invalidate_mapping_pages(&inode->i_data, 417 0, -1); 418 iput(inode); 419 spin_lock(&inode_lock); 420 421 if (inode != list_entry(inode_unused.next, 422 struct inode, i_list)) 423 continue; /* wrong inode or list_empty */ 424 if (!can_unuse(inode)) 425 continue; 426 } 427 list_move(&inode->i_list, &freeable); 428 inode->i_state |= I_FREEING; 429 nr_pruned++; 430 } 431 inodes_stat.nr_unused -= nr_pruned; 432 if (current_is_kswapd()) 433 __count_vm_events(KSWAPD_INODESTEAL, reap); 434 else 435 __count_vm_events(PGINODESTEAL, reap); 436 spin_unlock(&inode_lock); 437 438 dispose_list(&freeable); 439 mutex_unlock(&iprune_mutex); 440 } 441 442 /* 443 * shrink_icache_memory() will attempt to reclaim some unused inodes. Here, 444 * "unused" means that no dentries are referring to the inodes: the files are 445 * not open and the dcache references to those inodes have already been 446 * reclaimed. 447 * 448 * This function is passed the number of inodes to scan, and it returns the 449 * total number of remaining possibly-reclaimable inodes. 450 */ 451 static int shrink_icache_memory(int nr, gfp_t gfp_mask) 452 { 453 if (nr) { 454 /* 455 * Nasty deadlock avoidance. We may hold various FS locks, 456 * and we don't want to recurse into the FS that called us 457 * in clear_inode() and friends.. 458 */ 459 if (!(gfp_mask & __GFP_FS)) 460 return -1; 461 prune_icache(nr); 462 } 463 return (inodes_stat.nr_unused / 100) * sysctl_vfs_cache_pressure; 464 } 465 466 static void __wait_on_freeing_inode(struct inode *inode); 467 /* 468 * Called with the inode lock held. 469 * NOTE: we are not increasing the inode-refcount, you must call __iget() 470 * by hand after calling find_inode now! This simplifies iunique and won't 471 * add any additional branch in the common code. 472 */ 473 static struct inode * find_inode(struct super_block * sb, struct hlist_head *head, int (*test)(struct inode *, void *), void *data) 474 { 475 struct hlist_node *node; 476 struct inode * inode = NULL; 477 478 repeat: 479 hlist_for_each (node, head) { 480 inode = hlist_entry(node, struct inode, i_hash); 481 if (inode->i_sb != sb) 482 continue; 483 if (!test(inode, data)) 484 continue; 485 if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE)) { 486 __wait_on_freeing_inode(inode); 487 goto repeat; 488 } 489 break; 490 } 491 return node ? inode : NULL; 492 } 493 494 /* 495 * find_inode_fast is the fast path version of find_inode, see the comment at 496 * iget_locked for details. 497 */ 498 static struct inode * find_inode_fast(struct super_block * sb, struct hlist_head *head, unsigned long ino) 499 { 500 struct hlist_node *node; 501 struct inode * inode = NULL; 502 503 repeat: 504 hlist_for_each (node, head) { 505 inode = hlist_entry(node, struct inode, i_hash); 506 if (inode->i_ino != ino) 507 continue; 508 if (inode->i_sb != sb) 509 continue; 510 if (inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE)) { 511 __wait_on_freeing_inode(inode); 512 goto repeat; 513 } 514 break; 515 } 516 return node ? inode : NULL; 517 } 518 519 /** 520 * new_inode - obtain an inode 521 * @sb: superblock 522 * 523 * Allocates a new inode for given superblock. 524 */ 525 struct inode *new_inode(struct super_block *sb) 526 { 527 /* 528 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW 529 * error if st_ino won't fit in target struct field. Use 32bit counter 530 * here to attempt to avoid that. 531 */ 532 static unsigned int last_ino; 533 struct inode * inode; 534 535 spin_lock_prefetch(&inode_lock); 536 537 inode = alloc_inode(sb); 538 if (inode) { 539 spin_lock(&inode_lock); 540 inodes_stat.nr_inodes++; 541 list_add(&inode->i_list, &inode_in_use); 542 list_add(&inode->i_sb_list, &sb->s_inodes); 543 inode->i_ino = ++last_ino; 544 inode->i_state = 0; 545 spin_unlock(&inode_lock); 546 } 547 return inode; 548 } 549 550 EXPORT_SYMBOL(new_inode); 551 552 void unlock_new_inode(struct inode *inode) 553 { 554 /* 555 * This is special! We do not need the spinlock 556 * when clearing I_LOCK, because we're guaranteed 557 * that nobody else tries to do anything about the 558 * state of the inode when it is locked, as we 559 * just created it (so there can be no old holders 560 * that haven't tested I_LOCK). 561 */ 562 inode->i_state &= ~(I_LOCK|I_NEW); 563 wake_up_inode(inode); 564 } 565 566 EXPORT_SYMBOL(unlock_new_inode); 567 568 /* 569 * This is called without the inode lock held.. Be careful. 570 * 571 * We no longer cache the sb_flags in i_flags - see fs.h 572 * -- rmk@arm.uk.linux.org 573 */ 574 static struct inode * get_new_inode(struct super_block *sb, struct hlist_head *head, int (*test)(struct inode *, void *), int (*set)(struct inode *, void *), void *data) 575 { 576 struct inode * inode; 577 578 inode = alloc_inode(sb); 579 if (inode) { 580 struct inode * old; 581 582 spin_lock(&inode_lock); 583 /* We released the lock, so.. */ 584 old = find_inode(sb, head, test, data); 585 if (!old) { 586 if (set(inode, data)) 587 goto set_failed; 588 589 inodes_stat.nr_inodes++; 590 list_add(&inode->i_list, &inode_in_use); 591 list_add(&inode->i_sb_list, &sb->s_inodes); 592 hlist_add_head(&inode->i_hash, head); 593 inode->i_state = I_LOCK|I_NEW; 594 spin_unlock(&inode_lock); 595 596 /* Return the locked inode with I_NEW set, the 597 * caller is responsible for filling in the contents 598 */ 599 return inode; 600 } 601 602 /* 603 * Uhhuh, somebody else created the same inode under 604 * us. Use the old inode instead of the one we just 605 * allocated. 606 */ 607 __iget(old); 608 spin_unlock(&inode_lock); 609 destroy_inode(inode); 610 inode = old; 611 wait_on_inode(inode); 612 } 613 return inode; 614 615 set_failed: 616 spin_unlock(&inode_lock); 617 destroy_inode(inode); 618 return NULL; 619 } 620 621 /* 622 * get_new_inode_fast is the fast path version of get_new_inode, see the 623 * comment at iget_locked for details. 624 */ 625 static struct inode * get_new_inode_fast(struct super_block *sb, struct hlist_head *head, unsigned long ino) 626 { 627 struct inode * inode; 628 629 inode = alloc_inode(sb); 630 if (inode) { 631 struct inode * old; 632 633 spin_lock(&inode_lock); 634 /* We released the lock, so.. */ 635 old = find_inode_fast(sb, head, ino); 636 if (!old) { 637 inode->i_ino = ino; 638 inodes_stat.nr_inodes++; 639 list_add(&inode->i_list, &inode_in_use); 640 list_add(&inode->i_sb_list, &sb->s_inodes); 641 hlist_add_head(&inode->i_hash, head); 642 inode->i_state = I_LOCK|I_NEW; 643 spin_unlock(&inode_lock); 644 645 /* Return the locked inode with I_NEW set, the 646 * caller is responsible for filling in the contents 647 */ 648 return inode; 649 } 650 651 /* 652 * Uhhuh, somebody else created the same inode under 653 * us. Use the old inode instead of the one we just 654 * allocated. 655 */ 656 __iget(old); 657 spin_unlock(&inode_lock); 658 destroy_inode(inode); 659 inode = old; 660 wait_on_inode(inode); 661 } 662 return inode; 663 } 664 665 static unsigned long hash(struct super_block *sb, unsigned long hashval) 666 { 667 unsigned long tmp; 668 669 tmp = (hashval * (unsigned long)sb) ^ (GOLDEN_RATIO_PRIME + hashval) / 670 L1_CACHE_BYTES; 671 tmp = tmp ^ ((tmp ^ GOLDEN_RATIO_PRIME) >> I_HASHBITS); 672 return tmp & I_HASHMASK; 673 } 674 675 /** 676 * iunique - get a unique inode number 677 * @sb: superblock 678 * @max_reserved: highest reserved inode number 679 * 680 * Obtain an inode number that is unique on the system for a given 681 * superblock. This is used by file systems that have no natural 682 * permanent inode numbering system. An inode number is returned that 683 * is higher than the reserved limit but unique. 684 * 685 * BUGS: 686 * With a large number of inodes live on the file system this function 687 * currently becomes quite slow. 688 */ 689 ino_t iunique(struct super_block *sb, ino_t max_reserved) 690 { 691 /* 692 * On a 32bit, non LFS stat() call, glibc will generate an EOVERFLOW 693 * error if st_ino won't fit in target struct field. Use 32bit counter 694 * here to attempt to avoid that. 695 */ 696 static unsigned int counter; 697 struct inode *inode; 698 struct hlist_head *head; 699 ino_t res; 700 701 spin_lock(&inode_lock); 702 do { 703 if (counter <= max_reserved) 704 counter = max_reserved + 1; 705 res = counter++; 706 head = inode_hashtable + hash(sb, res); 707 inode = find_inode_fast(sb, head, res); 708 } while (inode != NULL); 709 spin_unlock(&inode_lock); 710 711 return res; 712 } 713 EXPORT_SYMBOL(iunique); 714 715 struct inode *igrab(struct inode *inode) 716 { 717 spin_lock(&inode_lock); 718 if (!(inode->i_state & (I_FREEING|I_CLEAR|I_WILL_FREE))) 719 __iget(inode); 720 else 721 /* 722 * Handle the case where s_op->clear_inode is not been 723 * called yet, and somebody is calling igrab 724 * while the inode is getting freed. 725 */ 726 inode = NULL; 727 spin_unlock(&inode_lock); 728 return inode; 729 } 730 731 EXPORT_SYMBOL(igrab); 732 733 /** 734 * ifind - internal function, you want ilookup5() or iget5(). 735 * @sb: super block of file system to search 736 * @head: the head of the list to search 737 * @test: callback used for comparisons between inodes 738 * @data: opaque data pointer to pass to @test 739 * @wait: if true wait for the inode to be unlocked, if false do not 740 * 741 * ifind() searches for the inode specified by @data in the inode 742 * cache. This is a generalized version of ifind_fast() for file systems where 743 * the inode number is not sufficient for unique identification of an inode. 744 * 745 * If the inode is in the cache, the inode is returned with an incremented 746 * reference count. 747 * 748 * Otherwise NULL is returned. 749 * 750 * Note, @test is called with the inode_lock held, so can't sleep. 751 */ 752 static struct inode *ifind(struct super_block *sb, 753 struct hlist_head *head, int (*test)(struct inode *, void *), 754 void *data, const int wait) 755 { 756 struct inode *inode; 757 758 spin_lock(&inode_lock); 759 inode = find_inode(sb, head, test, data); 760 if (inode) { 761 __iget(inode); 762 spin_unlock(&inode_lock); 763 if (likely(wait)) 764 wait_on_inode(inode); 765 return inode; 766 } 767 spin_unlock(&inode_lock); 768 return NULL; 769 } 770 771 /** 772 * ifind_fast - internal function, you want ilookup() or iget(). 773 * @sb: super block of file system to search 774 * @head: head of the list to search 775 * @ino: inode number to search for 776 * 777 * ifind_fast() searches for the inode @ino in the inode cache. This is for 778 * file systems where the inode number is sufficient for unique identification 779 * of an inode. 780 * 781 * If the inode is in the cache, the inode is returned with an incremented 782 * reference count. 783 * 784 * Otherwise NULL is returned. 785 */ 786 static struct inode *ifind_fast(struct super_block *sb, 787 struct hlist_head *head, unsigned long ino) 788 { 789 struct inode *inode; 790 791 spin_lock(&inode_lock); 792 inode = find_inode_fast(sb, head, ino); 793 if (inode) { 794 __iget(inode); 795 spin_unlock(&inode_lock); 796 wait_on_inode(inode); 797 return inode; 798 } 799 spin_unlock(&inode_lock); 800 return NULL; 801 } 802 803 /** 804 * ilookup5_nowait - search for an inode in the inode cache 805 * @sb: super block of file system to search 806 * @hashval: hash value (usually inode number) to search for 807 * @test: callback used for comparisons between inodes 808 * @data: opaque data pointer to pass to @test 809 * 810 * ilookup5() uses ifind() to search for the inode specified by @hashval and 811 * @data in the inode cache. This is a generalized version of ilookup() for 812 * file systems where the inode number is not sufficient for unique 813 * identification of an inode. 814 * 815 * If the inode is in the cache, the inode is returned with an incremented 816 * reference count. Note, the inode lock is not waited upon so you have to be 817 * very careful what you do with the returned inode. You probably should be 818 * using ilookup5() instead. 819 * 820 * Otherwise NULL is returned. 821 * 822 * Note, @test is called with the inode_lock held, so can't sleep. 823 */ 824 struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval, 825 int (*test)(struct inode *, void *), void *data) 826 { 827 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 828 829 return ifind(sb, head, test, data, 0); 830 } 831 832 EXPORT_SYMBOL(ilookup5_nowait); 833 834 /** 835 * ilookup5 - search for an inode in the inode cache 836 * @sb: super block of file system to search 837 * @hashval: hash value (usually inode number) to search for 838 * @test: callback used for comparisons between inodes 839 * @data: opaque data pointer to pass to @test 840 * 841 * ilookup5() uses ifind() to search for the inode specified by @hashval and 842 * @data in the inode cache. This is a generalized version of ilookup() for 843 * file systems where the inode number is not sufficient for unique 844 * identification of an inode. 845 * 846 * If the inode is in the cache, the inode lock is waited upon and the inode is 847 * returned with an incremented reference count. 848 * 849 * Otherwise NULL is returned. 850 * 851 * Note, @test is called with the inode_lock held, so can't sleep. 852 */ 853 struct inode *ilookup5(struct super_block *sb, unsigned long hashval, 854 int (*test)(struct inode *, void *), void *data) 855 { 856 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 857 858 return ifind(sb, head, test, data, 1); 859 } 860 861 EXPORT_SYMBOL(ilookup5); 862 863 /** 864 * ilookup - search for an inode in the inode cache 865 * @sb: super block of file system to search 866 * @ino: inode number to search for 867 * 868 * ilookup() uses ifind_fast() to search for the inode @ino in the inode cache. 869 * This is for file systems where the inode number is sufficient for unique 870 * identification of an inode. 871 * 872 * If the inode is in the cache, the inode is returned with an incremented 873 * reference count. 874 * 875 * Otherwise NULL is returned. 876 */ 877 struct inode *ilookup(struct super_block *sb, unsigned long ino) 878 { 879 struct hlist_head *head = inode_hashtable + hash(sb, ino); 880 881 return ifind_fast(sb, head, ino); 882 } 883 884 EXPORT_SYMBOL(ilookup); 885 886 /** 887 * iget5_locked - obtain an inode from a mounted file system 888 * @sb: super block of file system 889 * @hashval: hash value (usually inode number) to get 890 * @test: callback used for comparisons between inodes 891 * @set: callback used to initialize a new struct inode 892 * @data: opaque data pointer to pass to @test and @set 893 * 894 * This is iget() without the read_inode() portion of get_new_inode(). 895 * 896 * iget5_locked() uses ifind() to search for the inode specified by @hashval 897 * and @data in the inode cache and if present it is returned with an increased 898 * reference count. This is a generalized version of iget_locked() for file 899 * systems where the inode number is not sufficient for unique identification 900 * of an inode. 901 * 902 * If the inode is not in cache, get_new_inode() is called to allocate a new 903 * inode and this is returned locked, hashed, and with the I_NEW flag set. The 904 * file system gets to fill it in before unlocking it via unlock_new_inode(). 905 * 906 * Note both @test and @set are called with the inode_lock held, so can't sleep. 907 */ 908 struct inode *iget5_locked(struct super_block *sb, unsigned long hashval, 909 int (*test)(struct inode *, void *), 910 int (*set)(struct inode *, void *), void *data) 911 { 912 struct hlist_head *head = inode_hashtable + hash(sb, hashval); 913 struct inode *inode; 914 915 inode = ifind(sb, head, test, data, 1); 916 if (inode) 917 return inode; 918 /* 919 * get_new_inode() will do the right thing, re-trying the search 920 * in case it had to block at any point. 921 */ 922 return get_new_inode(sb, head, test, set, data); 923 } 924 925 EXPORT_SYMBOL(iget5_locked); 926 927 /** 928 * iget_locked - obtain an inode from a mounted file system 929 * @sb: super block of file system 930 * @ino: inode number to get 931 * 932 * This is iget() without the read_inode() portion of get_new_inode_fast(). 933 * 934 * iget_locked() uses ifind_fast() to search for the inode specified by @ino in 935 * the inode cache and if present it is returned with an increased reference 936 * count. This is for file systems where the inode number is sufficient for 937 * unique identification of an inode. 938 * 939 * If the inode is not in cache, get_new_inode_fast() is called to allocate a 940 * new inode and this is returned locked, hashed, and with the I_NEW flag set. 941 * The file system gets to fill it in before unlocking it via 942 * unlock_new_inode(). 943 */ 944 struct inode *iget_locked(struct super_block *sb, unsigned long ino) 945 { 946 struct hlist_head *head = inode_hashtable + hash(sb, ino); 947 struct inode *inode; 948 949 inode = ifind_fast(sb, head, ino); 950 if (inode) 951 return inode; 952 /* 953 * get_new_inode_fast() will do the right thing, re-trying the search 954 * in case it had to block at any point. 955 */ 956 return get_new_inode_fast(sb, head, ino); 957 } 958 959 EXPORT_SYMBOL(iget_locked); 960 961 /** 962 * __insert_inode_hash - hash an inode 963 * @inode: unhashed inode 964 * @hashval: unsigned long value used to locate this object in the 965 * inode_hashtable. 966 * 967 * Add an inode to the inode hash for this superblock. 968 */ 969 void __insert_inode_hash(struct inode *inode, unsigned long hashval) 970 { 971 struct hlist_head *head = inode_hashtable + hash(inode->i_sb, hashval); 972 spin_lock(&inode_lock); 973 hlist_add_head(&inode->i_hash, head); 974 spin_unlock(&inode_lock); 975 } 976 977 EXPORT_SYMBOL(__insert_inode_hash); 978 979 /** 980 * remove_inode_hash - remove an inode from the hash 981 * @inode: inode to unhash 982 * 983 * Remove an inode from the superblock. 984 */ 985 void remove_inode_hash(struct inode *inode) 986 { 987 spin_lock(&inode_lock); 988 hlist_del_init(&inode->i_hash); 989 spin_unlock(&inode_lock); 990 } 991 992 EXPORT_SYMBOL(remove_inode_hash); 993 994 /* 995 * Tell the filesystem that this inode is no longer of any interest and should 996 * be completely destroyed. 997 * 998 * We leave the inode in the inode hash table until *after* the filesystem's 999 * ->delete_inode completes. This ensures that an iget (such as nfsd might 1000 * instigate) will always find up-to-date information either in the hash or on 1001 * disk. 1002 * 1003 * I_FREEING is set so that no-one will take a new reference to the inode while 1004 * it is being deleted. 1005 */ 1006 void generic_delete_inode(struct inode *inode) 1007 { 1008 const struct super_operations *op = inode->i_sb->s_op; 1009 1010 list_del_init(&inode->i_list); 1011 list_del_init(&inode->i_sb_list); 1012 inode->i_state |= I_FREEING; 1013 inodes_stat.nr_inodes--; 1014 spin_unlock(&inode_lock); 1015 1016 security_inode_delete(inode); 1017 1018 if (op->delete_inode) { 1019 void (*delete)(struct inode *) = op->delete_inode; 1020 if (!is_bad_inode(inode)) 1021 DQUOT_INIT(inode); 1022 /* Filesystems implementing their own 1023 * s_op->delete_inode are required to call 1024 * truncate_inode_pages and clear_inode() 1025 * internally */ 1026 delete(inode); 1027 } else { 1028 truncate_inode_pages(&inode->i_data, 0); 1029 clear_inode(inode); 1030 } 1031 spin_lock(&inode_lock); 1032 hlist_del_init(&inode->i_hash); 1033 spin_unlock(&inode_lock); 1034 wake_up_inode(inode); 1035 BUG_ON(inode->i_state != I_CLEAR); 1036 destroy_inode(inode); 1037 } 1038 1039 EXPORT_SYMBOL(generic_delete_inode); 1040 1041 static void generic_forget_inode(struct inode *inode) 1042 { 1043 struct super_block *sb = inode->i_sb; 1044 1045 if (!hlist_unhashed(&inode->i_hash)) { 1046 if (!(inode->i_state & (I_DIRTY|I_LOCK))) 1047 list_move(&inode->i_list, &inode_unused); 1048 inodes_stat.nr_unused++; 1049 if (sb->s_flags & MS_ACTIVE) { 1050 spin_unlock(&inode_lock); 1051 return; 1052 } 1053 inode->i_state |= I_WILL_FREE; 1054 spin_unlock(&inode_lock); 1055 write_inode_now(inode, 1); 1056 spin_lock(&inode_lock); 1057 inode->i_state &= ~I_WILL_FREE; 1058 inodes_stat.nr_unused--; 1059 hlist_del_init(&inode->i_hash); 1060 } 1061 list_del_init(&inode->i_list); 1062 list_del_init(&inode->i_sb_list); 1063 inode->i_state |= I_FREEING; 1064 inodes_stat.nr_inodes--; 1065 spin_unlock(&inode_lock); 1066 if (inode->i_data.nrpages) 1067 truncate_inode_pages(&inode->i_data, 0); 1068 clear_inode(inode); 1069 wake_up_inode(inode); 1070 destroy_inode(inode); 1071 } 1072 1073 /* 1074 * Normal UNIX filesystem behaviour: delete the 1075 * inode when the usage count drops to zero, and 1076 * i_nlink is zero. 1077 */ 1078 void generic_drop_inode(struct inode *inode) 1079 { 1080 if (!inode->i_nlink) 1081 generic_delete_inode(inode); 1082 else 1083 generic_forget_inode(inode); 1084 } 1085 1086 EXPORT_SYMBOL_GPL(generic_drop_inode); 1087 1088 /* 1089 * Called when we're dropping the last reference 1090 * to an inode. 1091 * 1092 * Call the FS "drop()" function, defaulting to 1093 * the legacy UNIX filesystem behaviour.. 1094 * 1095 * NOTE! NOTE! NOTE! We're called with the inode lock 1096 * held, and the drop function is supposed to release 1097 * the lock! 1098 */ 1099 static inline void iput_final(struct inode *inode) 1100 { 1101 const struct super_operations *op = inode->i_sb->s_op; 1102 void (*drop)(struct inode *) = generic_drop_inode; 1103 1104 if (op && op->drop_inode) 1105 drop = op->drop_inode; 1106 drop(inode); 1107 } 1108 1109 /** 1110 * iput - put an inode 1111 * @inode: inode to put 1112 * 1113 * Puts an inode, dropping its usage count. If the inode use count hits 1114 * zero, the inode is then freed and may also be destroyed. 1115 * 1116 * Consequently, iput() can sleep. 1117 */ 1118 void iput(struct inode *inode) 1119 { 1120 if (inode) { 1121 const struct super_operations *op = inode->i_sb->s_op; 1122 1123 BUG_ON(inode->i_state == I_CLEAR); 1124 1125 if (op && op->put_inode) 1126 op->put_inode(inode); 1127 1128 if (atomic_dec_and_lock(&inode->i_count, &inode_lock)) 1129 iput_final(inode); 1130 } 1131 } 1132 1133 EXPORT_SYMBOL(iput); 1134 1135 /** 1136 * bmap - find a block number in a file 1137 * @inode: inode of file 1138 * @block: block to find 1139 * 1140 * Returns the block number on the device holding the inode that 1141 * is the disk block number for the block of the file requested. 1142 * That is, asked for block 4 of inode 1 the function will return the 1143 * disk block relative to the disk start that holds that block of the 1144 * file. 1145 */ 1146 sector_t bmap(struct inode * inode, sector_t block) 1147 { 1148 sector_t res = 0; 1149 if (inode->i_mapping->a_ops->bmap) 1150 res = inode->i_mapping->a_ops->bmap(inode->i_mapping, block); 1151 return res; 1152 } 1153 EXPORT_SYMBOL(bmap); 1154 1155 /** 1156 * touch_atime - update the access time 1157 * @mnt: mount the inode is accessed on 1158 * @dentry: dentry accessed 1159 * 1160 * Update the accessed time on an inode and mark it for writeback. 1161 * This function automatically handles read only file systems and media, 1162 * as well as the "noatime" flag and inode specific "noatime" markers. 1163 */ 1164 void touch_atime(struct vfsmount *mnt, struct dentry *dentry) 1165 { 1166 struct inode *inode = dentry->d_inode; 1167 struct timespec now; 1168 1169 if (inode->i_flags & S_NOATIME) 1170 return; 1171 if (IS_NOATIME(inode)) 1172 return; 1173 if ((inode->i_sb->s_flags & MS_NODIRATIME) && S_ISDIR(inode->i_mode)) 1174 return; 1175 1176 /* 1177 * We may have a NULL vfsmount when coming from NFSD 1178 */ 1179 if (mnt) { 1180 if (mnt->mnt_flags & MNT_NOATIME) 1181 return; 1182 if ((mnt->mnt_flags & MNT_NODIRATIME) && S_ISDIR(inode->i_mode)) 1183 return; 1184 1185 if (mnt->mnt_flags & MNT_RELATIME) { 1186 /* 1187 * With relative atime, only update atime if the 1188 * previous atime is earlier than either the ctime or 1189 * mtime. 1190 */ 1191 if (timespec_compare(&inode->i_mtime, 1192 &inode->i_atime) < 0 && 1193 timespec_compare(&inode->i_ctime, 1194 &inode->i_atime) < 0) 1195 return; 1196 } 1197 } 1198 1199 now = current_fs_time(inode->i_sb); 1200 if (timespec_equal(&inode->i_atime, &now)) 1201 return; 1202 1203 inode->i_atime = now; 1204 mark_inode_dirty_sync(inode); 1205 } 1206 EXPORT_SYMBOL(touch_atime); 1207 1208 /** 1209 * file_update_time - update mtime and ctime time 1210 * @file: file accessed 1211 * 1212 * Update the mtime and ctime members of an inode and mark the inode 1213 * for writeback. Note that this function is meant exclusively for 1214 * usage in the file write path of filesystems, and filesystems may 1215 * choose to explicitly ignore update via this function with the 1216 * S_NOCTIME inode flag, e.g. for network filesystem where these 1217 * timestamps are handled by the server. 1218 */ 1219 1220 void file_update_time(struct file *file) 1221 { 1222 struct inode *inode = file->f_path.dentry->d_inode; 1223 struct timespec now; 1224 int sync_it = 0; 1225 1226 if (IS_NOCMTIME(inode)) 1227 return; 1228 if (IS_RDONLY(inode)) 1229 return; 1230 1231 now = current_fs_time(inode->i_sb); 1232 if (!timespec_equal(&inode->i_mtime, &now)) { 1233 inode->i_mtime = now; 1234 sync_it = 1; 1235 } 1236 1237 if (!timespec_equal(&inode->i_ctime, &now)) { 1238 inode->i_ctime = now; 1239 sync_it = 1; 1240 } 1241 1242 if (sync_it) 1243 mark_inode_dirty_sync(inode); 1244 } 1245 1246 EXPORT_SYMBOL(file_update_time); 1247 1248 int inode_needs_sync(struct inode *inode) 1249 { 1250 if (IS_SYNC(inode)) 1251 return 1; 1252 if (S_ISDIR(inode->i_mode) && IS_DIRSYNC(inode)) 1253 return 1; 1254 return 0; 1255 } 1256 1257 EXPORT_SYMBOL(inode_needs_sync); 1258 1259 int inode_wait(void *word) 1260 { 1261 schedule(); 1262 return 0; 1263 } 1264 1265 /* 1266 * If we try to find an inode in the inode hash while it is being 1267 * deleted, we have to wait until the filesystem completes its 1268 * deletion before reporting that it isn't found. This function waits 1269 * until the deletion _might_ have completed. Callers are responsible 1270 * to recheck inode state. 1271 * 1272 * It doesn't matter if I_LOCK is not set initially, a call to 1273 * wake_up_inode() after removing from the hash list will DTRT. 1274 * 1275 * This is called with inode_lock held. 1276 */ 1277 static void __wait_on_freeing_inode(struct inode *inode) 1278 { 1279 wait_queue_head_t *wq; 1280 DEFINE_WAIT_BIT(wait, &inode->i_state, __I_LOCK); 1281 wq = bit_waitqueue(&inode->i_state, __I_LOCK); 1282 prepare_to_wait(wq, &wait.wait, TASK_UNINTERRUPTIBLE); 1283 spin_unlock(&inode_lock); 1284 schedule(); 1285 finish_wait(wq, &wait.wait); 1286 spin_lock(&inode_lock); 1287 } 1288 1289 void wake_up_inode(struct inode *inode) 1290 { 1291 /* 1292 * Prevent speculative execution through spin_unlock(&inode_lock); 1293 */ 1294 smp_mb(); 1295 wake_up_bit(&inode->i_state, __I_LOCK); 1296 } 1297 1298 /* 1299 * We rarely want to lock two inodes that do not have a parent/child 1300 * relationship (such as directory, child inode) simultaneously. The 1301 * vast majority of file systems should be able to get along fine 1302 * without this. Do not use these functions except as a last resort. 1303 */ 1304 void inode_double_lock(struct inode *inode1, struct inode *inode2) 1305 { 1306 if (inode1 == NULL || inode2 == NULL || inode1 == inode2) { 1307 if (inode1) 1308 mutex_lock(&inode1->i_mutex); 1309 else if (inode2) 1310 mutex_lock(&inode2->i_mutex); 1311 return; 1312 } 1313 1314 if (inode1 < inode2) { 1315 mutex_lock_nested(&inode1->i_mutex, I_MUTEX_PARENT); 1316 mutex_lock_nested(&inode2->i_mutex, I_MUTEX_CHILD); 1317 } else { 1318 mutex_lock_nested(&inode2->i_mutex, I_MUTEX_PARENT); 1319 mutex_lock_nested(&inode1->i_mutex, I_MUTEX_CHILD); 1320 } 1321 } 1322 EXPORT_SYMBOL(inode_double_lock); 1323 1324 void inode_double_unlock(struct inode *inode1, struct inode *inode2) 1325 { 1326 if (inode1) 1327 mutex_unlock(&inode1->i_mutex); 1328 1329 if (inode2 && inode2 != inode1) 1330 mutex_unlock(&inode2->i_mutex); 1331 } 1332 EXPORT_SYMBOL(inode_double_unlock); 1333 1334 static __initdata unsigned long ihash_entries; 1335 static int __init set_ihash_entries(char *str) 1336 { 1337 if (!str) 1338 return 0; 1339 ihash_entries = simple_strtoul(str, &str, 0); 1340 return 1; 1341 } 1342 __setup("ihash_entries=", set_ihash_entries); 1343 1344 /* 1345 * Initialize the waitqueues and inode hash table. 1346 */ 1347 void __init inode_init_early(void) 1348 { 1349 int loop; 1350 1351 /* If hashes are distributed across NUMA nodes, defer 1352 * hash allocation until vmalloc space is available. 1353 */ 1354 if (hashdist) 1355 return; 1356 1357 inode_hashtable = 1358 alloc_large_system_hash("Inode-cache", 1359 sizeof(struct hlist_head), 1360 ihash_entries, 1361 14, 1362 HASH_EARLY, 1363 &i_hash_shift, 1364 &i_hash_mask, 1365 0); 1366 1367 for (loop = 0; loop < (1 << i_hash_shift); loop++) 1368 INIT_HLIST_HEAD(&inode_hashtable[loop]); 1369 } 1370 1371 void __init inode_init(unsigned long mempages) 1372 { 1373 int loop; 1374 1375 /* inode slab cache */ 1376 inode_cachep = kmem_cache_create("inode_cache", 1377 sizeof(struct inode), 1378 0, 1379 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC| 1380 SLAB_MEM_SPREAD), 1381 init_once, 1382 NULL); 1383 set_shrinker(DEFAULT_SEEKS, shrink_icache_memory); 1384 1385 /* Hash may have been set up in inode_init_early */ 1386 if (!hashdist) 1387 return; 1388 1389 inode_hashtable = 1390 alloc_large_system_hash("Inode-cache", 1391 sizeof(struct hlist_head), 1392 ihash_entries, 1393 14, 1394 0, 1395 &i_hash_shift, 1396 &i_hash_mask, 1397 0); 1398 1399 for (loop = 0; loop < (1 << i_hash_shift); loop++) 1400 INIT_HLIST_HEAD(&inode_hashtable[loop]); 1401 } 1402 1403 void init_special_inode(struct inode *inode, umode_t mode, dev_t rdev) 1404 { 1405 inode->i_mode = mode; 1406 if (S_ISCHR(mode)) { 1407 inode->i_fop = &def_chr_fops; 1408 inode->i_rdev = rdev; 1409 } else if (S_ISBLK(mode)) { 1410 inode->i_fop = &def_blk_fops; 1411 inode->i_rdev = rdev; 1412 } else if (S_ISFIFO(mode)) 1413 inode->i_fop = &def_fifo_fops; 1414 else if (S_ISSOCK(mode)) 1415 inode->i_fop = &bad_sock_fops; 1416 else 1417 printk(KERN_DEBUG "init_special_inode: bogus i_mode (%o)\n", 1418 mode); 1419 } 1420 EXPORT_SYMBOL(init_special_inode); 1421