10dca4462SChristoph Hellwig // SPDX-License-Identifier: GPL-2.0-only 20dca4462SChristoph Hellwig /* 30dca4462SChristoph Hellwig * Copyright (C) 1991, 1992 Linus Torvalds 40dca4462SChristoph Hellwig * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 50dca4462SChristoph Hellwig * Copyright (C) 2016 - 2020 Christoph Hellwig 60dca4462SChristoph Hellwig */ 70dca4462SChristoph Hellwig 80dca4462SChristoph Hellwig #include <linux/init.h> 90dca4462SChristoph Hellwig #include <linux/mm.h> 100dca4462SChristoph Hellwig #include <linux/slab.h> 110dca4462SChristoph Hellwig #include <linux/kmod.h> 120dca4462SChristoph Hellwig #include <linux/major.h> 130dca4462SChristoph Hellwig #include <linux/device_cgroup.h> 140dca4462SChristoph Hellwig #include <linux/blkdev.h> 15fe45e630SChristoph Hellwig #include <linux/blk-integrity.h> 160dca4462SChristoph Hellwig #include <linux/backing-dev.h> 170dca4462SChristoph Hellwig #include <linux/module.h> 180dca4462SChristoph Hellwig #include <linux/blkpg.h> 190dca4462SChristoph Hellwig #include <linux/magic.h> 200dca4462SChristoph Hellwig #include <linux/buffer_head.h> 210dca4462SChristoph Hellwig #include <linux/swap.h> 220dca4462SChristoph Hellwig #include <linux/writeback.h> 230dca4462SChristoph Hellwig #include <linux/mount.h> 240dca4462SChristoph Hellwig #include <linux/pseudo_fs.h> 250dca4462SChristoph Hellwig #include <linux/uio.h> 260dca4462SChristoph Hellwig #include <linux/namei.h> 270dca4462SChristoph Hellwig #include <linux/part_stat.h> 280dca4462SChristoph Hellwig #include <linux/uaccess.h> 292d985f8cSEric Biggers #include <linux/stat.h> 300dca4462SChristoph Hellwig #include "../fs/internal.h" 310dca4462SChristoph Hellwig #include "blk.h" 320dca4462SChristoph Hellwig 330dca4462SChristoph Hellwig struct bdev_inode { 340dca4462SChristoph Hellwig struct block_device bdev; 350dca4462SChristoph Hellwig struct inode vfs_inode; 360dca4462SChristoph Hellwig }; 370dca4462SChristoph Hellwig 380dca4462SChristoph Hellwig static inline struct bdev_inode *BDEV_I(struct inode *inode) 390dca4462SChristoph Hellwig { 400dca4462SChristoph Hellwig return container_of(inode, struct bdev_inode, vfs_inode); 410dca4462SChristoph Hellwig } 420dca4462SChristoph Hellwig 430dca4462SChristoph Hellwig struct block_device *I_BDEV(struct inode *inode) 440dca4462SChristoph Hellwig { 450dca4462SChristoph Hellwig return &BDEV_I(inode)->bdev; 460dca4462SChristoph Hellwig } 470dca4462SChristoph Hellwig EXPORT_SYMBOL(I_BDEV); 480dca4462SChristoph Hellwig 490dca4462SChristoph Hellwig static void bdev_write_inode(struct block_device *bdev) 500dca4462SChristoph Hellwig { 510dca4462SChristoph Hellwig struct inode *inode = bdev->bd_inode; 520dca4462SChristoph Hellwig int ret; 530dca4462SChristoph Hellwig 540dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 550dca4462SChristoph Hellwig while (inode->i_state & I_DIRTY) { 560dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 570dca4462SChristoph Hellwig ret = write_inode_now(inode, true); 585bf83e9aSChristoph Hellwig if (ret) 595bf83e9aSChristoph Hellwig pr_warn_ratelimited( 605bf83e9aSChristoph Hellwig "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n", 615bf83e9aSChristoph Hellwig bdev, ret); 620dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 630dca4462SChristoph Hellwig } 640dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 650dca4462SChristoph Hellwig } 660dca4462SChristoph Hellwig 670dca4462SChristoph Hellwig /* Kill _all_ buffers and pagecache , dirty or not.. */ 680dca4462SChristoph Hellwig static void kill_bdev(struct block_device *bdev) 690dca4462SChristoph Hellwig { 700dca4462SChristoph Hellwig struct address_space *mapping = bdev->bd_inode->i_mapping; 710dca4462SChristoph Hellwig 720dca4462SChristoph Hellwig if (mapping_empty(mapping)) 730dca4462SChristoph Hellwig return; 740dca4462SChristoph Hellwig 750dca4462SChristoph Hellwig invalidate_bh_lrus(); 760dca4462SChristoph Hellwig truncate_inode_pages(mapping, 0); 770dca4462SChristoph Hellwig } 780dca4462SChristoph Hellwig 790dca4462SChristoph Hellwig /* Invalidate clean unused buffers and pagecache. */ 800dca4462SChristoph Hellwig void invalidate_bdev(struct block_device *bdev) 810dca4462SChristoph Hellwig { 820dca4462SChristoph Hellwig struct address_space *mapping = bdev->bd_inode->i_mapping; 830dca4462SChristoph Hellwig 840dca4462SChristoph Hellwig if (mapping->nrpages) { 850dca4462SChristoph Hellwig invalidate_bh_lrus(); 860dca4462SChristoph Hellwig lru_add_drain_all(); /* make sure all lru add caches are flushed */ 870dca4462SChristoph Hellwig invalidate_mapping_pages(mapping, 0, -1); 880dca4462SChristoph Hellwig } 890dca4462SChristoph Hellwig } 900dca4462SChristoph Hellwig EXPORT_SYMBOL(invalidate_bdev); 910dca4462SChristoph Hellwig 920dca4462SChristoph Hellwig /* 930dca4462SChristoph Hellwig * Drop all buffers & page cache for given bdev range. This function bails 940dca4462SChristoph Hellwig * with error if bdev has other exclusive owner (such as filesystem). 950dca4462SChristoph Hellwig */ 960dca4462SChristoph Hellwig int truncate_bdev_range(struct block_device *bdev, fmode_t mode, 970dca4462SChristoph Hellwig loff_t lstart, loff_t lend) 980dca4462SChristoph Hellwig { 990dca4462SChristoph Hellwig /* 1000dca4462SChristoph Hellwig * If we don't hold exclusive handle for the device, upgrade to it 1010dca4462SChristoph Hellwig * while we discard the buffer cache to avoid discarding buffers 1020dca4462SChristoph Hellwig * under live filesystem. 1030dca4462SChristoph Hellwig */ 1040dca4462SChristoph Hellwig if (!(mode & FMODE_EXCL)) { 105*0718afd4SChristoph Hellwig int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL); 1060dca4462SChristoph Hellwig if (err) 1070dca4462SChristoph Hellwig goto invalidate; 1080dca4462SChristoph Hellwig } 1090dca4462SChristoph Hellwig 1100dca4462SChristoph Hellwig truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); 1110dca4462SChristoph Hellwig if (!(mode & FMODE_EXCL)) 1120dca4462SChristoph Hellwig bd_abort_claiming(bdev, truncate_bdev_range); 1130dca4462SChristoph Hellwig return 0; 1140dca4462SChristoph Hellwig 1150dca4462SChristoph Hellwig invalidate: 1160dca4462SChristoph Hellwig /* 1170dca4462SChristoph Hellwig * Someone else has handle exclusively open. Try invalidating instead. 1180dca4462SChristoph Hellwig * The 'end' argument is inclusive so the rounding is safe. 1190dca4462SChristoph Hellwig */ 1200dca4462SChristoph Hellwig return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, 1210dca4462SChristoph Hellwig lstart >> PAGE_SHIFT, 1220dca4462SChristoph Hellwig lend >> PAGE_SHIFT); 1230dca4462SChristoph Hellwig } 1240dca4462SChristoph Hellwig 1250dca4462SChristoph Hellwig static void set_init_blocksize(struct block_device *bdev) 1260dca4462SChristoph Hellwig { 1270dca4462SChristoph Hellwig unsigned int bsize = bdev_logical_block_size(bdev); 1280dca4462SChristoph Hellwig loff_t size = i_size_read(bdev->bd_inode); 1290dca4462SChristoph Hellwig 1300dca4462SChristoph Hellwig while (bsize < PAGE_SIZE) { 1310dca4462SChristoph Hellwig if (size & bsize) 1320dca4462SChristoph Hellwig break; 1330dca4462SChristoph Hellwig bsize <<= 1; 1340dca4462SChristoph Hellwig } 1350dca4462SChristoph Hellwig bdev->bd_inode->i_blkbits = blksize_bits(bsize); 1360dca4462SChristoph Hellwig } 1370dca4462SChristoph Hellwig 1380dca4462SChristoph Hellwig int set_blocksize(struct block_device *bdev, int size) 1390dca4462SChristoph Hellwig { 1400dca4462SChristoph Hellwig /* Size must be a power of two, and between 512 and PAGE_SIZE */ 1410dca4462SChristoph Hellwig if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 1420dca4462SChristoph Hellwig return -EINVAL; 1430dca4462SChristoph Hellwig 1440dca4462SChristoph Hellwig /* Size cannot be smaller than the size supported by the device */ 1450dca4462SChristoph Hellwig if (size < bdev_logical_block_size(bdev)) 1460dca4462SChristoph Hellwig return -EINVAL; 1470dca4462SChristoph Hellwig 1480dca4462SChristoph Hellwig /* Don't change the size if it is same as current */ 1490dca4462SChristoph Hellwig if (bdev->bd_inode->i_blkbits != blksize_bits(size)) { 1500dca4462SChristoph Hellwig sync_blockdev(bdev); 1510dca4462SChristoph Hellwig bdev->bd_inode->i_blkbits = blksize_bits(size); 1520dca4462SChristoph Hellwig kill_bdev(bdev); 1530dca4462SChristoph Hellwig } 1540dca4462SChristoph Hellwig return 0; 1550dca4462SChristoph Hellwig } 1560dca4462SChristoph Hellwig 1570dca4462SChristoph Hellwig EXPORT_SYMBOL(set_blocksize); 1580dca4462SChristoph Hellwig 1590dca4462SChristoph Hellwig int sb_set_blocksize(struct super_block *sb, int size) 1600dca4462SChristoph Hellwig { 1610dca4462SChristoph Hellwig if (set_blocksize(sb->s_bdev, size)) 1620dca4462SChristoph Hellwig return 0; 1630dca4462SChristoph Hellwig /* If we get here, we know size is power of two 1640dca4462SChristoph Hellwig * and it's value is between 512 and PAGE_SIZE */ 1650dca4462SChristoph Hellwig sb->s_blocksize = size; 1660dca4462SChristoph Hellwig sb->s_blocksize_bits = blksize_bits(size); 1670dca4462SChristoph Hellwig return sb->s_blocksize; 1680dca4462SChristoph Hellwig } 1690dca4462SChristoph Hellwig 1700dca4462SChristoph Hellwig EXPORT_SYMBOL(sb_set_blocksize); 1710dca4462SChristoph Hellwig 1720dca4462SChristoph Hellwig int sb_min_blocksize(struct super_block *sb, int size) 1730dca4462SChristoph Hellwig { 1740dca4462SChristoph Hellwig int minsize = bdev_logical_block_size(sb->s_bdev); 1750dca4462SChristoph Hellwig if (size < minsize) 1760dca4462SChristoph Hellwig size = minsize; 1770dca4462SChristoph Hellwig return sb_set_blocksize(sb, size); 1780dca4462SChristoph Hellwig } 1790dca4462SChristoph Hellwig 1800dca4462SChristoph Hellwig EXPORT_SYMBOL(sb_min_blocksize); 1810dca4462SChristoph Hellwig 18270164eb6SChristoph Hellwig int sync_blockdev_nowait(struct block_device *bdev) 1830dca4462SChristoph Hellwig { 1840dca4462SChristoph Hellwig if (!bdev) 1850dca4462SChristoph Hellwig return 0; 1860dca4462SChristoph Hellwig return filemap_flush(bdev->bd_inode->i_mapping); 1870dca4462SChristoph Hellwig } 18870164eb6SChristoph Hellwig EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 1890dca4462SChristoph Hellwig 1900dca4462SChristoph Hellwig /* 1910dca4462SChristoph Hellwig * Write out and wait upon all the dirty data associated with a block 1920dca4462SChristoph Hellwig * device via its mapping. Does not take the superblock lock. 1930dca4462SChristoph Hellwig */ 1940dca4462SChristoph Hellwig int sync_blockdev(struct block_device *bdev) 1950dca4462SChristoph Hellwig { 19670164eb6SChristoph Hellwig if (!bdev) 19770164eb6SChristoph Hellwig return 0; 19870164eb6SChristoph Hellwig return filemap_write_and_wait(bdev->bd_inode->i_mapping); 1990dca4462SChristoph Hellwig } 2000dca4462SChristoph Hellwig EXPORT_SYMBOL(sync_blockdev); 2010dca4462SChristoph Hellwig 20297d6fb1bSYuezhang Mo int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend) 20397d6fb1bSYuezhang Mo { 20497d6fb1bSYuezhang Mo return filemap_write_and_wait_range(bdev->bd_inode->i_mapping, 20597d6fb1bSYuezhang Mo lstart, lend); 20697d6fb1bSYuezhang Mo } 20797d6fb1bSYuezhang Mo EXPORT_SYMBOL(sync_blockdev_range); 20897d6fb1bSYuezhang Mo 2090dca4462SChristoph Hellwig /* 2100dca4462SChristoph Hellwig * Write out and wait upon all dirty data associated with this 2110dca4462SChristoph Hellwig * device. Filesystem data as well as the underlying block 2120dca4462SChristoph Hellwig * device. Takes the superblock lock. 2130dca4462SChristoph Hellwig */ 2140dca4462SChristoph Hellwig int fsync_bdev(struct block_device *bdev) 2150dca4462SChristoph Hellwig { 2160dca4462SChristoph Hellwig struct super_block *sb = get_super(bdev); 2170dca4462SChristoph Hellwig if (sb) { 2180dca4462SChristoph Hellwig int res = sync_filesystem(sb); 2190dca4462SChristoph Hellwig drop_super(sb); 2200dca4462SChristoph Hellwig return res; 2210dca4462SChristoph Hellwig } 2220dca4462SChristoph Hellwig return sync_blockdev(bdev); 2230dca4462SChristoph Hellwig } 2240dca4462SChristoph Hellwig EXPORT_SYMBOL(fsync_bdev); 2250dca4462SChristoph Hellwig 2260dca4462SChristoph Hellwig /** 2272e833c8cSRandy Dunlap * freeze_bdev - lock a filesystem and force it into a consistent state 2280dca4462SChristoph Hellwig * @bdev: blockdevice to lock 2290dca4462SChristoph Hellwig * 2300dca4462SChristoph Hellwig * If a superblock is found on this device, we take the s_umount semaphore 2310dca4462SChristoph Hellwig * on it to make sure nobody unmounts until the snapshot creation is done. 2320dca4462SChristoph Hellwig * The reference counter (bd_fsfreeze_count) guarantees that only the last 2330dca4462SChristoph Hellwig * unfreeze process can unfreeze the frozen filesystem actually when multiple 2340dca4462SChristoph Hellwig * freeze requests arrive simultaneously. It counts up in freeze_bdev() and 2350dca4462SChristoph Hellwig * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze 2360dca4462SChristoph Hellwig * actually. 2370dca4462SChristoph Hellwig */ 2380dca4462SChristoph Hellwig int freeze_bdev(struct block_device *bdev) 2390dca4462SChristoph Hellwig { 2400dca4462SChristoph Hellwig struct super_block *sb; 2410dca4462SChristoph Hellwig int error = 0; 2420dca4462SChristoph Hellwig 2430dca4462SChristoph Hellwig mutex_lock(&bdev->bd_fsfreeze_mutex); 2440dca4462SChristoph Hellwig if (++bdev->bd_fsfreeze_count > 1) 2450dca4462SChristoph Hellwig goto done; 2460dca4462SChristoph Hellwig 2470dca4462SChristoph Hellwig sb = get_active_super(bdev); 2480dca4462SChristoph Hellwig if (!sb) 2490dca4462SChristoph Hellwig goto sync; 2500dca4462SChristoph Hellwig if (sb->s_op->freeze_super) 2510dca4462SChristoph Hellwig error = sb->s_op->freeze_super(sb); 2520dca4462SChristoph Hellwig else 2530dca4462SChristoph Hellwig error = freeze_super(sb); 2540dca4462SChristoph Hellwig deactivate_super(sb); 2550dca4462SChristoph Hellwig 2560dca4462SChristoph Hellwig if (error) { 2570dca4462SChristoph Hellwig bdev->bd_fsfreeze_count--; 2580dca4462SChristoph Hellwig goto done; 2590dca4462SChristoph Hellwig } 2600dca4462SChristoph Hellwig bdev->bd_fsfreeze_sb = sb; 2610dca4462SChristoph Hellwig 2620dca4462SChristoph Hellwig sync: 2630dca4462SChristoph Hellwig sync_blockdev(bdev); 2640dca4462SChristoph Hellwig done: 2650dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_fsfreeze_mutex); 2660dca4462SChristoph Hellwig return error; 2670dca4462SChristoph Hellwig } 2680dca4462SChristoph Hellwig EXPORT_SYMBOL(freeze_bdev); 2690dca4462SChristoph Hellwig 2700dca4462SChristoph Hellwig /** 2712e833c8cSRandy Dunlap * thaw_bdev - unlock filesystem 2720dca4462SChristoph Hellwig * @bdev: blockdevice to unlock 2730dca4462SChristoph Hellwig * 2740dca4462SChristoph Hellwig * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 2750dca4462SChristoph Hellwig */ 2760dca4462SChristoph Hellwig int thaw_bdev(struct block_device *bdev) 2770dca4462SChristoph Hellwig { 2780dca4462SChristoph Hellwig struct super_block *sb; 2790dca4462SChristoph Hellwig int error = -EINVAL; 2800dca4462SChristoph Hellwig 2810dca4462SChristoph Hellwig mutex_lock(&bdev->bd_fsfreeze_mutex); 2820dca4462SChristoph Hellwig if (!bdev->bd_fsfreeze_count) 2830dca4462SChristoph Hellwig goto out; 2840dca4462SChristoph Hellwig 2850dca4462SChristoph Hellwig error = 0; 2860dca4462SChristoph Hellwig if (--bdev->bd_fsfreeze_count > 0) 2870dca4462SChristoph Hellwig goto out; 2880dca4462SChristoph Hellwig 2890dca4462SChristoph Hellwig sb = bdev->bd_fsfreeze_sb; 2900dca4462SChristoph Hellwig if (!sb) 2910dca4462SChristoph Hellwig goto out; 2920dca4462SChristoph Hellwig 2930dca4462SChristoph Hellwig if (sb->s_op->thaw_super) 2940dca4462SChristoph Hellwig error = sb->s_op->thaw_super(sb); 2950dca4462SChristoph Hellwig else 2960dca4462SChristoph Hellwig error = thaw_super(sb); 2970dca4462SChristoph Hellwig if (error) 2980dca4462SChristoph Hellwig bdev->bd_fsfreeze_count++; 2990dca4462SChristoph Hellwig else 3000dca4462SChristoph Hellwig bdev->bd_fsfreeze_sb = NULL; 3010dca4462SChristoph Hellwig out: 3020dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_fsfreeze_mutex); 3030dca4462SChristoph Hellwig return error; 3040dca4462SChristoph Hellwig } 3050dca4462SChristoph Hellwig EXPORT_SYMBOL(thaw_bdev); 3060dca4462SChristoph Hellwig 3070dca4462SChristoph Hellwig /* 3080dca4462SChristoph Hellwig * pseudo-fs 3090dca4462SChristoph Hellwig */ 3100dca4462SChristoph Hellwig 31174e6464aSChristoph Hellwig static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock); 3120dca4462SChristoph Hellwig static struct kmem_cache * bdev_cachep __read_mostly; 3130dca4462SChristoph Hellwig 3140dca4462SChristoph Hellwig static struct inode *bdev_alloc_inode(struct super_block *sb) 3150dca4462SChristoph Hellwig { 316fd60b288SMuchun Song struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL); 3170dca4462SChristoph Hellwig 3180dca4462SChristoph Hellwig if (!ei) 3190dca4462SChristoph Hellwig return NULL; 3200dca4462SChristoph Hellwig memset(&ei->bdev, 0, sizeof(ei->bdev)); 3210dca4462SChristoph Hellwig return &ei->vfs_inode; 3220dca4462SChristoph Hellwig } 3230dca4462SChristoph Hellwig 3240dca4462SChristoph Hellwig static void bdev_free_inode(struct inode *inode) 3250dca4462SChristoph Hellwig { 3260dca4462SChristoph Hellwig struct block_device *bdev = I_BDEV(inode); 3270dca4462SChristoph Hellwig 3280dca4462SChristoph Hellwig free_percpu(bdev->bd_stats); 3290dca4462SChristoph Hellwig kfree(bdev->bd_meta_info); 3300dca4462SChristoph Hellwig 3310dca4462SChristoph Hellwig if (!bdev_is_partition(bdev)) { 3320dca4462SChristoph Hellwig if (bdev->bd_disk && bdev->bd_disk->bdi) 3330dca4462SChristoph Hellwig bdi_put(bdev->bd_disk->bdi); 3340dca4462SChristoph Hellwig kfree(bdev->bd_disk); 3350dca4462SChristoph Hellwig } 3360dca4462SChristoph Hellwig 3370dca4462SChristoph Hellwig if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 3380dca4462SChristoph Hellwig blk_free_ext_minor(MINOR(bdev->bd_dev)); 3390dca4462SChristoph Hellwig 3400dca4462SChristoph Hellwig kmem_cache_free(bdev_cachep, BDEV_I(inode)); 3410dca4462SChristoph Hellwig } 3420dca4462SChristoph Hellwig 3430dca4462SChristoph Hellwig static void init_once(void *data) 3440dca4462SChristoph Hellwig { 3450dca4462SChristoph Hellwig struct bdev_inode *ei = data; 3460dca4462SChristoph Hellwig 3470dca4462SChristoph Hellwig inode_init_once(&ei->vfs_inode); 3480dca4462SChristoph Hellwig } 3490dca4462SChristoph Hellwig 3500dca4462SChristoph Hellwig static void bdev_evict_inode(struct inode *inode) 3510dca4462SChristoph Hellwig { 3520dca4462SChristoph Hellwig truncate_inode_pages_final(&inode->i_data); 3530dca4462SChristoph Hellwig invalidate_inode_buffers(inode); /* is it needed here? */ 3540dca4462SChristoph Hellwig clear_inode(inode); 3550dca4462SChristoph Hellwig } 3560dca4462SChristoph Hellwig 3570dca4462SChristoph Hellwig static const struct super_operations bdev_sops = { 3580dca4462SChristoph Hellwig .statfs = simple_statfs, 3590dca4462SChristoph Hellwig .alloc_inode = bdev_alloc_inode, 3600dca4462SChristoph Hellwig .free_inode = bdev_free_inode, 3610dca4462SChristoph Hellwig .drop_inode = generic_delete_inode, 3620dca4462SChristoph Hellwig .evict_inode = bdev_evict_inode, 3630dca4462SChristoph Hellwig }; 3640dca4462SChristoph Hellwig 3650dca4462SChristoph Hellwig static int bd_init_fs_context(struct fs_context *fc) 3660dca4462SChristoph Hellwig { 3670dca4462SChristoph Hellwig struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 3680dca4462SChristoph Hellwig if (!ctx) 3690dca4462SChristoph Hellwig return -ENOMEM; 3700dca4462SChristoph Hellwig fc->s_iflags |= SB_I_CGROUPWB; 3710dca4462SChristoph Hellwig ctx->ops = &bdev_sops; 3720dca4462SChristoph Hellwig return 0; 3730dca4462SChristoph Hellwig } 3740dca4462SChristoph Hellwig 3750dca4462SChristoph Hellwig static struct file_system_type bd_type = { 3760dca4462SChristoph Hellwig .name = "bdev", 3770dca4462SChristoph Hellwig .init_fs_context = bd_init_fs_context, 3780dca4462SChristoph Hellwig .kill_sb = kill_anon_super, 3790dca4462SChristoph Hellwig }; 3800dca4462SChristoph Hellwig 3810dca4462SChristoph Hellwig struct super_block *blockdev_superblock __read_mostly; 3820dca4462SChristoph Hellwig EXPORT_SYMBOL_GPL(blockdev_superblock); 3830dca4462SChristoph Hellwig 3840dca4462SChristoph Hellwig void __init bdev_cache_init(void) 3850dca4462SChristoph Hellwig { 3860dca4462SChristoph Hellwig int err; 3870dca4462SChristoph Hellwig static struct vfsmount *bd_mnt; 3880dca4462SChristoph Hellwig 3890dca4462SChristoph Hellwig bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 3900dca4462SChristoph Hellwig 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 3910dca4462SChristoph Hellwig SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC), 3920dca4462SChristoph Hellwig init_once); 3930dca4462SChristoph Hellwig err = register_filesystem(&bd_type); 3940dca4462SChristoph Hellwig if (err) 3950dca4462SChristoph Hellwig panic("Cannot register bdev pseudo-fs"); 3960dca4462SChristoph Hellwig bd_mnt = kern_mount(&bd_type); 3970dca4462SChristoph Hellwig if (IS_ERR(bd_mnt)) 3980dca4462SChristoph Hellwig panic("Cannot create bdev pseudo-fs"); 3990dca4462SChristoph Hellwig blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */ 4000dca4462SChristoph Hellwig } 4010dca4462SChristoph Hellwig 4020dca4462SChristoph Hellwig struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 4030dca4462SChristoph Hellwig { 4040dca4462SChristoph Hellwig struct block_device *bdev; 4050dca4462SChristoph Hellwig struct inode *inode; 4060dca4462SChristoph Hellwig 4070dca4462SChristoph Hellwig inode = new_inode(blockdev_superblock); 4080dca4462SChristoph Hellwig if (!inode) 4090dca4462SChristoph Hellwig return NULL; 4100dca4462SChristoph Hellwig inode->i_mode = S_IFBLK; 4110dca4462SChristoph Hellwig inode->i_rdev = 0; 4120dca4462SChristoph Hellwig inode->i_data.a_ops = &def_blk_aops; 4130dca4462SChristoph Hellwig mapping_set_gfp_mask(&inode->i_data, GFP_USER); 4140dca4462SChristoph Hellwig 4150dca4462SChristoph Hellwig bdev = I_BDEV(inode); 4160dca4462SChristoph Hellwig mutex_init(&bdev->bd_fsfreeze_mutex); 4170dca4462SChristoph Hellwig spin_lock_init(&bdev->bd_size_lock); 418*0718afd4SChristoph Hellwig mutex_init(&bdev->bd_holder_lock); 4190dca4462SChristoph Hellwig bdev->bd_partno = partno; 4200dca4462SChristoph Hellwig bdev->bd_inode = inode; 42117220ca5SPavel Begunkov bdev->bd_queue = disk->queue; 42238c8e3dfSMing Lei if (partno) 42338c8e3dfSMing Lei bdev->bd_has_submit_bio = disk->part0->bd_has_submit_bio; 42438c8e3dfSMing Lei else 4259f4107b0SJens Axboe bdev->bd_has_submit_bio = false; 42638c8e3dfSMing Lei bdev->bd_stats = alloc_percpu(struct disk_stats); 4270dca4462SChristoph Hellwig if (!bdev->bd_stats) { 4280dca4462SChristoph Hellwig iput(inode); 4290dca4462SChristoph Hellwig return NULL; 4300dca4462SChristoph Hellwig } 43106cc978dSTetsuo Handa bdev->bd_disk = disk; 4320dca4462SChristoph Hellwig return bdev; 4330dca4462SChristoph Hellwig } 4340dca4462SChristoph Hellwig 43583794367SDamien Le Moal void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors) 43683794367SDamien Le Moal { 43783794367SDamien Le Moal spin_lock(&bdev->bd_size_lock); 43883794367SDamien Le Moal i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT); 43983794367SDamien Le Moal bdev->bd_nr_sectors = sectors; 44083794367SDamien Le Moal spin_unlock(&bdev->bd_size_lock); 44183794367SDamien Le Moal } 44283794367SDamien Le Moal 4430dca4462SChristoph Hellwig void bdev_add(struct block_device *bdev, dev_t dev) 4440dca4462SChristoph Hellwig { 4450dca4462SChristoph Hellwig bdev->bd_dev = dev; 4460dca4462SChristoph Hellwig bdev->bd_inode->i_rdev = dev; 4470dca4462SChristoph Hellwig bdev->bd_inode->i_ino = dev; 4480dca4462SChristoph Hellwig insert_inode_hash(bdev->bd_inode); 4490dca4462SChristoph Hellwig } 4500dca4462SChristoph Hellwig 4510dca4462SChristoph Hellwig long nr_blockdev_pages(void) 4520dca4462SChristoph Hellwig { 4530dca4462SChristoph Hellwig struct inode *inode; 4540dca4462SChristoph Hellwig long ret = 0; 4550dca4462SChristoph Hellwig 4560dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 4570dca4462SChristoph Hellwig list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 4580dca4462SChristoph Hellwig ret += inode->i_mapping->nrpages; 4590dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 4600dca4462SChristoph Hellwig 4610dca4462SChristoph Hellwig return ret; 4620dca4462SChristoph Hellwig } 4630dca4462SChristoph Hellwig 4640dca4462SChristoph Hellwig /** 4650dca4462SChristoph Hellwig * bd_may_claim - test whether a block device can be claimed 4660dca4462SChristoph Hellwig * @bdev: block device of interest 4670dca4462SChristoph Hellwig * @holder: holder trying to claim @bdev 468*0718afd4SChristoph Hellwig * @hops: holder ops 4690dca4462SChristoph Hellwig * 4700dca4462SChristoph Hellwig * Test whether @bdev can be claimed by @holder. 4710dca4462SChristoph Hellwig * 4720dca4462SChristoph Hellwig * RETURNS: 4730dca4462SChristoph Hellwig * %true if @bdev can be claimed, %false otherwise. 4740dca4462SChristoph Hellwig */ 475*0718afd4SChristoph Hellwig static bool bd_may_claim(struct block_device *bdev, void *holder, 476*0718afd4SChristoph Hellwig const struct blk_holder_ops *hops) 4770dca4462SChristoph Hellwig { 478ae5f855eSChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 4790dca4462SChristoph Hellwig 48074e6464aSChristoph Hellwig lockdep_assert_held(&bdev_lock); 48174e6464aSChristoph Hellwig 482ae5f855eSChristoph Hellwig if (bdev->bd_holder) { 483ae5f855eSChristoph Hellwig /* 484ae5f855eSChristoph Hellwig * The same holder can always re-claim. 485ae5f855eSChristoph Hellwig */ 486*0718afd4SChristoph Hellwig if (bdev->bd_holder == holder) { 487*0718afd4SChristoph Hellwig if (WARN_ON_ONCE(bdev->bd_holder_ops != hops)) 488*0718afd4SChristoph Hellwig return false; 489ae5f855eSChristoph Hellwig return true; 490*0718afd4SChristoph Hellwig } 491ae5f855eSChristoph Hellwig return false; 492ae5f855eSChristoph Hellwig } 493ae5f855eSChristoph Hellwig 494ae5f855eSChristoph Hellwig /* 495ae5f855eSChristoph Hellwig * If the whole devices holder is set to bd_may_claim, a partition on 496ae5f855eSChristoph Hellwig * the device is claimed, but not the whole device. 497ae5f855eSChristoph Hellwig */ 498ae5f855eSChristoph Hellwig if (whole != bdev && 499ae5f855eSChristoph Hellwig whole->bd_holder && whole->bd_holder != bd_may_claim) 500ae5f855eSChristoph Hellwig return false; 501ae5f855eSChristoph Hellwig return true; 5020dca4462SChristoph Hellwig } 5030dca4462SChristoph Hellwig 5040dca4462SChristoph Hellwig /** 5050dca4462SChristoph Hellwig * bd_prepare_to_claim - claim a block device 5060dca4462SChristoph Hellwig * @bdev: block device of interest 5070dca4462SChristoph Hellwig * @holder: holder trying to claim @bdev 508*0718afd4SChristoph Hellwig * @hops: holder ops. 5090dca4462SChristoph Hellwig * 5100dca4462SChristoph Hellwig * Claim @bdev. This function fails if @bdev is already claimed by another 5110dca4462SChristoph Hellwig * holder and waits if another claiming is in progress. return, the caller 5120dca4462SChristoph Hellwig * has ownership of bd_claiming and bd_holder[s]. 5130dca4462SChristoph Hellwig * 5140dca4462SChristoph Hellwig * RETURNS: 5150dca4462SChristoph Hellwig * 0 if @bdev can be claimed, -EBUSY otherwise. 5160dca4462SChristoph Hellwig */ 517*0718afd4SChristoph Hellwig int bd_prepare_to_claim(struct block_device *bdev, void *holder, 518*0718afd4SChristoph Hellwig const struct blk_holder_ops *hops) 5190dca4462SChristoph Hellwig { 5200dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 5210dca4462SChristoph Hellwig 5220dca4462SChristoph Hellwig if (WARN_ON_ONCE(!holder)) 5230dca4462SChristoph Hellwig return -EINVAL; 5240dca4462SChristoph Hellwig retry: 52574e6464aSChristoph Hellwig mutex_lock(&bdev_lock); 5260dca4462SChristoph Hellwig /* if someone else claimed, fail */ 527*0718afd4SChristoph Hellwig if (!bd_may_claim(bdev, holder, hops)) { 52874e6464aSChristoph Hellwig mutex_unlock(&bdev_lock); 5290dca4462SChristoph Hellwig return -EBUSY; 5300dca4462SChristoph Hellwig } 5310dca4462SChristoph Hellwig 5320dca4462SChristoph Hellwig /* if claiming is already in progress, wait for it to finish */ 5330dca4462SChristoph Hellwig if (whole->bd_claiming) { 5340dca4462SChristoph Hellwig wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 5350dca4462SChristoph Hellwig DEFINE_WAIT(wait); 5360dca4462SChristoph Hellwig 5370dca4462SChristoph Hellwig prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 53874e6464aSChristoph Hellwig mutex_unlock(&bdev_lock); 5390dca4462SChristoph Hellwig schedule(); 5400dca4462SChristoph Hellwig finish_wait(wq, &wait); 5410dca4462SChristoph Hellwig goto retry; 5420dca4462SChristoph Hellwig } 5430dca4462SChristoph Hellwig 5440dca4462SChristoph Hellwig /* yay, all mine */ 5450dca4462SChristoph Hellwig whole->bd_claiming = holder; 54674e6464aSChristoph Hellwig mutex_unlock(&bdev_lock); 5470dca4462SChristoph Hellwig return 0; 5480dca4462SChristoph Hellwig } 5490dca4462SChristoph Hellwig EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 5500dca4462SChristoph Hellwig 5510dca4462SChristoph Hellwig static void bd_clear_claiming(struct block_device *whole, void *holder) 5520dca4462SChristoph Hellwig { 5530dca4462SChristoph Hellwig lockdep_assert_held(&bdev_lock); 5540dca4462SChristoph Hellwig /* tell others that we're done */ 5550dca4462SChristoph Hellwig BUG_ON(whole->bd_claiming != holder); 5560dca4462SChristoph Hellwig whole->bd_claiming = NULL; 5570dca4462SChristoph Hellwig wake_up_bit(&whole->bd_claiming, 0); 5580dca4462SChristoph Hellwig } 5590dca4462SChristoph Hellwig 5600dca4462SChristoph Hellwig /** 5610dca4462SChristoph Hellwig * bd_finish_claiming - finish claiming of a block device 5620dca4462SChristoph Hellwig * @bdev: block device of interest 5630dca4462SChristoph Hellwig * @holder: holder that has claimed @bdev 5640dca4462SChristoph Hellwig * 5650dca4462SChristoph Hellwig * Finish exclusive open of a block device. Mark the device as exlusively 5660dca4462SChristoph Hellwig * open by the holder and wake up all waiters for exclusive open to finish. 5670dca4462SChristoph Hellwig */ 568*0718afd4SChristoph Hellwig static void bd_finish_claiming(struct block_device *bdev, void *holder, 569*0718afd4SChristoph Hellwig const struct blk_holder_ops *hops) 5700dca4462SChristoph Hellwig { 5710dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 5720dca4462SChristoph Hellwig 57374e6464aSChristoph Hellwig mutex_lock(&bdev_lock); 574*0718afd4SChristoph Hellwig BUG_ON(!bd_may_claim(bdev, holder, hops)); 5750dca4462SChristoph Hellwig /* 5760dca4462SChristoph Hellwig * Note that for a whole device bd_holders will be incremented twice, 5770dca4462SChristoph Hellwig * and bd_holder will be set to bd_may_claim before being set to holder 5780dca4462SChristoph Hellwig */ 5790dca4462SChristoph Hellwig whole->bd_holders++; 5800dca4462SChristoph Hellwig whole->bd_holder = bd_may_claim; 5810dca4462SChristoph Hellwig bdev->bd_holders++; 582*0718afd4SChristoph Hellwig mutex_lock(&bdev->bd_holder_lock); 5830dca4462SChristoph Hellwig bdev->bd_holder = holder; 584*0718afd4SChristoph Hellwig bdev->bd_holder_ops = hops; 585*0718afd4SChristoph Hellwig mutex_unlock(&bdev->bd_holder_lock); 5860dca4462SChristoph Hellwig bd_clear_claiming(whole, holder); 58774e6464aSChristoph Hellwig mutex_unlock(&bdev_lock); 5880dca4462SChristoph Hellwig } 5890dca4462SChristoph Hellwig 5900dca4462SChristoph Hellwig /** 5910dca4462SChristoph Hellwig * bd_abort_claiming - abort claiming of a block device 5920dca4462SChristoph Hellwig * @bdev: block device of interest 5930dca4462SChristoph Hellwig * @holder: holder that has claimed @bdev 5940dca4462SChristoph Hellwig * 5950dca4462SChristoph Hellwig * Abort claiming of a block device when the exclusive open failed. This can be 5960dca4462SChristoph Hellwig * also used when exclusive open is not actually desired and we just needed 5970dca4462SChristoph Hellwig * to block other exclusive openers for a while. 5980dca4462SChristoph Hellwig */ 5990dca4462SChristoph Hellwig void bd_abort_claiming(struct block_device *bdev, void *holder) 6000dca4462SChristoph Hellwig { 60174e6464aSChristoph Hellwig mutex_lock(&bdev_lock); 6020dca4462SChristoph Hellwig bd_clear_claiming(bdev_whole(bdev), holder); 60374e6464aSChristoph Hellwig mutex_unlock(&bdev_lock); 6040dca4462SChristoph Hellwig } 6050dca4462SChristoph Hellwig EXPORT_SYMBOL(bd_abort_claiming); 6060dca4462SChristoph Hellwig 6070783b1a7SChristoph Hellwig static void bd_end_claim(struct block_device *bdev) 6080783b1a7SChristoph Hellwig { 6090783b1a7SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 6100783b1a7SChristoph Hellwig bool unblock = false; 6110783b1a7SChristoph Hellwig 6120783b1a7SChristoph Hellwig /* 6130783b1a7SChristoph Hellwig * Release a claim on the device. The holder fields are protected with 6140783b1a7SChristoph Hellwig * bdev_lock. open_mutex is used to synchronize disk_holder unlinking. 6150783b1a7SChristoph Hellwig */ 61674e6464aSChristoph Hellwig mutex_lock(&bdev_lock); 6170783b1a7SChristoph Hellwig WARN_ON_ONCE(--bdev->bd_holders < 0); 6180783b1a7SChristoph Hellwig WARN_ON_ONCE(--whole->bd_holders < 0); 6190783b1a7SChristoph Hellwig if (!bdev->bd_holders) { 620*0718afd4SChristoph Hellwig mutex_lock(&bdev->bd_holder_lock); 6210783b1a7SChristoph Hellwig bdev->bd_holder = NULL; 622*0718afd4SChristoph Hellwig bdev->bd_holder_ops = NULL; 623*0718afd4SChristoph Hellwig mutex_unlock(&bdev->bd_holder_lock); 6240783b1a7SChristoph Hellwig if (bdev->bd_write_holder) 6250783b1a7SChristoph Hellwig unblock = true; 6260783b1a7SChristoph Hellwig } 6270783b1a7SChristoph Hellwig if (!whole->bd_holders) 6280783b1a7SChristoph Hellwig whole->bd_holder = NULL; 62974e6464aSChristoph Hellwig mutex_unlock(&bdev_lock); 6300783b1a7SChristoph Hellwig 6310783b1a7SChristoph Hellwig /* 6320783b1a7SChristoph Hellwig * If this was the last claim, remove holder link and unblock evpoll if 6330783b1a7SChristoph Hellwig * it was a write holder. 6340783b1a7SChristoph Hellwig */ 6350783b1a7SChristoph Hellwig if (unblock) { 6360783b1a7SChristoph Hellwig disk_unblock_events(bdev->bd_disk); 6370783b1a7SChristoph Hellwig bdev->bd_write_holder = false; 6380783b1a7SChristoph Hellwig } 6390783b1a7SChristoph Hellwig } 6400783b1a7SChristoph Hellwig 6410dca4462SChristoph Hellwig static void blkdev_flush_mapping(struct block_device *bdev) 6420dca4462SChristoph Hellwig { 6430dca4462SChristoph Hellwig WARN_ON_ONCE(bdev->bd_holders); 6440dca4462SChristoph Hellwig sync_blockdev(bdev); 6450dca4462SChristoph Hellwig kill_bdev(bdev); 6460dca4462SChristoph Hellwig bdev_write_inode(bdev); 6470dca4462SChristoph Hellwig } 6480dca4462SChristoph Hellwig 6490dca4462SChristoph Hellwig static int blkdev_get_whole(struct block_device *bdev, fmode_t mode) 6500dca4462SChristoph Hellwig { 6510dca4462SChristoph Hellwig struct gendisk *disk = bdev->bd_disk; 652af22fef3SColin Ian King int ret; 6530dca4462SChristoph Hellwig 6540dca4462SChristoph Hellwig if (disk->fops->open) { 6550dca4462SChristoph Hellwig ret = disk->fops->open(bdev, mode); 6560dca4462SChristoph Hellwig if (ret) { 6570dca4462SChristoph Hellwig /* avoid ghost partitions on a removed medium */ 6580dca4462SChristoph Hellwig if (ret == -ENOMEDIUM && 6590dca4462SChristoph Hellwig test_bit(GD_NEED_PART_SCAN, &disk->state)) 6600dca4462SChristoph Hellwig bdev_disk_changed(disk, true); 6610dca4462SChristoph Hellwig return ret; 6620dca4462SChristoph Hellwig } 6630dca4462SChristoph Hellwig } 6640dca4462SChristoph Hellwig 6659acf381fSChristoph Hellwig if (!atomic_read(&bdev->bd_openers)) 6660dca4462SChristoph Hellwig set_init_blocksize(bdev); 6670dca4462SChristoph Hellwig if (test_bit(GD_NEED_PART_SCAN, &disk->state)) 6680dca4462SChristoph Hellwig bdev_disk_changed(disk, false); 6699acf381fSChristoph Hellwig atomic_inc(&bdev->bd_openers); 670483546c1SNian Yanchuan return 0; 6710dca4462SChristoph Hellwig } 6720dca4462SChristoph Hellwig 6730dca4462SChristoph Hellwig static void blkdev_put_whole(struct block_device *bdev, fmode_t mode) 6740dca4462SChristoph Hellwig { 6759acf381fSChristoph Hellwig if (atomic_dec_and_test(&bdev->bd_openers)) 6760dca4462SChristoph Hellwig blkdev_flush_mapping(bdev); 6770dca4462SChristoph Hellwig if (bdev->bd_disk->fops->release) 6780dca4462SChristoph Hellwig bdev->bd_disk->fops->release(bdev->bd_disk, mode); 6790dca4462SChristoph Hellwig } 6800dca4462SChristoph Hellwig 6810dca4462SChristoph Hellwig static int blkdev_get_part(struct block_device *part, fmode_t mode) 6820dca4462SChristoph Hellwig { 6830dca4462SChristoph Hellwig struct gendisk *disk = part->bd_disk; 6840dca4462SChristoph Hellwig int ret; 6850dca4462SChristoph Hellwig 6869acf381fSChristoph Hellwig if (atomic_read(&part->bd_openers)) 6870dca4462SChristoph Hellwig goto done; 6880dca4462SChristoph Hellwig 6890dca4462SChristoph Hellwig ret = blkdev_get_whole(bdev_whole(part), mode); 6900dca4462SChristoph Hellwig if (ret) 6910dca4462SChristoph Hellwig return ret; 6920dca4462SChristoph Hellwig 6930dca4462SChristoph Hellwig ret = -ENXIO; 6940dca4462SChristoph Hellwig if (!bdev_nr_sectors(part)) 6950dca4462SChristoph Hellwig goto out_blkdev_put; 6960dca4462SChristoph Hellwig 6970dca4462SChristoph Hellwig disk->open_partitions++; 6980dca4462SChristoph Hellwig set_init_blocksize(part); 6990dca4462SChristoph Hellwig done: 7009acf381fSChristoph Hellwig atomic_inc(&part->bd_openers); 7010dca4462SChristoph Hellwig return 0; 7020dca4462SChristoph Hellwig 7030dca4462SChristoph Hellwig out_blkdev_put: 7040dca4462SChristoph Hellwig blkdev_put_whole(bdev_whole(part), mode); 7050dca4462SChristoph Hellwig return ret; 7060dca4462SChristoph Hellwig } 7070dca4462SChristoph Hellwig 7080dca4462SChristoph Hellwig static void blkdev_put_part(struct block_device *part, fmode_t mode) 7090dca4462SChristoph Hellwig { 7100dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(part); 7110dca4462SChristoph Hellwig 7129acf381fSChristoph Hellwig if (!atomic_dec_and_test(&part->bd_openers)) 7130dca4462SChristoph Hellwig return; 7140dca4462SChristoph Hellwig blkdev_flush_mapping(part); 7150dca4462SChristoph Hellwig whole->bd_disk->open_partitions--; 7160dca4462SChristoph Hellwig blkdev_put_whole(whole, mode); 7170dca4462SChristoph Hellwig } 7180dca4462SChristoph Hellwig 7190dca4462SChristoph Hellwig struct block_device *blkdev_get_no_open(dev_t dev) 7200dca4462SChristoph Hellwig { 7210dca4462SChristoph Hellwig struct block_device *bdev; 7220dca4462SChristoph Hellwig struct inode *inode; 7230dca4462SChristoph Hellwig 7240dca4462SChristoph Hellwig inode = ilookup(blockdev_superblock, dev); 725fbdee71bSChristoph Hellwig if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { 7260dca4462SChristoph Hellwig blk_request_module(dev); 7270dca4462SChristoph Hellwig inode = ilookup(blockdev_superblock, dev); 728fbdee71bSChristoph Hellwig if (inode) 729fbdee71bSChristoph Hellwig pr_warn_ratelimited( 730451f0b6fSChristoph Hellwig "block device autoloading is deprecated and will be removed.\n"); 731fbdee71bSChristoph Hellwig } 7320dca4462SChristoph Hellwig if (!inode) 7330dca4462SChristoph Hellwig return NULL; 7340dca4462SChristoph Hellwig 7350dca4462SChristoph Hellwig /* switch from the inode reference to a device mode one: */ 7360dca4462SChristoph Hellwig bdev = &BDEV_I(inode)->bdev; 7370dca4462SChristoph Hellwig if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 7380dca4462SChristoph Hellwig bdev = NULL; 7390dca4462SChristoph Hellwig iput(inode); 7400dca4462SChristoph Hellwig return bdev; 7410dca4462SChristoph Hellwig } 7420dca4462SChristoph Hellwig 7430dca4462SChristoph Hellwig void blkdev_put_no_open(struct block_device *bdev) 7440dca4462SChristoph Hellwig { 7450dca4462SChristoph Hellwig put_device(&bdev->bd_device); 7460dca4462SChristoph Hellwig } 7470dca4462SChristoph Hellwig 7480dca4462SChristoph Hellwig /** 7490dca4462SChristoph Hellwig * blkdev_get_by_dev - open a block device by device number 7500dca4462SChristoph Hellwig * @dev: device number of block device to open 7510dca4462SChristoph Hellwig * @mode: FMODE_* mask 7520dca4462SChristoph Hellwig * @holder: exclusive holder identifier 753*0718afd4SChristoph Hellwig * @hops: holder operations 7540dca4462SChristoph Hellwig * 7550dca4462SChristoph Hellwig * Open the block device described by device number @dev. If @mode includes 7560dca4462SChristoph Hellwig * %FMODE_EXCL, the block device is opened with exclusive access. Specifying 7570dca4462SChristoph Hellwig * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for 7580dca4462SChristoph Hellwig * the same @holder. 7590dca4462SChristoph Hellwig * 7600dca4462SChristoph Hellwig * Use this interface ONLY if you really do not have anything better - i.e. when 7610dca4462SChristoph Hellwig * you are behind a truly sucky interface and all you are given is a device 7620dca4462SChristoph Hellwig * number. Everything else should use blkdev_get_by_path(). 7630dca4462SChristoph Hellwig * 7640dca4462SChristoph Hellwig * CONTEXT: 7650dca4462SChristoph Hellwig * Might sleep. 7660dca4462SChristoph Hellwig * 7670dca4462SChristoph Hellwig * RETURNS: 7680dca4462SChristoph Hellwig * Reference to the block_device on success, ERR_PTR(-errno) on failure. 7690dca4462SChristoph Hellwig */ 770*0718afd4SChristoph Hellwig struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder, 771*0718afd4SChristoph Hellwig const struct blk_holder_ops *hops) 7720dca4462SChristoph Hellwig { 7730dca4462SChristoph Hellwig bool unblock_events = true; 7740dca4462SChristoph Hellwig struct block_device *bdev; 7750dca4462SChristoph Hellwig struct gendisk *disk; 7760dca4462SChristoph Hellwig int ret; 7770dca4462SChristoph Hellwig 7780dca4462SChristoph Hellwig ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 7790dca4462SChristoph Hellwig MAJOR(dev), MINOR(dev), 7800dca4462SChristoph Hellwig ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) | 7810dca4462SChristoph Hellwig ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0)); 7820dca4462SChristoph Hellwig if (ret) 7830dca4462SChristoph Hellwig return ERR_PTR(ret); 7840dca4462SChristoph Hellwig 7850dca4462SChristoph Hellwig bdev = blkdev_get_no_open(dev); 7860dca4462SChristoph Hellwig if (!bdev) 7870dca4462SChristoph Hellwig return ERR_PTR(-ENXIO); 7880dca4462SChristoph Hellwig disk = bdev->bd_disk; 7890dca4462SChristoph Hellwig 7900dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 791*0718afd4SChristoph Hellwig ret = bd_prepare_to_claim(bdev, holder, hops); 7920dca4462SChristoph Hellwig if (ret) 7930dca4462SChristoph Hellwig goto put_blkdev; 7940dca4462SChristoph Hellwig } 7950dca4462SChristoph Hellwig 7960dca4462SChristoph Hellwig disk_block_events(disk); 7970dca4462SChristoph Hellwig 7980dca4462SChristoph Hellwig mutex_lock(&disk->open_mutex); 7990dca4462SChristoph Hellwig ret = -ENXIO; 8000dca4462SChristoph Hellwig if (!disk_live(disk)) 8010dca4462SChristoph Hellwig goto abort_claiming; 802efcf5932SMing Lei if (!try_module_get(disk->fops->owner)) 803efcf5932SMing Lei goto abort_claiming; 8040dca4462SChristoph Hellwig if (bdev_is_partition(bdev)) 8050dca4462SChristoph Hellwig ret = blkdev_get_part(bdev, mode); 8060dca4462SChristoph Hellwig else 8070dca4462SChristoph Hellwig ret = blkdev_get_whole(bdev, mode); 8080dca4462SChristoph Hellwig if (ret) 809efcf5932SMing Lei goto put_module; 8100dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 811*0718afd4SChristoph Hellwig bd_finish_claiming(bdev, holder, hops); 8120dca4462SChristoph Hellwig 8130dca4462SChristoph Hellwig /* 8140dca4462SChristoph Hellwig * Block event polling for write claims if requested. Any write 8150dca4462SChristoph Hellwig * holder makes the write_holder state stick until all are 8160dca4462SChristoph Hellwig * released. This is good enough and tracking individual 8170dca4462SChristoph Hellwig * writeable reference is too fragile given the way @mode is 8180dca4462SChristoph Hellwig * used in blkdev_get/put(). 8190dca4462SChristoph Hellwig */ 8200dca4462SChristoph Hellwig if ((mode & FMODE_WRITE) && !bdev->bd_write_holder && 8211545e0b4SChristoph Hellwig (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) { 8220dca4462SChristoph Hellwig bdev->bd_write_holder = true; 8230dca4462SChristoph Hellwig unblock_events = false; 8240dca4462SChristoph Hellwig } 8250dca4462SChristoph Hellwig } 8260dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 8270dca4462SChristoph Hellwig 8280dca4462SChristoph Hellwig if (unblock_events) 8290dca4462SChristoph Hellwig disk_unblock_events(disk); 8300dca4462SChristoph Hellwig return bdev; 831efcf5932SMing Lei put_module: 832efcf5932SMing Lei module_put(disk->fops->owner); 8330dca4462SChristoph Hellwig abort_claiming: 8340dca4462SChristoph Hellwig if (mode & FMODE_EXCL) 8350dca4462SChristoph Hellwig bd_abort_claiming(bdev, holder); 8360dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 8370dca4462SChristoph Hellwig disk_unblock_events(disk); 8380dca4462SChristoph Hellwig put_blkdev: 8390dca4462SChristoph Hellwig blkdev_put_no_open(bdev); 8400dca4462SChristoph Hellwig return ERR_PTR(ret); 8410dca4462SChristoph Hellwig } 8420dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_get_by_dev); 8430dca4462SChristoph Hellwig 8440dca4462SChristoph Hellwig /** 8450dca4462SChristoph Hellwig * blkdev_get_by_path - open a block device by name 8460dca4462SChristoph Hellwig * @path: path to the block device to open 8470dca4462SChristoph Hellwig * @mode: FMODE_* mask 8480dca4462SChristoph Hellwig * @holder: exclusive holder identifier 8490dca4462SChristoph Hellwig * 8500dca4462SChristoph Hellwig * Open the block device described by the device file at @path. If @mode 8510dca4462SChristoph Hellwig * includes %FMODE_EXCL, the block device is opened with exclusive access. 8520dca4462SChristoph Hellwig * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may 8530dca4462SChristoph Hellwig * nest for the same @holder. 8540dca4462SChristoph Hellwig * 8550dca4462SChristoph Hellwig * CONTEXT: 8560dca4462SChristoph Hellwig * Might sleep. 8570dca4462SChristoph Hellwig * 8580dca4462SChristoph Hellwig * RETURNS: 8590dca4462SChristoph Hellwig * Reference to the block_device on success, ERR_PTR(-errno) on failure. 8600dca4462SChristoph Hellwig */ 8610dca4462SChristoph Hellwig struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, 862*0718afd4SChristoph Hellwig void *holder, const struct blk_holder_ops *hops) 8630dca4462SChristoph Hellwig { 8640dca4462SChristoph Hellwig struct block_device *bdev; 8650dca4462SChristoph Hellwig dev_t dev; 8660dca4462SChristoph Hellwig int error; 8670dca4462SChristoph Hellwig 8680dca4462SChristoph Hellwig error = lookup_bdev(path, &dev); 8690dca4462SChristoph Hellwig if (error) 8700dca4462SChristoph Hellwig return ERR_PTR(error); 8710dca4462SChristoph Hellwig 872*0718afd4SChristoph Hellwig bdev = blkdev_get_by_dev(dev, mode, holder, hops); 8730dca4462SChristoph Hellwig if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) { 8740dca4462SChristoph Hellwig blkdev_put(bdev, mode); 8750dca4462SChristoph Hellwig return ERR_PTR(-EACCES); 8760dca4462SChristoph Hellwig } 8770dca4462SChristoph Hellwig 8780dca4462SChristoph Hellwig return bdev; 8790dca4462SChristoph Hellwig } 8800dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_get_by_path); 8810dca4462SChristoph Hellwig 8820dca4462SChristoph Hellwig void blkdev_put(struct block_device *bdev, fmode_t mode) 8830dca4462SChristoph Hellwig { 8840dca4462SChristoph Hellwig struct gendisk *disk = bdev->bd_disk; 8850dca4462SChristoph Hellwig 8860dca4462SChristoph Hellwig /* 8870dca4462SChristoph Hellwig * Sync early if it looks like we're the last one. If someone else 8880dca4462SChristoph Hellwig * opens the block device between now and the decrement of bd_openers 8890dca4462SChristoph Hellwig * then we did a sync that we didn't need to, but that's not the end 8900dca4462SChristoph Hellwig * of the world and we want to avoid long (could be several minute) 8910dca4462SChristoph Hellwig * syncs while holding the mutex. 8920dca4462SChristoph Hellwig */ 8939acf381fSChristoph Hellwig if (atomic_read(&bdev->bd_openers) == 1) 8940dca4462SChristoph Hellwig sync_blockdev(bdev); 8950dca4462SChristoph Hellwig 8960dca4462SChristoph Hellwig mutex_lock(&disk->open_mutex); 8970783b1a7SChristoph Hellwig if (mode & FMODE_EXCL) 8980783b1a7SChristoph Hellwig bd_end_claim(bdev); 8990dca4462SChristoph Hellwig 9000dca4462SChristoph Hellwig /* 9010dca4462SChristoph Hellwig * Trigger event checking and tell drivers to flush MEDIA_CHANGE 9020dca4462SChristoph Hellwig * event. This is to ensure detection of media removal commanded 9030dca4462SChristoph Hellwig * from userland - e.g. eject(1). 9040dca4462SChristoph Hellwig */ 9050dca4462SChristoph Hellwig disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 9060dca4462SChristoph Hellwig 9070dca4462SChristoph Hellwig if (bdev_is_partition(bdev)) 9080dca4462SChristoph Hellwig blkdev_put_part(bdev, mode); 9090dca4462SChristoph Hellwig else 9100dca4462SChristoph Hellwig blkdev_put_whole(bdev, mode); 9110dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 9120dca4462SChristoph Hellwig 913efcf5932SMing Lei module_put(disk->fops->owner); 9140dca4462SChristoph Hellwig blkdev_put_no_open(bdev); 9150dca4462SChristoph Hellwig } 9160dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_put); 9170dca4462SChristoph Hellwig 9180dca4462SChristoph Hellwig /** 9190ba4566cSMatthew Wilcox (Oracle) * lookup_bdev() - Look up a struct block_device by name. 9200ba4566cSMatthew Wilcox (Oracle) * @pathname: Name of the block device in the filesystem. 9210ba4566cSMatthew Wilcox (Oracle) * @dev: Pointer to the block device's dev_t, if found. 9220dca4462SChristoph Hellwig * 923057178cfSJackie Liu * Lookup the block device's dev_t at @pathname in the current 9240ba4566cSMatthew Wilcox (Oracle) * namespace if possible and return it in @dev. 925057178cfSJackie Liu * 9260ba4566cSMatthew Wilcox (Oracle) * Context: May sleep. 9270ba4566cSMatthew Wilcox (Oracle) * Return: 0 if succeeded, negative errno otherwise. 9280dca4462SChristoph Hellwig */ 9290dca4462SChristoph Hellwig int lookup_bdev(const char *pathname, dev_t *dev) 9300dca4462SChristoph Hellwig { 9310dca4462SChristoph Hellwig struct inode *inode; 9320dca4462SChristoph Hellwig struct path path; 9330dca4462SChristoph Hellwig int error; 9340dca4462SChristoph Hellwig 9350dca4462SChristoph Hellwig if (!pathname || !*pathname) 9360dca4462SChristoph Hellwig return -EINVAL; 9370dca4462SChristoph Hellwig 9380dca4462SChristoph Hellwig error = kern_path(pathname, LOOKUP_FOLLOW, &path); 9390dca4462SChristoph Hellwig if (error) 9400dca4462SChristoph Hellwig return error; 9410dca4462SChristoph Hellwig 9420dca4462SChristoph Hellwig inode = d_backing_inode(path.dentry); 9430dca4462SChristoph Hellwig error = -ENOTBLK; 9440dca4462SChristoph Hellwig if (!S_ISBLK(inode->i_mode)) 9450dca4462SChristoph Hellwig goto out_path_put; 9460dca4462SChristoph Hellwig error = -EACCES; 9470dca4462SChristoph Hellwig if (!may_open_dev(&path)) 9480dca4462SChristoph Hellwig goto out_path_put; 9490dca4462SChristoph Hellwig 9500dca4462SChristoph Hellwig *dev = inode->i_rdev; 9510dca4462SChristoph Hellwig error = 0; 9520dca4462SChristoph Hellwig out_path_put: 9530dca4462SChristoph Hellwig path_put(&path); 9540dca4462SChristoph Hellwig return error; 9550dca4462SChristoph Hellwig } 9560dca4462SChristoph Hellwig EXPORT_SYMBOL(lookup_bdev); 9570dca4462SChristoph Hellwig 9580dca4462SChristoph Hellwig int __invalidate_device(struct block_device *bdev, bool kill_dirty) 9590dca4462SChristoph Hellwig { 9600dca4462SChristoph Hellwig struct super_block *sb = get_super(bdev); 9610dca4462SChristoph Hellwig int res = 0; 9620dca4462SChristoph Hellwig 9630dca4462SChristoph Hellwig if (sb) { 9640dca4462SChristoph Hellwig /* 9650dca4462SChristoph Hellwig * no need to lock the super, get_super holds the 9660dca4462SChristoph Hellwig * read mutex so the filesystem cannot go away 9670dca4462SChristoph Hellwig * under us (->put_super runs with the write lock 9680dca4462SChristoph Hellwig * hold). 9690dca4462SChristoph Hellwig */ 9700dca4462SChristoph Hellwig shrink_dcache_sb(sb); 9710dca4462SChristoph Hellwig res = invalidate_inodes(sb, kill_dirty); 9720dca4462SChristoph Hellwig drop_super(sb); 9730dca4462SChristoph Hellwig } 9740dca4462SChristoph Hellwig invalidate_bdev(bdev); 9750dca4462SChristoph Hellwig return res; 9760dca4462SChristoph Hellwig } 9770dca4462SChristoph Hellwig EXPORT_SYMBOL(__invalidate_device); 9780dca4462SChristoph Hellwig 9791e03a36bSChristoph Hellwig void sync_bdevs(bool wait) 9800dca4462SChristoph Hellwig { 9810dca4462SChristoph Hellwig struct inode *inode, *old_inode = NULL; 9820dca4462SChristoph Hellwig 9830dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 9840dca4462SChristoph Hellwig list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 9850dca4462SChristoph Hellwig struct address_space *mapping = inode->i_mapping; 9860dca4462SChristoph Hellwig struct block_device *bdev; 9870dca4462SChristoph Hellwig 9880dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 9890dca4462SChristoph Hellwig if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 9900dca4462SChristoph Hellwig mapping->nrpages == 0) { 9910dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 9920dca4462SChristoph Hellwig continue; 9930dca4462SChristoph Hellwig } 9940dca4462SChristoph Hellwig __iget(inode); 9950dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 9960dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 9970dca4462SChristoph Hellwig /* 9980dca4462SChristoph Hellwig * We hold a reference to 'inode' so it couldn't have been 9990dca4462SChristoph Hellwig * removed from s_inodes list while we dropped the 10000dca4462SChristoph Hellwig * s_inode_list_lock We cannot iput the inode now as we can 10010dca4462SChristoph Hellwig * be holding the last reference and we cannot iput it under 10020dca4462SChristoph Hellwig * s_inode_list_lock. So we keep the reference and iput it 10030dca4462SChristoph Hellwig * later. 10040dca4462SChristoph Hellwig */ 10050dca4462SChristoph Hellwig iput(old_inode); 10060dca4462SChristoph Hellwig old_inode = inode; 10070dca4462SChristoph Hellwig bdev = I_BDEV(inode); 10080dca4462SChristoph Hellwig 10090dca4462SChristoph Hellwig mutex_lock(&bdev->bd_disk->open_mutex); 10109acf381fSChristoph Hellwig if (!atomic_read(&bdev->bd_openers)) { 10111e03a36bSChristoph Hellwig ; /* skip */ 10121e03a36bSChristoph Hellwig } else if (wait) { 10131e03a36bSChristoph Hellwig /* 10141e03a36bSChristoph Hellwig * We keep the error status of individual mapping so 10151e03a36bSChristoph Hellwig * that applications can catch the writeback error using 10161e03a36bSChristoph Hellwig * fsync(2). See filemap_fdatawait_keep_errors() for 10171e03a36bSChristoph Hellwig * details. 10181e03a36bSChristoph Hellwig */ 10191e03a36bSChristoph Hellwig filemap_fdatawait_keep_errors(inode->i_mapping); 10201e03a36bSChristoph Hellwig } else { 10211e03a36bSChristoph Hellwig filemap_fdatawrite(inode->i_mapping); 10221e03a36bSChristoph Hellwig } 10230dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_disk->open_mutex); 10240dca4462SChristoph Hellwig 10250dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 10260dca4462SChristoph Hellwig } 10270dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 10280dca4462SChristoph Hellwig iput(old_inode); 10290dca4462SChristoph Hellwig } 10302d985f8cSEric Biggers 10312d985f8cSEric Biggers /* 10322d985f8cSEric Biggers * Handle STATX_DIOALIGN for block devices. 10332d985f8cSEric Biggers * 10342d985f8cSEric Biggers * Note that the inode passed to this is the inode of a block device node file, 10352d985f8cSEric Biggers * not the block device's internal inode. Therefore it is *not* valid to use 10362d985f8cSEric Biggers * I_BDEV() here; the block device has to be looked up by i_rdev instead. 10372d985f8cSEric Biggers */ 10382d985f8cSEric Biggers void bdev_statx_dioalign(struct inode *inode, struct kstat *stat) 10392d985f8cSEric Biggers { 10402d985f8cSEric Biggers struct block_device *bdev; 10412d985f8cSEric Biggers 10422d985f8cSEric Biggers bdev = blkdev_get_no_open(inode->i_rdev); 10432d985f8cSEric Biggers if (!bdev) 10442d985f8cSEric Biggers return; 10452d985f8cSEric Biggers 10462d985f8cSEric Biggers stat->dio_mem_align = bdev_dma_alignment(bdev) + 1; 10472d985f8cSEric Biggers stat->dio_offset_align = bdev_logical_block_size(bdev); 10482d985f8cSEric Biggers stat->result_mask |= STATX_DIOALIGN; 10492d985f8cSEric Biggers 10502d985f8cSEric Biggers blkdev_put_no_open(bdev); 10512d985f8cSEric Biggers } 1052