10dca4462SChristoph Hellwig // SPDX-License-Identifier: GPL-2.0-only 20dca4462SChristoph Hellwig /* 30dca4462SChristoph Hellwig * Copyright (C) 1991, 1992 Linus Torvalds 40dca4462SChristoph Hellwig * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 50dca4462SChristoph Hellwig * Copyright (C) 2016 - 2020 Christoph Hellwig 60dca4462SChristoph Hellwig */ 70dca4462SChristoph Hellwig 80dca4462SChristoph Hellwig #include <linux/init.h> 90dca4462SChristoph Hellwig #include <linux/mm.h> 100dca4462SChristoph Hellwig #include <linux/slab.h> 110dca4462SChristoph Hellwig #include <linux/kmod.h> 120dca4462SChristoph Hellwig #include <linux/major.h> 130dca4462SChristoph Hellwig #include <linux/device_cgroup.h> 140dca4462SChristoph Hellwig #include <linux/blkdev.h> 15fe45e630SChristoph Hellwig #include <linux/blk-integrity.h> 160dca4462SChristoph Hellwig #include <linux/backing-dev.h> 170dca4462SChristoph Hellwig #include <linux/module.h> 180dca4462SChristoph Hellwig #include <linux/blkpg.h> 190dca4462SChristoph Hellwig #include <linux/magic.h> 200dca4462SChristoph Hellwig #include <linux/buffer_head.h> 210dca4462SChristoph Hellwig #include <linux/swap.h> 220dca4462SChristoph Hellwig #include <linux/writeback.h> 230dca4462SChristoph Hellwig #include <linux/mount.h> 240dca4462SChristoph Hellwig #include <linux/pseudo_fs.h> 250dca4462SChristoph Hellwig #include <linux/uio.h> 260dca4462SChristoph Hellwig #include <linux/namei.h> 270dca4462SChristoph Hellwig #include <linux/cleancache.h> 280dca4462SChristoph Hellwig #include <linux/part_stat.h> 290dca4462SChristoph Hellwig #include <linux/uaccess.h> 300dca4462SChristoph Hellwig #include "../fs/internal.h" 310dca4462SChristoph Hellwig #include "blk.h" 320dca4462SChristoph Hellwig 330dca4462SChristoph Hellwig struct bdev_inode { 340dca4462SChristoph Hellwig struct block_device bdev; 350dca4462SChristoph Hellwig struct inode vfs_inode; 360dca4462SChristoph Hellwig }; 370dca4462SChristoph Hellwig 380dca4462SChristoph Hellwig static inline struct bdev_inode *BDEV_I(struct inode *inode) 390dca4462SChristoph Hellwig { 400dca4462SChristoph Hellwig return container_of(inode, struct bdev_inode, vfs_inode); 410dca4462SChristoph Hellwig } 420dca4462SChristoph Hellwig 430dca4462SChristoph Hellwig struct block_device *I_BDEV(struct inode *inode) 440dca4462SChristoph Hellwig { 450dca4462SChristoph Hellwig return &BDEV_I(inode)->bdev; 460dca4462SChristoph Hellwig } 470dca4462SChristoph Hellwig EXPORT_SYMBOL(I_BDEV); 480dca4462SChristoph Hellwig 490dca4462SChristoph Hellwig static void bdev_write_inode(struct block_device *bdev) 500dca4462SChristoph Hellwig { 510dca4462SChristoph Hellwig struct inode *inode = bdev->bd_inode; 520dca4462SChristoph Hellwig int ret; 530dca4462SChristoph Hellwig 540dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 550dca4462SChristoph Hellwig while (inode->i_state & I_DIRTY) { 560dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 570dca4462SChristoph Hellwig ret = write_inode_now(inode, true); 580dca4462SChristoph Hellwig if (ret) { 590dca4462SChristoph Hellwig char name[BDEVNAME_SIZE]; 600dca4462SChristoph Hellwig pr_warn_ratelimited("VFS: Dirty inode writeback failed " 610dca4462SChristoph Hellwig "for block device %s (err=%d).\n", 620dca4462SChristoph Hellwig bdevname(bdev, name), ret); 630dca4462SChristoph Hellwig } 640dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 650dca4462SChristoph Hellwig } 660dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 670dca4462SChristoph Hellwig } 680dca4462SChristoph Hellwig 690dca4462SChristoph Hellwig /* Kill _all_ buffers and pagecache , dirty or not.. */ 700dca4462SChristoph Hellwig static void kill_bdev(struct block_device *bdev) 710dca4462SChristoph Hellwig { 720dca4462SChristoph Hellwig struct address_space *mapping = bdev->bd_inode->i_mapping; 730dca4462SChristoph Hellwig 740dca4462SChristoph Hellwig if (mapping_empty(mapping)) 750dca4462SChristoph Hellwig return; 760dca4462SChristoph Hellwig 770dca4462SChristoph Hellwig invalidate_bh_lrus(); 780dca4462SChristoph Hellwig truncate_inode_pages(mapping, 0); 790dca4462SChristoph Hellwig } 800dca4462SChristoph Hellwig 810dca4462SChristoph Hellwig /* Invalidate clean unused buffers and pagecache. */ 820dca4462SChristoph Hellwig void invalidate_bdev(struct block_device *bdev) 830dca4462SChristoph Hellwig { 840dca4462SChristoph Hellwig struct address_space *mapping = bdev->bd_inode->i_mapping; 850dca4462SChristoph Hellwig 860dca4462SChristoph Hellwig if (mapping->nrpages) { 870dca4462SChristoph Hellwig invalidate_bh_lrus(); 880dca4462SChristoph Hellwig lru_add_drain_all(); /* make sure all lru add caches are flushed */ 890dca4462SChristoph Hellwig invalidate_mapping_pages(mapping, 0, -1); 900dca4462SChristoph Hellwig } 910dca4462SChristoph Hellwig /* 99% of the time, we don't need to flush the cleancache on the bdev. 920dca4462SChristoph Hellwig * But, for the strange corners, lets be cautious 930dca4462SChristoph Hellwig */ 940dca4462SChristoph Hellwig cleancache_invalidate_inode(mapping); 950dca4462SChristoph Hellwig } 960dca4462SChristoph Hellwig EXPORT_SYMBOL(invalidate_bdev); 970dca4462SChristoph Hellwig 980dca4462SChristoph Hellwig /* 990dca4462SChristoph Hellwig * Drop all buffers & page cache for given bdev range. This function bails 1000dca4462SChristoph Hellwig * with error if bdev has other exclusive owner (such as filesystem). 1010dca4462SChristoph Hellwig */ 1020dca4462SChristoph Hellwig int truncate_bdev_range(struct block_device *bdev, fmode_t mode, 1030dca4462SChristoph Hellwig loff_t lstart, loff_t lend) 1040dca4462SChristoph Hellwig { 1050dca4462SChristoph Hellwig /* 1060dca4462SChristoph Hellwig * If we don't hold exclusive handle for the device, upgrade to it 1070dca4462SChristoph Hellwig * while we discard the buffer cache to avoid discarding buffers 1080dca4462SChristoph Hellwig * under live filesystem. 1090dca4462SChristoph Hellwig */ 1100dca4462SChristoph Hellwig if (!(mode & FMODE_EXCL)) { 1110dca4462SChristoph Hellwig int err = bd_prepare_to_claim(bdev, truncate_bdev_range); 1120dca4462SChristoph Hellwig if (err) 1130dca4462SChristoph Hellwig goto invalidate; 1140dca4462SChristoph Hellwig } 1150dca4462SChristoph Hellwig 1160dca4462SChristoph Hellwig truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); 1170dca4462SChristoph Hellwig if (!(mode & FMODE_EXCL)) 1180dca4462SChristoph Hellwig bd_abort_claiming(bdev, truncate_bdev_range); 1190dca4462SChristoph Hellwig return 0; 1200dca4462SChristoph Hellwig 1210dca4462SChristoph Hellwig invalidate: 1220dca4462SChristoph Hellwig /* 1230dca4462SChristoph Hellwig * Someone else has handle exclusively open. Try invalidating instead. 1240dca4462SChristoph Hellwig * The 'end' argument is inclusive so the rounding is safe. 1250dca4462SChristoph Hellwig */ 1260dca4462SChristoph Hellwig return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, 1270dca4462SChristoph Hellwig lstart >> PAGE_SHIFT, 1280dca4462SChristoph Hellwig lend >> PAGE_SHIFT); 1290dca4462SChristoph Hellwig } 1300dca4462SChristoph Hellwig 1310dca4462SChristoph Hellwig static void set_init_blocksize(struct block_device *bdev) 1320dca4462SChristoph Hellwig { 1330dca4462SChristoph Hellwig unsigned int bsize = bdev_logical_block_size(bdev); 1340dca4462SChristoph Hellwig loff_t size = i_size_read(bdev->bd_inode); 1350dca4462SChristoph Hellwig 1360dca4462SChristoph Hellwig while (bsize < PAGE_SIZE) { 1370dca4462SChristoph Hellwig if (size & bsize) 1380dca4462SChristoph Hellwig break; 1390dca4462SChristoph Hellwig bsize <<= 1; 1400dca4462SChristoph Hellwig } 1410dca4462SChristoph Hellwig bdev->bd_inode->i_blkbits = blksize_bits(bsize); 1420dca4462SChristoph Hellwig } 1430dca4462SChristoph Hellwig 1440dca4462SChristoph Hellwig int set_blocksize(struct block_device *bdev, int size) 1450dca4462SChristoph Hellwig { 1460dca4462SChristoph Hellwig /* Size must be a power of two, and between 512 and PAGE_SIZE */ 1470dca4462SChristoph Hellwig if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 1480dca4462SChristoph Hellwig return -EINVAL; 1490dca4462SChristoph Hellwig 1500dca4462SChristoph Hellwig /* Size cannot be smaller than the size supported by the device */ 1510dca4462SChristoph Hellwig if (size < bdev_logical_block_size(bdev)) 1520dca4462SChristoph Hellwig return -EINVAL; 1530dca4462SChristoph Hellwig 1540dca4462SChristoph Hellwig /* Don't change the size if it is same as current */ 1550dca4462SChristoph Hellwig if (bdev->bd_inode->i_blkbits != blksize_bits(size)) { 1560dca4462SChristoph Hellwig sync_blockdev(bdev); 1570dca4462SChristoph Hellwig bdev->bd_inode->i_blkbits = blksize_bits(size); 1580dca4462SChristoph Hellwig kill_bdev(bdev); 1590dca4462SChristoph Hellwig } 1600dca4462SChristoph Hellwig return 0; 1610dca4462SChristoph Hellwig } 1620dca4462SChristoph Hellwig 1630dca4462SChristoph Hellwig EXPORT_SYMBOL(set_blocksize); 1640dca4462SChristoph Hellwig 1650dca4462SChristoph Hellwig int sb_set_blocksize(struct super_block *sb, int size) 1660dca4462SChristoph Hellwig { 1670dca4462SChristoph Hellwig if (set_blocksize(sb->s_bdev, size)) 1680dca4462SChristoph Hellwig return 0; 1690dca4462SChristoph Hellwig /* If we get here, we know size is power of two 1700dca4462SChristoph Hellwig * and it's value is between 512 and PAGE_SIZE */ 1710dca4462SChristoph Hellwig sb->s_blocksize = size; 1720dca4462SChristoph Hellwig sb->s_blocksize_bits = blksize_bits(size); 1730dca4462SChristoph Hellwig return sb->s_blocksize; 1740dca4462SChristoph Hellwig } 1750dca4462SChristoph Hellwig 1760dca4462SChristoph Hellwig EXPORT_SYMBOL(sb_set_blocksize); 1770dca4462SChristoph Hellwig 1780dca4462SChristoph Hellwig int sb_min_blocksize(struct super_block *sb, int size) 1790dca4462SChristoph Hellwig { 1800dca4462SChristoph Hellwig int minsize = bdev_logical_block_size(sb->s_bdev); 1810dca4462SChristoph Hellwig if (size < minsize) 1820dca4462SChristoph Hellwig size = minsize; 1830dca4462SChristoph Hellwig return sb_set_blocksize(sb, size); 1840dca4462SChristoph Hellwig } 1850dca4462SChristoph Hellwig 1860dca4462SChristoph Hellwig EXPORT_SYMBOL(sb_min_blocksize); 1870dca4462SChristoph Hellwig 18870164eb6SChristoph Hellwig int sync_blockdev_nowait(struct block_device *bdev) 1890dca4462SChristoph Hellwig { 1900dca4462SChristoph Hellwig if (!bdev) 1910dca4462SChristoph Hellwig return 0; 1920dca4462SChristoph Hellwig return filemap_flush(bdev->bd_inode->i_mapping); 1930dca4462SChristoph Hellwig } 19470164eb6SChristoph Hellwig EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 1950dca4462SChristoph Hellwig 1960dca4462SChristoph Hellwig /* 1970dca4462SChristoph Hellwig * Write out and wait upon all the dirty data associated with a block 1980dca4462SChristoph Hellwig * device via its mapping. Does not take the superblock lock. 1990dca4462SChristoph Hellwig */ 2000dca4462SChristoph Hellwig int sync_blockdev(struct block_device *bdev) 2010dca4462SChristoph Hellwig { 20270164eb6SChristoph Hellwig if (!bdev) 20370164eb6SChristoph Hellwig return 0; 20470164eb6SChristoph Hellwig return filemap_write_and_wait(bdev->bd_inode->i_mapping); 2050dca4462SChristoph Hellwig } 2060dca4462SChristoph Hellwig EXPORT_SYMBOL(sync_blockdev); 2070dca4462SChristoph Hellwig 2080dca4462SChristoph Hellwig /* 2090dca4462SChristoph Hellwig * Write out and wait upon all dirty data associated with this 2100dca4462SChristoph Hellwig * device. Filesystem data as well as the underlying block 2110dca4462SChristoph Hellwig * device. Takes the superblock lock. 2120dca4462SChristoph Hellwig */ 2130dca4462SChristoph Hellwig int fsync_bdev(struct block_device *bdev) 2140dca4462SChristoph Hellwig { 2150dca4462SChristoph Hellwig struct super_block *sb = get_super(bdev); 2160dca4462SChristoph Hellwig if (sb) { 2170dca4462SChristoph Hellwig int res = sync_filesystem(sb); 2180dca4462SChristoph Hellwig drop_super(sb); 2190dca4462SChristoph Hellwig return res; 2200dca4462SChristoph Hellwig } 2210dca4462SChristoph Hellwig return sync_blockdev(bdev); 2220dca4462SChristoph Hellwig } 2230dca4462SChristoph Hellwig EXPORT_SYMBOL(fsync_bdev); 2240dca4462SChristoph Hellwig 2250dca4462SChristoph Hellwig /** 2260dca4462SChristoph Hellwig * freeze_bdev -- lock a filesystem and force it into a consistent state 2270dca4462SChristoph Hellwig * @bdev: blockdevice to lock 2280dca4462SChristoph Hellwig * 2290dca4462SChristoph Hellwig * If a superblock is found on this device, we take the s_umount semaphore 2300dca4462SChristoph Hellwig * on it to make sure nobody unmounts until the snapshot creation is done. 2310dca4462SChristoph Hellwig * The reference counter (bd_fsfreeze_count) guarantees that only the last 2320dca4462SChristoph Hellwig * unfreeze process can unfreeze the frozen filesystem actually when multiple 2330dca4462SChristoph Hellwig * freeze requests arrive simultaneously. It counts up in freeze_bdev() and 2340dca4462SChristoph Hellwig * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze 2350dca4462SChristoph Hellwig * actually. 2360dca4462SChristoph Hellwig */ 2370dca4462SChristoph Hellwig int freeze_bdev(struct block_device *bdev) 2380dca4462SChristoph Hellwig { 2390dca4462SChristoph Hellwig struct super_block *sb; 2400dca4462SChristoph Hellwig int error = 0; 2410dca4462SChristoph Hellwig 2420dca4462SChristoph Hellwig mutex_lock(&bdev->bd_fsfreeze_mutex); 2430dca4462SChristoph Hellwig if (++bdev->bd_fsfreeze_count > 1) 2440dca4462SChristoph Hellwig goto done; 2450dca4462SChristoph Hellwig 2460dca4462SChristoph Hellwig sb = get_active_super(bdev); 2470dca4462SChristoph Hellwig if (!sb) 2480dca4462SChristoph Hellwig goto sync; 2490dca4462SChristoph Hellwig if (sb->s_op->freeze_super) 2500dca4462SChristoph Hellwig error = sb->s_op->freeze_super(sb); 2510dca4462SChristoph Hellwig else 2520dca4462SChristoph Hellwig error = freeze_super(sb); 2530dca4462SChristoph Hellwig deactivate_super(sb); 2540dca4462SChristoph Hellwig 2550dca4462SChristoph Hellwig if (error) { 2560dca4462SChristoph Hellwig bdev->bd_fsfreeze_count--; 2570dca4462SChristoph Hellwig goto done; 2580dca4462SChristoph Hellwig } 2590dca4462SChristoph Hellwig bdev->bd_fsfreeze_sb = sb; 2600dca4462SChristoph Hellwig 2610dca4462SChristoph Hellwig sync: 2620dca4462SChristoph Hellwig sync_blockdev(bdev); 2630dca4462SChristoph Hellwig done: 2640dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_fsfreeze_mutex); 2650dca4462SChristoph Hellwig return error; 2660dca4462SChristoph Hellwig } 2670dca4462SChristoph Hellwig EXPORT_SYMBOL(freeze_bdev); 2680dca4462SChristoph Hellwig 2690dca4462SChristoph Hellwig /** 2700dca4462SChristoph Hellwig * thaw_bdev -- unlock filesystem 2710dca4462SChristoph Hellwig * @bdev: blockdevice to unlock 2720dca4462SChristoph Hellwig * 2730dca4462SChristoph Hellwig * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 2740dca4462SChristoph Hellwig */ 2750dca4462SChristoph Hellwig int thaw_bdev(struct block_device *bdev) 2760dca4462SChristoph Hellwig { 2770dca4462SChristoph Hellwig struct super_block *sb; 2780dca4462SChristoph Hellwig int error = -EINVAL; 2790dca4462SChristoph Hellwig 2800dca4462SChristoph Hellwig mutex_lock(&bdev->bd_fsfreeze_mutex); 2810dca4462SChristoph Hellwig if (!bdev->bd_fsfreeze_count) 2820dca4462SChristoph Hellwig goto out; 2830dca4462SChristoph Hellwig 2840dca4462SChristoph Hellwig error = 0; 2850dca4462SChristoph Hellwig if (--bdev->bd_fsfreeze_count > 0) 2860dca4462SChristoph Hellwig goto out; 2870dca4462SChristoph Hellwig 2880dca4462SChristoph Hellwig sb = bdev->bd_fsfreeze_sb; 2890dca4462SChristoph Hellwig if (!sb) 2900dca4462SChristoph Hellwig goto out; 2910dca4462SChristoph Hellwig 2920dca4462SChristoph Hellwig if (sb->s_op->thaw_super) 2930dca4462SChristoph Hellwig error = sb->s_op->thaw_super(sb); 2940dca4462SChristoph Hellwig else 2950dca4462SChristoph Hellwig error = thaw_super(sb); 2960dca4462SChristoph Hellwig if (error) 2970dca4462SChristoph Hellwig bdev->bd_fsfreeze_count++; 2980dca4462SChristoph Hellwig else 2990dca4462SChristoph Hellwig bdev->bd_fsfreeze_sb = NULL; 3000dca4462SChristoph Hellwig out: 3010dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_fsfreeze_mutex); 3020dca4462SChristoph Hellwig return error; 3030dca4462SChristoph Hellwig } 3040dca4462SChristoph Hellwig EXPORT_SYMBOL(thaw_bdev); 3050dca4462SChristoph Hellwig 3060dca4462SChristoph Hellwig /** 3070dca4462SChristoph Hellwig * bdev_read_page() - Start reading a page from a block device 3080dca4462SChristoph Hellwig * @bdev: The device to read the page from 3090dca4462SChristoph Hellwig * @sector: The offset on the device to read the page to (need not be aligned) 3100dca4462SChristoph Hellwig * @page: The page to read 3110dca4462SChristoph Hellwig * 3120dca4462SChristoph Hellwig * On entry, the page should be locked. It will be unlocked when the page 3130dca4462SChristoph Hellwig * has been read. If the block driver implements rw_page synchronously, 3140dca4462SChristoph Hellwig * that will be true on exit from this function, but it need not be. 3150dca4462SChristoph Hellwig * 3160dca4462SChristoph Hellwig * Errors returned by this function are usually "soft", eg out of memory, or 3170dca4462SChristoph Hellwig * queue full; callers should try a different route to read this page rather 3180dca4462SChristoph Hellwig * than propagate an error back up the stack. 3190dca4462SChristoph Hellwig * 3200dca4462SChristoph Hellwig * Return: negative errno if an error occurs, 0 if submission was successful. 3210dca4462SChristoph Hellwig */ 3220dca4462SChristoph Hellwig int bdev_read_page(struct block_device *bdev, sector_t sector, 3230dca4462SChristoph Hellwig struct page *page) 3240dca4462SChristoph Hellwig { 3250dca4462SChristoph Hellwig const struct block_device_operations *ops = bdev->bd_disk->fops; 3260dca4462SChristoph Hellwig int result = -EOPNOTSUPP; 3270dca4462SChristoph Hellwig 3280dca4462SChristoph Hellwig if (!ops->rw_page || bdev_get_integrity(bdev)) 3290dca4462SChristoph Hellwig return result; 3300dca4462SChristoph Hellwig 331025a3865SPavel Begunkov result = blk_queue_enter(bdev_get_queue(bdev), 0); 3320dca4462SChristoph Hellwig if (result) 3330dca4462SChristoph Hellwig return result; 3340dca4462SChristoph Hellwig result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, 3350dca4462SChristoph Hellwig REQ_OP_READ); 336025a3865SPavel Begunkov blk_queue_exit(bdev_get_queue(bdev)); 3370dca4462SChristoph Hellwig return result; 3380dca4462SChristoph Hellwig } 3390dca4462SChristoph Hellwig 3400dca4462SChristoph Hellwig /** 3410dca4462SChristoph Hellwig * bdev_write_page() - Start writing a page to a block device 3420dca4462SChristoph Hellwig * @bdev: The device to write the page to 3430dca4462SChristoph Hellwig * @sector: The offset on the device to write the page to (need not be aligned) 3440dca4462SChristoph Hellwig * @page: The page to write 3450dca4462SChristoph Hellwig * @wbc: The writeback_control for the write 3460dca4462SChristoph Hellwig * 3470dca4462SChristoph Hellwig * On entry, the page should be locked and not currently under writeback. 3480dca4462SChristoph Hellwig * On exit, if the write started successfully, the page will be unlocked and 3490dca4462SChristoph Hellwig * under writeback. If the write failed already (eg the driver failed to 3500dca4462SChristoph Hellwig * queue the page to the device), the page will still be locked. If the 3510dca4462SChristoph Hellwig * caller is a ->writepage implementation, it will need to unlock the page. 3520dca4462SChristoph Hellwig * 3530dca4462SChristoph Hellwig * Errors returned by this function are usually "soft", eg out of memory, or 3540dca4462SChristoph Hellwig * queue full; callers should try a different route to write this page rather 3550dca4462SChristoph Hellwig * than propagate an error back up the stack. 3560dca4462SChristoph Hellwig * 3570dca4462SChristoph Hellwig * Return: negative errno if an error occurs, 0 if submission was successful. 3580dca4462SChristoph Hellwig */ 3590dca4462SChristoph Hellwig int bdev_write_page(struct block_device *bdev, sector_t sector, 3600dca4462SChristoph Hellwig struct page *page, struct writeback_control *wbc) 3610dca4462SChristoph Hellwig { 3620dca4462SChristoph Hellwig int result; 3630dca4462SChristoph Hellwig const struct block_device_operations *ops = bdev->bd_disk->fops; 3640dca4462SChristoph Hellwig 3650dca4462SChristoph Hellwig if (!ops->rw_page || bdev_get_integrity(bdev)) 3660dca4462SChristoph Hellwig return -EOPNOTSUPP; 367025a3865SPavel Begunkov result = blk_queue_enter(bdev_get_queue(bdev), 0); 3680dca4462SChristoph Hellwig if (result) 3690dca4462SChristoph Hellwig return result; 3700dca4462SChristoph Hellwig 3710dca4462SChristoph Hellwig set_page_writeback(page); 3720dca4462SChristoph Hellwig result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, 3730dca4462SChristoph Hellwig REQ_OP_WRITE); 3740dca4462SChristoph Hellwig if (result) { 3750dca4462SChristoph Hellwig end_page_writeback(page); 3760dca4462SChristoph Hellwig } else { 3770dca4462SChristoph Hellwig clean_page_buffers(page); 3780dca4462SChristoph Hellwig unlock_page(page); 3790dca4462SChristoph Hellwig } 380025a3865SPavel Begunkov blk_queue_exit(bdev_get_queue(bdev)); 3810dca4462SChristoph Hellwig return result; 3820dca4462SChristoph Hellwig } 3830dca4462SChristoph Hellwig 3840dca4462SChristoph Hellwig /* 3850dca4462SChristoph Hellwig * pseudo-fs 3860dca4462SChristoph Hellwig */ 3870dca4462SChristoph Hellwig 3880dca4462SChristoph Hellwig static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock); 3890dca4462SChristoph Hellwig static struct kmem_cache * bdev_cachep __read_mostly; 3900dca4462SChristoph Hellwig 3910dca4462SChristoph Hellwig static struct inode *bdev_alloc_inode(struct super_block *sb) 3920dca4462SChristoph Hellwig { 3930dca4462SChristoph Hellwig struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL); 3940dca4462SChristoph Hellwig 3950dca4462SChristoph Hellwig if (!ei) 3960dca4462SChristoph Hellwig return NULL; 3970dca4462SChristoph Hellwig memset(&ei->bdev, 0, sizeof(ei->bdev)); 3980dca4462SChristoph Hellwig return &ei->vfs_inode; 3990dca4462SChristoph Hellwig } 4000dca4462SChristoph Hellwig 4010dca4462SChristoph Hellwig static void bdev_free_inode(struct inode *inode) 4020dca4462SChristoph Hellwig { 4030dca4462SChristoph Hellwig struct block_device *bdev = I_BDEV(inode); 4040dca4462SChristoph Hellwig 4050dca4462SChristoph Hellwig free_percpu(bdev->bd_stats); 4060dca4462SChristoph Hellwig kfree(bdev->bd_meta_info); 4070dca4462SChristoph Hellwig 4080dca4462SChristoph Hellwig if (!bdev_is_partition(bdev)) { 4090dca4462SChristoph Hellwig if (bdev->bd_disk && bdev->bd_disk->bdi) 4100dca4462SChristoph Hellwig bdi_put(bdev->bd_disk->bdi); 4110dca4462SChristoph Hellwig kfree(bdev->bd_disk); 4120dca4462SChristoph Hellwig } 4130dca4462SChristoph Hellwig 4140dca4462SChristoph Hellwig if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 4150dca4462SChristoph Hellwig blk_free_ext_minor(MINOR(bdev->bd_dev)); 4160dca4462SChristoph Hellwig 4170dca4462SChristoph Hellwig kmem_cache_free(bdev_cachep, BDEV_I(inode)); 4180dca4462SChristoph Hellwig } 4190dca4462SChristoph Hellwig 4200dca4462SChristoph Hellwig static void init_once(void *data) 4210dca4462SChristoph Hellwig { 4220dca4462SChristoph Hellwig struct bdev_inode *ei = data; 4230dca4462SChristoph Hellwig 4240dca4462SChristoph Hellwig inode_init_once(&ei->vfs_inode); 4250dca4462SChristoph Hellwig } 4260dca4462SChristoph Hellwig 4270dca4462SChristoph Hellwig static void bdev_evict_inode(struct inode *inode) 4280dca4462SChristoph Hellwig { 4290dca4462SChristoph Hellwig truncate_inode_pages_final(&inode->i_data); 4300dca4462SChristoph Hellwig invalidate_inode_buffers(inode); /* is it needed here? */ 4310dca4462SChristoph Hellwig clear_inode(inode); 4320dca4462SChristoph Hellwig } 4330dca4462SChristoph Hellwig 4340dca4462SChristoph Hellwig static const struct super_operations bdev_sops = { 4350dca4462SChristoph Hellwig .statfs = simple_statfs, 4360dca4462SChristoph Hellwig .alloc_inode = bdev_alloc_inode, 4370dca4462SChristoph Hellwig .free_inode = bdev_free_inode, 4380dca4462SChristoph Hellwig .drop_inode = generic_delete_inode, 4390dca4462SChristoph Hellwig .evict_inode = bdev_evict_inode, 4400dca4462SChristoph Hellwig }; 4410dca4462SChristoph Hellwig 4420dca4462SChristoph Hellwig static int bd_init_fs_context(struct fs_context *fc) 4430dca4462SChristoph Hellwig { 4440dca4462SChristoph Hellwig struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 4450dca4462SChristoph Hellwig if (!ctx) 4460dca4462SChristoph Hellwig return -ENOMEM; 4470dca4462SChristoph Hellwig fc->s_iflags |= SB_I_CGROUPWB; 4480dca4462SChristoph Hellwig ctx->ops = &bdev_sops; 4490dca4462SChristoph Hellwig return 0; 4500dca4462SChristoph Hellwig } 4510dca4462SChristoph Hellwig 4520dca4462SChristoph Hellwig static struct file_system_type bd_type = { 4530dca4462SChristoph Hellwig .name = "bdev", 4540dca4462SChristoph Hellwig .init_fs_context = bd_init_fs_context, 4550dca4462SChristoph Hellwig .kill_sb = kill_anon_super, 4560dca4462SChristoph Hellwig }; 4570dca4462SChristoph Hellwig 4580dca4462SChristoph Hellwig struct super_block *blockdev_superblock __read_mostly; 4590dca4462SChristoph Hellwig EXPORT_SYMBOL_GPL(blockdev_superblock); 4600dca4462SChristoph Hellwig 4610dca4462SChristoph Hellwig void __init bdev_cache_init(void) 4620dca4462SChristoph Hellwig { 4630dca4462SChristoph Hellwig int err; 4640dca4462SChristoph Hellwig static struct vfsmount *bd_mnt; 4650dca4462SChristoph Hellwig 4660dca4462SChristoph Hellwig bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 4670dca4462SChristoph Hellwig 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 4680dca4462SChristoph Hellwig SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC), 4690dca4462SChristoph Hellwig init_once); 4700dca4462SChristoph Hellwig err = register_filesystem(&bd_type); 4710dca4462SChristoph Hellwig if (err) 4720dca4462SChristoph Hellwig panic("Cannot register bdev pseudo-fs"); 4730dca4462SChristoph Hellwig bd_mnt = kern_mount(&bd_type); 4740dca4462SChristoph Hellwig if (IS_ERR(bd_mnt)) 4750dca4462SChristoph Hellwig panic("Cannot create bdev pseudo-fs"); 4760dca4462SChristoph Hellwig blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */ 4770dca4462SChristoph Hellwig } 4780dca4462SChristoph Hellwig 4790dca4462SChristoph Hellwig struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 4800dca4462SChristoph Hellwig { 4810dca4462SChristoph Hellwig struct block_device *bdev; 4820dca4462SChristoph Hellwig struct inode *inode; 4830dca4462SChristoph Hellwig 4840dca4462SChristoph Hellwig inode = new_inode(blockdev_superblock); 4850dca4462SChristoph Hellwig if (!inode) 4860dca4462SChristoph Hellwig return NULL; 4870dca4462SChristoph Hellwig inode->i_mode = S_IFBLK; 4880dca4462SChristoph Hellwig inode->i_rdev = 0; 4890dca4462SChristoph Hellwig inode->i_data.a_ops = &def_blk_aops; 4900dca4462SChristoph Hellwig mapping_set_gfp_mask(&inode->i_data, GFP_USER); 4910dca4462SChristoph Hellwig 4920dca4462SChristoph Hellwig bdev = I_BDEV(inode); 4930dca4462SChristoph Hellwig mutex_init(&bdev->bd_fsfreeze_mutex); 4940dca4462SChristoph Hellwig spin_lock_init(&bdev->bd_size_lock); 4950dca4462SChristoph Hellwig bdev->bd_partno = partno; 4960dca4462SChristoph Hellwig bdev->bd_inode = inode; 49717220ca5SPavel Begunkov bdev->bd_queue = disk->queue; 4980dca4462SChristoph Hellwig bdev->bd_stats = alloc_percpu(struct disk_stats); 4990dca4462SChristoph Hellwig if (!bdev->bd_stats) { 5000dca4462SChristoph Hellwig iput(inode); 5010dca4462SChristoph Hellwig return NULL; 5020dca4462SChristoph Hellwig } 50306cc978dSTetsuo Handa bdev->bd_disk = disk; 5040dca4462SChristoph Hellwig return bdev; 5050dca4462SChristoph Hellwig } 5060dca4462SChristoph Hellwig 5070dca4462SChristoph Hellwig void bdev_add(struct block_device *bdev, dev_t dev) 5080dca4462SChristoph Hellwig { 5090dca4462SChristoph Hellwig bdev->bd_dev = dev; 5100dca4462SChristoph Hellwig bdev->bd_inode->i_rdev = dev; 5110dca4462SChristoph Hellwig bdev->bd_inode->i_ino = dev; 5120dca4462SChristoph Hellwig insert_inode_hash(bdev->bd_inode); 5130dca4462SChristoph Hellwig } 5140dca4462SChristoph Hellwig 5150dca4462SChristoph Hellwig long nr_blockdev_pages(void) 5160dca4462SChristoph Hellwig { 5170dca4462SChristoph Hellwig struct inode *inode; 5180dca4462SChristoph Hellwig long ret = 0; 5190dca4462SChristoph Hellwig 5200dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 5210dca4462SChristoph Hellwig list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 5220dca4462SChristoph Hellwig ret += inode->i_mapping->nrpages; 5230dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 5240dca4462SChristoph Hellwig 5250dca4462SChristoph Hellwig return ret; 5260dca4462SChristoph Hellwig } 5270dca4462SChristoph Hellwig 5280dca4462SChristoph Hellwig /** 5290dca4462SChristoph Hellwig * bd_may_claim - test whether a block device can be claimed 5300dca4462SChristoph Hellwig * @bdev: block device of interest 5310dca4462SChristoph Hellwig * @whole: whole block device containing @bdev, may equal @bdev 5320dca4462SChristoph Hellwig * @holder: holder trying to claim @bdev 5330dca4462SChristoph Hellwig * 5340dca4462SChristoph Hellwig * Test whether @bdev can be claimed by @holder. 5350dca4462SChristoph Hellwig * 5360dca4462SChristoph Hellwig * CONTEXT: 5370dca4462SChristoph Hellwig * spin_lock(&bdev_lock). 5380dca4462SChristoph Hellwig * 5390dca4462SChristoph Hellwig * RETURNS: 5400dca4462SChristoph Hellwig * %true if @bdev can be claimed, %false otherwise. 5410dca4462SChristoph Hellwig */ 5420dca4462SChristoph Hellwig static bool bd_may_claim(struct block_device *bdev, struct block_device *whole, 5430dca4462SChristoph Hellwig void *holder) 5440dca4462SChristoph Hellwig { 5450dca4462SChristoph Hellwig if (bdev->bd_holder == holder) 5460dca4462SChristoph Hellwig return true; /* already a holder */ 5470dca4462SChristoph Hellwig else if (bdev->bd_holder != NULL) 5480dca4462SChristoph Hellwig return false; /* held by someone else */ 5490dca4462SChristoph Hellwig else if (whole == bdev) 5500dca4462SChristoph Hellwig return true; /* is a whole device which isn't held */ 5510dca4462SChristoph Hellwig 5520dca4462SChristoph Hellwig else if (whole->bd_holder == bd_may_claim) 5530dca4462SChristoph Hellwig return true; /* is a partition of a device that is being partitioned */ 5540dca4462SChristoph Hellwig else if (whole->bd_holder != NULL) 5550dca4462SChristoph Hellwig return false; /* is a partition of a held device */ 5560dca4462SChristoph Hellwig else 5570dca4462SChristoph Hellwig return true; /* is a partition of an un-held device */ 5580dca4462SChristoph Hellwig } 5590dca4462SChristoph Hellwig 5600dca4462SChristoph Hellwig /** 5610dca4462SChristoph Hellwig * bd_prepare_to_claim - claim a block device 5620dca4462SChristoph Hellwig * @bdev: block device of interest 5630dca4462SChristoph Hellwig * @holder: holder trying to claim @bdev 5640dca4462SChristoph Hellwig * 5650dca4462SChristoph Hellwig * Claim @bdev. This function fails if @bdev is already claimed by another 5660dca4462SChristoph Hellwig * holder and waits if another claiming is in progress. return, the caller 5670dca4462SChristoph Hellwig * has ownership of bd_claiming and bd_holder[s]. 5680dca4462SChristoph Hellwig * 5690dca4462SChristoph Hellwig * RETURNS: 5700dca4462SChristoph Hellwig * 0 if @bdev can be claimed, -EBUSY otherwise. 5710dca4462SChristoph Hellwig */ 5720dca4462SChristoph Hellwig int bd_prepare_to_claim(struct block_device *bdev, void *holder) 5730dca4462SChristoph Hellwig { 5740dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 5750dca4462SChristoph Hellwig 5760dca4462SChristoph Hellwig if (WARN_ON_ONCE(!holder)) 5770dca4462SChristoph Hellwig return -EINVAL; 5780dca4462SChristoph Hellwig retry: 5790dca4462SChristoph Hellwig spin_lock(&bdev_lock); 5800dca4462SChristoph Hellwig /* if someone else claimed, fail */ 5810dca4462SChristoph Hellwig if (!bd_may_claim(bdev, whole, holder)) { 5820dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 5830dca4462SChristoph Hellwig return -EBUSY; 5840dca4462SChristoph Hellwig } 5850dca4462SChristoph Hellwig 5860dca4462SChristoph Hellwig /* if claiming is already in progress, wait for it to finish */ 5870dca4462SChristoph Hellwig if (whole->bd_claiming) { 5880dca4462SChristoph Hellwig wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 5890dca4462SChristoph Hellwig DEFINE_WAIT(wait); 5900dca4462SChristoph Hellwig 5910dca4462SChristoph Hellwig prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 5920dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 5930dca4462SChristoph Hellwig schedule(); 5940dca4462SChristoph Hellwig finish_wait(wq, &wait); 5950dca4462SChristoph Hellwig goto retry; 5960dca4462SChristoph Hellwig } 5970dca4462SChristoph Hellwig 5980dca4462SChristoph Hellwig /* yay, all mine */ 5990dca4462SChristoph Hellwig whole->bd_claiming = holder; 6000dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 6010dca4462SChristoph Hellwig return 0; 6020dca4462SChristoph Hellwig } 6030dca4462SChristoph Hellwig EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 6040dca4462SChristoph Hellwig 6050dca4462SChristoph Hellwig static void bd_clear_claiming(struct block_device *whole, void *holder) 6060dca4462SChristoph Hellwig { 6070dca4462SChristoph Hellwig lockdep_assert_held(&bdev_lock); 6080dca4462SChristoph Hellwig /* tell others that we're done */ 6090dca4462SChristoph Hellwig BUG_ON(whole->bd_claiming != holder); 6100dca4462SChristoph Hellwig whole->bd_claiming = NULL; 6110dca4462SChristoph Hellwig wake_up_bit(&whole->bd_claiming, 0); 6120dca4462SChristoph Hellwig } 6130dca4462SChristoph Hellwig 6140dca4462SChristoph Hellwig /** 6150dca4462SChristoph Hellwig * bd_finish_claiming - finish claiming of a block device 6160dca4462SChristoph Hellwig * @bdev: block device of interest 6170dca4462SChristoph Hellwig * @holder: holder that has claimed @bdev 6180dca4462SChristoph Hellwig * 6190dca4462SChristoph Hellwig * Finish exclusive open of a block device. Mark the device as exlusively 6200dca4462SChristoph Hellwig * open by the holder and wake up all waiters for exclusive open to finish. 6210dca4462SChristoph Hellwig */ 6220dca4462SChristoph Hellwig static void bd_finish_claiming(struct block_device *bdev, void *holder) 6230dca4462SChristoph Hellwig { 6240dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 6250dca4462SChristoph Hellwig 6260dca4462SChristoph Hellwig spin_lock(&bdev_lock); 6270dca4462SChristoph Hellwig BUG_ON(!bd_may_claim(bdev, whole, holder)); 6280dca4462SChristoph Hellwig /* 6290dca4462SChristoph Hellwig * Note that for a whole device bd_holders will be incremented twice, 6300dca4462SChristoph Hellwig * and bd_holder will be set to bd_may_claim before being set to holder 6310dca4462SChristoph Hellwig */ 6320dca4462SChristoph Hellwig whole->bd_holders++; 6330dca4462SChristoph Hellwig whole->bd_holder = bd_may_claim; 6340dca4462SChristoph Hellwig bdev->bd_holders++; 6350dca4462SChristoph Hellwig bdev->bd_holder = holder; 6360dca4462SChristoph Hellwig bd_clear_claiming(whole, holder); 6370dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 6380dca4462SChristoph Hellwig } 6390dca4462SChristoph Hellwig 6400dca4462SChristoph Hellwig /** 6410dca4462SChristoph Hellwig * bd_abort_claiming - abort claiming of a block device 6420dca4462SChristoph Hellwig * @bdev: block device of interest 6430dca4462SChristoph Hellwig * @holder: holder that has claimed @bdev 6440dca4462SChristoph Hellwig * 6450dca4462SChristoph Hellwig * Abort claiming of a block device when the exclusive open failed. This can be 6460dca4462SChristoph Hellwig * also used when exclusive open is not actually desired and we just needed 6470dca4462SChristoph Hellwig * to block other exclusive openers for a while. 6480dca4462SChristoph Hellwig */ 6490dca4462SChristoph Hellwig void bd_abort_claiming(struct block_device *bdev, void *holder) 6500dca4462SChristoph Hellwig { 6510dca4462SChristoph Hellwig spin_lock(&bdev_lock); 6520dca4462SChristoph Hellwig bd_clear_claiming(bdev_whole(bdev), holder); 6530dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 6540dca4462SChristoph Hellwig } 6550dca4462SChristoph Hellwig EXPORT_SYMBOL(bd_abort_claiming); 6560dca4462SChristoph Hellwig 6570dca4462SChristoph Hellwig static void blkdev_flush_mapping(struct block_device *bdev) 6580dca4462SChristoph Hellwig { 6590dca4462SChristoph Hellwig WARN_ON_ONCE(bdev->bd_holders); 6600dca4462SChristoph Hellwig sync_blockdev(bdev); 6610dca4462SChristoph Hellwig kill_bdev(bdev); 6620dca4462SChristoph Hellwig bdev_write_inode(bdev); 6630dca4462SChristoph Hellwig } 6640dca4462SChristoph Hellwig 6650dca4462SChristoph Hellwig static int blkdev_get_whole(struct block_device *bdev, fmode_t mode) 6660dca4462SChristoph Hellwig { 6670dca4462SChristoph Hellwig struct gendisk *disk = bdev->bd_disk; 6680dca4462SChristoph Hellwig int ret = 0; 6690dca4462SChristoph Hellwig 6700dca4462SChristoph Hellwig if (disk->fops->open) { 6710dca4462SChristoph Hellwig ret = disk->fops->open(bdev, mode); 6720dca4462SChristoph Hellwig if (ret) { 6730dca4462SChristoph Hellwig /* avoid ghost partitions on a removed medium */ 6740dca4462SChristoph Hellwig if (ret == -ENOMEDIUM && 6750dca4462SChristoph Hellwig test_bit(GD_NEED_PART_SCAN, &disk->state)) 6760dca4462SChristoph Hellwig bdev_disk_changed(disk, true); 6770dca4462SChristoph Hellwig return ret; 6780dca4462SChristoph Hellwig } 6790dca4462SChristoph Hellwig } 6800dca4462SChristoph Hellwig 6810dca4462SChristoph Hellwig if (!bdev->bd_openers) 6820dca4462SChristoph Hellwig set_init_blocksize(bdev); 6830dca4462SChristoph Hellwig if (test_bit(GD_NEED_PART_SCAN, &disk->state)) 6840dca4462SChristoph Hellwig bdev_disk_changed(disk, false); 6850dca4462SChristoph Hellwig bdev->bd_openers++; 6860dca4462SChristoph Hellwig return 0;; 6870dca4462SChristoph Hellwig } 6880dca4462SChristoph Hellwig 6890dca4462SChristoph Hellwig static void blkdev_put_whole(struct block_device *bdev, fmode_t mode) 6900dca4462SChristoph Hellwig { 6910dca4462SChristoph Hellwig if (!--bdev->bd_openers) 6920dca4462SChristoph Hellwig blkdev_flush_mapping(bdev); 6930dca4462SChristoph Hellwig if (bdev->bd_disk->fops->release) 6940dca4462SChristoph Hellwig bdev->bd_disk->fops->release(bdev->bd_disk, mode); 6950dca4462SChristoph Hellwig } 6960dca4462SChristoph Hellwig 6970dca4462SChristoph Hellwig static int blkdev_get_part(struct block_device *part, fmode_t mode) 6980dca4462SChristoph Hellwig { 6990dca4462SChristoph Hellwig struct gendisk *disk = part->bd_disk; 7000dca4462SChristoph Hellwig int ret; 7010dca4462SChristoph Hellwig 7020dca4462SChristoph Hellwig if (part->bd_openers) 7030dca4462SChristoph Hellwig goto done; 7040dca4462SChristoph Hellwig 7050dca4462SChristoph Hellwig ret = blkdev_get_whole(bdev_whole(part), mode); 7060dca4462SChristoph Hellwig if (ret) 7070dca4462SChristoph Hellwig return ret; 7080dca4462SChristoph Hellwig 7090dca4462SChristoph Hellwig ret = -ENXIO; 7100dca4462SChristoph Hellwig if (!bdev_nr_sectors(part)) 7110dca4462SChristoph Hellwig goto out_blkdev_put; 7120dca4462SChristoph Hellwig 7130dca4462SChristoph Hellwig disk->open_partitions++; 7140dca4462SChristoph Hellwig set_init_blocksize(part); 7150dca4462SChristoph Hellwig done: 7160dca4462SChristoph Hellwig part->bd_openers++; 7170dca4462SChristoph Hellwig return 0; 7180dca4462SChristoph Hellwig 7190dca4462SChristoph Hellwig out_blkdev_put: 7200dca4462SChristoph Hellwig blkdev_put_whole(bdev_whole(part), mode); 7210dca4462SChristoph Hellwig return ret; 7220dca4462SChristoph Hellwig } 7230dca4462SChristoph Hellwig 7240dca4462SChristoph Hellwig static void blkdev_put_part(struct block_device *part, fmode_t mode) 7250dca4462SChristoph Hellwig { 7260dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(part); 7270dca4462SChristoph Hellwig 7280dca4462SChristoph Hellwig if (--part->bd_openers) 7290dca4462SChristoph Hellwig return; 7300dca4462SChristoph Hellwig blkdev_flush_mapping(part); 7310dca4462SChristoph Hellwig whole->bd_disk->open_partitions--; 7320dca4462SChristoph Hellwig blkdev_put_whole(whole, mode); 7330dca4462SChristoph Hellwig } 7340dca4462SChristoph Hellwig 7350dca4462SChristoph Hellwig struct block_device *blkdev_get_no_open(dev_t dev) 7360dca4462SChristoph Hellwig { 7370dca4462SChristoph Hellwig struct block_device *bdev; 7380dca4462SChristoph Hellwig struct inode *inode; 7390dca4462SChristoph Hellwig 7400dca4462SChristoph Hellwig inode = ilookup(blockdev_superblock, dev); 7410dca4462SChristoph Hellwig if (!inode) { 7420dca4462SChristoph Hellwig blk_request_module(dev); 7430dca4462SChristoph Hellwig inode = ilookup(blockdev_superblock, dev); 7440dca4462SChristoph Hellwig if (!inode) 7450dca4462SChristoph Hellwig return NULL; 7460dca4462SChristoph Hellwig } 7470dca4462SChristoph Hellwig 7480dca4462SChristoph Hellwig /* switch from the inode reference to a device mode one: */ 7490dca4462SChristoph Hellwig bdev = &BDEV_I(inode)->bdev; 7500dca4462SChristoph Hellwig if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 7510dca4462SChristoph Hellwig bdev = NULL; 7520dca4462SChristoph Hellwig iput(inode); 7530dca4462SChristoph Hellwig 7540dca4462SChristoph Hellwig if (!bdev) 7550dca4462SChristoph Hellwig return NULL; 756*efcf5932SMing Lei if ((bdev->bd_disk->flags & GENHD_FL_HIDDEN)) { 7570dca4462SChristoph Hellwig put_device(&bdev->bd_device); 7580dca4462SChristoph Hellwig return NULL; 7590dca4462SChristoph Hellwig } 7600dca4462SChristoph Hellwig 7610dca4462SChristoph Hellwig return bdev; 7620dca4462SChristoph Hellwig } 7630dca4462SChristoph Hellwig 7640dca4462SChristoph Hellwig void blkdev_put_no_open(struct block_device *bdev) 7650dca4462SChristoph Hellwig { 7660dca4462SChristoph Hellwig put_device(&bdev->bd_device); 7670dca4462SChristoph Hellwig } 7680dca4462SChristoph Hellwig 7690dca4462SChristoph Hellwig /** 7700dca4462SChristoph Hellwig * blkdev_get_by_dev - open a block device by device number 7710dca4462SChristoph Hellwig * @dev: device number of block device to open 7720dca4462SChristoph Hellwig * @mode: FMODE_* mask 7730dca4462SChristoph Hellwig * @holder: exclusive holder identifier 7740dca4462SChristoph Hellwig * 7750dca4462SChristoph Hellwig * Open the block device described by device number @dev. If @mode includes 7760dca4462SChristoph Hellwig * %FMODE_EXCL, the block device is opened with exclusive access. Specifying 7770dca4462SChristoph Hellwig * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for 7780dca4462SChristoph Hellwig * the same @holder. 7790dca4462SChristoph Hellwig * 7800dca4462SChristoph Hellwig * Use this interface ONLY if you really do not have anything better - i.e. when 7810dca4462SChristoph Hellwig * you are behind a truly sucky interface and all you are given is a device 7820dca4462SChristoph Hellwig * number. Everything else should use blkdev_get_by_path(). 7830dca4462SChristoph Hellwig * 7840dca4462SChristoph Hellwig * CONTEXT: 7850dca4462SChristoph Hellwig * Might sleep. 7860dca4462SChristoph Hellwig * 7870dca4462SChristoph Hellwig * RETURNS: 7880dca4462SChristoph Hellwig * Reference to the block_device on success, ERR_PTR(-errno) on failure. 7890dca4462SChristoph Hellwig */ 7900dca4462SChristoph Hellwig struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder) 7910dca4462SChristoph Hellwig { 7920dca4462SChristoph Hellwig bool unblock_events = true; 7930dca4462SChristoph Hellwig struct block_device *bdev; 7940dca4462SChristoph Hellwig struct gendisk *disk; 7950dca4462SChristoph Hellwig int ret; 7960dca4462SChristoph Hellwig 7970dca4462SChristoph Hellwig ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 7980dca4462SChristoph Hellwig MAJOR(dev), MINOR(dev), 7990dca4462SChristoph Hellwig ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) | 8000dca4462SChristoph Hellwig ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0)); 8010dca4462SChristoph Hellwig if (ret) 8020dca4462SChristoph Hellwig return ERR_PTR(ret); 8030dca4462SChristoph Hellwig 8040dca4462SChristoph Hellwig bdev = blkdev_get_no_open(dev); 8050dca4462SChristoph Hellwig if (!bdev) 8060dca4462SChristoph Hellwig return ERR_PTR(-ENXIO); 8070dca4462SChristoph Hellwig disk = bdev->bd_disk; 8080dca4462SChristoph Hellwig 8090dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 8100dca4462SChristoph Hellwig ret = bd_prepare_to_claim(bdev, holder); 8110dca4462SChristoph Hellwig if (ret) 8120dca4462SChristoph Hellwig goto put_blkdev; 8130dca4462SChristoph Hellwig } 8140dca4462SChristoph Hellwig 8150dca4462SChristoph Hellwig disk_block_events(disk); 8160dca4462SChristoph Hellwig 8170dca4462SChristoph Hellwig mutex_lock(&disk->open_mutex); 8180dca4462SChristoph Hellwig ret = -ENXIO; 8190dca4462SChristoph Hellwig if (!disk_live(disk)) 8200dca4462SChristoph Hellwig goto abort_claiming; 821*efcf5932SMing Lei if (!try_module_get(disk->fops->owner)) 822*efcf5932SMing Lei goto abort_claiming; 8230dca4462SChristoph Hellwig if (bdev_is_partition(bdev)) 8240dca4462SChristoph Hellwig ret = blkdev_get_part(bdev, mode); 8250dca4462SChristoph Hellwig else 8260dca4462SChristoph Hellwig ret = blkdev_get_whole(bdev, mode); 8270dca4462SChristoph Hellwig if (ret) 828*efcf5932SMing Lei goto put_module; 8290dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 8300dca4462SChristoph Hellwig bd_finish_claiming(bdev, holder); 8310dca4462SChristoph Hellwig 8320dca4462SChristoph Hellwig /* 8330dca4462SChristoph Hellwig * Block event polling for write claims if requested. Any write 8340dca4462SChristoph Hellwig * holder makes the write_holder state stick until all are 8350dca4462SChristoph Hellwig * released. This is good enough and tracking individual 8360dca4462SChristoph Hellwig * writeable reference is too fragile given the way @mode is 8370dca4462SChristoph Hellwig * used in blkdev_get/put(). 8380dca4462SChristoph Hellwig */ 8390dca4462SChristoph Hellwig if ((mode & FMODE_WRITE) && !bdev->bd_write_holder && 8400dca4462SChristoph Hellwig (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) { 8410dca4462SChristoph Hellwig bdev->bd_write_holder = true; 8420dca4462SChristoph Hellwig unblock_events = false; 8430dca4462SChristoph Hellwig } 8440dca4462SChristoph Hellwig } 8450dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 8460dca4462SChristoph Hellwig 8470dca4462SChristoph Hellwig if (unblock_events) 8480dca4462SChristoph Hellwig disk_unblock_events(disk); 8490dca4462SChristoph Hellwig return bdev; 850*efcf5932SMing Lei put_module: 851*efcf5932SMing Lei module_put(disk->fops->owner); 8520dca4462SChristoph Hellwig abort_claiming: 8530dca4462SChristoph Hellwig if (mode & FMODE_EXCL) 8540dca4462SChristoph Hellwig bd_abort_claiming(bdev, holder); 8550dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 8560dca4462SChristoph Hellwig disk_unblock_events(disk); 8570dca4462SChristoph Hellwig put_blkdev: 8580dca4462SChristoph Hellwig blkdev_put_no_open(bdev); 8590dca4462SChristoph Hellwig return ERR_PTR(ret); 8600dca4462SChristoph Hellwig } 8610dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_get_by_dev); 8620dca4462SChristoph Hellwig 8630dca4462SChristoph Hellwig /** 8640dca4462SChristoph Hellwig * blkdev_get_by_path - open a block device by name 8650dca4462SChristoph Hellwig * @path: path to the block device to open 8660dca4462SChristoph Hellwig * @mode: FMODE_* mask 8670dca4462SChristoph Hellwig * @holder: exclusive holder identifier 8680dca4462SChristoph Hellwig * 8690dca4462SChristoph Hellwig * Open the block device described by the device file at @path. If @mode 8700dca4462SChristoph Hellwig * includes %FMODE_EXCL, the block device is opened with exclusive access. 8710dca4462SChristoph Hellwig * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may 8720dca4462SChristoph Hellwig * nest for the same @holder. 8730dca4462SChristoph Hellwig * 8740dca4462SChristoph Hellwig * CONTEXT: 8750dca4462SChristoph Hellwig * Might sleep. 8760dca4462SChristoph Hellwig * 8770dca4462SChristoph Hellwig * RETURNS: 8780dca4462SChristoph Hellwig * Reference to the block_device on success, ERR_PTR(-errno) on failure. 8790dca4462SChristoph Hellwig */ 8800dca4462SChristoph Hellwig struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, 8810dca4462SChristoph Hellwig void *holder) 8820dca4462SChristoph Hellwig { 8830dca4462SChristoph Hellwig struct block_device *bdev; 8840dca4462SChristoph Hellwig dev_t dev; 8850dca4462SChristoph Hellwig int error; 8860dca4462SChristoph Hellwig 8870dca4462SChristoph Hellwig error = lookup_bdev(path, &dev); 8880dca4462SChristoph Hellwig if (error) 8890dca4462SChristoph Hellwig return ERR_PTR(error); 8900dca4462SChristoph Hellwig 8910dca4462SChristoph Hellwig bdev = blkdev_get_by_dev(dev, mode, holder); 8920dca4462SChristoph Hellwig if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) { 8930dca4462SChristoph Hellwig blkdev_put(bdev, mode); 8940dca4462SChristoph Hellwig return ERR_PTR(-EACCES); 8950dca4462SChristoph Hellwig } 8960dca4462SChristoph Hellwig 8970dca4462SChristoph Hellwig return bdev; 8980dca4462SChristoph Hellwig } 8990dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_get_by_path); 9000dca4462SChristoph Hellwig 9010dca4462SChristoph Hellwig void blkdev_put(struct block_device *bdev, fmode_t mode) 9020dca4462SChristoph Hellwig { 9030dca4462SChristoph Hellwig struct gendisk *disk = bdev->bd_disk; 9040dca4462SChristoph Hellwig 9050dca4462SChristoph Hellwig /* 9060dca4462SChristoph Hellwig * Sync early if it looks like we're the last one. If someone else 9070dca4462SChristoph Hellwig * opens the block device between now and the decrement of bd_openers 9080dca4462SChristoph Hellwig * then we did a sync that we didn't need to, but that's not the end 9090dca4462SChristoph Hellwig * of the world and we want to avoid long (could be several minute) 9100dca4462SChristoph Hellwig * syncs while holding the mutex. 9110dca4462SChristoph Hellwig */ 9120dca4462SChristoph Hellwig if (bdev->bd_openers == 1) 9130dca4462SChristoph Hellwig sync_blockdev(bdev); 9140dca4462SChristoph Hellwig 9150dca4462SChristoph Hellwig mutex_lock(&disk->open_mutex); 9160dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 9170dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 9180dca4462SChristoph Hellwig bool bdev_free; 9190dca4462SChristoph Hellwig 9200dca4462SChristoph Hellwig /* 9210dca4462SChristoph Hellwig * Release a claim on the device. The holder fields 9220dca4462SChristoph Hellwig * are protected with bdev_lock. open_mutex is to 9230dca4462SChristoph Hellwig * synchronize disk_holder unlinking. 9240dca4462SChristoph Hellwig */ 9250dca4462SChristoph Hellwig spin_lock(&bdev_lock); 9260dca4462SChristoph Hellwig 9270dca4462SChristoph Hellwig WARN_ON_ONCE(--bdev->bd_holders < 0); 9280dca4462SChristoph Hellwig WARN_ON_ONCE(--whole->bd_holders < 0); 9290dca4462SChristoph Hellwig 9300dca4462SChristoph Hellwig if ((bdev_free = !bdev->bd_holders)) 9310dca4462SChristoph Hellwig bdev->bd_holder = NULL; 9320dca4462SChristoph Hellwig if (!whole->bd_holders) 9330dca4462SChristoph Hellwig whole->bd_holder = NULL; 9340dca4462SChristoph Hellwig 9350dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 9360dca4462SChristoph Hellwig 9370dca4462SChristoph Hellwig /* 9380dca4462SChristoph Hellwig * If this was the last claim, remove holder link and 9390dca4462SChristoph Hellwig * unblock evpoll if it was a write holder. 9400dca4462SChristoph Hellwig */ 9410dca4462SChristoph Hellwig if (bdev_free && bdev->bd_write_holder) { 9420dca4462SChristoph Hellwig disk_unblock_events(disk); 9430dca4462SChristoph Hellwig bdev->bd_write_holder = false; 9440dca4462SChristoph Hellwig } 9450dca4462SChristoph Hellwig } 9460dca4462SChristoph Hellwig 9470dca4462SChristoph Hellwig /* 9480dca4462SChristoph Hellwig * Trigger event checking and tell drivers to flush MEDIA_CHANGE 9490dca4462SChristoph Hellwig * event. This is to ensure detection of media removal commanded 9500dca4462SChristoph Hellwig * from userland - e.g. eject(1). 9510dca4462SChristoph Hellwig */ 9520dca4462SChristoph Hellwig disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 9530dca4462SChristoph Hellwig 9540dca4462SChristoph Hellwig if (bdev_is_partition(bdev)) 9550dca4462SChristoph Hellwig blkdev_put_part(bdev, mode); 9560dca4462SChristoph Hellwig else 9570dca4462SChristoph Hellwig blkdev_put_whole(bdev, mode); 9580dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 9590dca4462SChristoph Hellwig 960*efcf5932SMing Lei module_put(disk->fops->owner); 9610dca4462SChristoph Hellwig blkdev_put_no_open(bdev); 9620dca4462SChristoph Hellwig } 9630dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_put); 9640dca4462SChristoph Hellwig 9650dca4462SChristoph Hellwig /** 9660dca4462SChristoph Hellwig * lookup_bdev - lookup a struct block_device by name 9670dca4462SChristoph Hellwig * @pathname: special file representing the block device 9680dca4462SChristoph Hellwig * @dev: return value of the block device's dev_t 9690dca4462SChristoph Hellwig * 970057178cfSJackie Liu * Lookup the block device's dev_t at @pathname in the current 971057178cfSJackie Liu * namespace if possible and return it by @dev. 972057178cfSJackie Liu * 973057178cfSJackie Liu * RETURNS: 974057178cfSJackie Liu * 0 if succeeded, errno otherwise. 9750dca4462SChristoph Hellwig */ 9760dca4462SChristoph Hellwig int lookup_bdev(const char *pathname, dev_t *dev) 9770dca4462SChristoph Hellwig { 9780dca4462SChristoph Hellwig struct inode *inode; 9790dca4462SChristoph Hellwig struct path path; 9800dca4462SChristoph Hellwig int error; 9810dca4462SChristoph Hellwig 9820dca4462SChristoph Hellwig if (!pathname || !*pathname) 9830dca4462SChristoph Hellwig return -EINVAL; 9840dca4462SChristoph Hellwig 9850dca4462SChristoph Hellwig error = kern_path(pathname, LOOKUP_FOLLOW, &path); 9860dca4462SChristoph Hellwig if (error) 9870dca4462SChristoph Hellwig return error; 9880dca4462SChristoph Hellwig 9890dca4462SChristoph Hellwig inode = d_backing_inode(path.dentry); 9900dca4462SChristoph Hellwig error = -ENOTBLK; 9910dca4462SChristoph Hellwig if (!S_ISBLK(inode->i_mode)) 9920dca4462SChristoph Hellwig goto out_path_put; 9930dca4462SChristoph Hellwig error = -EACCES; 9940dca4462SChristoph Hellwig if (!may_open_dev(&path)) 9950dca4462SChristoph Hellwig goto out_path_put; 9960dca4462SChristoph Hellwig 9970dca4462SChristoph Hellwig *dev = inode->i_rdev; 9980dca4462SChristoph Hellwig error = 0; 9990dca4462SChristoph Hellwig out_path_put: 10000dca4462SChristoph Hellwig path_put(&path); 10010dca4462SChristoph Hellwig return error; 10020dca4462SChristoph Hellwig } 10030dca4462SChristoph Hellwig EXPORT_SYMBOL(lookup_bdev); 10040dca4462SChristoph Hellwig 10050dca4462SChristoph Hellwig int __invalidate_device(struct block_device *bdev, bool kill_dirty) 10060dca4462SChristoph Hellwig { 10070dca4462SChristoph Hellwig struct super_block *sb = get_super(bdev); 10080dca4462SChristoph Hellwig int res = 0; 10090dca4462SChristoph Hellwig 10100dca4462SChristoph Hellwig if (sb) { 10110dca4462SChristoph Hellwig /* 10120dca4462SChristoph Hellwig * no need to lock the super, get_super holds the 10130dca4462SChristoph Hellwig * read mutex so the filesystem cannot go away 10140dca4462SChristoph Hellwig * under us (->put_super runs with the write lock 10150dca4462SChristoph Hellwig * hold). 10160dca4462SChristoph Hellwig */ 10170dca4462SChristoph Hellwig shrink_dcache_sb(sb); 10180dca4462SChristoph Hellwig res = invalidate_inodes(sb, kill_dirty); 10190dca4462SChristoph Hellwig drop_super(sb); 10200dca4462SChristoph Hellwig } 10210dca4462SChristoph Hellwig invalidate_bdev(bdev); 10220dca4462SChristoph Hellwig return res; 10230dca4462SChristoph Hellwig } 10240dca4462SChristoph Hellwig EXPORT_SYMBOL(__invalidate_device); 10250dca4462SChristoph Hellwig 10261e03a36bSChristoph Hellwig void sync_bdevs(bool wait) 10270dca4462SChristoph Hellwig { 10280dca4462SChristoph Hellwig struct inode *inode, *old_inode = NULL; 10290dca4462SChristoph Hellwig 10300dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 10310dca4462SChristoph Hellwig list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 10320dca4462SChristoph Hellwig struct address_space *mapping = inode->i_mapping; 10330dca4462SChristoph Hellwig struct block_device *bdev; 10340dca4462SChristoph Hellwig 10350dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 10360dca4462SChristoph Hellwig if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 10370dca4462SChristoph Hellwig mapping->nrpages == 0) { 10380dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 10390dca4462SChristoph Hellwig continue; 10400dca4462SChristoph Hellwig } 10410dca4462SChristoph Hellwig __iget(inode); 10420dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 10430dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 10440dca4462SChristoph Hellwig /* 10450dca4462SChristoph Hellwig * We hold a reference to 'inode' so it couldn't have been 10460dca4462SChristoph Hellwig * removed from s_inodes list while we dropped the 10470dca4462SChristoph Hellwig * s_inode_list_lock We cannot iput the inode now as we can 10480dca4462SChristoph Hellwig * be holding the last reference and we cannot iput it under 10490dca4462SChristoph Hellwig * s_inode_list_lock. So we keep the reference and iput it 10500dca4462SChristoph Hellwig * later. 10510dca4462SChristoph Hellwig */ 10520dca4462SChristoph Hellwig iput(old_inode); 10530dca4462SChristoph Hellwig old_inode = inode; 10540dca4462SChristoph Hellwig bdev = I_BDEV(inode); 10550dca4462SChristoph Hellwig 10560dca4462SChristoph Hellwig mutex_lock(&bdev->bd_disk->open_mutex); 10571e03a36bSChristoph Hellwig if (!bdev->bd_openers) { 10581e03a36bSChristoph Hellwig ; /* skip */ 10591e03a36bSChristoph Hellwig } else if (wait) { 10601e03a36bSChristoph Hellwig /* 10611e03a36bSChristoph Hellwig * We keep the error status of individual mapping so 10621e03a36bSChristoph Hellwig * that applications can catch the writeback error using 10631e03a36bSChristoph Hellwig * fsync(2). See filemap_fdatawait_keep_errors() for 10641e03a36bSChristoph Hellwig * details. 10651e03a36bSChristoph Hellwig */ 10661e03a36bSChristoph Hellwig filemap_fdatawait_keep_errors(inode->i_mapping); 10671e03a36bSChristoph Hellwig } else { 10681e03a36bSChristoph Hellwig filemap_fdatawrite(inode->i_mapping); 10691e03a36bSChristoph Hellwig } 10700dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_disk->open_mutex); 10710dca4462SChristoph Hellwig 10720dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 10730dca4462SChristoph Hellwig } 10740dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 10750dca4462SChristoph Hellwig iput(old_inode); 10760dca4462SChristoph Hellwig } 1077