10dca4462SChristoph Hellwig // SPDX-License-Identifier: GPL-2.0-only 20dca4462SChristoph Hellwig /* 30dca4462SChristoph Hellwig * Copyright (C) 1991, 1992 Linus Torvalds 40dca4462SChristoph Hellwig * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE 50dca4462SChristoph Hellwig * Copyright (C) 2016 - 2020 Christoph Hellwig 60dca4462SChristoph Hellwig */ 70dca4462SChristoph Hellwig 80dca4462SChristoph Hellwig #include <linux/init.h> 90dca4462SChristoph Hellwig #include <linux/mm.h> 100dca4462SChristoph Hellwig #include <linux/slab.h> 110dca4462SChristoph Hellwig #include <linux/kmod.h> 120dca4462SChristoph Hellwig #include <linux/major.h> 130dca4462SChristoph Hellwig #include <linux/device_cgroup.h> 140dca4462SChristoph Hellwig #include <linux/blkdev.h> 15fe45e630SChristoph Hellwig #include <linux/blk-integrity.h> 160dca4462SChristoph Hellwig #include <linux/backing-dev.h> 170dca4462SChristoph Hellwig #include <linux/module.h> 180dca4462SChristoph Hellwig #include <linux/blkpg.h> 190dca4462SChristoph Hellwig #include <linux/magic.h> 200dca4462SChristoph Hellwig #include <linux/buffer_head.h> 210dca4462SChristoph Hellwig #include <linux/swap.h> 220dca4462SChristoph Hellwig #include <linux/writeback.h> 230dca4462SChristoph Hellwig #include <linux/mount.h> 240dca4462SChristoph Hellwig #include <linux/pseudo_fs.h> 250dca4462SChristoph Hellwig #include <linux/uio.h> 260dca4462SChristoph Hellwig #include <linux/namei.h> 270dca4462SChristoph Hellwig #include <linux/part_stat.h> 280dca4462SChristoph Hellwig #include <linux/uaccess.h> 290dca4462SChristoph Hellwig #include "../fs/internal.h" 300dca4462SChristoph Hellwig #include "blk.h" 310dca4462SChristoph Hellwig 320dca4462SChristoph Hellwig struct bdev_inode { 330dca4462SChristoph Hellwig struct block_device bdev; 340dca4462SChristoph Hellwig struct inode vfs_inode; 350dca4462SChristoph Hellwig }; 360dca4462SChristoph Hellwig 370dca4462SChristoph Hellwig static inline struct bdev_inode *BDEV_I(struct inode *inode) 380dca4462SChristoph Hellwig { 390dca4462SChristoph Hellwig return container_of(inode, struct bdev_inode, vfs_inode); 400dca4462SChristoph Hellwig } 410dca4462SChristoph Hellwig 420dca4462SChristoph Hellwig struct block_device *I_BDEV(struct inode *inode) 430dca4462SChristoph Hellwig { 440dca4462SChristoph Hellwig return &BDEV_I(inode)->bdev; 450dca4462SChristoph Hellwig } 460dca4462SChristoph Hellwig EXPORT_SYMBOL(I_BDEV); 470dca4462SChristoph Hellwig 480dca4462SChristoph Hellwig static void bdev_write_inode(struct block_device *bdev) 490dca4462SChristoph Hellwig { 500dca4462SChristoph Hellwig struct inode *inode = bdev->bd_inode; 510dca4462SChristoph Hellwig int ret; 520dca4462SChristoph Hellwig 530dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 540dca4462SChristoph Hellwig while (inode->i_state & I_DIRTY) { 550dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 560dca4462SChristoph Hellwig ret = write_inode_now(inode, true); 570dca4462SChristoph Hellwig if (ret) { 580dca4462SChristoph Hellwig char name[BDEVNAME_SIZE]; 590dca4462SChristoph Hellwig pr_warn_ratelimited("VFS: Dirty inode writeback failed " 600dca4462SChristoph Hellwig "for block device %s (err=%d).\n", 610dca4462SChristoph Hellwig bdevname(bdev, name), ret); 620dca4462SChristoph Hellwig } 630dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 640dca4462SChristoph Hellwig } 650dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 660dca4462SChristoph Hellwig } 670dca4462SChristoph Hellwig 680dca4462SChristoph Hellwig /* Kill _all_ buffers and pagecache , dirty or not.. */ 690dca4462SChristoph Hellwig static void kill_bdev(struct block_device *bdev) 700dca4462SChristoph Hellwig { 710dca4462SChristoph Hellwig struct address_space *mapping = bdev->bd_inode->i_mapping; 720dca4462SChristoph Hellwig 730dca4462SChristoph Hellwig if (mapping_empty(mapping)) 740dca4462SChristoph Hellwig return; 750dca4462SChristoph Hellwig 760dca4462SChristoph Hellwig invalidate_bh_lrus(); 770dca4462SChristoph Hellwig truncate_inode_pages(mapping, 0); 780dca4462SChristoph Hellwig } 790dca4462SChristoph Hellwig 800dca4462SChristoph Hellwig /* Invalidate clean unused buffers and pagecache. */ 810dca4462SChristoph Hellwig void invalidate_bdev(struct block_device *bdev) 820dca4462SChristoph Hellwig { 830dca4462SChristoph Hellwig struct address_space *mapping = bdev->bd_inode->i_mapping; 840dca4462SChristoph Hellwig 850dca4462SChristoph Hellwig if (mapping->nrpages) { 860dca4462SChristoph Hellwig invalidate_bh_lrus(); 870dca4462SChristoph Hellwig lru_add_drain_all(); /* make sure all lru add caches are flushed */ 880dca4462SChristoph Hellwig invalidate_mapping_pages(mapping, 0, -1); 890dca4462SChristoph Hellwig } 900dca4462SChristoph Hellwig } 910dca4462SChristoph Hellwig EXPORT_SYMBOL(invalidate_bdev); 920dca4462SChristoph Hellwig 930dca4462SChristoph Hellwig /* 940dca4462SChristoph Hellwig * Drop all buffers & page cache for given bdev range. This function bails 950dca4462SChristoph Hellwig * with error if bdev has other exclusive owner (such as filesystem). 960dca4462SChristoph Hellwig */ 970dca4462SChristoph Hellwig int truncate_bdev_range(struct block_device *bdev, fmode_t mode, 980dca4462SChristoph Hellwig loff_t lstart, loff_t lend) 990dca4462SChristoph Hellwig { 1000dca4462SChristoph Hellwig /* 1010dca4462SChristoph Hellwig * If we don't hold exclusive handle for the device, upgrade to it 1020dca4462SChristoph Hellwig * while we discard the buffer cache to avoid discarding buffers 1030dca4462SChristoph Hellwig * under live filesystem. 1040dca4462SChristoph Hellwig */ 1050dca4462SChristoph Hellwig if (!(mode & FMODE_EXCL)) { 1060dca4462SChristoph Hellwig int err = bd_prepare_to_claim(bdev, truncate_bdev_range); 1070dca4462SChristoph Hellwig if (err) 1080dca4462SChristoph Hellwig goto invalidate; 1090dca4462SChristoph Hellwig } 1100dca4462SChristoph Hellwig 1110dca4462SChristoph Hellwig truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend); 1120dca4462SChristoph Hellwig if (!(mode & FMODE_EXCL)) 1130dca4462SChristoph Hellwig bd_abort_claiming(bdev, truncate_bdev_range); 1140dca4462SChristoph Hellwig return 0; 1150dca4462SChristoph Hellwig 1160dca4462SChristoph Hellwig invalidate: 1170dca4462SChristoph Hellwig /* 1180dca4462SChristoph Hellwig * Someone else has handle exclusively open. Try invalidating instead. 1190dca4462SChristoph Hellwig * The 'end' argument is inclusive so the rounding is safe. 1200dca4462SChristoph Hellwig */ 1210dca4462SChristoph Hellwig return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping, 1220dca4462SChristoph Hellwig lstart >> PAGE_SHIFT, 1230dca4462SChristoph Hellwig lend >> PAGE_SHIFT); 1240dca4462SChristoph Hellwig } 1250dca4462SChristoph Hellwig 1260dca4462SChristoph Hellwig static void set_init_blocksize(struct block_device *bdev) 1270dca4462SChristoph Hellwig { 1280dca4462SChristoph Hellwig unsigned int bsize = bdev_logical_block_size(bdev); 1290dca4462SChristoph Hellwig loff_t size = i_size_read(bdev->bd_inode); 1300dca4462SChristoph Hellwig 1310dca4462SChristoph Hellwig while (bsize < PAGE_SIZE) { 1320dca4462SChristoph Hellwig if (size & bsize) 1330dca4462SChristoph Hellwig break; 1340dca4462SChristoph Hellwig bsize <<= 1; 1350dca4462SChristoph Hellwig } 1360dca4462SChristoph Hellwig bdev->bd_inode->i_blkbits = blksize_bits(bsize); 1370dca4462SChristoph Hellwig } 1380dca4462SChristoph Hellwig 1390dca4462SChristoph Hellwig int set_blocksize(struct block_device *bdev, int size) 1400dca4462SChristoph Hellwig { 1410dca4462SChristoph Hellwig /* Size must be a power of two, and between 512 and PAGE_SIZE */ 1420dca4462SChristoph Hellwig if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size)) 1430dca4462SChristoph Hellwig return -EINVAL; 1440dca4462SChristoph Hellwig 1450dca4462SChristoph Hellwig /* Size cannot be smaller than the size supported by the device */ 1460dca4462SChristoph Hellwig if (size < bdev_logical_block_size(bdev)) 1470dca4462SChristoph Hellwig return -EINVAL; 1480dca4462SChristoph Hellwig 1490dca4462SChristoph Hellwig /* Don't change the size if it is same as current */ 1500dca4462SChristoph Hellwig if (bdev->bd_inode->i_blkbits != blksize_bits(size)) { 1510dca4462SChristoph Hellwig sync_blockdev(bdev); 1520dca4462SChristoph Hellwig bdev->bd_inode->i_blkbits = blksize_bits(size); 1530dca4462SChristoph Hellwig kill_bdev(bdev); 1540dca4462SChristoph Hellwig } 1550dca4462SChristoph Hellwig return 0; 1560dca4462SChristoph Hellwig } 1570dca4462SChristoph Hellwig 1580dca4462SChristoph Hellwig EXPORT_SYMBOL(set_blocksize); 1590dca4462SChristoph Hellwig 1600dca4462SChristoph Hellwig int sb_set_blocksize(struct super_block *sb, int size) 1610dca4462SChristoph Hellwig { 1620dca4462SChristoph Hellwig if (set_blocksize(sb->s_bdev, size)) 1630dca4462SChristoph Hellwig return 0; 1640dca4462SChristoph Hellwig /* If we get here, we know size is power of two 1650dca4462SChristoph Hellwig * and it's value is between 512 and PAGE_SIZE */ 1660dca4462SChristoph Hellwig sb->s_blocksize = size; 1670dca4462SChristoph Hellwig sb->s_blocksize_bits = blksize_bits(size); 1680dca4462SChristoph Hellwig return sb->s_blocksize; 1690dca4462SChristoph Hellwig } 1700dca4462SChristoph Hellwig 1710dca4462SChristoph Hellwig EXPORT_SYMBOL(sb_set_blocksize); 1720dca4462SChristoph Hellwig 1730dca4462SChristoph Hellwig int sb_min_blocksize(struct super_block *sb, int size) 1740dca4462SChristoph Hellwig { 1750dca4462SChristoph Hellwig int minsize = bdev_logical_block_size(sb->s_bdev); 1760dca4462SChristoph Hellwig if (size < minsize) 1770dca4462SChristoph Hellwig size = minsize; 1780dca4462SChristoph Hellwig return sb_set_blocksize(sb, size); 1790dca4462SChristoph Hellwig } 1800dca4462SChristoph Hellwig 1810dca4462SChristoph Hellwig EXPORT_SYMBOL(sb_min_blocksize); 1820dca4462SChristoph Hellwig 18370164eb6SChristoph Hellwig int sync_blockdev_nowait(struct block_device *bdev) 1840dca4462SChristoph Hellwig { 1850dca4462SChristoph Hellwig if (!bdev) 1860dca4462SChristoph Hellwig return 0; 1870dca4462SChristoph Hellwig return filemap_flush(bdev->bd_inode->i_mapping); 1880dca4462SChristoph Hellwig } 18970164eb6SChristoph Hellwig EXPORT_SYMBOL_GPL(sync_blockdev_nowait); 1900dca4462SChristoph Hellwig 1910dca4462SChristoph Hellwig /* 1920dca4462SChristoph Hellwig * Write out and wait upon all the dirty data associated with a block 1930dca4462SChristoph Hellwig * device via its mapping. Does not take the superblock lock. 1940dca4462SChristoph Hellwig */ 1950dca4462SChristoph Hellwig int sync_blockdev(struct block_device *bdev) 1960dca4462SChristoph Hellwig { 19770164eb6SChristoph Hellwig if (!bdev) 19870164eb6SChristoph Hellwig return 0; 19970164eb6SChristoph Hellwig return filemap_write_and_wait(bdev->bd_inode->i_mapping); 2000dca4462SChristoph Hellwig } 2010dca4462SChristoph Hellwig EXPORT_SYMBOL(sync_blockdev); 2020dca4462SChristoph Hellwig 2030dca4462SChristoph Hellwig /* 2040dca4462SChristoph Hellwig * Write out and wait upon all dirty data associated with this 2050dca4462SChristoph Hellwig * device. Filesystem data as well as the underlying block 2060dca4462SChristoph Hellwig * device. Takes the superblock lock. 2070dca4462SChristoph Hellwig */ 2080dca4462SChristoph Hellwig int fsync_bdev(struct block_device *bdev) 2090dca4462SChristoph Hellwig { 2100dca4462SChristoph Hellwig struct super_block *sb = get_super(bdev); 2110dca4462SChristoph Hellwig if (sb) { 2120dca4462SChristoph Hellwig int res = sync_filesystem(sb); 2130dca4462SChristoph Hellwig drop_super(sb); 2140dca4462SChristoph Hellwig return res; 2150dca4462SChristoph Hellwig } 2160dca4462SChristoph Hellwig return sync_blockdev(bdev); 2170dca4462SChristoph Hellwig } 2180dca4462SChristoph Hellwig EXPORT_SYMBOL(fsync_bdev); 2190dca4462SChristoph Hellwig 2200dca4462SChristoph Hellwig /** 2210dca4462SChristoph Hellwig * freeze_bdev -- lock a filesystem and force it into a consistent state 2220dca4462SChristoph Hellwig * @bdev: blockdevice to lock 2230dca4462SChristoph Hellwig * 2240dca4462SChristoph Hellwig * If a superblock is found on this device, we take the s_umount semaphore 2250dca4462SChristoph Hellwig * on it to make sure nobody unmounts until the snapshot creation is done. 2260dca4462SChristoph Hellwig * The reference counter (bd_fsfreeze_count) guarantees that only the last 2270dca4462SChristoph Hellwig * unfreeze process can unfreeze the frozen filesystem actually when multiple 2280dca4462SChristoph Hellwig * freeze requests arrive simultaneously. It counts up in freeze_bdev() and 2290dca4462SChristoph Hellwig * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze 2300dca4462SChristoph Hellwig * actually. 2310dca4462SChristoph Hellwig */ 2320dca4462SChristoph Hellwig int freeze_bdev(struct block_device *bdev) 2330dca4462SChristoph Hellwig { 2340dca4462SChristoph Hellwig struct super_block *sb; 2350dca4462SChristoph Hellwig int error = 0; 2360dca4462SChristoph Hellwig 2370dca4462SChristoph Hellwig mutex_lock(&bdev->bd_fsfreeze_mutex); 2380dca4462SChristoph Hellwig if (++bdev->bd_fsfreeze_count > 1) 2390dca4462SChristoph Hellwig goto done; 2400dca4462SChristoph Hellwig 2410dca4462SChristoph Hellwig sb = get_active_super(bdev); 2420dca4462SChristoph Hellwig if (!sb) 2430dca4462SChristoph Hellwig goto sync; 2440dca4462SChristoph Hellwig if (sb->s_op->freeze_super) 2450dca4462SChristoph Hellwig error = sb->s_op->freeze_super(sb); 2460dca4462SChristoph Hellwig else 2470dca4462SChristoph Hellwig error = freeze_super(sb); 2480dca4462SChristoph Hellwig deactivate_super(sb); 2490dca4462SChristoph Hellwig 2500dca4462SChristoph Hellwig if (error) { 2510dca4462SChristoph Hellwig bdev->bd_fsfreeze_count--; 2520dca4462SChristoph Hellwig goto done; 2530dca4462SChristoph Hellwig } 2540dca4462SChristoph Hellwig bdev->bd_fsfreeze_sb = sb; 2550dca4462SChristoph Hellwig 2560dca4462SChristoph Hellwig sync: 2570dca4462SChristoph Hellwig sync_blockdev(bdev); 2580dca4462SChristoph Hellwig done: 2590dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_fsfreeze_mutex); 2600dca4462SChristoph Hellwig return error; 2610dca4462SChristoph Hellwig } 2620dca4462SChristoph Hellwig EXPORT_SYMBOL(freeze_bdev); 2630dca4462SChristoph Hellwig 2640dca4462SChristoph Hellwig /** 2650dca4462SChristoph Hellwig * thaw_bdev -- unlock filesystem 2660dca4462SChristoph Hellwig * @bdev: blockdevice to unlock 2670dca4462SChristoph Hellwig * 2680dca4462SChristoph Hellwig * Unlocks the filesystem and marks it writeable again after freeze_bdev(). 2690dca4462SChristoph Hellwig */ 2700dca4462SChristoph Hellwig int thaw_bdev(struct block_device *bdev) 2710dca4462SChristoph Hellwig { 2720dca4462SChristoph Hellwig struct super_block *sb; 2730dca4462SChristoph Hellwig int error = -EINVAL; 2740dca4462SChristoph Hellwig 2750dca4462SChristoph Hellwig mutex_lock(&bdev->bd_fsfreeze_mutex); 2760dca4462SChristoph Hellwig if (!bdev->bd_fsfreeze_count) 2770dca4462SChristoph Hellwig goto out; 2780dca4462SChristoph Hellwig 2790dca4462SChristoph Hellwig error = 0; 2800dca4462SChristoph Hellwig if (--bdev->bd_fsfreeze_count > 0) 2810dca4462SChristoph Hellwig goto out; 2820dca4462SChristoph Hellwig 2830dca4462SChristoph Hellwig sb = bdev->bd_fsfreeze_sb; 2840dca4462SChristoph Hellwig if (!sb) 2850dca4462SChristoph Hellwig goto out; 2860dca4462SChristoph Hellwig 2870dca4462SChristoph Hellwig if (sb->s_op->thaw_super) 2880dca4462SChristoph Hellwig error = sb->s_op->thaw_super(sb); 2890dca4462SChristoph Hellwig else 2900dca4462SChristoph Hellwig error = thaw_super(sb); 2910dca4462SChristoph Hellwig if (error) 2920dca4462SChristoph Hellwig bdev->bd_fsfreeze_count++; 2930dca4462SChristoph Hellwig else 2940dca4462SChristoph Hellwig bdev->bd_fsfreeze_sb = NULL; 2950dca4462SChristoph Hellwig out: 2960dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_fsfreeze_mutex); 2970dca4462SChristoph Hellwig return error; 2980dca4462SChristoph Hellwig } 2990dca4462SChristoph Hellwig EXPORT_SYMBOL(thaw_bdev); 3000dca4462SChristoph Hellwig 3010dca4462SChristoph Hellwig /** 3020dca4462SChristoph Hellwig * bdev_read_page() - Start reading a page from a block device 3030dca4462SChristoph Hellwig * @bdev: The device to read the page from 3040dca4462SChristoph Hellwig * @sector: The offset on the device to read the page to (need not be aligned) 3050dca4462SChristoph Hellwig * @page: The page to read 3060dca4462SChristoph Hellwig * 3070dca4462SChristoph Hellwig * On entry, the page should be locked. It will be unlocked when the page 3080dca4462SChristoph Hellwig * has been read. If the block driver implements rw_page synchronously, 3090dca4462SChristoph Hellwig * that will be true on exit from this function, but it need not be. 3100dca4462SChristoph Hellwig * 3110dca4462SChristoph Hellwig * Errors returned by this function are usually "soft", eg out of memory, or 3120dca4462SChristoph Hellwig * queue full; callers should try a different route to read this page rather 3130dca4462SChristoph Hellwig * than propagate an error back up the stack. 3140dca4462SChristoph Hellwig * 3150dca4462SChristoph Hellwig * Return: negative errno if an error occurs, 0 if submission was successful. 3160dca4462SChristoph Hellwig */ 3170dca4462SChristoph Hellwig int bdev_read_page(struct block_device *bdev, sector_t sector, 3180dca4462SChristoph Hellwig struct page *page) 3190dca4462SChristoph Hellwig { 3200dca4462SChristoph Hellwig const struct block_device_operations *ops = bdev->bd_disk->fops; 3210dca4462SChristoph Hellwig int result = -EOPNOTSUPP; 3220dca4462SChristoph Hellwig 3230dca4462SChristoph Hellwig if (!ops->rw_page || bdev_get_integrity(bdev)) 3240dca4462SChristoph Hellwig return result; 3250dca4462SChristoph Hellwig 326025a3865SPavel Begunkov result = blk_queue_enter(bdev_get_queue(bdev), 0); 3270dca4462SChristoph Hellwig if (result) 3280dca4462SChristoph Hellwig return result; 3290dca4462SChristoph Hellwig result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, 3300dca4462SChristoph Hellwig REQ_OP_READ); 331025a3865SPavel Begunkov blk_queue_exit(bdev_get_queue(bdev)); 3320dca4462SChristoph Hellwig return result; 3330dca4462SChristoph Hellwig } 3340dca4462SChristoph Hellwig 3350dca4462SChristoph Hellwig /** 3360dca4462SChristoph Hellwig * bdev_write_page() - Start writing a page to a block device 3370dca4462SChristoph Hellwig * @bdev: The device to write the page to 3380dca4462SChristoph Hellwig * @sector: The offset on the device to write the page to (need not be aligned) 3390dca4462SChristoph Hellwig * @page: The page to write 3400dca4462SChristoph Hellwig * @wbc: The writeback_control for the write 3410dca4462SChristoph Hellwig * 3420dca4462SChristoph Hellwig * On entry, the page should be locked and not currently under writeback. 3430dca4462SChristoph Hellwig * On exit, if the write started successfully, the page will be unlocked and 3440dca4462SChristoph Hellwig * under writeback. If the write failed already (eg the driver failed to 3450dca4462SChristoph Hellwig * queue the page to the device), the page will still be locked. If the 3460dca4462SChristoph Hellwig * caller is a ->writepage implementation, it will need to unlock the page. 3470dca4462SChristoph Hellwig * 3480dca4462SChristoph Hellwig * Errors returned by this function are usually "soft", eg out of memory, or 3490dca4462SChristoph Hellwig * queue full; callers should try a different route to write this page rather 3500dca4462SChristoph Hellwig * than propagate an error back up the stack. 3510dca4462SChristoph Hellwig * 3520dca4462SChristoph Hellwig * Return: negative errno if an error occurs, 0 if submission was successful. 3530dca4462SChristoph Hellwig */ 3540dca4462SChristoph Hellwig int bdev_write_page(struct block_device *bdev, sector_t sector, 3550dca4462SChristoph Hellwig struct page *page, struct writeback_control *wbc) 3560dca4462SChristoph Hellwig { 3570dca4462SChristoph Hellwig int result; 3580dca4462SChristoph Hellwig const struct block_device_operations *ops = bdev->bd_disk->fops; 3590dca4462SChristoph Hellwig 3600dca4462SChristoph Hellwig if (!ops->rw_page || bdev_get_integrity(bdev)) 3610dca4462SChristoph Hellwig return -EOPNOTSUPP; 362025a3865SPavel Begunkov result = blk_queue_enter(bdev_get_queue(bdev), 0); 3630dca4462SChristoph Hellwig if (result) 3640dca4462SChristoph Hellwig return result; 3650dca4462SChristoph Hellwig 3660dca4462SChristoph Hellwig set_page_writeback(page); 3670dca4462SChristoph Hellwig result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, 3680dca4462SChristoph Hellwig REQ_OP_WRITE); 3690dca4462SChristoph Hellwig if (result) { 3700dca4462SChristoph Hellwig end_page_writeback(page); 3710dca4462SChristoph Hellwig } else { 3720dca4462SChristoph Hellwig clean_page_buffers(page); 3730dca4462SChristoph Hellwig unlock_page(page); 3740dca4462SChristoph Hellwig } 375025a3865SPavel Begunkov blk_queue_exit(bdev_get_queue(bdev)); 3760dca4462SChristoph Hellwig return result; 3770dca4462SChristoph Hellwig } 3780dca4462SChristoph Hellwig 3790dca4462SChristoph Hellwig /* 3800dca4462SChristoph Hellwig * pseudo-fs 3810dca4462SChristoph Hellwig */ 3820dca4462SChristoph Hellwig 3830dca4462SChristoph Hellwig static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock); 3840dca4462SChristoph Hellwig static struct kmem_cache * bdev_cachep __read_mostly; 3850dca4462SChristoph Hellwig 3860dca4462SChristoph Hellwig static struct inode *bdev_alloc_inode(struct super_block *sb) 3870dca4462SChristoph Hellwig { 388fd60b288SMuchun Song struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL); 3890dca4462SChristoph Hellwig 3900dca4462SChristoph Hellwig if (!ei) 3910dca4462SChristoph Hellwig return NULL; 3920dca4462SChristoph Hellwig memset(&ei->bdev, 0, sizeof(ei->bdev)); 3930dca4462SChristoph Hellwig return &ei->vfs_inode; 3940dca4462SChristoph Hellwig } 3950dca4462SChristoph Hellwig 3960dca4462SChristoph Hellwig static void bdev_free_inode(struct inode *inode) 3970dca4462SChristoph Hellwig { 3980dca4462SChristoph Hellwig struct block_device *bdev = I_BDEV(inode); 3990dca4462SChristoph Hellwig 4000dca4462SChristoph Hellwig free_percpu(bdev->bd_stats); 4010dca4462SChristoph Hellwig kfree(bdev->bd_meta_info); 4020dca4462SChristoph Hellwig 4030dca4462SChristoph Hellwig if (!bdev_is_partition(bdev)) { 4040dca4462SChristoph Hellwig if (bdev->bd_disk && bdev->bd_disk->bdi) 4050dca4462SChristoph Hellwig bdi_put(bdev->bd_disk->bdi); 4060dca4462SChristoph Hellwig kfree(bdev->bd_disk); 4070dca4462SChristoph Hellwig } 4080dca4462SChristoph Hellwig 4090dca4462SChristoph Hellwig if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR) 4100dca4462SChristoph Hellwig blk_free_ext_minor(MINOR(bdev->bd_dev)); 4110dca4462SChristoph Hellwig 4120dca4462SChristoph Hellwig kmem_cache_free(bdev_cachep, BDEV_I(inode)); 4130dca4462SChristoph Hellwig } 4140dca4462SChristoph Hellwig 4150dca4462SChristoph Hellwig static void init_once(void *data) 4160dca4462SChristoph Hellwig { 4170dca4462SChristoph Hellwig struct bdev_inode *ei = data; 4180dca4462SChristoph Hellwig 4190dca4462SChristoph Hellwig inode_init_once(&ei->vfs_inode); 4200dca4462SChristoph Hellwig } 4210dca4462SChristoph Hellwig 4220dca4462SChristoph Hellwig static void bdev_evict_inode(struct inode *inode) 4230dca4462SChristoph Hellwig { 4240dca4462SChristoph Hellwig truncate_inode_pages_final(&inode->i_data); 4250dca4462SChristoph Hellwig invalidate_inode_buffers(inode); /* is it needed here? */ 4260dca4462SChristoph Hellwig clear_inode(inode); 4270dca4462SChristoph Hellwig } 4280dca4462SChristoph Hellwig 4290dca4462SChristoph Hellwig static const struct super_operations bdev_sops = { 4300dca4462SChristoph Hellwig .statfs = simple_statfs, 4310dca4462SChristoph Hellwig .alloc_inode = bdev_alloc_inode, 4320dca4462SChristoph Hellwig .free_inode = bdev_free_inode, 4330dca4462SChristoph Hellwig .drop_inode = generic_delete_inode, 4340dca4462SChristoph Hellwig .evict_inode = bdev_evict_inode, 4350dca4462SChristoph Hellwig }; 4360dca4462SChristoph Hellwig 4370dca4462SChristoph Hellwig static int bd_init_fs_context(struct fs_context *fc) 4380dca4462SChristoph Hellwig { 4390dca4462SChristoph Hellwig struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC); 4400dca4462SChristoph Hellwig if (!ctx) 4410dca4462SChristoph Hellwig return -ENOMEM; 4420dca4462SChristoph Hellwig fc->s_iflags |= SB_I_CGROUPWB; 4430dca4462SChristoph Hellwig ctx->ops = &bdev_sops; 4440dca4462SChristoph Hellwig return 0; 4450dca4462SChristoph Hellwig } 4460dca4462SChristoph Hellwig 4470dca4462SChristoph Hellwig static struct file_system_type bd_type = { 4480dca4462SChristoph Hellwig .name = "bdev", 4490dca4462SChristoph Hellwig .init_fs_context = bd_init_fs_context, 4500dca4462SChristoph Hellwig .kill_sb = kill_anon_super, 4510dca4462SChristoph Hellwig }; 4520dca4462SChristoph Hellwig 4530dca4462SChristoph Hellwig struct super_block *blockdev_superblock __read_mostly; 4540dca4462SChristoph Hellwig EXPORT_SYMBOL_GPL(blockdev_superblock); 4550dca4462SChristoph Hellwig 4560dca4462SChristoph Hellwig void __init bdev_cache_init(void) 4570dca4462SChristoph Hellwig { 4580dca4462SChristoph Hellwig int err; 4590dca4462SChristoph Hellwig static struct vfsmount *bd_mnt; 4600dca4462SChristoph Hellwig 4610dca4462SChristoph Hellwig bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode), 4620dca4462SChristoph Hellwig 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT| 4630dca4462SChristoph Hellwig SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC), 4640dca4462SChristoph Hellwig init_once); 4650dca4462SChristoph Hellwig err = register_filesystem(&bd_type); 4660dca4462SChristoph Hellwig if (err) 4670dca4462SChristoph Hellwig panic("Cannot register bdev pseudo-fs"); 4680dca4462SChristoph Hellwig bd_mnt = kern_mount(&bd_type); 4690dca4462SChristoph Hellwig if (IS_ERR(bd_mnt)) 4700dca4462SChristoph Hellwig panic("Cannot create bdev pseudo-fs"); 4710dca4462SChristoph Hellwig blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */ 4720dca4462SChristoph Hellwig } 4730dca4462SChristoph Hellwig 4740dca4462SChristoph Hellwig struct block_device *bdev_alloc(struct gendisk *disk, u8 partno) 4750dca4462SChristoph Hellwig { 4760dca4462SChristoph Hellwig struct block_device *bdev; 4770dca4462SChristoph Hellwig struct inode *inode; 4780dca4462SChristoph Hellwig 4790dca4462SChristoph Hellwig inode = new_inode(blockdev_superblock); 4800dca4462SChristoph Hellwig if (!inode) 4810dca4462SChristoph Hellwig return NULL; 4820dca4462SChristoph Hellwig inode->i_mode = S_IFBLK; 4830dca4462SChristoph Hellwig inode->i_rdev = 0; 4840dca4462SChristoph Hellwig inode->i_data.a_ops = &def_blk_aops; 4850dca4462SChristoph Hellwig mapping_set_gfp_mask(&inode->i_data, GFP_USER); 4860dca4462SChristoph Hellwig 4870dca4462SChristoph Hellwig bdev = I_BDEV(inode); 4880dca4462SChristoph Hellwig mutex_init(&bdev->bd_fsfreeze_mutex); 4890dca4462SChristoph Hellwig spin_lock_init(&bdev->bd_size_lock); 4900dca4462SChristoph Hellwig bdev->bd_partno = partno; 4910dca4462SChristoph Hellwig bdev->bd_inode = inode; 49217220ca5SPavel Begunkov bdev->bd_queue = disk->queue; 4930dca4462SChristoph Hellwig bdev->bd_stats = alloc_percpu(struct disk_stats); 4940dca4462SChristoph Hellwig if (!bdev->bd_stats) { 4950dca4462SChristoph Hellwig iput(inode); 4960dca4462SChristoph Hellwig return NULL; 4970dca4462SChristoph Hellwig } 49806cc978dSTetsuo Handa bdev->bd_disk = disk; 4990dca4462SChristoph Hellwig return bdev; 5000dca4462SChristoph Hellwig } 5010dca4462SChristoph Hellwig 5020dca4462SChristoph Hellwig void bdev_add(struct block_device *bdev, dev_t dev) 5030dca4462SChristoph Hellwig { 5040dca4462SChristoph Hellwig bdev->bd_dev = dev; 5050dca4462SChristoph Hellwig bdev->bd_inode->i_rdev = dev; 5060dca4462SChristoph Hellwig bdev->bd_inode->i_ino = dev; 5070dca4462SChristoph Hellwig insert_inode_hash(bdev->bd_inode); 5080dca4462SChristoph Hellwig } 5090dca4462SChristoph Hellwig 5100dca4462SChristoph Hellwig long nr_blockdev_pages(void) 5110dca4462SChristoph Hellwig { 5120dca4462SChristoph Hellwig struct inode *inode; 5130dca4462SChristoph Hellwig long ret = 0; 5140dca4462SChristoph Hellwig 5150dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 5160dca4462SChristoph Hellwig list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) 5170dca4462SChristoph Hellwig ret += inode->i_mapping->nrpages; 5180dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 5190dca4462SChristoph Hellwig 5200dca4462SChristoph Hellwig return ret; 5210dca4462SChristoph Hellwig } 5220dca4462SChristoph Hellwig 5230dca4462SChristoph Hellwig /** 5240dca4462SChristoph Hellwig * bd_may_claim - test whether a block device can be claimed 5250dca4462SChristoph Hellwig * @bdev: block device of interest 5260dca4462SChristoph Hellwig * @whole: whole block device containing @bdev, may equal @bdev 5270dca4462SChristoph Hellwig * @holder: holder trying to claim @bdev 5280dca4462SChristoph Hellwig * 5290dca4462SChristoph Hellwig * Test whether @bdev can be claimed by @holder. 5300dca4462SChristoph Hellwig * 5310dca4462SChristoph Hellwig * CONTEXT: 5320dca4462SChristoph Hellwig * spin_lock(&bdev_lock). 5330dca4462SChristoph Hellwig * 5340dca4462SChristoph Hellwig * RETURNS: 5350dca4462SChristoph Hellwig * %true if @bdev can be claimed, %false otherwise. 5360dca4462SChristoph Hellwig */ 5370dca4462SChristoph Hellwig static bool bd_may_claim(struct block_device *bdev, struct block_device *whole, 5380dca4462SChristoph Hellwig void *holder) 5390dca4462SChristoph Hellwig { 5400dca4462SChristoph Hellwig if (bdev->bd_holder == holder) 5410dca4462SChristoph Hellwig return true; /* already a holder */ 5420dca4462SChristoph Hellwig else if (bdev->bd_holder != NULL) 5430dca4462SChristoph Hellwig return false; /* held by someone else */ 5440dca4462SChristoph Hellwig else if (whole == bdev) 5450dca4462SChristoph Hellwig return true; /* is a whole device which isn't held */ 5460dca4462SChristoph Hellwig 5470dca4462SChristoph Hellwig else if (whole->bd_holder == bd_may_claim) 5480dca4462SChristoph Hellwig return true; /* is a partition of a device that is being partitioned */ 5490dca4462SChristoph Hellwig else if (whole->bd_holder != NULL) 5500dca4462SChristoph Hellwig return false; /* is a partition of a held device */ 5510dca4462SChristoph Hellwig else 5520dca4462SChristoph Hellwig return true; /* is a partition of an un-held device */ 5530dca4462SChristoph Hellwig } 5540dca4462SChristoph Hellwig 5550dca4462SChristoph Hellwig /** 5560dca4462SChristoph Hellwig * bd_prepare_to_claim - claim a block device 5570dca4462SChristoph Hellwig * @bdev: block device of interest 5580dca4462SChristoph Hellwig * @holder: holder trying to claim @bdev 5590dca4462SChristoph Hellwig * 5600dca4462SChristoph Hellwig * Claim @bdev. This function fails if @bdev is already claimed by another 5610dca4462SChristoph Hellwig * holder and waits if another claiming is in progress. return, the caller 5620dca4462SChristoph Hellwig * has ownership of bd_claiming and bd_holder[s]. 5630dca4462SChristoph Hellwig * 5640dca4462SChristoph Hellwig * RETURNS: 5650dca4462SChristoph Hellwig * 0 if @bdev can be claimed, -EBUSY otherwise. 5660dca4462SChristoph Hellwig */ 5670dca4462SChristoph Hellwig int bd_prepare_to_claim(struct block_device *bdev, void *holder) 5680dca4462SChristoph Hellwig { 5690dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 5700dca4462SChristoph Hellwig 5710dca4462SChristoph Hellwig if (WARN_ON_ONCE(!holder)) 5720dca4462SChristoph Hellwig return -EINVAL; 5730dca4462SChristoph Hellwig retry: 5740dca4462SChristoph Hellwig spin_lock(&bdev_lock); 5750dca4462SChristoph Hellwig /* if someone else claimed, fail */ 5760dca4462SChristoph Hellwig if (!bd_may_claim(bdev, whole, holder)) { 5770dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 5780dca4462SChristoph Hellwig return -EBUSY; 5790dca4462SChristoph Hellwig } 5800dca4462SChristoph Hellwig 5810dca4462SChristoph Hellwig /* if claiming is already in progress, wait for it to finish */ 5820dca4462SChristoph Hellwig if (whole->bd_claiming) { 5830dca4462SChristoph Hellwig wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0); 5840dca4462SChristoph Hellwig DEFINE_WAIT(wait); 5850dca4462SChristoph Hellwig 5860dca4462SChristoph Hellwig prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE); 5870dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 5880dca4462SChristoph Hellwig schedule(); 5890dca4462SChristoph Hellwig finish_wait(wq, &wait); 5900dca4462SChristoph Hellwig goto retry; 5910dca4462SChristoph Hellwig } 5920dca4462SChristoph Hellwig 5930dca4462SChristoph Hellwig /* yay, all mine */ 5940dca4462SChristoph Hellwig whole->bd_claiming = holder; 5950dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 5960dca4462SChristoph Hellwig return 0; 5970dca4462SChristoph Hellwig } 5980dca4462SChristoph Hellwig EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */ 5990dca4462SChristoph Hellwig 6000dca4462SChristoph Hellwig static void bd_clear_claiming(struct block_device *whole, void *holder) 6010dca4462SChristoph Hellwig { 6020dca4462SChristoph Hellwig lockdep_assert_held(&bdev_lock); 6030dca4462SChristoph Hellwig /* tell others that we're done */ 6040dca4462SChristoph Hellwig BUG_ON(whole->bd_claiming != holder); 6050dca4462SChristoph Hellwig whole->bd_claiming = NULL; 6060dca4462SChristoph Hellwig wake_up_bit(&whole->bd_claiming, 0); 6070dca4462SChristoph Hellwig } 6080dca4462SChristoph Hellwig 6090dca4462SChristoph Hellwig /** 6100dca4462SChristoph Hellwig * bd_finish_claiming - finish claiming of a block device 6110dca4462SChristoph Hellwig * @bdev: block device of interest 6120dca4462SChristoph Hellwig * @holder: holder that has claimed @bdev 6130dca4462SChristoph Hellwig * 6140dca4462SChristoph Hellwig * Finish exclusive open of a block device. Mark the device as exlusively 6150dca4462SChristoph Hellwig * open by the holder and wake up all waiters for exclusive open to finish. 6160dca4462SChristoph Hellwig */ 6170dca4462SChristoph Hellwig static void bd_finish_claiming(struct block_device *bdev, void *holder) 6180dca4462SChristoph Hellwig { 6190dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 6200dca4462SChristoph Hellwig 6210dca4462SChristoph Hellwig spin_lock(&bdev_lock); 6220dca4462SChristoph Hellwig BUG_ON(!bd_may_claim(bdev, whole, holder)); 6230dca4462SChristoph Hellwig /* 6240dca4462SChristoph Hellwig * Note that for a whole device bd_holders will be incremented twice, 6250dca4462SChristoph Hellwig * and bd_holder will be set to bd_may_claim before being set to holder 6260dca4462SChristoph Hellwig */ 6270dca4462SChristoph Hellwig whole->bd_holders++; 6280dca4462SChristoph Hellwig whole->bd_holder = bd_may_claim; 6290dca4462SChristoph Hellwig bdev->bd_holders++; 6300dca4462SChristoph Hellwig bdev->bd_holder = holder; 6310dca4462SChristoph Hellwig bd_clear_claiming(whole, holder); 6320dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 6330dca4462SChristoph Hellwig } 6340dca4462SChristoph Hellwig 6350dca4462SChristoph Hellwig /** 6360dca4462SChristoph Hellwig * bd_abort_claiming - abort claiming of a block device 6370dca4462SChristoph Hellwig * @bdev: block device of interest 6380dca4462SChristoph Hellwig * @holder: holder that has claimed @bdev 6390dca4462SChristoph Hellwig * 6400dca4462SChristoph Hellwig * Abort claiming of a block device when the exclusive open failed. This can be 6410dca4462SChristoph Hellwig * also used when exclusive open is not actually desired and we just needed 6420dca4462SChristoph Hellwig * to block other exclusive openers for a while. 6430dca4462SChristoph Hellwig */ 6440dca4462SChristoph Hellwig void bd_abort_claiming(struct block_device *bdev, void *holder) 6450dca4462SChristoph Hellwig { 6460dca4462SChristoph Hellwig spin_lock(&bdev_lock); 6470dca4462SChristoph Hellwig bd_clear_claiming(bdev_whole(bdev), holder); 6480dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 6490dca4462SChristoph Hellwig } 6500dca4462SChristoph Hellwig EXPORT_SYMBOL(bd_abort_claiming); 6510dca4462SChristoph Hellwig 6520dca4462SChristoph Hellwig static void blkdev_flush_mapping(struct block_device *bdev) 6530dca4462SChristoph Hellwig { 6540dca4462SChristoph Hellwig WARN_ON_ONCE(bdev->bd_holders); 6550dca4462SChristoph Hellwig sync_blockdev(bdev); 6560dca4462SChristoph Hellwig kill_bdev(bdev); 6570dca4462SChristoph Hellwig bdev_write_inode(bdev); 6580dca4462SChristoph Hellwig } 6590dca4462SChristoph Hellwig 6600dca4462SChristoph Hellwig static int blkdev_get_whole(struct block_device *bdev, fmode_t mode) 6610dca4462SChristoph Hellwig { 6620dca4462SChristoph Hellwig struct gendisk *disk = bdev->bd_disk; 663af22fef3SColin Ian King int ret; 6640dca4462SChristoph Hellwig 6650dca4462SChristoph Hellwig if (disk->fops->open) { 6660dca4462SChristoph Hellwig ret = disk->fops->open(bdev, mode); 6670dca4462SChristoph Hellwig if (ret) { 6680dca4462SChristoph Hellwig /* avoid ghost partitions on a removed medium */ 6690dca4462SChristoph Hellwig if (ret == -ENOMEDIUM && 6700dca4462SChristoph Hellwig test_bit(GD_NEED_PART_SCAN, &disk->state)) 6710dca4462SChristoph Hellwig bdev_disk_changed(disk, true); 6720dca4462SChristoph Hellwig return ret; 6730dca4462SChristoph Hellwig } 6740dca4462SChristoph Hellwig } 6750dca4462SChristoph Hellwig 676*9acf381fSChristoph Hellwig if (!atomic_read(&bdev->bd_openers)) 6770dca4462SChristoph Hellwig set_init_blocksize(bdev); 6780dca4462SChristoph Hellwig if (test_bit(GD_NEED_PART_SCAN, &disk->state)) 6790dca4462SChristoph Hellwig bdev_disk_changed(disk, false); 680*9acf381fSChristoph Hellwig atomic_inc(&bdev->bd_openers); 681483546c1SNian Yanchuan return 0; 6820dca4462SChristoph Hellwig } 6830dca4462SChristoph Hellwig 6840dca4462SChristoph Hellwig static void blkdev_put_whole(struct block_device *bdev, fmode_t mode) 6850dca4462SChristoph Hellwig { 686*9acf381fSChristoph Hellwig if (atomic_dec_and_test(&bdev->bd_openers)) 6870dca4462SChristoph Hellwig blkdev_flush_mapping(bdev); 6880dca4462SChristoph Hellwig if (bdev->bd_disk->fops->release) 6890dca4462SChristoph Hellwig bdev->bd_disk->fops->release(bdev->bd_disk, mode); 6900dca4462SChristoph Hellwig } 6910dca4462SChristoph Hellwig 6920dca4462SChristoph Hellwig static int blkdev_get_part(struct block_device *part, fmode_t mode) 6930dca4462SChristoph Hellwig { 6940dca4462SChristoph Hellwig struct gendisk *disk = part->bd_disk; 6950dca4462SChristoph Hellwig int ret; 6960dca4462SChristoph Hellwig 697*9acf381fSChristoph Hellwig if (atomic_read(&part->bd_openers)) 6980dca4462SChristoph Hellwig goto done; 6990dca4462SChristoph Hellwig 7000dca4462SChristoph Hellwig ret = blkdev_get_whole(bdev_whole(part), mode); 7010dca4462SChristoph Hellwig if (ret) 7020dca4462SChristoph Hellwig return ret; 7030dca4462SChristoph Hellwig 7040dca4462SChristoph Hellwig ret = -ENXIO; 7050dca4462SChristoph Hellwig if (!bdev_nr_sectors(part)) 7060dca4462SChristoph Hellwig goto out_blkdev_put; 7070dca4462SChristoph Hellwig 7080dca4462SChristoph Hellwig disk->open_partitions++; 7090dca4462SChristoph Hellwig set_init_blocksize(part); 7100dca4462SChristoph Hellwig done: 711*9acf381fSChristoph Hellwig atomic_inc(&part->bd_openers); 7120dca4462SChristoph Hellwig return 0; 7130dca4462SChristoph Hellwig 7140dca4462SChristoph Hellwig out_blkdev_put: 7150dca4462SChristoph Hellwig blkdev_put_whole(bdev_whole(part), mode); 7160dca4462SChristoph Hellwig return ret; 7170dca4462SChristoph Hellwig } 7180dca4462SChristoph Hellwig 7190dca4462SChristoph Hellwig static void blkdev_put_part(struct block_device *part, fmode_t mode) 7200dca4462SChristoph Hellwig { 7210dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(part); 7220dca4462SChristoph Hellwig 723*9acf381fSChristoph Hellwig if (!atomic_dec_and_test(&part->bd_openers)) 7240dca4462SChristoph Hellwig return; 7250dca4462SChristoph Hellwig blkdev_flush_mapping(part); 7260dca4462SChristoph Hellwig whole->bd_disk->open_partitions--; 7270dca4462SChristoph Hellwig blkdev_put_whole(whole, mode); 7280dca4462SChristoph Hellwig } 7290dca4462SChristoph Hellwig 7300dca4462SChristoph Hellwig struct block_device *blkdev_get_no_open(dev_t dev) 7310dca4462SChristoph Hellwig { 7320dca4462SChristoph Hellwig struct block_device *bdev; 7330dca4462SChristoph Hellwig struct inode *inode; 7340dca4462SChristoph Hellwig 7350dca4462SChristoph Hellwig inode = ilookup(blockdev_superblock, dev); 736fbdee71bSChristoph Hellwig if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) { 7370dca4462SChristoph Hellwig blk_request_module(dev); 7380dca4462SChristoph Hellwig inode = ilookup(blockdev_superblock, dev); 739fbdee71bSChristoph Hellwig if (inode) 740fbdee71bSChristoph Hellwig pr_warn_ratelimited( 741451f0b6fSChristoph Hellwig "block device autoloading is deprecated and will be removed.\n"); 742fbdee71bSChristoph Hellwig } 7430dca4462SChristoph Hellwig if (!inode) 7440dca4462SChristoph Hellwig return NULL; 7450dca4462SChristoph Hellwig 7460dca4462SChristoph Hellwig /* switch from the inode reference to a device mode one: */ 7470dca4462SChristoph Hellwig bdev = &BDEV_I(inode)->bdev; 7480dca4462SChristoph Hellwig if (!kobject_get_unless_zero(&bdev->bd_device.kobj)) 7490dca4462SChristoph Hellwig bdev = NULL; 7500dca4462SChristoph Hellwig iput(inode); 7510dca4462SChristoph Hellwig return bdev; 7520dca4462SChristoph Hellwig } 7530dca4462SChristoph Hellwig 7540dca4462SChristoph Hellwig void blkdev_put_no_open(struct block_device *bdev) 7550dca4462SChristoph Hellwig { 7560dca4462SChristoph Hellwig put_device(&bdev->bd_device); 7570dca4462SChristoph Hellwig } 7580dca4462SChristoph Hellwig 7590dca4462SChristoph Hellwig /** 7600dca4462SChristoph Hellwig * blkdev_get_by_dev - open a block device by device number 7610dca4462SChristoph Hellwig * @dev: device number of block device to open 7620dca4462SChristoph Hellwig * @mode: FMODE_* mask 7630dca4462SChristoph Hellwig * @holder: exclusive holder identifier 7640dca4462SChristoph Hellwig * 7650dca4462SChristoph Hellwig * Open the block device described by device number @dev. If @mode includes 7660dca4462SChristoph Hellwig * %FMODE_EXCL, the block device is opened with exclusive access. Specifying 7670dca4462SChristoph Hellwig * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for 7680dca4462SChristoph Hellwig * the same @holder. 7690dca4462SChristoph Hellwig * 7700dca4462SChristoph Hellwig * Use this interface ONLY if you really do not have anything better - i.e. when 7710dca4462SChristoph Hellwig * you are behind a truly sucky interface and all you are given is a device 7720dca4462SChristoph Hellwig * number. Everything else should use blkdev_get_by_path(). 7730dca4462SChristoph Hellwig * 7740dca4462SChristoph Hellwig * CONTEXT: 7750dca4462SChristoph Hellwig * Might sleep. 7760dca4462SChristoph Hellwig * 7770dca4462SChristoph Hellwig * RETURNS: 7780dca4462SChristoph Hellwig * Reference to the block_device on success, ERR_PTR(-errno) on failure. 7790dca4462SChristoph Hellwig */ 7800dca4462SChristoph Hellwig struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder) 7810dca4462SChristoph Hellwig { 7820dca4462SChristoph Hellwig bool unblock_events = true; 7830dca4462SChristoph Hellwig struct block_device *bdev; 7840dca4462SChristoph Hellwig struct gendisk *disk; 7850dca4462SChristoph Hellwig int ret; 7860dca4462SChristoph Hellwig 7870dca4462SChristoph Hellwig ret = devcgroup_check_permission(DEVCG_DEV_BLOCK, 7880dca4462SChristoph Hellwig MAJOR(dev), MINOR(dev), 7890dca4462SChristoph Hellwig ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) | 7900dca4462SChristoph Hellwig ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0)); 7910dca4462SChristoph Hellwig if (ret) 7920dca4462SChristoph Hellwig return ERR_PTR(ret); 7930dca4462SChristoph Hellwig 7940dca4462SChristoph Hellwig bdev = blkdev_get_no_open(dev); 7950dca4462SChristoph Hellwig if (!bdev) 7960dca4462SChristoph Hellwig return ERR_PTR(-ENXIO); 7970dca4462SChristoph Hellwig disk = bdev->bd_disk; 7980dca4462SChristoph Hellwig 7990dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 8000dca4462SChristoph Hellwig ret = bd_prepare_to_claim(bdev, holder); 8010dca4462SChristoph Hellwig if (ret) 8020dca4462SChristoph Hellwig goto put_blkdev; 8030dca4462SChristoph Hellwig } 8040dca4462SChristoph Hellwig 8050dca4462SChristoph Hellwig disk_block_events(disk); 8060dca4462SChristoph Hellwig 8070dca4462SChristoph Hellwig mutex_lock(&disk->open_mutex); 8080dca4462SChristoph Hellwig ret = -ENXIO; 8090dca4462SChristoph Hellwig if (!disk_live(disk)) 8100dca4462SChristoph Hellwig goto abort_claiming; 811efcf5932SMing Lei if (!try_module_get(disk->fops->owner)) 812efcf5932SMing Lei goto abort_claiming; 8130dca4462SChristoph Hellwig if (bdev_is_partition(bdev)) 8140dca4462SChristoph Hellwig ret = blkdev_get_part(bdev, mode); 8150dca4462SChristoph Hellwig else 8160dca4462SChristoph Hellwig ret = blkdev_get_whole(bdev, mode); 8170dca4462SChristoph Hellwig if (ret) 818efcf5932SMing Lei goto put_module; 8190dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 8200dca4462SChristoph Hellwig bd_finish_claiming(bdev, holder); 8210dca4462SChristoph Hellwig 8220dca4462SChristoph Hellwig /* 8230dca4462SChristoph Hellwig * Block event polling for write claims if requested. Any write 8240dca4462SChristoph Hellwig * holder makes the write_holder state stick until all are 8250dca4462SChristoph Hellwig * released. This is good enough and tracking individual 8260dca4462SChristoph Hellwig * writeable reference is too fragile given the way @mode is 8270dca4462SChristoph Hellwig * used in blkdev_get/put(). 8280dca4462SChristoph Hellwig */ 8290dca4462SChristoph Hellwig if ((mode & FMODE_WRITE) && !bdev->bd_write_holder && 8301545e0b4SChristoph Hellwig (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) { 8310dca4462SChristoph Hellwig bdev->bd_write_holder = true; 8320dca4462SChristoph Hellwig unblock_events = false; 8330dca4462SChristoph Hellwig } 8340dca4462SChristoph Hellwig } 8350dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 8360dca4462SChristoph Hellwig 8370dca4462SChristoph Hellwig if (unblock_events) 8380dca4462SChristoph Hellwig disk_unblock_events(disk); 8390dca4462SChristoph Hellwig return bdev; 840efcf5932SMing Lei put_module: 841efcf5932SMing Lei module_put(disk->fops->owner); 8420dca4462SChristoph Hellwig abort_claiming: 8430dca4462SChristoph Hellwig if (mode & FMODE_EXCL) 8440dca4462SChristoph Hellwig bd_abort_claiming(bdev, holder); 8450dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 8460dca4462SChristoph Hellwig disk_unblock_events(disk); 8470dca4462SChristoph Hellwig put_blkdev: 8480dca4462SChristoph Hellwig blkdev_put_no_open(bdev); 8490dca4462SChristoph Hellwig return ERR_PTR(ret); 8500dca4462SChristoph Hellwig } 8510dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_get_by_dev); 8520dca4462SChristoph Hellwig 8530dca4462SChristoph Hellwig /** 8540dca4462SChristoph Hellwig * blkdev_get_by_path - open a block device by name 8550dca4462SChristoph Hellwig * @path: path to the block device to open 8560dca4462SChristoph Hellwig * @mode: FMODE_* mask 8570dca4462SChristoph Hellwig * @holder: exclusive holder identifier 8580dca4462SChristoph Hellwig * 8590dca4462SChristoph Hellwig * Open the block device described by the device file at @path. If @mode 8600dca4462SChristoph Hellwig * includes %FMODE_EXCL, the block device is opened with exclusive access. 8610dca4462SChristoph Hellwig * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may 8620dca4462SChristoph Hellwig * nest for the same @holder. 8630dca4462SChristoph Hellwig * 8640dca4462SChristoph Hellwig * CONTEXT: 8650dca4462SChristoph Hellwig * Might sleep. 8660dca4462SChristoph Hellwig * 8670dca4462SChristoph Hellwig * RETURNS: 8680dca4462SChristoph Hellwig * Reference to the block_device on success, ERR_PTR(-errno) on failure. 8690dca4462SChristoph Hellwig */ 8700dca4462SChristoph Hellwig struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, 8710dca4462SChristoph Hellwig void *holder) 8720dca4462SChristoph Hellwig { 8730dca4462SChristoph Hellwig struct block_device *bdev; 8740dca4462SChristoph Hellwig dev_t dev; 8750dca4462SChristoph Hellwig int error; 8760dca4462SChristoph Hellwig 8770dca4462SChristoph Hellwig error = lookup_bdev(path, &dev); 8780dca4462SChristoph Hellwig if (error) 8790dca4462SChristoph Hellwig return ERR_PTR(error); 8800dca4462SChristoph Hellwig 8810dca4462SChristoph Hellwig bdev = blkdev_get_by_dev(dev, mode, holder); 8820dca4462SChristoph Hellwig if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) { 8830dca4462SChristoph Hellwig blkdev_put(bdev, mode); 8840dca4462SChristoph Hellwig return ERR_PTR(-EACCES); 8850dca4462SChristoph Hellwig } 8860dca4462SChristoph Hellwig 8870dca4462SChristoph Hellwig return bdev; 8880dca4462SChristoph Hellwig } 8890dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_get_by_path); 8900dca4462SChristoph Hellwig 8910dca4462SChristoph Hellwig void blkdev_put(struct block_device *bdev, fmode_t mode) 8920dca4462SChristoph Hellwig { 8930dca4462SChristoph Hellwig struct gendisk *disk = bdev->bd_disk; 8940dca4462SChristoph Hellwig 8950dca4462SChristoph Hellwig /* 8960dca4462SChristoph Hellwig * Sync early if it looks like we're the last one. If someone else 8970dca4462SChristoph Hellwig * opens the block device between now and the decrement of bd_openers 8980dca4462SChristoph Hellwig * then we did a sync that we didn't need to, but that's not the end 8990dca4462SChristoph Hellwig * of the world and we want to avoid long (could be several minute) 9000dca4462SChristoph Hellwig * syncs while holding the mutex. 9010dca4462SChristoph Hellwig */ 902*9acf381fSChristoph Hellwig if (atomic_read(&bdev->bd_openers) == 1) 9030dca4462SChristoph Hellwig sync_blockdev(bdev); 9040dca4462SChristoph Hellwig 9050dca4462SChristoph Hellwig mutex_lock(&disk->open_mutex); 9060dca4462SChristoph Hellwig if (mode & FMODE_EXCL) { 9070dca4462SChristoph Hellwig struct block_device *whole = bdev_whole(bdev); 9080dca4462SChristoph Hellwig bool bdev_free; 9090dca4462SChristoph Hellwig 9100dca4462SChristoph Hellwig /* 9110dca4462SChristoph Hellwig * Release a claim on the device. The holder fields 9120dca4462SChristoph Hellwig * are protected with bdev_lock. open_mutex is to 9130dca4462SChristoph Hellwig * synchronize disk_holder unlinking. 9140dca4462SChristoph Hellwig */ 9150dca4462SChristoph Hellwig spin_lock(&bdev_lock); 9160dca4462SChristoph Hellwig 9170dca4462SChristoph Hellwig WARN_ON_ONCE(--bdev->bd_holders < 0); 9180dca4462SChristoph Hellwig WARN_ON_ONCE(--whole->bd_holders < 0); 9190dca4462SChristoph Hellwig 9200dca4462SChristoph Hellwig if ((bdev_free = !bdev->bd_holders)) 9210dca4462SChristoph Hellwig bdev->bd_holder = NULL; 9220dca4462SChristoph Hellwig if (!whole->bd_holders) 9230dca4462SChristoph Hellwig whole->bd_holder = NULL; 9240dca4462SChristoph Hellwig 9250dca4462SChristoph Hellwig spin_unlock(&bdev_lock); 9260dca4462SChristoph Hellwig 9270dca4462SChristoph Hellwig /* 9280dca4462SChristoph Hellwig * If this was the last claim, remove holder link and 9290dca4462SChristoph Hellwig * unblock evpoll if it was a write holder. 9300dca4462SChristoph Hellwig */ 9310dca4462SChristoph Hellwig if (bdev_free && bdev->bd_write_holder) { 9320dca4462SChristoph Hellwig disk_unblock_events(disk); 9330dca4462SChristoph Hellwig bdev->bd_write_holder = false; 9340dca4462SChristoph Hellwig } 9350dca4462SChristoph Hellwig } 9360dca4462SChristoph Hellwig 9370dca4462SChristoph Hellwig /* 9380dca4462SChristoph Hellwig * Trigger event checking and tell drivers to flush MEDIA_CHANGE 9390dca4462SChristoph Hellwig * event. This is to ensure detection of media removal commanded 9400dca4462SChristoph Hellwig * from userland - e.g. eject(1). 9410dca4462SChristoph Hellwig */ 9420dca4462SChristoph Hellwig disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE); 9430dca4462SChristoph Hellwig 9440dca4462SChristoph Hellwig if (bdev_is_partition(bdev)) 9450dca4462SChristoph Hellwig blkdev_put_part(bdev, mode); 9460dca4462SChristoph Hellwig else 9470dca4462SChristoph Hellwig blkdev_put_whole(bdev, mode); 9480dca4462SChristoph Hellwig mutex_unlock(&disk->open_mutex); 9490dca4462SChristoph Hellwig 950efcf5932SMing Lei module_put(disk->fops->owner); 9510dca4462SChristoph Hellwig blkdev_put_no_open(bdev); 9520dca4462SChristoph Hellwig } 9530dca4462SChristoph Hellwig EXPORT_SYMBOL(blkdev_put); 9540dca4462SChristoph Hellwig 9550dca4462SChristoph Hellwig /** 9560ba4566cSMatthew Wilcox (Oracle) * lookup_bdev() - Look up a struct block_device by name. 9570ba4566cSMatthew Wilcox (Oracle) * @pathname: Name of the block device in the filesystem. 9580ba4566cSMatthew Wilcox (Oracle) * @dev: Pointer to the block device's dev_t, if found. 9590dca4462SChristoph Hellwig * 960057178cfSJackie Liu * Lookup the block device's dev_t at @pathname in the current 9610ba4566cSMatthew Wilcox (Oracle) * namespace if possible and return it in @dev. 962057178cfSJackie Liu * 9630ba4566cSMatthew Wilcox (Oracle) * Context: May sleep. 9640ba4566cSMatthew Wilcox (Oracle) * Return: 0 if succeeded, negative errno otherwise. 9650dca4462SChristoph Hellwig */ 9660dca4462SChristoph Hellwig int lookup_bdev(const char *pathname, dev_t *dev) 9670dca4462SChristoph Hellwig { 9680dca4462SChristoph Hellwig struct inode *inode; 9690dca4462SChristoph Hellwig struct path path; 9700dca4462SChristoph Hellwig int error; 9710dca4462SChristoph Hellwig 9720dca4462SChristoph Hellwig if (!pathname || !*pathname) 9730dca4462SChristoph Hellwig return -EINVAL; 9740dca4462SChristoph Hellwig 9750dca4462SChristoph Hellwig error = kern_path(pathname, LOOKUP_FOLLOW, &path); 9760dca4462SChristoph Hellwig if (error) 9770dca4462SChristoph Hellwig return error; 9780dca4462SChristoph Hellwig 9790dca4462SChristoph Hellwig inode = d_backing_inode(path.dentry); 9800dca4462SChristoph Hellwig error = -ENOTBLK; 9810dca4462SChristoph Hellwig if (!S_ISBLK(inode->i_mode)) 9820dca4462SChristoph Hellwig goto out_path_put; 9830dca4462SChristoph Hellwig error = -EACCES; 9840dca4462SChristoph Hellwig if (!may_open_dev(&path)) 9850dca4462SChristoph Hellwig goto out_path_put; 9860dca4462SChristoph Hellwig 9870dca4462SChristoph Hellwig *dev = inode->i_rdev; 9880dca4462SChristoph Hellwig error = 0; 9890dca4462SChristoph Hellwig out_path_put: 9900dca4462SChristoph Hellwig path_put(&path); 9910dca4462SChristoph Hellwig return error; 9920dca4462SChristoph Hellwig } 9930dca4462SChristoph Hellwig EXPORT_SYMBOL(lookup_bdev); 9940dca4462SChristoph Hellwig 9950dca4462SChristoph Hellwig int __invalidate_device(struct block_device *bdev, bool kill_dirty) 9960dca4462SChristoph Hellwig { 9970dca4462SChristoph Hellwig struct super_block *sb = get_super(bdev); 9980dca4462SChristoph Hellwig int res = 0; 9990dca4462SChristoph Hellwig 10000dca4462SChristoph Hellwig if (sb) { 10010dca4462SChristoph Hellwig /* 10020dca4462SChristoph Hellwig * no need to lock the super, get_super holds the 10030dca4462SChristoph Hellwig * read mutex so the filesystem cannot go away 10040dca4462SChristoph Hellwig * under us (->put_super runs with the write lock 10050dca4462SChristoph Hellwig * hold). 10060dca4462SChristoph Hellwig */ 10070dca4462SChristoph Hellwig shrink_dcache_sb(sb); 10080dca4462SChristoph Hellwig res = invalidate_inodes(sb, kill_dirty); 10090dca4462SChristoph Hellwig drop_super(sb); 10100dca4462SChristoph Hellwig } 10110dca4462SChristoph Hellwig invalidate_bdev(bdev); 10120dca4462SChristoph Hellwig return res; 10130dca4462SChristoph Hellwig } 10140dca4462SChristoph Hellwig EXPORT_SYMBOL(__invalidate_device); 10150dca4462SChristoph Hellwig 10161e03a36bSChristoph Hellwig void sync_bdevs(bool wait) 10170dca4462SChristoph Hellwig { 10180dca4462SChristoph Hellwig struct inode *inode, *old_inode = NULL; 10190dca4462SChristoph Hellwig 10200dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 10210dca4462SChristoph Hellwig list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) { 10220dca4462SChristoph Hellwig struct address_space *mapping = inode->i_mapping; 10230dca4462SChristoph Hellwig struct block_device *bdev; 10240dca4462SChristoph Hellwig 10250dca4462SChristoph Hellwig spin_lock(&inode->i_lock); 10260dca4462SChristoph Hellwig if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) || 10270dca4462SChristoph Hellwig mapping->nrpages == 0) { 10280dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 10290dca4462SChristoph Hellwig continue; 10300dca4462SChristoph Hellwig } 10310dca4462SChristoph Hellwig __iget(inode); 10320dca4462SChristoph Hellwig spin_unlock(&inode->i_lock); 10330dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 10340dca4462SChristoph Hellwig /* 10350dca4462SChristoph Hellwig * We hold a reference to 'inode' so it couldn't have been 10360dca4462SChristoph Hellwig * removed from s_inodes list while we dropped the 10370dca4462SChristoph Hellwig * s_inode_list_lock We cannot iput the inode now as we can 10380dca4462SChristoph Hellwig * be holding the last reference and we cannot iput it under 10390dca4462SChristoph Hellwig * s_inode_list_lock. So we keep the reference and iput it 10400dca4462SChristoph Hellwig * later. 10410dca4462SChristoph Hellwig */ 10420dca4462SChristoph Hellwig iput(old_inode); 10430dca4462SChristoph Hellwig old_inode = inode; 10440dca4462SChristoph Hellwig bdev = I_BDEV(inode); 10450dca4462SChristoph Hellwig 10460dca4462SChristoph Hellwig mutex_lock(&bdev->bd_disk->open_mutex); 1047*9acf381fSChristoph Hellwig if (!atomic_read(&bdev->bd_openers)) { 10481e03a36bSChristoph Hellwig ; /* skip */ 10491e03a36bSChristoph Hellwig } else if (wait) { 10501e03a36bSChristoph Hellwig /* 10511e03a36bSChristoph Hellwig * We keep the error status of individual mapping so 10521e03a36bSChristoph Hellwig * that applications can catch the writeback error using 10531e03a36bSChristoph Hellwig * fsync(2). See filemap_fdatawait_keep_errors() for 10541e03a36bSChristoph Hellwig * details. 10551e03a36bSChristoph Hellwig */ 10561e03a36bSChristoph Hellwig filemap_fdatawait_keep_errors(inode->i_mapping); 10571e03a36bSChristoph Hellwig } else { 10581e03a36bSChristoph Hellwig filemap_fdatawrite(inode->i_mapping); 10591e03a36bSChristoph Hellwig } 10600dca4462SChristoph Hellwig mutex_unlock(&bdev->bd_disk->open_mutex); 10610dca4462SChristoph Hellwig 10620dca4462SChristoph Hellwig spin_lock(&blockdev_superblock->s_inode_list_lock); 10630dca4462SChristoph Hellwig } 10640dca4462SChristoph Hellwig spin_unlock(&blockdev_superblock->s_inode_list_lock); 10650dca4462SChristoph Hellwig iput(old_inode); 10660dca4462SChristoph Hellwig } 1067