11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/drivers/block/loop.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Written by Theodore Ts'o, 3/29/93 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Copyright 1993 by Theodore Ts'o. Redistribution of this file is 71da177e4SLinus Torvalds * permitted under the GNU General Public License. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993 101da177e4SLinus Torvalds * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994 131da177e4SLinus Torvalds * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997 161da177e4SLinus Torvalds * 171da177e4SLinus Torvalds * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998 201da177e4SLinus Torvalds * 211da177e4SLinus Torvalds * Loadable modules and other fixes by AK, 1998 221da177e4SLinus Torvalds * 231da177e4SLinus Torvalds * Make real block number available to downstream transfer functions, enables 241da177e4SLinus Torvalds * CBC (and relatives) mode encryption requiring unique IVs per data block. 251da177e4SLinus Torvalds * Reed H. Petty, rhp@draper.net 261da177e4SLinus Torvalds * 271da177e4SLinus Torvalds * Maximum number of loop devices now dynamic via max_loop module parameter. 281da177e4SLinus Torvalds * Russell Kroll <rkroll@exploits.org> 19990701 291da177e4SLinus Torvalds * 301da177e4SLinus Torvalds * Maximum number of loop devices when compiled-in now selectable by passing 311da177e4SLinus Torvalds * max_loop=<1-255> to the kernel on boot. 3296de0e25SJan Engelhardt * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999 331da177e4SLinus Torvalds * 341da177e4SLinus Torvalds * Completely rewrite request handling to be make_request_fn style and 351da177e4SLinus Torvalds * non blocking, pushing work to a helper thread. Lots of fixes from 361da177e4SLinus Torvalds * Al Viro too. 371da177e4SLinus Torvalds * Jens Axboe <axboe@suse.de>, Nov 2000 381da177e4SLinus Torvalds * 391da177e4SLinus Torvalds * Support up to 256 loop devices 401da177e4SLinus Torvalds * Heinz Mauelshagen <mge@sistina.com>, Feb 2002 411da177e4SLinus Torvalds * 421da177e4SLinus Torvalds * Support for falling back on the write file operation when the address space 434e02ed4bSNick Piggin * operations write_begin is not available on the backing filesystem. 441da177e4SLinus Torvalds * Anton Altaparmakov, 16 Feb 2005 451da177e4SLinus Torvalds * 461da177e4SLinus Torvalds * Still To Fix: 471da177e4SLinus Torvalds * - Advisory locking is ignored here. 481da177e4SLinus Torvalds * - Should use an own CAP_* category instead of CAP_SYS_ADMIN 491da177e4SLinus Torvalds * 501da177e4SLinus Torvalds */ 511da177e4SLinus Torvalds 521da177e4SLinus Torvalds #include <linux/module.h> 531da177e4SLinus Torvalds #include <linux/moduleparam.h> 541da177e4SLinus Torvalds #include <linux/sched.h> 551da177e4SLinus Torvalds #include <linux/fs.h> 561da177e4SLinus Torvalds #include <linux/file.h> 571da177e4SLinus Torvalds #include <linux/stat.h> 581da177e4SLinus Torvalds #include <linux/errno.h> 591da177e4SLinus Torvalds #include <linux/major.h> 601da177e4SLinus Torvalds #include <linux/wait.h> 611da177e4SLinus Torvalds #include <linux/blkdev.h> 621da177e4SLinus Torvalds #include <linux/blkpg.h> 631da177e4SLinus Torvalds #include <linux/init.h> 641da177e4SLinus Torvalds #include <linux/swap.h> 651da177e4SLinus Torvalds #include <linux/slab.h> 66863d5b82SDavid Howells #include <linux/compat.h> 671da177e4SLinus Torvalds #include <linux/suspend.h> 6883144186SRafael J. Wysocki #include <linux/freezer.h> 692a48fc0aSArnd Bergmann #include <linux/mutex.h> 701da177e4SLinus Torvalds #include <linux/writeback.h> 711da177e4SLinus Torvalds #include <linux/completion.h> 721da177e4SLinus Torvalds #include <linux/highmem.h> 736c997918SSerge E. Hallyn #include <linux/kthread.h> 74d6b29d7cSJens Axboe #include <linux/splice.h> 75ee862730SMilan Broz #include <linux/sysfs.h> 76770fe30aSKay Sievers #include <linux/miscdevice.h> 77dfaa2ef6SLukas Czerner #include <linux/falloc.h> 78283e7e5dSAl Viro #include <linux/uio.h> 79d9a08a9eSAdam Manzanares #include <linux/ioprio.h> 80db6638d7SDennis Zhou #include <linux/blk-cgroup.h> 81d9a08a9eSAdam Manzanares 8283a87611SAl Viro #include "loop.h" 831da177e4SLinus Torvalds 847c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 851da177e4SLinus Torvalds 8634dd82afSKay Sievers static DEFINE_IDR(loop_index_idr); 87310ca162STetsuo Handa static DEFINE_MUTEX(loop_ctl_mutex); 881da177e4SLinus Torvalds 89476a4813SLaurent Vivier static int max_part; 90476a4813SLaurent Vivier static int part_shift; 91476a4813SLaurent Vivier 921da177e4SLinus Torvalds static int transfer_xor(struct loop_device *lo, int cmd, 931da177e4SLinus Torvalds struct page *raw_page, unsigned raw_off, 941da177e4SLinus Torvalds struct page *loop_page, unsigned loop_off, 951da177e4SLinus Torvalds int size, sector_t real_block) 961da177e4SLinus Torvalds { 97cfd8005cSCong Wang char *raw_buf = kmap_atomic(raw_page) + raw_off; 98cfd8005cSCong Wang char *loop_buf = kmap_atomic(loop_page) + loop_off; 991da177e4SLinus Torvalds char *in, *out, *key; 1001da177e4SLinus Torvalds int i, keysize; 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds if (cmd == READ) { 1031da177e4SLinus Torvalds in = raw_buf; 1041da177e4SLinus Torvalds out = loop_buf; 1051da177e4SLinus Torvalds } else { 1061da177e4SLinus Torvalds in = loop_buf; 1071da177e4SLinus Torvalds out = raw_buf; 1081da177e4SLinus Torvalds } 1091da177e4SLinus Torvalds 1101da177e4SLinus Torvalds key = lo->lo_encrypt_key; 1111da177e4SLinus Torvalds keysize = lo->lo_encrypt_key_size; 1121da177e4SLinus Torvalds for (i = 0; i < size; i++) 1131da177e4SLinus Torvalds *out++ = *in++ ^ key[(i & 511) % keysize]; 1141da177e4SLinus Torvalds 115cfd8005cSCong Wang kunmap_atomic(loop_buf); 116cfd8005cSCong Wang kunmap_atomic(raw_buf); 1171da177e4SLinus Torvalds cond_resched(); 1181da177e4SLinus Torvalds return 0; 1191da177e4SLinus Torvalds } 1201da177e4SLinus Torvalds 1211da177e4SLinus Torvalds static int xor_init(struct loop_device *lo, const struct loop_info64 *info) 1221da177e4SLinus Torvalds { 1231da177e4SLinus Torvalds if (unlikely(info->lo_encrypt_key_size <= 0)) 1241da177e4SLinus Torvalds return -EINVAL; 1251da177e4SLinus Torvalds return 0; 1261da177e4SLinus Torvalds } 1271da177e4SLinus Torvalds 1281da177e4SLinus Torvalds static struct loop_func_table none_funcs = { 1291da177e4SLinus Torvalds .number = LO_CRYPT_NONE, 1301da177e4SLinus Torvalds }; 1311da177e4SLinus Torvalds 1321da177e4SLinus Torvalds static struct loop_func_table xor_funcs = { 1331da177e4SLinus Torvalds .number = LO_CRYPT_XOR, 1341da177e4SLinus Torvalds .transfer = transfer_xor, 1351da177e4SLinus Torvalds .init = xor_init 1361da177e4SLinus Torvalds }; 1371da177e4SLinus Torvalds 1381da177e4SLinus Torvalds /* xfer_funcs[0] is special - its release function is never called */ 1391da177e4SLinus Torvalds static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = { 1401da177e4SLinus Torvalds &none_funcs, 1411da177e4SLinus Torvalds &xor_funcs 1421da177e4SLinus Torvalds }; 1431da177e4SLinus Torvalds 1447035b5dfSDmitry Monakhov static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file) 1451da177e4SLinus Torvalds { 146b7a1da69SGuo Chao loff_t loopsize; 1471da177e4SLinus Torvalds 1481da177e4SLinus Torvalds /* Compute loopsize in bytes */ 149b7a1da69SGuo Chao loopsize = i_size_read(file->f_mapping->host); 150b7a1da69SGuo Chao if (offset > 0) 151b7a1da69SGuo Chao loopsize -= offset; 152b7a1da69SGuo Chao /* offset is beyond i_size, weird but possible */ 1537035b5dfSDmitry Monakhov if (loopsize < 0) 1547035b5dfSDmitry Monakhov return 0; 1551da177e4SLinus Torvalds 1567035b5dfSDmitry Monakhov if (sizelimit > 0 && sizelimit < loopsize) 1577035b5dfSDmitry Monakhov loopsize = sizelimit; 1581da177e4SLinus Torvalds /* 1591da177e4SLinus Torvalds * Unfortunately, if we want to do I/O on the device, 1601da177e4SLinus Torvalds * the number of 512-byte sectors has to fit into a sector_t. 1611da177e4SLinus Torvalds */ 1621da177e4SLinus Torvalds return loopsize >> 9; 1631da177e4SLinus Torvalds } 1641da177e4SLinus Torvalds 1657035b5dfSDmitry Monakhov static loff_t get_loop_size(struct loop_device *lo, struct file *file) 1661da177e4SLinus Torvalds { 1677035b5dfSDmitry Monakhov return get_size(lo->lo_offset, lo->lo_sizelimit, file); 1687035b5dfSDmitry Monakhov } 1697035b5dfSDmitry Monakhov 1702e5ab5f3SMing Lei static void __loop_update_dio(struct loop_device *lo, bool dio) 1712e5ab5f3SMing Lei { 1722e5ab5f3SMing Lei struct file *file = lo->lo_backing_file; 1732e5ab5f3SMing Lei struct address_space *mapping = file->f_mapping; 1742e5ab5f3SMing Lei struct inode *inode = mapping->host; 1752e5ab5f3SMing Lei unsigned short sb_bsize = 0; 1762e5ab5f3SMing Lei unsigned dio_align = 0; 1772e5ab5f3SMing Lei bool use_dio; 1782e5ab5f3SMing Lei 1792e5ab5f3SMing Lei if (inode->i_sb->s_bdev) { 1802e5ab5f3SMing Lei sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 1812e5ab5f3SMing Lei dio_align = sb_bsize - 1; 1822e5ab5f3SMing Lei } 1832e5ab5f3SMing Lei 1842e5ab5f3SMing Lei /* 1852e5ab5f3SMing Lei * We support direct I/O only if lo_offset is aligned with the 1862e5ab5f3SMing Lei * logical I/O size of backing device, and the logical block 1872e5ab5f3SMing Lei * size of loop is bigger than the backing device's and the loop 1882e5ab5f3SMing Lei * needn't transform transfer. 1892e5ab5f3SMing Lei * 1902e5ab5f3SMing Lei * TODO: the above condition may be loosed in the future, and 1912e5ab5f3SMing Lei * direct I/O may be switched runtime at that time because most 19289d790abSMasahiro Yamada * of requests in sane applications should be PAGE_SIZE aligned 1932e5ab5f3SMing Lei */ 1942e5ab5f3SMing Lei if (dio) { 1952e5ab5f3SMing Lei if (queue_logical_block_size(lo->lo_queue) >= sb_bsize && 1962e5ab5f3SMing Lei !(lo->lo_offset & dio_align) && 1972e5ab5f3SMing Lei mapping->a_ops->direct_IO && 1982e5ab5f3SMing Lei !lo->transfer) 1992e5ab5f3SMing Lei use_dio = true; 2002e5ab5f3SMing Lei else 2012e5ab5f3SMing Lei use_dio = false; 2022e5ab5f3SMing Lei } else { 2032e5ab5f3SMing Lei use_dio = false; 2042e5ab5f3SMing Lei } 2052e5ab5f3SMing Lei 2062e5ab5f3SMing Lei if (lo->use_dio == use_dio) 2072e5ab5f3SMing Lei return; 2082e5ab5f3SMing Lei 2092e5ab5f3SMing Lei /* flush dirty pages before changing direct IO */ 2102e5ab5f3SMing Lei vfs_fsync(file, 0); 2112e5ab5f3SMing Lei 2122e5ab5f3SMing Lei /* 2132e5ab5f3SMing Lei * The flag of LO_FLAGS_DIRECT_IO is handled similarly with 2142e5ab5f3SMing Lei * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup 2152e5ab5f3SMing Lei * will get updated by ioctl(LOOP_GET_STATUS) 2162e5ab5f3SMing Lei */ 2170fbcf579SMartijn Coenen if (lo->lo_state == Lo_bound) 2182e5ab5f3SMing Lei blk_mq_freeze_queue(lo->lo_queue); 2192e5ab5f3SMing Lei lo->use_dio = use_dio; 22040326d8aSShaohua Li if (use_dio) { 2218b904b5bSBart Van Assche blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2222e5ab5f3SMing Lei lo->lo_flags |= LO_FLAGS_DIRECT_IO; 22340326d8aSShaohua Li } else { 2248b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2252e5ab5f3SMing Lei lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; 22640326d8aSShaohua Li } 2270fbcf579SMartijn Coenen if (lo->lo_state == Lo_bound) 2282e5ab5f3SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 2292e5ab5f3SMing Lei } 2302e5ab5f3SMing Lei 231*083a6a50SMartijn Coenen static void 2321e6ec9eaSOmar Sandoval figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) 2337035b5dfSDmitry Monakhov { 2347035b5dfSDmitry Monakhov loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); 2357b0576a3SGuo Chao struct block_device *bdev = lo->lo_device; 2361da177e4SLinus Torvalds 2377035b5dfSDmitry Monakhov if (lo->lo_offset != offset) 2387035b5dfSDmitry Monakhov lo->lo_offset = offset; 2397035b5dfSDmitry Monakhov if (lo->lo_sizelimit != sizelimit) 2407035b5dfSDmitry Monakhov lo->lo_sizelimit = sizelimit; 241*083a6a50SMartijn Coenen set_capacity(lo->lo_disk, size); 2427b0576a3SGuo Chao bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9); 2437b0576a3SGuo Chao /* let user-space know about the new size */ 2447b0576a3SGuo Chao kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 2451da177e4SLinus Torvalds } 2461da177e4SLinus Torvalds 2471da177e4SLinus Torvalds static inline int 2481da177e4SLinus Torvalds lo_do_transfer(struct loop_device *lo, int cmd, 2491da177e4SLinus Torvalds struct page *rpage, unsigned roffs, 2501da177e4SLinus Torvalds struct page *lpage, unsigned loffs, 2511da177e4SLinus Torvalds int size, sector_t rblock) 2521da177e4SLinus Torvalds { 253aa4d8616SChristoph Hellwig int ret; 254aa4d8616SChristoph Hellwig 255aa4d8616SChristoph Hellwig ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); 256aa4d8616SChristoph Hellwig if (likely(!ret)) 2571da177e4SLinus Torvalds return 0; 2581da177e4SLinus Torvalds 259aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR 260aa4d8616SChristoph Hellwig "loop: Transfer error at byte offset %llu, length %i.\n", 261aa4d8616SChristoph Hellwig (unsigned long long)rblock << 9, size); 262aa4d8616SChristoph Hellwig return ret; 2631da177e4SLinus Torvalds } 2641da177e4SLinus Torvalds 265aa4d8616SChristoph Hellwig static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos) 2661da177e4SLinus Torvalds { 267aa4d8616SChristoph Hellwig struct iov_iter i; 2681da177e4SLinus Torvalds ssize_t bw; 269283e7e5dSAl Viro 270b6207430SChristoph Hellwig iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len); 2711da177e4SLinus Torvalds 27203d95eb2SAl Viro file_start_write(file); 273abbb6589SChristoph Hellwig bw = vfs_iter_write(file, &i, ppos, 0); 27403d95eb2SAl Viro file_end_write(file); 275aa4d8616SChristoph Hellwig 276aa4d8616SChristoph Hellwig if (likely(bw == bvec->bv_len)) 2771da177e4SLinus Torvalds return 0; 278aa4d8616SChristoph Hellwig 279aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR 280aa4d8616SChristoph Hellwig "loop: Write error at byte offset %llu, length %i.\n", 281aa4d8616SChristoph Hellwig (unsigned long long)*ppos, bvec->bv_len); 2821da177e4SLinus Torvalds if (bw >= 0) 2831da177e4SLinus Torvalds bw = -EIO; 2841da177e4SLinus Torvalds return bw; 2851da177e4SLinus Torvalds } 2861da177e4SLinus Torvalds 287aa4d8616SChristoph Hellwig static int lo_write_simple(struct loop_device *lo, struct request *rq, 288aa4d8616SChristoph Hellwig loff_t pos) 2891da177e4SLinus Torvalds { 290aa4d8616SChristoph Hellwig struct bio_vec bvec; 291aa4d8616SChristoph Hellwig struct req_iterator iter; 292aa4d8616SChristoph Hellwig int ret = 0; 293aa4d8616SChristoph Hellwig 294aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) { 295aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos); 296aa4d8616SChristoph Hellwig if (ret < 0) 297aa4d8616SChristoph Hellwig break; 2981da177e4SLinus Torvalds cond_resched(); 2991da177e4SLinus Torvalds } 3001da177e4SLinus Torvalds 301aa4d8616SChristoph Hellwig return ret; 302aa4d8616SChristoph Hellwig } 303aa4d8616SChristoph Hellwig 304aa4d8616SChristoph Hellwig /* 305456be148SChristoph Hellwig * This is the slow, transforming version that needs to double buffer the 306456be148SChristoph Hellwig * data as it cannot do the transformations in place without having direct 307456be148SChristoph Hellwig * access to the destination pages of the backing file. 3081da177e4SLinus Torvalds */ 309aa4d8616SChristoph Hellwig static int lo_write_transfer(struct loop_device *lo, struct request *rq, 310aa4d8616SChristoph Hellwig loff_t pos) 3111da177e4SLinus Torvalds { 312aa4d8616SChristoph Hellwig struct bio_vec bvec, b; 31330112013SMing Lei struct req_iterator iter; 314aa4d8616SChristoph Hellwig struct page *page; 3157988613bSKent Overstreet int ret = 0; 3161da177e4SLinus Torvalds 317aa4d8616SChristoph Hellwig page = alloc_page(GFP_NOIO); 3181da177e4SLinus Torvalds if (unlikely(!page)) 319aa4d8616SChristoph Hellwig return -ENOMEM; 320456be148SChristoph Hellwig 32130112013SMing Lei rq_for_each_segment(bvec, rq, iter) { 322aa4d8616SChristoph Hellwig ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page, 323aa4d8616SChristoph Hellwig bvec.bv_offset, bvec.bv_len, pos >> 9); 324aa4d8616SChristoph Hellwig if (unlikely(ret)) 325aa4d8616SChristoph Hellwig break; 326aa4d8616SChristoph Hellwig 327aa4d8616SChristoph Hellwig b.bv_page = page; 328aa4d8616SChristoph Hellwig b.bv_offset = 0; 329aa4d8616SChristoph Hellwig b.bv_len = bvec.bv_len; 330aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &b, &pos); 3311da177e4SLinus Torvalds if (ret < 0) 3321da177e4SLinus Torvalds break; 3331da177e4SLinus Torvalds } 334aa4d8616SChristoph Hellwig 3351da177e4SLinus Torvalds __free_page(page); 3361da177e4SLinus Torvalds return ret; 3371da177e4SLinus Torvalds } 3381da177e4SLinus Torvalds 339aa4d8616SChristoph Hellwig static int lo_read_simple(struct loop_device *lo, struct request *rq, 340aa4d8616SChristoph Hellwig loff_t pos) 3411da177e4SLinus Torvalds { 3427988613bSKent Overstreet struct bio_vec bvec; 34330112013SMing Lei struct req_iterator iter; 344aa4d8616SChristoph Hellwig struct iov_iter i; 345aa4d8616SChristoph Hellwig ssize_t len; 3461da177e4SLinus Torvalds 34730112013SMing Lei rq_for_each_segment(bvec, rq, iter) { 348b6207430SChristoph Hellwig iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len); 34918e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 350aa4d8616SChristoph Hellwig if (len < 0) 351aa4d8616SChristoph Hellwig return len; 352306df071SDave Young 353aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page); 354aa4d8616SChristoph Hellwig 355aa4d8616SChristoph Hellwig if (len != bvec.bv_len) { 35630112013SMing Lei struct bio *bio; 35730112013SMing Lei 35830112013SMing Lei __rq_for_each_bio(bio, rq) 359306df071SDave Young zero_fill_bio(bio); 3601da177e4SLinus Torvalds break; 361306df071SDave Young } 362aa4d8616SChristoph Hellwig cond_resched(); 3631da177e4SLinus Torvalds } 364aa4d8616SChristoph Hellwig 365306df071SDave Young return 0; 3661da177e4SLinus Torvalds } 3671da177e4SLinus Torvalds 368aa4d8616SChristoph Hellwig static int lo_read_transfer(struct loop_device *lo, struct request *rq, 369aa4d8616SChristoph Hellwig loff_t pos) 370aa4d8616SChristoph Hellwig { 371aa4d8616SChristoph Hellwig struct bio_vec bvec, b; 372aa4d8616SChristoph Hellwig struct req_iterator iter; 373aa4d8616SChristoph Hellwig struct iov_iter i; 374aa4d8616SChristoph Hellwig struct page *page; 375aa4d8616SChristoph Hellwig ssize_t len; 376aa4d8616SChristoph Hellwig int ret = 0; 377aa4d8616SChristoph Hellwig 378aa4d8616SChristoph Hellwig page = alloc_page(GFP_NOIO); 379aa4d8616SChristoph Hellwig if (unlikely(!page)) 380aa4d8616SChristoph Hellwig return -ENOMEM; 381aa4d8616SChristoph Hellwig 382aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) { 383aa4d8616SChristoph Hellwig loff_t offset = pos; 384aa4d8616SChristoph Hellwig 385aa4d8616SChristoph Hellwig b.bv_page = page; 386aa4d8616SChristoph Hellwig b.bv_offset = 0; 387aa4d8616SChristoph Hellwig b.bv_len = bvec.bv_len; 388aa4d8616SChristoph Hellwig 389b6207430SChristoph Hellwig iov_iter_bvec(&i, READ, &b, 1, b.bv_len); 39018e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 391aa4d8616SChristoph Hellwig if (len < 0) { 392aa4d8616SChristoph Hellwig ret = len; 393aa4d8616SChristoph Hellwig goto out_free_page; 394aa4d8616SChristoph Hellwig } 395aa4d8616SChristoph Hellwig 396aa4d8616SChristoph Hellwig ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page, 397aa4d8616SChristoph Hellwig bvec.bv_offset, len, offset >> 9); 398aa4d8616SChristoph Hellwig if (ret) 399aa4d8616SChristoph Hellwig goto out_free_page; 400aa4d8616SChristoph Hellwig 401aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page); 402aa4d8616SChristoph Hellwig 403aa4d8616SChristoph Hellwig if (len != bvec.bv_len) { 404aa4d8616SChristoph Hellwig struct bio *bio; 405aa4d8616SChristoph Hellwig 406aa4d8616SChristoph Hellwig __rq_for_each_bio(bio, rq) 407aa4d8616SChristoph Hellwig zero_fill_bio(bio); 408aa4d8616SChristoph Hellwig break; 409aa4d8616SChristoph Hellwig } 410aa4d8616SChristoph Hellwig } 411aa4d8616SChristoph Hellwig 412aa4d8616SChristoph Hellwig ret = 0; 413aa4d8616SChristoph Hellwig out_free_page: 414aa4d8616SChristoph Hellwig __free_page(page); 415aa4d8616SChristoph Hellwig return ret; 416aa4d8616SChristoph Hellwig } 417aa4d8616SChristoph Hellwig 418efcfec57SDarrick J. Wong static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos, 419efcfec57SDarrick J. Wong int mode) 420cf655d95SMing Lei { 421cf655d95SMing Lei /* 422efcfec57SDarrick J. Wong * We use fallocate to manipulate the space mappings used by the image 423efcfec57SDarrick J. Wong * a.k.a. discard/zerorange. However we do not support this if 424efcfec57SDarrick J. Wong * encryption is enabled, because it may give an attacker useful 425efcfec57SDarrick J. Wong * information. 426cf655d95SMing Lei */ 427cf655d95SMing Lei struct file *file = lo->lo_backing_file; 428c52abf56SEvan Green struct request_queue *q = lo->lo_queue; 429cf655d95SMing Lei int ret; 430cf655d95SMing Lei 431efcfec57SDarrick J. Wong mode |= FALLOC_FL_KEEP_SIZE; 432efcfec57SDarrick J. Wong 433c52abf56SEvan Green if (!blk_queue_discard(q)) { 434cf655d95SMing Lei ret = -EOPNOTSUPP; 435cf655d95SMing Lei goto out; 436cf655d95SMing Lei } 437cf655d95SMing Lei 438cf655d95SMing Lei ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq)); 439cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP)) 440cf655d95SMing Lei ret = -EIO; 441cf655d95SMing Lei out: 442cf655d95SMing Lei return ret; 443cf655d95SMing Lei } 444cf655d95SMing Lei 445cf655d95SMing Lei static int lo_req_flush(struct loop_device *lo, struct request *rq) 446cf655d95SMing Lei { 447cf655d95SMing Lei struct file *file = lo->lo_backing_file; 448cf655d95SMing Lei int ret = vfs_fsync(file, 0); 449cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL)) 450cf655d95SMing Lei ret = -EIO; 451cf655d95SMing Lei 452cf655d95SMing Lei return ret; 453cf655d95SMing Lei } 454cf655d95SMing Lei 455fe2cb290SChristoph Hellwig static void lo_complete_rq(struct request *rq) 456bc07c10aSMing Lei { 457fe2cb290SChristoph Hellwig struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 458f9de14bcSJens Axboe blk_status_t ret = BLK_STS_OK; 459bc07c10aSMing Lei 460f9de14bcSJens Axboe if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) || 461f9de14bcSJens Axboe req_op(rq) != REQ_OP_READ) { 462f9de14bcSJens Axboe if (cmd->ret < 0) 4638cd55087SEvan Green ret = errno_to_blk_status(cmd->ret); 464f9de14bcSJens Axboe goto end_io; 465bc07c10aSMing Lei } 466fe2cb290SChristoph Hellwig 467f9de14bcSJens Axboe /* 468f9de14bcSJens Axboe * Short READ - if we got some data, advance our request and 469f9de14bcSJens Axboe * retry it. If we got no data, end the rest with EIO. 470f9de14bcSJens Axboe */ 471f9de14bcSJens Axboe if (cmd->ret) { 472f9de14bcSJens Axboe blk_update_request(rq, BLK_STS_OK, cmd->ret); 473f9de14bcSJens Axboe cmd->ret = 0; 474f9de14bcSJens Axboe blk_mq_requeue_request(rq, true); 475f9de14bcSJens Axboe } else { 476f9de14bcSJens Axboe if (cmd->use_aio) { 477f9de14bcSJens Axboe struct bio *bio = rq->bio; 478f9de14bcSJens Axboe 479f9de14bcSJens Axboe while (bio) { 480f9de14bcSJens Axboe zero_fill_bio(bio); 481f9de14bcSJens Axboe bio = bio->bi_next; 482f9de14bcSJens Axboe } 483f9de14bcSJens Axboe } 484f9de14bcSJens Axboe ret = BLK_STS_IOERR; 485f9de14bcSJens Axboe end_io: 486f9de14bcSJens Axboe blk_mq_end_request(rq, ret); 487f9de14bcSJens Axboe } 488bc07c10aSMing Lei } 489bc07c10aSMing Lei 49092d77332SShaohua Li static void lo_rw_aio_do_completion(struct loop_cmd *cmd) 49192d77332SShaohua Li { 4921894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 4931894e916SJens Axboe 49492d77332SShaohua Li if (!atomic_dec_and_test(&cmd->ref)) 49592d77332SShaohua Li return; 49692d77332SShaohua Li kfree(cmd->bvec); 49792d77332SShaohua Li cmd->bvec = NULL; 4981894e916SJens Axboe blk_mq_complete_request(rq); 49992d77332SShaohua Li } 50092d77332SShaohua Li 501bc07c10aSMing Lei static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2) 502bc07c10aSMing Lei { 503bc07c10aSMing Lei struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb); 504bc07c10aSMing Lei 505d4478e92SShaohua Li if (cmd->css) 506d4478e92SShaohua Li css_put(cmd->css); 507fe2cb290SChristoph Hellwig cmd->ret = ret; 50892d77332SShaohua Li lo_rw_aio_do_completion(cmd); 509bc07c10aSMing Lei } 510bc07c10aSMing Lei 511bc07c10aSMing Lei static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, 512bc07c10aSMing Lei loff_t pos, bool rw) 513bc07c10aSMing Lei { 514bc07c10aSMing Lei struct iov_iter iter; 51586af5952SMing Lei struct req_iterator rq_iter; 516bc07c10aSMing Lei struct bio_vec *bvec; 5171894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 51840326d8aSShaohua Li struct bio *bio = rq->bio; 519bc07c10aSMing Lei struct file *file = lo->lo_backing_file; 52086af5952SMing Lei struct bio_vec tmp; 52140326d8aSShaohua Li unsigned int offset; 52286af5952SMing Lei int nr_bvec = 0; 523bc07c10aSMing Lei int ret; 524bc07c10aSMing Lei 52586af5952SMing Lei rq_for_each_bvec(tmp, rq, rq_iter) 52686af5952SMing Lei nr_bvec++; 527bc07c10aSMing Lei 52886af5952SMing Lei if (rq->bio != rq->biotail) { 52986af5952SMing Lei 53086af5952SMing Lei bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec), 5316da2ec56SKees Cook GFP_NOIO); 53240326d8aSShaohua Li if (!bvec) 53340326d8aSShaohua Li return -EIO; 53440326d8aSShaohua Li cmd->bvec = bvec; 53540326d8aSShaohua Li 536a7297a6aSMing Lei /* 53740326d8aSShaohua Li * The bios of the request may be started from the middle of 53840326d8aSShaohua Li * the 'bvec' because of bio splitting, so we can't directly 53986af5952SMing Lei * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec 54040326d8aSShaohua Li * API will take care of all details for us. 541a7297a6aSMing Lei */ 54286af5952SMing Lei rq_for_each_bvec(tmp, rq, rq_iter) { 54340326d8aSShaohua Li *bvec = tmp; 54440326d8aSShaohua Li bvec++; 54540326d8aSShaohua Li } 54640326d8aSShaohua Li bvec = cmd->bvec; 54740326d8aSShaohua Li offset = 0; 54840326d8aSShaohua Li } else { 54940326d8aSShaohua Li /* 55040326d8aSShaohua Li * Same here, this bio may be started from the middle of the 55140326d8aSShaohua Li * 'bvec' because of bio splitting, so offset from the bvec 55240326d8aSShaohua Li * must be passed to iov iterator 55340326d8aSShaohua Li */ 55440326d8aSShaohua Li offset = bio->bi_iter.bi_bvec_done; 55540326d8aSShaohua Li bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 55640326d8aSShaohua Li } 55792d77332SShaohua Li atomic_set(&cmd->ref, 2); 55840326d8aSShaohua Li 559b6207430SChristoph Hellwig iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq)); 56040326d8aSShaohua Li iter.iov_offset = offset; 561bc07c10aSMing Lei 562bc07c10aSMing Lei cmd->iocb.ki_pos = pos; 563bc07c10aSMing Lei cmd->iocb.ki_filp = file; 564bc07c10aSMing Lei cmd->iocb.ki_complete = lo_rw_aio_complete; 565bc07c10aSMing Lei cmd->iocb.ki_flags = IOCB_DIRECT; 566d9a08a9eSAdam Manzanares cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); 567d4478e92SShaohua Li if (cmd->css) 568d4478e92SShaohua Li kthread_associate_blkcg(cmd->css); 569bc07c10aSMing Lei 570bc07c10aSMing Lei if (rw == WRITE) 571bb7462b6SMiklos Szeredi ret = call_write_iter(file, &cmd->iocb, &iter); 572bc07c10aSMing Lei else 573bb7462b6SMiklos Szeredi ret = call_read_iter(file, &cmd->iocb, &iter); 574bc07c10aSMing Lei 57592d77332SShaohua Li lo_rw_aio_do_completion(cmd); 576d4478e92SShaohua Li kthread_associate_blkcg(NULL); 57792d77332SShaohua Li 578bc07c10aSMing Lei if (ret != -EIOCBQUEUED) 579bc07c10aSMing Lei cmd->iocb.ki_complete(&cmd->iocb, ret, 0); 580bc07c10aSMing Lei return 0; 581bc07c10aSMing Lei } 582bc07c10aSMing Lei 583c1c87c2bSChristoph Hellwig static int do_req_filebacked(struct loop_device *lo, struct request *rq) 584bc07c10aSMing Lei { 585bc07c10aSMing Lei struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 586c1c87c2bSChristoph Hellwig loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; 587bc07c10aSMing Lei 588bc07c10aSMing Lei /* 589bc07c10aSMing Lei * lo_write_simple and lo_read_simple should have been covered 590bc07c10aSMing Lei * by io submit style function like lo_rw_aio(), one blocker 591bc07c10aSMing Lei * is that lo_read_simple() need to call flush_dcache_page after 592bc07c10aSMing Lei * the page is written from kernel, and it isn't easy to handle 593bc07c10aSMing Lei * this in io submit style function which submits all segments 594bc07c10aSMing Lei * of the req at one time. And direct read IO doesn't need to 595bc07c10aSMing Lei * run flush_dcache_page(). 596bc07c10aSMing Lei */ 597c1c87c2bSChristoph Hellwig switch (req_op(rq)) { 598c1c87c2bSChristoph Hellwig case REQ_OP_FLUSH: 599c1c87c2bSChristoph Hellwig return lo_req_flush(lo, rq); 60019372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 601efcfec57SDarrick J. Wong /* 602efcfec57SDarrick J. Wong * If the caller doesn't want deallocation, call zeroout to 603efcfec57SDarrick J. Wong * write zeroes the range. Otherwise, punch them out. 604efcfec57SDarrick J. Wong */ 605efcfec57SDarrick J. Wong return lo_fallocate(lo, rq, pos, 606efcfec57SDarrick J. Wong (rq->cmd_flags & REQ_NOUNMAP) ? 607efcfec57SDarrick J. Wong FALLOC_FL_ZERO_RANGE : 608efcfec57SDarrick J. Wong FALLOC_FL_PUNCH_HOLE); 609efcfec57SDarrick J. Wong case REQ_OP_DISCARD: 610efcfec57SDarrick J. Wong return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE); 611c1c87c2bSChristoph Hellwig case REQ_OP_WRITE: 612c1c87c2bSChristoph Hellwig if (lo->transfer) 613c1c87c2bSChristoph Hellwig return lo_write_transfer(lo, rq, pos); 614c1c87c2bSChristoph Hellwig else if (cmd->use_aio) 615c1c87c2bSChristoph Hellwig return lo_rw_aio(lo, cmd, pos, WRITE); 616c1c87c2bSChristoph Hellwig else 617bc07c10aSMing Lei return lo_write_simple(lo, rq, pos); 618c1c87c2bSChristoph Hellwig case REQ_OP_READ: 619c1c87c2bSChristoph Hellwig if (lo->transfer) 620c1c87c2bSChristoph Hellwig return lo_read_transfer(lo, rq, pos); 621c1c87c2bSChristoph Hellwig else if (cmd->use_aio) 622c1c87c2bSChristoph Hellwig return lo_rw_aio(lo, cmd, pos, READ); 623bc07c10aSMing Lei else 624bc07c10aSMing Lei return lo_read_simple(lo, rq, pos); 625c1c87c2bSChristoph Hellwig default: 626c1c87c2bSChristoph Hellwig WARN_ON_ONCE(1); 627c1c87c2bSChristoph Hellwig return -EIO; 628bc07c10aSMing Lei } 6291da177e4SLinus Torvalds } 6301da177e4SLinus Torvalds 6312e5ab5f3SMing Lei static inline void loop_update_dio(struct loop_device *lo) 6322e5ab5f3SMing Lei { 6332e5ab5f3SMing Lei __loop_update_dio(lo, io_is_direct(lo->lo_backing_file) | 6342e5ab5f3SMing Lei lo->use_dio); 6352e5ab5f3SMing Lei } 6362e5ab5f3SMing Lei 63706f0e9e6SMing Lei static void loop_reread_partitions(struct loop_device *lo, 63806f0e9e6SMing Lei struct block_device *bdev) 63906f0e9e6SMing Lei { 64006f0e9e6SMing Lei int rc; 64106f0e9e6SMing Lei 642f0b870dfSChristoph Hellwig mutex_lock(&bdev->bd_mutex); 643f0b870dfSChristoph Hellwig rc = bdev_disk_changed(bdev, false); 644f0b870dfSChristoph Hellwig mutex_unlock(&bdev->bd_mutex); 64506f0e9e6SMing Lei if (rc) 64606f0e9e6SMing Lei pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n", 64706f0e9e6SMing Lei __func__, lo->lo_number, lo->lo_file_name, rc); 64806f0e9e6SMing Lei } 64906f0e9e6SMing Lei 650d2ac838eSTheodore Ts'o static inline int is_loop_device(struct file *file) 651d2ac838eSTheodore Ts'o { 652d2ac838eSTheodore Ts'o struct inode *i = file->f_mapping->host; 653d2ac838eSTheodore Ts'o 654d2ac838eSTheodore Ts'o return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; 655d2ac838eSTheodore Ts'o } 656d2ac838eSTheodore Ts'o 657d2ac838eSTheodore Ts'o static int loop_validate_file(struct file *file, struct block_device *bdev) 658d2ac838eSTheodore Ts'o { 659d2ac838eSTheodore Ts'o struct inode *inode = file->f_mapping->host; 660d2ac838eSTheodore Ts'o struct file *f = file; 661d2ac838eSTheodore Ts'o 662d2ac838eSTheodore Ts'o /* Avoid recursion */ 663d2ac838eSTheodore Ts'o while (is_loop_device(f)) { 664d2ac838eSTheodore Ts'o struct loop_device *l; 665d2ac838eSTheodore Ts'o 666d2ac838eSTheodore Ts'o if (f->f_mapping->host->i_bdev == bdev) 667d2ac838eSTheodore Ts'o return -EBADF; 668d2ac838eSTheodore Ts'o 669d2ac838eSTheodore Ts'o l = f->f_mapping->host->i_bdev->bd_disk->private_data; 670f7c8a412SDongli Zhang if (l->lo_state != Lo_bound) { 671d2ac838eSTheodore Ts'o return -EINVAL; 672d2ac838eSTheodore Ts'o } 673d2ac838eSTheodore Ts'o f = l->lo_backing_file; 674d2ac838eSTheodore Ts'o } 675d2ac838eSTheodore Ts'o if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode)) 676d2ac838eSTheodore Ts'o return -EINVAL; 677d2ac838eSTheodore Ts'o return 0; 678d2ac838eSTheodore Ts'o } 679d2ac838eSTheodore Ts'o 6801da177e4SLinus Torvalds /* 6811da177e4SLinus Torvalds * loop_change_fd switched the backing store of a loopback device to 6821da177e4SLinus Torvalds * a new file. This is useful for operating system installers to free up 6831da177e4SLinus Torvalds * the original file and in High Availability environments to switch to 6841da177e4SLinus Torvalds * an alternative location for the content in case of server meltdown. 6851da177e4SLinus Torvalds * This can only work if the loop device is used read-only, and if the 6861da177e4SLinus Torvalds * new backing store is the same size and type as the old backing store. 6871da177e4SLinus Torvalds */ 688bb214884SAl Viro static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, 689bb214884SAl Viro unsigned int arg) 6901da177e4SLinus Torvalds { 6911dded9acSJan Kara struct file *file = NULL, *old_file; 6921da177e4SLinus Torvalds int error; 69385b0a54aSJan Kara bool partscan; 6941da177e4SLinus Torvalds 695c28445faSJan Kara error = mutex_lock_killable(&loop_ctl_mutex); 696c3710770SJan Kara if (error) 697c3710770SJan Kara return error; 6981da177e4SLinus Torvalds error = -ENXIO; 6991da177e4SLinus Torvalds if (lo->lo_state != Lo_bound) 7001dded9acSJan Kara goto out_err; 7011da177e4SLinus Torvalds 7021da177e4SLinus Torvalds /* the loop device has to be read-only */ 7031da177e4SLinus Torvalds error = -EINVAL; 7041da177e4SLinus Torvalds if (!(lo->lo_flags & LO_FLAGS_READ_ONLY)) 7051dded9acSJan Kara goto out_err; 7061da177e4SLinus Torvalds 7071da177e4SLinus Torvalds error = -EBADF; 7081da177e4SLinus Torvalds file = fget(arg); 7091da177e4SLinus Torvalds if (!file) 7101dded9acSJan Kara goto out_err; 7111da177e4SLinus Torvalds 712d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev); 713d2ac838eSTheodore Ts'o if (error) 7141dded9acSJan Kara goto out_err; 715d2ac838eSTheodore Ts'o 7161da177e4SLinus Torvalds old_file = lo->lo_backing_file; 7171da177e4SLinus Torvalds 7181da177e4SLinus Torvalds error = -EINVAL; 7191da177e4SLinus Torvalds 7201da177e4SLinus Torvalds /* size of the new backing store needs to be the same */ 7211da177e4SLinus Torvalds if (get_loop_size(lo, file) != get_loop_size(lo, old_file)) 7221dded9acSJan Kara goto out_err; 7231da177e4SLinus Torvalds 7241da177e4SLinus Torvalds /* and ... switch */ 72543cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 72643cade80SOmar Sandoval mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask); 72743cade80SOmar Sandoval lo->lo_backing_file = file; 72843cade80SOmar Sandoval lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping); 72943cade80SOmar Sandoval mapping_set_gfp_mask(file->f_mapping, 73043cade80SOmar Sandoval lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 73143cade80SOmar Sandoval loop_update_dio(lo); 73243cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 73385b0a54aSJan Kara partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 734c3710770SJan Kara mutex_unlock(&loop_ctl_mutex); 7351dded9acSJan Kara /* 7361dded9acSJan Kara * We must drop file reference outside of loop_ctl_mutex as dropping 7371dded9acSJan Kara * the file ref can take bd_mutex which creates circular locking 7381dded9acSJan Kara * dependency. 7391dded9acSJan Kara */ 7401dded9acSJan Kara fput(old_file); 74185b0a54aSJan Kara if (partscan) 74285b0a54aSJan Kara loop_reread_partitions(lo, bdev); 7431da177e4SLinus Torvalds return 0; 7441da177e4SLinus Torvalds 7451dded9acSJan Kara out_err: 746c3710770SJan Kara mutex_unlock(&loop_ctl_mutex); 7471dded9acSJan Kara if (file) 7481dded9acSJan Kara fput(file); 7491da177e4SLinus Torvalds return error; 7501da177e4SLinus Torvalds } 7511da177e4SLinus Torvalds 752ee862730SMilan Broz /* loop sysfs attributes */ 753ee862730SMilan Broz 754ee862730SMilan Broz static ssize_t loop_attr_show(struct device *dev, char *page, 755ee862730SMilan Broz ssize_t (*callback)(struct loop_device *, char *)) 756ee862730SMilan Broz { 75734dd82afSKay Sievers struct gendisk *disk = dev_to_disk(dev); 75834dd82afSKay Sievers struct loop_device *lo = disk->private_data; 759ee862730SMilan Broz 76034dd82afSKay Sievers return callback(lo, page); 761ee862730SMilan Broz } 762ee862730SMilan Broz 763ee862730SMilan Broz #define LOOP_ATTR_RO(_name) \ 764ee862730SMilan Broz static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \ 765ee862730SMilan Broz static ssize_t loop_attr_do_show_##_name(struct device *d, \ 766ee862730SMilan Broz struct device_attribute *attr, char *b) \ 767ee862730SMilan Broz { \ 768ee862730SMilan Broz return loop_attr_show(d, b, loop_attr_##_name##_show); \ 769ee862730SMilan Broz } \ 770ee862730SMilan Broz static struct device_attribute loop_attr_##_name = \ 7715657a819SJoe Perches __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL); 772ee862730SMilan Broz 773ee862730SMilan Broz static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf) 774ee862730SMilan Broz { 775ee862730SMilan Broz ssize_t ret; 776ee862730SMilan Broz char *p = NULL; 777ee862730SMilan Broz 77805eb0f25SKay Sievers spin_lock_irq(&lo->lo_lock); 779ee862730SMilan Broz if (lo->lo_backing_file) 7809bf39ab2SMiklos Szeredi p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1); 78105eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock); 782ee862730SMilan Broz 783ee862730SMilan Broz if (IS_ERR_OR_NULL(p)) 784ee862730SMilan Broz ret = PTR_ERR(p); 785ee862730SMilan Broz else { 786ee862730SMilan Broz ret = strlen(p); 787ee862730SMilan Broz memmove(buf, p, ret); 788ee862730SMilan Broz buf[ret++] = '\n'; 789ee862730SMilan Broz buf[ret] = 0; 790ee862730SMilan Broz } 791ee862730SMilan Broz 792ee862730SMilan Broz return ret; 793ee862730SMilan Broz } 794ee862730SMilan Broz 795ee862730SMilan Broz static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf) 796ee862730SMilan Broz { 797ee862730SMilan Broz return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset); 798ee862730SMilan Broz } 799ee862730SMilan Broz 800ee862730SMilan Broz static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf) 801ee862730SMilan Broz { 802ee862730SMilan Broz return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit); 803ee862730SMilan Broz } 804ee862730SMilan Broz 805ee862730SMilan Broz static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf) 806ee862730SMilan Broz { 807ee862730SMilan Broz int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR); 808ee862730SMilan Broz 809ee862730SMilan Broz return sprintf(buf, "%s\n", autoclear ? "1" : "0"); 810ee862730SMilan Broz } 811ee862730SMilan Broz 812e03c8dd1SKay Sievers static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf) 813e03c8dd1SKay Sievers { 814e03c8dd1SKay Sievers int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN); 815e03c8dd1SKay Sievers 816e03c8dd1SKay Sievers return sprintf(buf, "%s\n", partscan ? "1" : "0"); 817e03c8dd1SKay Sievers } 818e03c8dd1SKay Sievers 8192e5ab5f3SMing Lei static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf) 8202e5ab5f3SMing Lei { 8212e5ab5f3SMing Lei int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO); 8222e5ab5f3SMing Lei 8232e5ab5f3SMing Lei return sprintf(buf, "%s\n", dio ? "1" : "0"); 8242e5ab5f3SMing Lei } 8252e5ab5f3SMing Lei 826ee862730SMilan Broz LOOP_ATTR_RO(backing_file); 827ee862730SMilan Broz LOOP_ATTR_RO(offset); 828ee862730SMilan Broz LOOP_ATTR_RO(sizelimit); 829ee862730SMilan Broz LOOP_ATTR_RO(autoclear); 830e03c8dd1SKay Sievers LOOP_ATTR_RO(partscan); 8312e5ab5f3SMing Lei LOOP_ATTR_RO(dio); 832ee862730SMilan Broz 833ee862730SMilan Broz static struct attribute *loop_attrs[] = { 834ee862730SMilan Broz &loop_attr_backing_file.attr, 835ee862730SMilan Broz &loop_attr_offset.attr, 836ee862730SMilan Broz &loop_attr_sizelimit.attr, 837ee862730SMilan Broz &loop_attr_autoclear.attr, 838e03c8dd1SKay Sievers &loop_attr_partscan.attr, 8392e5ab5f3SMing Lei &loop_attr_dio.attr, 840ee862730SMilan Broz NULL, 841ee862730SMilan Broz }; 842ee862730SMilan Broz 843ee862730SMilan Broz static struct attribute_group loop_attribute_group = { 844ee862730SMilan Broz .name = "loop", 845ee862730SMilan Broz .attrs= loop_attrs, 846ee862730SMilan Broz }; 847ee862730SMilan Broz 848d3349b6bSTetsuo Handa static void loop_sysfs_init(struct loop_device *lo) 849ee862730SMilan Broz { 850d3349b6bSTetsuo Handa lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj, 851ee862730SMilan Broz &loop_attribute_group); 852ee862730SMilan Broz } 853ee862730SMilan Broz 854ee862730SMilan Broz static void loop_sysfs_exit(struct loop_device *lo) 855ee862730SMilan Broz { 856d3349b6bSTetsuo Handa if (lo->sysfs_inited) 857ee862730SMilan Broz sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj, 858ee862730SMilan Broz &loop_attribute_group); 859ee862730SMilan Broz } 860ee862730SMilan Broz 861dfaa2ef6SLukas Czerner static void loop_config_discard(struct loop_device *lo) 862dfaa2ef6SLukas Czerner { 863dfaa2ef6SLukas Czerner struct file *file = lo->lo_backing_file; 864dfaa2ef6SLukas Czerner struct inode *inode = file->f_mapping->host; 865dfaa2ef6SLukas Czerner struct request_queue *q = lo->lo_queue; 866dfaa2ef6SLukas Czerner 867dfaa2ef6SLukas Czerner /* 868c52abf56SEvan Green * If the backing device is a block device, mirror its zeroing 869c52abf56SEvan Green * capability. Set the discard sectors to the block device's zeroing 870c52abf56SEvan Green * capabilities because loop discards result in blkdev_issue_zeroout(), 871c52abf56SEvan Green * not blkdev_issue_discard(). This maintains consistent behavior with 872c52abf56SEvan Green * file-backed loop devices: discarded regions read back as zero. 873c52abf56SEvan Green */ 874c52abf56SEvan Green if (S_ISBLK(inode->i_mode) && !lo->lo_encrypt_key_size) { 875c52abf56SEvan Green struct request_queue *backingq; 876c52abf56SEvan Green 877c52abf56SEvan Green backingq = bdev_get_queue(inode->i_bdev); 878c52abf56SEvan Green blk_queue_max_discard_sectors(q, 879c52abf56SEvan Green backingq->limits.max_write_zeroes_sectors); 880c52abf56SEvan Green 881c52abf56SEvan Green blk_queue_max_write_zeroes_sectors(q, 882c52abf56SEvan Green backingq->limits.max_write_zeroes_sectors); 883c52abf56SEvan Green 884c52abf56SEvan Green /* 885dfaa2ef6SLukas Czerner * We use punch hole to reclaim the free space used by the 88612a64d2fSOlaf Hering * image a.k.a. discard. However we do not support discard if 887dfaa2ef6SLukas Czerner * encryption is enabled, because it may give an attacker 888dfaa2ef6SLukas Czerner * useful information. 889dfaa2ef6SLukas Czerner */ 890c52abf56SEvan Green } else if (!file->f_op->fallocate || lo->lo_encrypt_key_size) { 891dfaa2ef6SLukas Czerner q->limits.discard_granularity = 0; 892dfaa2ef6SLukas Czerner q->limits.discard_alignment = 0; 8932bb4cd5cSJens Axboe blk_queue_max_discard_sectors(q, 0); 89419372e27SChristoph Hellwig blk_queue_max_write_zeroes_sectors(q, 0); 895dfaa2ef6SLukas Czerner 896c52abf56SEvan Green } else { 897dfaa2ef6SLukas Czerner q->limits.discard_granularity = inode->i_sb->s_blocksize; 898dfaf3c03SLukas Czerner q->limits.discard_alignment = 0; 899f2c6df7dSHannes Reinecke 9001e6ec9eaSOmar Sandoval blk_queue_max_discard_sectors(q, UINT_MAX >> 9); 9011e6ec9eaSOmar Sandoval blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9); 902c52abf56SEvan Green } 903c52abf56SEvan Green 904c52abf56SEvan Green if (q->limits.max_write_zeroes_sectors) 9058b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 906c52abf56SEvan Green else 907c52abf56SEvan Green blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q); 908dfaa2ef6SLukas Czerner } 909dfaa2ef6SLukas Czerner 910e03a3d7aSMing Lei static void loop_unprepare_queue(struct loop_device *lo) 911e03a3d7aSMing Lei { 9123989144fSPetr Mladek kthread_flush_worker(&lo->worker); 913e03a3d7aSMing Lei kthread_stop(lo->worker_task); 914e03a3d7aSMing Lei } 915e03a3d7aSMing Lei 916b2ee7d46SNeilBrown static int loop_kthread_worker_fn(void *worker_ptr) 917b2ee7d46SNeilBrown { 918d0a255e7SMikulas Patocka current->flags |= PF_LESS_THROTTLE | PF_MEMALLOC_NOIO; 919b2ee7d46SNeilBrown return kthread_worker_fn(worker_ptr); 920b2ee7d46SNeilBrown } 921b2ee7d46SNeilBrown 922e03a3d7aSMing Lei static int loop_prepare_queue(struct loop_device *lo) 923e03a3d7aSMing Lei { 9243989144fSPetr Mladek kthread_init_worker(&lo->worker); 925b2ee7d46SNeilBrown lo->worker_task = kthread_run(loop_kthread_worker_fn, 926e03a3d7aSMing Lei &lo->worker, "loop%d", lo->lo_number); 927e03a3d7aSMing Lei if (IS_ERR(lo->worker_task)) 928e03a3d7aSMing Lei return -ENOMEM; 929e03a3d7aSMing Lei set_user_nice(lo->worker_task, MIN_NICE); 930e03a3d7aSMing Lei return 0; 931e03a3d7aSMing Lei } 932e03a3d7aSMing Lei 93356a85fd8SHolger Hoffstätte static void loop_update_rotational(struct loop_device *lo) 93456a85fd8SHolger Hoffstätte { 93556a85fd8SHolger Hoffstätte struct file *file = lo->lo_backing_file; 93656a85fd8SHolger Hoffstätte struct inode *file_inode = file->f_mapping->host; 93756a85fd8SHolger Hoffstätte struct block_device *file_bdev = file_inode->i_sb->s_bdev; 93856a85fd8SHolger Hoffstätte struct request_queue *q = lo->lo_queue; 93956a85fd8SHolger Hoffstätte bool nonrot = true; 94056a85fd8SHolger Hoffstätte 94156a85fd8SHolger Hoffstätte /* not all filesystems (e.g. tmpfs) have a sb->s_bdev */ 94256a85fd8SHolger Hoffstätte if (file_bdev) 94356a85fd8SHolger Hoffstätte nonrot = blk_queue_nonrot(bdev_get_queue(file_bdev)); 94456a85fd8SHolger Hoffstätte 94556a85fd8SHolger Hoffstätte if (nonrot) 94656a85fd8SHolger Hoffstätte blk_queue_flag_set(QUEUE_FLAG_NONROT, q); 94756a85fd8SHolger Hoffstätte else 94856a85fd8SHolger Hoffstätte blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); 94956a85fd8SHolger Hoffstätte } 95056a85fd8SHolger Hoffstätte 951bb214884SAl Viro static int loop_set_fd(struct loop_device *lo, fmode_t mode, 9521da177e4SLinus Torvalds struct block_device *bdev, unsigned int arg) 9531da177e4SLinus Torvalds { 954d2ac838eSTheodore Ts'o struct file *file; 9551da177e4SLinus Torvalds struct inode *inode; 9561da177e4SLinus Torvalds struct address_space *mapping; 95789e524c0SJan Kara struct block_device *claimed_bdev = NULL; 9581da177e4SLinus Torvalds int lo_flags = 0; 9591da177e4SLinus Torvalds int error; 9601da177e4SLinus Torvalds loff_t size; 96185b0a54aSJan Kara bool partscan; 9621da177e4SLinus Torvalds 9631da177e4SLinus Torvalds /* This is safe, since we have a reference from open(). */ 9641da177e4SLinus Torvalds __module_get(THIS_MODULE); 9651da177e4SLinus Torvalds 9661da177e4SLinus Torvalds error = -EBADF; 9671da177e4SLinus Torvalds file = fget(arg); 9681da177e4SLinus Torvalds if (!file) 9691da177e4SLinus Torvalds goto out; 9701da177e4SLinus Torvalds 97133ec3e53SJan Kara /* 97233ec3e53SJan Kara * If we don't hold exclusive handle for the device, upgrade to it 97333ec3e53SJan Kara * here to avoid changing device under exclusive owner. 97433ec3e53SJan Kara */ 97533ec3e53SJan Kara if (!(mode & FMODE_EXCL)) { 97689e524c0SJan Kara claimed_bdev = bd_start_claiming(bdev, loop_set_fd); 97789e524c0SJan Kara if (IS_ERR(claimed_bdev)) { 97889e524c0SJan Kara error = PTR_ERR(claimed_bdev); 979757ecf40SJan Kara goto out_putf; 98033ec3e53SJan Kara } 98189e524c0SJan Kara } 98233ec3e53SJan Kara 98333ec3e53SJan Kara error = mutex_lock_killable(&loop_ctl_mutex); 98433ec3e53SJan Kara if (error) 98533ec3e53SJan Kara goto out_bdev; 986757ecf40SJan Kara 9871da177e4SLinus Torvalds error = -EBUSY; 9881da177e4SLinus Torvalds if (lo->lo_state != Lo_unbound) 989757ecf40SJan Kara goto out_unlock; 9901da177e4SLinus Torvalds 991d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev); 992d2ac838eSTheodore Ts'o if (error) 993757ecf40SJan Kara goto out_unlock; 9941da177e4SLinus Torvalds 9951da177e4SLinus Torvalds mapping = file->f_mapping; 9961da177e4SLinus Torvalds inode = mapping->host; 9971da177e4SLinus Torvalds 998456be148SChristoph Hellwig if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) || 999283e7e5dSAl Viro !file->f_op->write_iter) 10001da177e4SLinus Torvalds lo_flags |= LO_FLAGS_READ_ONLY; 10011da177e4SLinus Torvalds 1002456be148SChristoph Hellwig size = get_loop_size(lo, file); 1003*083a6a50SMartijn Coenen 1004e03a3d7aSMing Lei error = loop_prepare_queue(lo); 1005e03a3d7aSMing Lei if (error) 1006757ecf40SJan Kara goto out_unlock; 10071da177e4SLinus Torvalds 1008456be148SChristoph Hellwig error = 0; 10091da177e4SLinus Torvalds 10101da177e4SLinus Torvalds set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0); 10111da177e4SLinus Torvalds 10122e5ab5f3SMing Lei lo->use_dio = false; 10131da177e4SLinus Torvalds lo->lo_device = bdev; 10141da177e4SLinus Torvalds lo->lo_flags = lo_flags; 10151da177e4SLinus Torvalds lo->lo_backing_file = file; 1016aa4d8616SChristoph Hellwig lo->transfer = NULL; 10171da177e4SLinus Torvalds lo->ioctl = NULL; 10181da177e4SLinus Torvalds lo->lo_sizelimit = 0; 10191da177e4SLinus Torvalds lo->old_gfp_mask = mapping_gfp_mask(mapping); 10201da177e4SLinus Torvalds mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 10211da177e4SLinus Torvalds 102268db1961SNikanth Karthikesan if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) 102321d0727fSJens Axboe blk_queue_write_cache(lo->lo_queue, true, false); 102468db1961SNikanth Karthikesan 102585560117SMartijn Coenen if (io_is_direct(lo->lo_backing_file) && inode->i_sb->s_bdev) { 102685560117SMartijn Coenen /* In case of direct I/O, match underlying block size */ 102785560117SMartijn Coenen unsigned short bsize = bdev_logical_block_size( 102885560117SMartijn Coenen inode->i_sb->s_bdev); 102985560117SMartijn Coenen 103085560117SMartijn Coenen blk_queue_logical_block_size(lo->lo_queue, bsize); 103185560117SMartijn Coenen blk_queue_physical_block_size(lo->lo_queue, bsize); 103285560117SMartijn Coenen blk_queue_io_min(lo->lo_queue, bsize); 103385560117SMartijn Coenen } 103485560117SMartijn Coenen 103556a85fd8SHolger Hoffstätte loop_update_rotational(lo); 10362e5ab5f3SMing Lei loop_update_dio(lo); 103773285082SKen Chen set_capacity(lo->lo_disk, size); 10381da177e4SLinus Torvalds bd_set_size(bdev, size << 9); 1039ee862730SMilan Broz loop_sysfs_init(lo); 1040c3473c63SDavid Zeuthen /* let user-space know about the new size */ 1041c3473c63SDavid Zeuthen kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 10421da177e4SLinus Torvalds 10438a0740c4SOmar Sandoval set_blocksize(bdev, S_ISBLK(inode->i_mode) ? 10448a0740c4SOmar Sandoval block_size(inode->i_bdev) : PAGE_SIZE); 10451da177e4SLinus Torvalds 10466c997918SSerge E. Hallyn lo->lo_state = Lo_bound; 1047e03c8dd1SKay Sievers if (part_shift) 1048e03c8dd1SKay Sievers lo->lo_flags |= LO_FLAGS_PARTSCAN; 104985b0a54aSJan Kara partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 1050c1681bf8SAnatol Pomozov 1051c1681bf8SAnatol Pomozov /* Grab the block_device to prevent its destruction after we 1052a2505b79SJan Kara * put /dev/loopXX inode. Later in __loop_clr_fd() we bdput(bdev). 1053c1681bf8SAnatol Pomozov */ 1054c1681bf8SAnatol Pomozov bdgrab(bdev); 1055757ecf40SJan Kara mutex_unlock(&loop_ctl_mutex); 105685b0a54aSJan Kara if (partscan) 105785b0a54aSJan Kara loop_reread_partitions(lo, bdev); 105889e524c0SJan Kara if (claimed_bdev) 105989e524c0SJan Kara bd_abort_claiming(bdev, claimed_bdev, loop_set_fd); 10601da177e4SLinus Torvalds return 0; 10611da177e4SLinus Torvalds 1062757ecf40SJan Kara out_unlock: 1063757ecf40SJan Kara mutex_unlock(&loop_ctl_mutex); 106433ec3e53SJan Kara out_bdev: 106589e524c0SJan Kara if (claimed_bdev) 106689e524c0SJan Kara bd_abort_claiming(bdev, claimed_bdev, loop_set_fd); 10671da177e4SLinus Torvalds out_putf: 10681da177e4SLinus Torvalds fput(file); 10691da177e4SLinus Torvalds out: 10701da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */ 10711da177e4SLinus Torvalds module_put(THIS_MODULE); 10721da177e4SLinus Torvalds return error; 10731da177e4SLinus Torvalds } 10741da177e4SLinus Torvalds 10751da177e4SLinus Torvalds static int 10761da177e4SLinus Torvalds loop_release_xfer(struct loop_device *lo) 10771da177e4SLinus Torvalds { 10781da177e4SLinus Torvalds int err = 0; 10791da177e4SLinus Torvalds struct loop_func_table *xfer = lo->lo_encryption; 10801da177e4SLinus Torvalds 10811da177e4SLinus Torvalds if (xfer) { 10821da177e4SLinus Torvalds if (xfer->release) 10831da177e4SLinus Torvalds err = xfer->release(lo); 10841da177e4SLinus Torvalds lo->transfer = NULL; 10851da177e4SLinus Torvalds lo->lo_encryption = NULL; 10861da177e4SLinus Torvalds module_put(xfer->owner); 10871da177e4SLinus Torvalds } 10881da177e4SLinus Torvalds return err; 10891da177e4SLinus Torvalds } 10901da177e4SLinus Torvalds 10911da177e4SLinus Torvalds static int 10921da177e4SLinus Torvalds loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, 10931da177e4SLinus Torvalds const struct loop_info64 *i) 10941da177e4SLinus Torvalds { 10951da177e4SLinus Torvalds int err = 0; 10961da177e4SLinus Torvalds 10971da177e4SLinus Torvalds if (xfer) { 10981da177e4SLinus Torvalds struct module *owner = xfer->owner; 10991da177e4SLinus Torvalds 11001da177e4SLinus Torvalds if (!try_module_get(owner)) 11011da177e4SLinus Torvalds return -EINVAL; 11021da177e4SLinus Torvalds if (xfer->init) 11031da177e4SLinus Torvalds err = xfer->init(lo, i); 11041da177e4SLinus Torvalds if (err) 11051da177e4SLinus Torvalds module_put(owner); 11061da177e4SLinus Torvalds else 11071da177e4SLinus Torvalds lo->lo_encryption = xfer; 11081da177e4SLinus Torvalds } 11091da177e4SLinus Torvalds return err; 11101da177e4SLinus Torvalds } 11111da177e4SLinus Torvalds 11120da03cabSJan Kara static int __loop_clr_fd(struct loop_device *lo, bool release) 11131da177e4SLinus Torvalds { 11147ccd0791SJan Kara struct file *filp = NULL; 1115b4e3ca1aSAl Viro gfp_t gfp = lo->old_gfp_mask; 11164c823cc3SAyan George struct block_device *bdev = lo->lo_device; 11177ccd0791SJan Kara int err = 0; 11180da03cabSJan Kara bool partscan = false; 11190da03cabSJan Kara int lo_number; 11201da177e4SLinus Torvalds 11217ccd0791SJan Kara mutex_lock(&loop_ctl_mutex); 11227ccd0791SJan Kara if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) { 11237ccd0791SJan Kara err = -ENXIO; 11247ccd0791SJan Kara goto out_unlock; 11257ccd0791SJan Kara } 11261da177e4SLinus Torvalds 11277ccd0791SJan Kara filp = lo->lo_backing_file; 11287ccd0791SJan Kara if (filp == NULL) { 11297ccd0791SJan Kara err = -EINVAL; 11307ccd0791SJan Kara goto out_unlock; 11317ccd0791SJan Kara } 11321da177e4SLinus Torvalds 1133f8933667SMing Lei /* freeze request queue during the transition */ 1134f8933667SMing Lei blk_mq_freeze_queue(lo->lo_queue); 1135f8933667SMing Lei 11361da177e4SLinus Torvalds spin_lock_irq(&lo->lo_lock); 11371da177e4SLinus Torvalds lo->lo_backing_file = NULL; 113805eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock); 11391da177e4SLinus Torvalds 11401da177e4SLinus Torvalds loop_release_xfer(lo); 11411da177e4SLinus Torvalds lo->transfer = NULL; 11421da177e4SLinus Torvalds lo->ioctl = NULL; 11431da177e4SLinus Torvalds lo->lo_device = NULL; 11441da177e4SLinus Torvalds lo->lo_encryption = NULL; 11451da177e4SLinus Torvalds lo->lo_offset = 0; 11461da177e4SLinus Torvalds lo->lo_sizelimit = 0; 11471da177e4SLinus Torvalds lo->lo_encrypt_key_size = 0; 11481da177e4SLinus Torvalds memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE); 11491da177e4SLinus Torvalds memset(lo->lo_crypt_name, 0, LO_NAME_SIZE); 11501da177e4SLinus Torvalds memset(lo->lo_file_name, 0, LO_NAME_SIZE); 115189e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, 512); 1152bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, 512); 1153bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, 512); 1154c1681bf8SAnatol Pomozov if (bdev) { 1155c1681bf8SAnatol Pomozov bdput(bdev); 1156f98393a6SPeter Zijlstra invalidate_bdev(bdev); 1157eedffa28SJeff Layton bdev->bd_inode->i_mapping->wb_err = 0; 1158c1681bf8SAnatol Pomozov } 115973285082SKen Chen set_capacity(lo->lo_disk, 0); 116051a0bb0cSMilan Broz loop_sysfs_exit(lo); 1161c3473c63SDavid Zeuthen if (bdev) { 11621da177e4SLinus Torvalds bd_set_size(bdev, 0); 1163c3473c63SDavid Zeuthen /* let user-space know about this change */ 1164c3473c63SDavid Zeuthen kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 1165c3473c63SDavid Zeuthen } 11661da177e4SLinus Torvalds mapping_set_gfp_mask(filp->f_mapping, gfp); 11671da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */ 11681da177e4SLinus Torvalds module_put(THIS_MODULE); 1169f8933667SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 1170f8933667SMing Lei 11710da03cabSJan Kara partscan = lo->lo_flags & LO_FLAGS_PARTSCAN && bdev; 11720da03cabSJan Kara lo_number = lo->lo_number; 11730da03cabSJan Kara loop_unprepare_queue(lo); 11740da03cabSJan Kara out_unlock: 11750da03cabSJan Kara mutex_unlock(&loop_ctl_mutex); 11760da03cabSJan Kara if (partscan) { 1177d57f3374SJan Kara /* 1178d57f3374SJan Kara * bd_mutex has been held already in release path, so don't 1179d57f3374SJan Kara * acquire it if this function is called in such case. 1180d57f3374SJan Kara * 1181d57f3374SJan Kara * If the reread partition isn't from release path, lo_refcnt 1182d57f3374SJan Kara * must be at least one and it can only become zero when the 1183d57f3374SJan Kara * current holder is released. 1184d57f3374SJan Kara */ 1185f0b870dfSChristoph Hellwig if (!release) 1186f0b870dfSChristoph Hellwig mutex_lock(&bdev->bd_mutex); 1187f0b870dfSChristoph Hellwig err = bdev_disk_changed(bdev, false); 1188f0b870dfSChristoph Hellwig if (!release) 1189f0b870dfSChristoph Hellwig mutex_unlock(&bdev->bd_mutex); 119040853d6fSDongli Zhang if (err) 1191d57f3374SJan Kara pr_warn("%s: partition scan of loop%d failed (rc=%d)\n", 11920da03cabSJan Kara __func__, lo_number, err); 1193d57f3374SJan Kara /* Device is gone, no point in returning error */ 1194d57f3374SJan Kara err = 0; 1195d57f3374SJan Kara } 1196758a58d0SDongli Zhang 1197758a58d0SDongli Zhang /* 1198758a58d0SDongli Zhang * lo->lo_state is set to Lo_unbound here after above partscan has 1199758a58d0SDongli Zhang * finished. 1200758a58d0SDongli Zhang * 1201758a58d0SDongli Zhang * There cannot be anybody else entering __loop_clr_fd() as 1202758a58d0SDongli Zhang * lo->lo_backing_file is already cleared and Lo_rundown state 1203758a58d0SDongli Zhang * protects us from all the other places trying to change the 'lo' 1204758a58d0SDongli Zhang * device. 1205758a58d0SDongli Zhang */ 1206758a58d0SDongli Zhang mutex_lock(&loop_ctl_mutex); 1207758a58d0SDongli Zhang lo->lo_flags = 0; 1208758a58d0SDongli Zhang if (!part_shift) 1209758a58d0SDongli Zhang lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN; 1210758a58d0SDongli Zhang lo->lo_state = Lo_unbound; 1211758a58d0SDongli Zhang mutex_unlock(&loop_ctl_mutex); 1212758a58d0SDongli Zhang 1213f028f3b2SNikanth Karthikesan /* 1214310ca162STetsuo Handa * Need not hold loop_ctl_mutex to fput backing file. 1215310ca162STetsuo Handa * Calling fput holding loop_ctl_mutex triggers a circular 1216f028f3b2SNikanth Karthikesan * lock dependency possibility warning as fput can take 1217310ca162STetsuo Handa * bd_mutex which is usually taken before loop_ctl_mutex. 1218f028f3b2SNikanth Karthikesan */ 12197ccd0791SJan Kara if (filp) 1220f028f3b2SNikanth Karthikesan fput(filp); 12217ccd0791SJan Kara return err; 12221da177e4SLinus Torvalds } 12231da177e4SLinus Torvalds 1224a2505b79SJan Kara static int loop_clr_fd(struct loop_device *lo) 1225a2505b79SJan Kara { 12267ccd0791SJan Kara int err; 12277ccd0791SJan Kara 1228c28445faSJan Kara err = mutex_lock_killable(&loop_ctl_mutex); 12297ccd0791SJan Kara if (err) 12307ccd0791SJan Kara return err; 12317ccd0791SJan Kara if (lo->lo_state != Lo_bound) { 12327ccd0791SJan Kara mutex_unlock(&loop_ctl_mutex); 1233a2505b79SJan Kara return -ENXIO; 12347ccd0791SJan Kara } 1235a2505b79SJan Kara /* 1236a2505b79SJan Kara * If we've explicitly asked to tear down the loop device, 1237a2505b79SJan Kara * and it has an elevated reference count, set it for auto-teardown when 1238a2505b79SJan Kara * the last reference goes away. This stops $!~#$@ udev from 1239a2505b79SJan Kara * preventing teardown because it decided that it needs to run blkid on 1240a2505b79SJan Kara * the loopback device whenever they appear. xfstests is notorious for 1241a2505b79SJan Kara * failing tests because blkid via udev races with a losetup 1242a2505b79SJan Kara * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d 1243a2505b79SJan Kara * command to fail with EBUSY. 1244a2505b79SJan Kara */ 1245a2505b79SJan Kara if (atomic_read(&lo->lo_refcnt) > 1) { 1246a2505b79SJan Kara lo->lo_flags |= LO_FLAGS_AUTOCLEAR; 1247a2505b79SJan Kara mutex_unlock(&loop_ctl_mutex); 1248a2505b79SJan Kara return 0; 1249a2505b79SJan Kara } 1250a2505b79SJan Kara lo->lo_state = Lo_rundown; 12517ccd0791SJan Kara mutex_unlock(&loop_ctl_mutex); 1252a2505b79SJan Kara 12530da03cabSJan Kara return __loop_clr_fd(lo, false); 1254a2505b79SJan Kara } 1255a2505b79SJan Kara 12561da177e4SLinus Torvalds static int 12571da177e4SLinus Torvalds loop_set_status(struct loop_device *lo, const struct loop_info64 *info) 12581da177e4SLinus Torvalds { 12591da177e4SLinus Torvalds int err; 12601da177e4SLinus Torvalds struct loop_func_table *xfer; 1261e4849737SEric W. Biederman kuid_t uid = current_uid(); 126285b0a54aSJan Kara struct block_device *bdev; 126385b0a54aSJan Kara bool partscan = false; 12641da177e4SLinus Torvalds 1265c28445faSJan Kara err = mutex_lock_killable(&loop_ctl_mutex); 1266550df5fdSJan Kara if (err) 1267550df5fdSJan Kara return err; 1268b0fafa81SDavid Howells if (lo->lo_encrypt_key_size && 1269e4849737SEric W. Biederman !uid_eq(lo->lo_key_owner, uid) && 1270550df5fdSJan Kara !capable(CAP_SYS_ADMIN)) { 1271550df5fdSJan Kara err = -EPERM; 1272550df5fdSJan Kara goto out_unlock; 1273550df5fdSJan Kara } 1274550df5fdSJan Kara if (lo->lo_state != Lo_bound) { 1275550df5fdSJan Kara err = -ENXIO; 1276550df5fdSJan Kara goto out_unlock; 1277550df5fdSJan Kara } 1278550df5fdSJan Kara if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) { 1279550df5fdSJan Kara err = -EINVAL; 1280550df5fdSJan Kara goto out_unlock; 1281550df5fdSJan Kara } 12821da177e4SLinus Torvalds 12835db470e2SJaegeuk Kim if (lo->lo_offset != info->lo_offset || 12845db470e2SJaegeuk Kim lo->lo_sizelimit != info->lo_sizelimit) { 12855db470e2SJaegeuk Kim sync_blockdev(lo->lo_device); 12865db470e2SJaegeuk Kim kill_bdev(lo->lo_device); 12875db470e2SJaegeuk Kim } 12885db470e2SJaegeuk Kim 1289ecdd0959SMing Lei /* I/O need to be drained during transfer transition */ 1290ecdd0959SMing Lei blk_mq_freeze_queue(lo->lo_queue); 1291ecdd0959SMing Lei 12921da177e4SLinus Torvalds err = loop_release_xfer(lo); 12931da177e4SLinus Torvalds if (err) 1294550df5fdSJan Kara goto out_unfreeze; 12951da177e4SLinus Torvalds 12961da177e4SLinus Torvalds if (info->lo_encrypt_type) { 12971da177e4SLinus Torvalds unsigned int type = info->lo_encrypt_type; 12981da177e4SLinus Torvalds 12991e047eaaSTetsuo Handa if (type >= MAX_LO_CRYPT) { 13001e047eaaSTetsuo Handa err = -EINVAL; 1301550df5fdSJan Kara goto out_unfreeze; 13021e047eaaSTetsuo Handa } 13031da177e4SLinus Torvalds xfer = xfer_funcs[type]; 13041e047eaaSTetsuo Handa if (xfer == NULL) { 13051e047eaaSTetsuo Handa err = -EINVAL; 1306550df5fdSJan Kara goto out_unfreeze; 13071e047eaaSTetsuo Handa } 13081da177e4SLinus Torvalds } else 13091da177e4SLinus Torvalds xfer = NULL; 13101da177e4SLinus Torvalds 13111da177e4SLinus Torvalds err = loop_init_xfer(lo, xfer, info); 13121da177e4SLinus Torvalds if (err) 1313550df5fdSJan Kara goto out_unfreeze; 13141da177e4SLinus Torvalds 13151da177e4SLinus Torvalds if (lo->lo_offset != info->lo_offset || 13161e6ec9eaSOmar Sandoval lo->lo_sizelimit != info->lo_sizelimit) { 13175db470e2SJaegeuk Kim /* kill_bdev should have truncated all the pages */ 13185db470e2SJaegeuk Kim if (lo->lo_device->bd_inode->i_mapping->nrpages) { 13195db470e2SJaegeuk Kim err = -EAGAIN; 13205db470e2SJaegeuk Kim pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", 13215db470e2SJaegeuk Kim __func__, lo->lo_number, lo->lo_file_name, 13225db470e2SJaegeuk Kim lo->lo_device->bd_inode->i_mapping->nrpages); 13235db470e2SJaegeuk Kim goto out_unfreeze; 13245db470e2SJaegeuk Kim } 1325*083a6a50SMartijn Coenen figure_loop_size(lo, info->lo_offset, info->lo_sizelimit); 1326b040ad9cSArnd Bergmann } 1327541c742aSGuo Chao 13281da177e4SLinus Torvalds memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); 13291da177e4SLinus Torvalds memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); 13301da177e4SLinus Torvalds lo->lo_file_name[LO_NAME_SIZE-1] = 0; 13311da177e4SLinus Torvalds lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; 13321da177e4SLinus Torvalds 13331da177e4SLinus Torvalds if (!xfer) 13341da177e4SLinus Torvalds xfer = &none_funcs; 13351da177e4SLinus Torvalds lo->transfer = xfer->transfer; 13361da177e4SLinus Torvalds lo->ioctl = xfer->ioctl; 13371da177e4SLinus Torvalds 133896c58655SDavid Woodhouse if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) != 133996c58655SDavid Woodhouse (info->lo_flags & LO_FLAGS_AUTOCLEAR)) 134096c58655SDavid Woodhouse lo->lo_flags ^= LO_FLAGS_AUTOCLEAR; 134196c58655SDavid Woodhouse 13421da177e4SLinus Torvalds lo->lo_encrypt_key_size = info->lo_encrypt_key_size; 13431da177e4SLinus Torvalds lo->lo_init[0] = info->lo_init[0]; 13441da177e4SLinus Torvalds lo->lo_init[1] = info->lo_init[1]; 13451da177e4SLinus Torvalds if (info->lo_encrypt_key_size) { 13461da177e4SLinus Torvalds memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, 13471da177e4SLinus Torvalds info->lo_encrypt_key_size); 1348b0fafa81SDavid Howells lo->lo_key_owner = uid; 13491da177e4SLinus Torvalds } 13501da177e4SLinus Torvalds 13517c5014b0SMartijn Coenen loop_config_discard(lo); 13527c5014b0SMartijn Coenen 13532e5ab5f3SMing Lei /* update dio if lo_offset or transfer is changed */ 13542e5ab5f3SMing Lei __loop_update_dio(lo, lo->use_dio); 13552e5ab5f3SMing Lei 1356550df5fdSJan Kara out_unfreeze: 1357ecdd0959SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 1358e02898b4SOmar Sandoval 1359e02898b4SOmar Sandoval if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) && 1360e02898b4SOmar Sandoval !(lo->lo_flags & LO_FLAGS_PARTSCAN)) { 1361e02898b4SOmar Sandoval lo->lo_flags |= LO_FLAGS_PARTSCAN; 1362e02898b4SOmar Sandoval lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; 136385b0a54aSJan Kara bdev = lo->lo_device; 136485b0a54aSJan Kara partscan = true; 1365e02898b4SOmar Sandoval } 1366550df5fdSJan Kara out_unlock: 1367550df5fdSJan Kara mutex_unlock(&loop_ctl_mutex); 136885b0a54aSJan Kara if (partscan) 136985b0a54aSJan Kara loop_reread_partitions(lo, bdev); 1370e02898b4SOmar Sandoval 1371ecdd0959SMing Lei return err; 13721da177e4SLinus Torvalds } 13731da177e4SLinus Torvalds 13741da177e4SLinus Torvalds static int 13751da177e4SLinus Torvalds loop_get_status(struct loop_device *lo, struct loop_info64 *info) 13761da177e4SLinus Torvalds { 1377b1ab5fa3STetsuo Handa struct path path; 13781da177e4SLinus Torvalds struct kstat stat; 13792d1d4c1eSOmar Sandoval int ret; 13801da177e4SLinus Torvalds 1381c28445faSJan Kara ret = mutex_lock_killable(&loop_ctl_mutex); 13824a5ce9baSJan Kara if (ret) 13834a5ce9baSJan Kara return ret; 13842d1d4c1eSOmar Sandoval if (lo->lo_state != Lo_bound) { 1385310ca162STetsuo Handa mutex_unlock(&loop_ctl_mutex); 13861da177e4SLinus Torvalds return -ENXIO; 13872d1d4c1eSOmar Sandoval } 13882d1d4c1eSOmar Sandoval 13891da177e4SLinus Torvalds memset(info, 0, sizeof(*info)); 13901da177e4SLinus Torvalds info->lo_number = lo->lo_number; 13911da177e4SLinus Torvalds info->lo_offset = lo->lo_offset; 13921da177e4SLinus Torvalds info->lo_sizelimit = lo->lo_sizelimit; 13931da177e4SLinus Torvalds info->lo_flags = lo->lo_flags; 13941da177e4SLinus Torvalds memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE); 13951da177e4SLinus Torvalds memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE); 13961da177e4SLinus Torvalds info->lo_encrypt_type = 13971da177e4SLinus Torvalds lo->lo_encryption ? lo->lo_encryption->number : 0; 13981da177e4SLinus Torvalds if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) { 13991da177e4SLinus Torvalds info->lo_encrypt_key_size = lo->lo_encrypt_key_size; 14001da177e4SLinus Torvalds memcpy(info->lo_encrypt_key, lo->lo_encrypt_key, 14011da177e4SLinus Torvalds lo->lo_encrypt_key_size); 14021da177e4SLinus Torvalds } 14032d1d4c1eSOmar Sandoval 1404310ca162STetsuo Handa /* Drop loop_ctl_mutex while we call into the filesystem. */ 1405b1ab5fa3STetsuo Handa path = lo->lo_backing_file->f_path; 1406b1ab5fa3STetsuo Handa path_get(&path); 1407310ca162STetsuo Handa mutex_unlock(&loop_ctl_mutex); 1408b1ab5fa3STetsuo Handa ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT); 14092d1d4c1eSOmar Sandoval if (!ret) { 14102d1d4c1eSOmar Sandoval info->lo_device = huge_encode_dev(stat.dev); 14112d1d4c1eSOmar Sandoval info->lo_inode = stat.ino; 14122d1d4c1eSOmar Sandoval info->lo_rdevice = huge_encode_dev(stat.rdev); 14132d1d4c1eSOmar Sandoval } 1414b1ab5fa3STetsuo Handa path_put(&path); 14152d1d4c1eSOmar Sandoval return ret; 14161da177e4SLinus Torvalds } 14171da177e4SLinus Torvalds 14181da177e4SLinus Torvalds static void 14191da177e4SLinus Torvalds loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64) 14201da177e4SLinus Torvalds { 14211da177e4SLinus Torvalds memset(info64, 0, sizeof(*info64)); 14221da177e4SLinus Torvalds info64->lo_number = info->lo_number; 14231da177e4SLinus Torvalds info64->lo_device = info->lo_device; 14241da177e4SLinus Torvalds info64->lo_inode = info->lo_inode; 14251da177e4SLinus Torvalds info64->lo_rdevice = info->lo_rdevice; 14261da177e4SLinus Torvalds info64->lo_offset = info->lo_offset; 14271da177e4SLinus Torvalds info64->lo_sizelimit = 0; 14281da177e4SLinus Torvalds info64->lo_encrypt_type = info->lo_encrypt_type; 14291da177e4SLinus Torvalds info64->lo_encrypt_key_size = info->lo_encrypt_key_size; 14301da177e4SLinus Torvalds info64->lo_flags = info->lo_flags; 14311da177e4SLinus Torvalds info64->lo_init[0] = info->lo_init[0]; 14321da177e4SLinus Torvalds info64->lo_init[1] = info->lo_init[1]; 14331da177e4SLinus Torvalds if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 14341da177e4SLinus Torvalds memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE); 14351da177e4SLinus Torvalds else 14361da177e4SLinus Torvalds memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); 14371da177e4SLinus Torvalds memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE); 14381da177e4SLinus Torvalds } 14391da177e4SLinus Torvalds 14401da177e4SLinus Torvalds static int 14411da177e4SLinus Torvalds loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info) 14421da177e4SLinus Torvalds { 14431da177e4SLinus Torvalds memset(info, 0, sizeof(*info)); 14441da177e4SLinus Torvalds info->lo_number = info64->lo_number; 14451da177e4SLinus Torvalds info->lo_device = info64->lo_device; 14461da177e4SLinus Torvalds info->lo_inode = info64->lo_inode; 14471da177e4SLinus Torvalds info->lo_rdevice = info64->lo_rdevice; 14481da177e4SLinus Torvalds info->lo_offset = info64->lo_offset; 14491da177e4SLinus Torvalds info->lo_encrypt_type = info64->lo_encrypt_type; 14501da177e4SLinus Torvalds info->lo_encrypt_key_size = info64->lo_encrypt_key_size; 14511da177e4SLinus Torvalds info->lo_flags = info64->lo_flags; 14521da177e4SLinus Torvalds info->lo_init[0] = info64->lo_init[0]; 14531da177e4SLinus Torvalds info->lo_init[1] = info64->lo_init[1]; 14541da177e4SLinus Torvalds if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 14551da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 14561da177e4SLinus Torvalds else 14571da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); 14581da177e4SLinus Torvalds memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 14591da177e4SLinus Torvalds 14601da177e4SLinus Torvalds /* error in case values were truncated */ 14611da177e4SLinus Torvalds if (info->lo_device != info64->lo_device || 14621da177e4SLinus Torvalds info->lo_rdevice != info64->lo_rdevice || 14631da177e4SLinus Torvalds info->lo_inode != info64->lo_inode || 14641da177e4SLinus Torvalds info->lo_offset != info64->lo_offset) 14651da177e4SLinus Torvalds return -EOVERFLOW; 14661da177e4SLinus Torvalds 14671da177e4SLinus Torvalds return 0; 14681da177e4SLinus Torvalds } 14691da177e4SLinus Torvalds 14701da177e4SLinus Torvalds static int 14711da177e4SLinus Torvalds loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg) 14721da177e4SLinus Torvalds { 14731da177e4SLinus Torvalds struct loop_info info; 14741da177e4SLinus Torvalds struct loop_info64 info64; 14751da177e4SLinus Torvalds 14761da177e4SLinus Torvalds if (copy_from_user(&info, arg, sizeof (struct loop_info))) 14771da177e4SLinus Torvalds return -EFAULT; 14781da177e4SLinus Torvalds loop_info64_from_old(&info, &info64); 14791da177e4SLinus Torvalds return loop_set_status(lo, &info64); 14801da177e4SLinus Torvalds } 14811da177e4SLinus Torvalds 14821da177e4SLinus Torvalds static int 14831da177e4SLinus Torvalds loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg) 14841da177e4SLinus Torvalds { 14851da177e4SLinus Torvalds struct loop_info64 info64; 14861da177e4SLinus Torvalds 14871da177e4SLinus Torvalds if (copy_from_user(&info64, arg, sizeof (struct loop_info64))) 14881da177e4SLinus Torvalds return -EFAULT; 14891da177e4SLinus Torvalds return loop_set_status(lo, &info64); 14901da177e4SLinus Torvalds } 14911da177e4SLinus Torvalds 14921da177e4SLinus Torvalds static int 14931da177e4SLinus Torvalds loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) { 14941da177e4SLinus Torvalds struct loop_info info; 14951da177e4SLinus Torvalds struct loop_info64 info64; 1496bdac616dSOmar Sandoval int err; 14971da177e4SLinus Torvalds 14984a5ce9baSJan Kara if (!arg) 1499bdac616dSOmar Sandoval return -EINVAL; 15001da177e4SLinus Torvalds err = loop_get_status(lo, &info64); 15011da177e4SLinus Torvalds if (!err) 15021da177e4SLinus Torvalds err = loop_info64_to_old(&info64, &info); 15031da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info, sizeof(info))) 15041da177e4SLinus Torvalds err = -EFAULT; 15051da177e4SLinus Torvalds 15061da177e4SLinus Torvalds return err; 15071da177e4SLinus Torvalds } 15081da177e4SLinus Torvalds 15091da177e4SLinus Torvalds static int 15101da177e4SLinus Torvalds loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { 15111da177e4SLinus Torvalds struct loop_info64 info64; 1512bdac616dSOmar Sandoval int err; 15131da177e4SLinus Torvalds 15144a5ce9baSJan Kara if (!arg) 1515bdac616dSOmar Sandoval return -EINVAL; 15161da177e4SLinus Torvalds err = loop_get_status(lo, &info64); 15171da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info64, sizeof(info64))) 15181da177e4SLinus Torvalds err = -EFAULT; 15191da177e4SLinus Torvalds 15201da177e4SLinus Torvalds return err; 15211da177e4SLinus Torvalds } 15221da177e4SLinus Torvalds 152351001b7dSHannes Reinecke static int loop_set_capacity(struct loop_device *lo) 152453d66608SJ. R. Okajima { 152553d66608SJ. R. Okajima if (unlikely(lo->lo_state != Lo_bound)) 15267b0576a3SGuo Chao return -ENXIO; 152753d66608SJ. R. Okajima 1528*083a6a50SMartijn Coenen figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit); 1529*083a6a50SMartijn Coenen 1530*083a6a50SMartijn Coenen return 0; 153153d66608SJ. R. Okajima } 153253d66608SJ. R. Okajima 1533ab1cb278SMing Lei static int loop_set_dio(struct loop_device *lo, unsigned long arg) 1534ab1cb278SMing Lei { 1535ab1cb278SMing Lei int error = -ENXIO; 1536ab1cb278SMing Lei if (lo->lo_state != Lo_bound) 1537ab1cb278SMing Lei goto out; 1538ab1cb278SMing Lei 1539ab1cb278SMing Lei __loop_update_dio(lo, !!arg); 1540ab1cb278SMing Lei if (lo->use_dio == !!arg) 1541ab1cb278SMing Lei return 0; 1542ab1cb278SMing Lei error = -EINVAL; 1543ab1cb278SMing Lei out: 1544ab1cb278SMing Lei return error; 1545ab1cb278SMing Lei } 1546ab1cb278SMing Lei 154789e4fdecSOmar Sandoval static int loop_set_block_size(struct loop_device *lo, unsigned long arg) 154889e4fdecSOmar Sandoval { 15495db470e2SJaegeuk Kim int err = 0; 15505db470e2SJaegeuk Kim 155189e4fdecSOmar Sandoval if (lo->lo_state != Lo_bound) 155289e4fdecSOmar Sandoval return -ENXIO; 155389e4fdecSOmar Sandoval 155489e4fdecSOmar Sandoval if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg)) 155589e4fdecSOmar Sandoval return -EINVAL; 155689e4fdecSOmar Sandoval 15577e81f99aSMartijn Coenen if (lo->lo_queue->limits.logical_block_size == arg) 15587e81f99aSMartijn Coenen return 0; 15597e81f99aSMartijn Coenen 15605db470e2SJaegeuk Kim sync_blockdev(lo->lo_device); 15615db470e2SJaegeuk Kim kill_bdev(lo->lo_device); 15625db470e2SJaegeuk Kim 156389e4fdecSOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 156489e4fdecSOmar Sandoval 15655db470e2SJaegeuk Kim /* kill_bdev should have truncated all the pages */ 15667e81f99aSMartijn Coenen if (lo->lo_device->bd_inode->i_mapping->nrpages) { 15675db470e2SJaegeuk Kim err = -EAGAIN; 15685db470e2SJaegeuk Kim pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", 15695db470e2SJaegeuk Kim __func__, lo->lo_number, lo->lo_file_name, 15705db470e2SJaegeuk Kim lo->lo_device->bd_inode->i_mapping->nrpages); 15715db470e2SJaegeuk Kim goto out_unfreeze; 15725db470e2SJaegeuk Kim } 15735db470e2SJaegeuk Kim 157489e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, arg); 1575bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, arg); 1576bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, arg); 157789e4fdecSOmar Sandoval loop_update_dio(lo); 15785db470e2SJaegeuk Kim out_unfreeze: 157989e4fdecSOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 158089e4fdecSOmar Sandoval 15815db470e2SJaegeuk Kim return err; 158289e4fdecSOmar Sandoval } 158389e4fdecSOmar Sandoval 1584a1316544SJan Kara static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd, 1585a1316544SJan Kara unsigned long arg) 15861da177e4SLinus Torvalds { 15871da177e4SLinus Torvalds int err; 15881da177e4SLinus Torvalds 1589c28445faSJan Kara err = mutex_lock_killable(&loop_ctl_mutex); 15903148ffbdSOmar Sandoval if (err) 1591a1316544SJan Kara return err; 15921da177e4SLinus Torvalds switch (cmd) { 159353d66608SJ. R. Okajima case LOOP_SET_CAPACITY: 159451001b7dSHannes Reinecke err = loop_set_capacity(lo); 159553d66608SJ. R. Okajima break; 1596ab1cb278SMing Lei case LOOP_SET_DIRECT_IO: 1597ab1cb278SMing Lei err = loop_set_dio(lo, arg); 1598ab1cb278SMing Lei break; 159989e4fdecSOmar Sandoval case LOOP_SET_BLOCK_SIZE: 160089e4fdecSOmar Sandoval err = loop_set_block_size(lo, arg); 160189e4fdecSOmar Sandoval break; 16021da177e4SLinus Torvalds default: 16031da177e4SLinus Torvalds err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL; 16041da177e4SLinus Torvalds } 1605310ca162STetsuo Handa mutex_unlock(&loop_ctl_mutex); 1606a1316544SJan Kara return err; 1607a1316544SJan Kara } 1608f028f3b2SNikanth Karthikesan 1609a1316544SJan Kara static int lo_ioctl(struct block_device *bdev, fmode_t mode, 1610a1316544SJan Kara unsigned int cmd, unsigned long arg) 1611a1316544SJan Kara { 1612a1316544SJan Kara struct loop_device *lo = bdev->bd_disk->private_data; 1613a1316544SJan Kara int err; 1614a1316544SJan Kara 1615a1316544SJan Kara switch (cmd) { 1616a1316544SJan Kara case LOOP_SET_FD: 1617757ecf40SJan Kara return loop_set_fd(lo, mode, bdev, arg); 1618a1316544SJan Kara case LOOP_CHANGE_FD: 1619c3710770SJan Kara return loop_change_fd(lo, bdev, arg); 1620a1316544SJan Kara case LOOP_CLR_FD: 16217ccd0791SJan Kara return loop_clr_fd(lo); 1622a1316544SJan Kara case LOOP_SET_STATUS: 1623a1316544SJan Kara err = -EPERM; 1624a1316544SJan Kara if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { 1625a1316544SJan Kara err = loop_set_status_old(lo, 1626a1316544SJan Kara (struct loop_info __user *)arg); 1627a1316544SJan Kara } 1628a1316544SJan Kara break; 1629a1316544SJan Kara case LOOP_GET_STATUS: 16304a5ce9baSJan Kara return loop_get_status_old(lo, (struct loop_info __user *) arg); 1631a1316544SJan Kara case LOOP_SET_STATUS64: 1632a1316544SJan Kara err = -EPERM; 1633a1316544SJan Kara if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { 1634a1316544SJan Kara err = loop_set_status64(lo, 1635a1316544SJan Kara (struct loop_info64 __user *) arg); 1636a1316544SJan Kara } 1637a1316544SJan Kara break; 1638a1316544SJan Kara case LOOP_GET_STATUS64: 16394a5ce9baSJan Kara return loop_get_status64(lo, (struct loop_info64 __user *) arg); 1640a1316544SJan Kara case LOOP_SET_CAPACITY: 1641a1316544SJan Kara case LOOP_SET_DIRECT_IO: 1642a1316544SJan Kara case LOOP_SET_BLOCK_SIZE: 1643a1316544SJan Kara if (!(mode & FMODE_WRITE) && !capable(CAP_SYS_ADMIN)) 1644a1316544SJan Kara return -EPERM; 1645a1316544SJan Kara /* Fall through */ 1646a1316544SJan Kara default: 1647a1316544SJan Kara err = lo_simple_ioctl(lo, cmd, arg); 1648a1316544SJan Kara break; 1649a1316544SJan Kara } 1650a1316544SJan Kara 16511da177e4SLinus Torvalds return err; 16521da177e4SLinus Torvalds } 16531da177e4SLinus Torvalds 1654863d5b82SDavid Howells #ifdef CONFIG_COMPAT 1655863d5b82SDavid Howells struct compat_loop_info { 1656863d5b82SDavid Howells compat_int_t lo_number; /* ioctl r/o */ 1657863d5b82SDavid Howells compat_dev_t lo_device; /* ioctl r/o */ 1658863d5b82SDavid Howells compat_ulong_t lo_inode; /* ioctl r/o */ 1659863d5b82SDavid Howells compat_dev_t lo_rdevice; /* ioctl r/o */ 1660863d5b82SDavid Howells compat_int_t lo_offset; 1661863d5b82SDavid Howells compat_int_t lo_encrypt_type; 1662863d5b82SDavid Howells compat_int_t lo_encrypt_key_size; /* ioctl w/o */ 1663863d5b82SDavid Howells compat_int_t lo_flags; /* ioctl r/o */ 1664863d5b82SDavid Howells char lo_name[LO_NAME_SIZE]; 1665863d5b82SDavid Howells unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ 1666863d5b82SDavid Howells compat_ulong_t lo_init[2]; 1667863d5b82SDavid Howells char reserved[4]; 1668863d5b82SDavid Howells }; 1669863d5b82SDavid Howells 1670863d5b82SDavid Howells /* 1671863d5b82SDavid Howells * Transfer 32-bit compatibility structure in userspace to 64-bit loop info 1672863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver 1673863d5b82SDavid Howells */ 1674863d5b82SDavid Howells static noinline int 1675ba674cfcSAl Viro loop_info64_from_compat(const struct compat_loop_info __user *arg, 1676863d5b82SDavid Howells struct loop_info64 *info64) 1677863d5b82SDavid Howells { 1678863d5b82SDavid Howells struct compat_loop_info info; 1679863d5b82SDavid Howells 1680863d5b82SDavid Howells if (copy_from_user(&info, arg, sizeof(info))) 1681863d5b82SDavid Howells return -EFAULT; 1682863d5b82SDavid Howells 1683863d5b82SDavid Howells memset(info64, 0, sizeof(*info64)); 1684863d5b82SDavid Howells info64->lo_number = info.lo_number; 1685863d5b82SDavid Howells info64->lo_device = info.lo_device; 1686863d5b82SDavid Howells info64->lo_inode = info.lo_inode; 1687863d5b82SDavid Howells info64->lo_rdevice = info.lo_rdevice; 1688863d5b82SDavid Howells info64->lo_offset = info.lo_offset; 1689863d5b82SDavid Howells info64->lo_sizelimit = 0; 1690863d5b82SDavid Howells info64->lo_encrypt_type = info.lo_encrypt_type; 1691863d5b82SDavid Howells info64->lo_encrypt_key_size = info.lo_encrypt_key_size; 1692863d5b82SDavid Howells info64->lo_flags = info.lo_flags; 1693863d5b82SDavid Howells info64->lo_init[0] = info.lo_init[0]; 1694863d5b82SDavid Howells info64->lo_init[1] = info.lo_init[1]; 1695863d5b82SDavid Howells if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1696863d5b82SDavid Howells memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE); 1697863d5b82SDavid Howells else 1698863d5b82SDavid Howells memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); 1699863d5b82SDavid Howells memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE); 1700863d5b82SDavid Howells return 0; 1701863d5b82SDavid Howells } 1702863d5b82SDavid Howells 1703863d5b82SDavid Howells /* 1704863d5b82SDavid Howells * Transfer 64-bit loop info to 32-bit compatibility structure in userspace 1705863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver 1706863d5b82SDavid Howells */ 1707863d5b82SDavid Howells static noinline int 1708863d5b82SDavid Howells loop_info64_to_compat(const struct loop_info64 *info64, 1709863d5b82SDavid Howells struct compat_loop_info __user *arg) 1710863d5b82SDavid Howells { 1711863d5b82SDavid Howells struct compat_loop_info info; 1712863d5b82SDavid Howells 1713863d5b82SDavid Howells memset(&info, 0, sizeof(info)); 1714863d5b82SDavid Howells info.lo_number = info64->lo_number; 1715863d5b82SDavid Howells info.lo_device = info64->lo_device; 1716863d5b82SDavid Howells info.lo_inode = info64->lo_inode; 1717863d5b82SDavid Howells info.lo_rdevice = info64->lo_rdevice; 1718863d5b82SDavid Howells info.lo_offset = info64->lo_offset; 1719863d5b82SDavid Howells info.lo_encrypt_type = info64->lo_encrypt_type; 1720863d5b82SDavid Howells info.lo_encrypt_key_size = info64->lo_encrypt_key_size; 1721863d5b82SDavid Howells info.lo_flags = info64->lo_flags; 1722863d5b82SDavid Howells info.lo_init[0] = info64->lo_init[0]; 1723863d5b82SDavid Howells info.lo_init[1] = info64->lo_init[1]; 1724863d5b82SDavid Howells if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1725863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 1726863d5b82SDavid Howells else 1727863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); 1728863d5b82SDavid Howells memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 1729863d5b82SDavid Howells 1730863d5b82SDavid Howells /* error in case values were truncated */ 1731863d5b82SDavid Howells if (info.lo_device != info64->lo_device || 1732863d5b82SDavid Howells info.lo_rdevice != info64->lo_rdevice || 1733863d5b82SDavid Howells info.lo_inode != info64->lo_inode || 1734863d5b82SDavid Howells info.lo_offset != info64->lo_offset || 1735863d5b82SDavid Howells info.lo_init[0] != info64->lo_init[0] || 1736863d5b82SDavid Howells info.lo_init[1] != info64->lo_init[1]) 1737863d5b82SDavid Howells return -EOVERFLOW; 1738863d5b82SDavid Howells 1739863d5b82SDavid Howells if (copy_to_user(arg, &info, sizeof(info))) 1740863d5b82SDavid Howells return -EFAULT; 1741863d5b82SDavid Howells return 0; 1742863d5b82SDavid Howells } 1743863d5b82SDavid Howells 1744863d5b82SDavid Howells static int 1745863d5b82SDavid Howells loop_set_status_compat(struct loop_device *lo, 1746863d5b82SDavid Howells const struct compat_loop_info __user *arg) 1747863d5b82SDavid Howells { 1748863d5b82SDavid Howells struct loop_info64 info64; 1749863d5b82SDavid Howells int ret; 1750863d5b82SDavid Howells 1751863d5b82SDavid Howells ret = loop_info64_from_compat(arg, &info64); 1752863d5b82SDavid Howells if (ret < 0) 1753863d5b82SDavid Howells return ret; 1754863d5b82SDavid Howells return loop_set_status(lo, &info64); 1755863d5b82SDavid Howells } 1756863d5b82SDavid Howells 1757863d5b82SDavid Howells static int 1758863d5b82SDavid Howells loop_get_status_compat(struct loop_device *lo, 1759863d5b82SDavid Howells struct compat_loop_info __user *arg) 1760863d5b82SDavid Howells { 1761863d5b82SDavid Howells struct loop_info64 info64; 1762bdac616dSOmar Sandoval int err; 1763863d5b82SDavid Howells 17644a5ce9baSJan Kara if (!arg) 1765bdac616dSOmar Sandoval return -EINVAL; 1766863d5b82SDavid Howells err = loop_get_status(lo, &info64); 1767863d5b82SDavid Howells if (!err) 1768863d5b82SDavid Howells err = loop_info64_to_compat(&info64, arg); 1769863d5b82SDavid Howells return err; 1770863d5b82SDavid Howells } 1771863d5b82SDavid Howells 1772bb214884SAl Viro static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode, 1773bb214884SAl Viro unsigned int cmd, unsigned long arg) 1774863d5b82SDavid Howells { 1775bb214884SAl Viro struct loop_device *lo = bdev->bd_disk->private_data; 1776863d5b82SDavid Howells int err; 1777863d5b82SDavid Howells 1778863d5b82SDavid Howells switch(cmd) { 1779863d5b82SDavid Howells case LOOP_SET_STATUS: 17803148ffbdSOmar Sandoval err = loop_set_status_compat(lo, 17813148ffbdSOmar Sandoval (const struct compat_loop_info __user *)arg); 1782863d5b82SDavid Howells break; 1783863d5b82SDavid Howells case LOOP_GET_STATUS: 17843148ffbdSOmar Sandoval err = loop_get_status_compat(lo, 17853148ffbdSOmar Sandoval (struct compat_loop_info __user *)arg); 1786863d5b82SDavid Howells break; 178753d66608SJ. R. Okajima case LOOP_SET_CAPACITY: 1788863d5b82SDavid Howells case LOOP_CLR_FD: 1789863d5b82SDavid Howells case LOOP_GET_STATUS64: 1790863d5b82SDavid Howells case LOOP_SET_STATUS64: 1791863d5b82SDavid Howells arg = (unsigned long) compat_ptr(arg); 1792d893ff86SGustavo A. R. Silva /* fall through */ 1793863d5b82SDavid Howells case LOOP_SET_FD: 1794863d5b82SDavid Howells case LOOP_CHANGE_FD: 17959fea4b39SEvan Green case LOOP_SET_BLOCK_SIZE: 1796fdbe4eeeSAlessio Balsini case LOOP_SET_DIRECT_IO: 1797bb214884SAl Viro err = lo_ioctl(bdev, mode, cmd, arg); 1798863d5b82SDavid Howells break; 1799863d5b82SDavid Howells default: 1800863d5b82SDavid Howells err = -ENOIOCTLCMD; 1801863d5b82SDavid Howells break; 1802863d5b82SDavid Howells } 1803863d5b82SDavid Howells return err; 1804863d5b82SDavid Howells } 1805863d5b82SDavid Howells #endif 1806863d5b82SDavid Howells 1807bb214884SAl Viro static int lo_open(struct block_device *bdev, fmode_t mode) 18081da177e4SLinus Torvalds { 1809770fe30aSKay Sievers struct loop_device *lo; 18100a42e99bSJan Kara int err; 1811770fe30aSKay Sievers 18120a42e99bSJan Kara err = mutex_lock_killable(&loop_ctl_mutex); 18130a42e99bSJan Kara if (err) 18140a42e99bSJan Kara return err; 1815770fe30aSKay Sievers lo = bdev->bd_disk->private_data; 1816770fe30aSKay Sievers if (!lo) { 1817770fe30aSKay Sievers err = -ENXIO; 1818770fe30aSKay Sievers goto out; 1819770fe30aSKay Sievers } 18201da177e4SLinus Torvalds 1821f8933667SMing Lei atomic_inc(&lo->lo_refcnt); 1822770fe30aSKay Sievers out: 18230a42e99bSJan Kara mutex_unlock(&loop_ctl_mutex); 1824770fe30aSKay Sievers return err; 18251da177e4SLinus Torvalds } 18261da177e4SLinus Torvalds 1827967d1dc1SJan Kara static void lo_release(struct gendisk *disk, fmode_t mode) 18281da177e4SLinus Torvalds { 1829967d1dc1SJan Kara struct loop_device *lo; 18301da177e4SLinus Torvalds 18310a42e99bSJan Kara mutex_lock(&loop_ctl_mutex); 1832967d1dc1SJan Kara lo = disk->private_data; 1833f8933667SMing Lei if (atomic_dec_return(&lo->lo_refcnt)) 18340a42e99bSJan Kara goto out_unlock; 1835f8933667SMing Lei 183614f27939SMilan Broz if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) { 1837a2505b79SJan Kara if (lo->lo_state != Lo_bound) 1838a2505b79SJan Kara goto out_unlock; 1839a2505b79SJan Kara lo->lo_state = Lo_rundown; 18407ccd0791SJan Kara mutex_unlock(&loop_ctl_mutex); 184114f27939SMilan Broz /* 184214f27939SMilan Broz * In autoclear mode, stop the loop thread 184314f27939SMilan Broz * and remove configuration after last close. 184414f27939SMilan Broz */ 18450da03cabSJan Kara __loop_clr_fd(lo, true); 18460a42e99bSJan Kara return; 184743cade80SOmar Sandoval } else if (lo->lo_state == Lo_bound) { 184814f27939SMilan Broz /* 184914f27939SMilan Broz * Otherwise keep thread (if running) and config, 185014f27939SMilan Broz * but flush possible ongoing bios in thread. 185114f27939SMilan Broz */ 185243cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 185343cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 185414f27939SMilan Broz } 185596c58655SDavid Woodhouse 18560a42e99bSJan Kara out_unlock: 1857310ca162STetsuo Handa mutex_unlock(&loop_ctl_mutex); 1858ae665016SLinus Torvalds } 1859ae665016SLinus Torvalds 186083d5cde4SAlexey Dobriyan static const struct block_device_operations lo_fops = { 18611da177e4SLinus Torvalds .owner = THIS_MODULE, 1862bb214884SAl Viro .open = lo_open, 1863bb214884SAl Viro .release = lo_release, 1864bb214884SAl Viro .ioctl = lo_ioctl, 1865863d5b82SDavid Howells #ifdef CONFIG_COMPAT 1866bb214884SAl Viro .compat_ioctl = lo_compat_ioctl, 1867863d5b82SDavid Howells #endif 18681da177e4SLinus Torvalds }; 18691da177e4SLinus Torvalds 18701da177e4SLinus Torvalds /* 18711da177e4SLinus Torvalds * And now the modules code and kernel interface. 18721da177e4SLinus Torvalds */ 187373285082SKen Chen static int max_loop; 18745657a819SJoe Perches module_param(max_loop, int, 0444); 1875a47653fcSKen Chen MODULE_PARM_DESC(max_loop, "Maximum number of loop devices"); 18765657a819SJoe Perches module_param(max_part, int, 0444); 1877476a4813SLaurent Vivier MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device"); 18781da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 18791da177e4SLinus Torvalds MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR); 18801da177e4SLinus Torvalds 18811da177e4SLinus Torvalds int loop_register_transfer(struct loop_func_table *funcs) 18821da177e4SLinus Torvalds { 18831da177e4SLinus Torvalds unsigned int n = funcs->number; 18841da177e4SLinus Torvalds 18851da177e4SLinus Torvalds if (n >= MAX_LO_CRYPT || xfer_funcs[n]) 18861da177e4SLinus Torvalds return -EINVAL; 18871da177e4SLinus Torvalds xfer_funcs[n] = funcs; 18881da177e4SLinus Torvalds return 0; 18891da177e4SLinus Torvalds } 18901da177e4SLinus Torvalds 189134dd82afSKay Sievers static int unregister_transfer_cb(int id, void *ptr, void *data) 189234dd82afSKay Sievers { 189334dd82afSKay Sievers struct loop_device *lo = ptr; 189434dd82afSKay Sievers struct loop_func_table *xfer = data; 189534dd82afSKay Sievers 1896310ca162STetsuo Handa mutex_lock(&loop_ctl_mutex); 189734dd82afSKay Sievers if (lo->lo_encryption == xfer) 189834dd82afSKay Sievers loop_release_xfer(lo); 1899310ca162STetsuo Handa mutex_unlock(&loop_ctl_mutex); 190034dd82afSKay Sievers return 0; 190134dd82afSKay Sievers } 190234dd82afSKay Sievers 19031da177e4SLinus Torvalds int loop_unregister_transfer(int number) 19041da177e4SLinus Torvalds { 19051da177e4SLinus Torvalds unsigned int n = number; 19061da177e4SLinus Torvalds struct loop_func_table *xfer; 19071da177e4SLinus Torvalds 19081da177e4SLinus Torvalds if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL) 19091da177e4SLinus Torvalds return -EINVAL; 19101da177e4SLinus Torvalds 19111da177e4SLinus Torvalds xfer_funcs[n] = NULL; 191234dd82afSKay Sievers idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer); 19131da177e4SLinus Torvalds return 0; 19141da177e4SLinus Torvalds } 19151da177e4SLinus Torvalds 19161da177e4SLinus Torvalds EXPORT_SYMBOL(loop_register_transfer); 19171da177e4SLinus Torvalds EXPORT_SYMBOL(loop_unregister_transfer); 19181da177e4SLinus Torvalds 1919fc17b653SChristoph Hellwig static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, 1920b5dd2f60SMing Lei const struct blk_mq_queue_data *bd) 1921b5dd2f60SMing Lei { 19221894e916SJens Axboe struct request *rq = bd->rq; 19231894e916SJens Axboe struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 19241894e916SJens Axboe struct loop_device *lo = rq->q->queuedata; 1925b5dd2f60SMing Lei 19261894e916SJens Axboe blk_mq_start_request(rq); 1927b5dd2f60SMing Lei 1928f4aa4c7bSMing Lei if (lo->lo_state != Lo_bound) 1929fc17b653SChristoph Hellwig return BLK_STS_IOERR; 1930f4aa4c7bSMing Lei 19311894e916SJens Axboe switch (req_op(rq)) { 1932f0225cacSChristoph Hellwig case REQ_OP_FLUSH: 1933f0225cacSChristoph Hellwig case REQ_OP_DISCARD: 193419372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 1935bc07c10aSMing Lei cmd->use_aio = false; 1936f0225cacSChristoph Hellwig break; 1937f0225cacSChristoph Hellwig default: 1938f0225cacSChristoph Hellwig cmd->use_aio = lo->use_dio; 1939f0225cacSChristoph Hellwig break; 1940f0225cacSChristoph Hellwig } 1941bc07c10aSMing Lei 1942d4478e92SShaohua Li /* always use the first bio's css */ 19430b508bc9SShaohua Li #ifdef CONFIG_BLK_CGROUP 1944db6638d7SDennis Zhou if (cmd->use_aio && rq->bio && rq->bio->bi_blkg) { 1945db6638d7SDennis Zhou cmd->css = &bio_blkcg(rq->bio)->css; 1946d4478e92SShaohua Li css_get(cmd->css); 1947d4478e92SShaohua Li } else 1948d4478e92SShaohua Li #endif 1949d4478e92SShaohua Li cmd->css = NULL; 19503989144fSPetr Mladek kthread_queue_work(&lo->worker, &cmd->work); 1951b5dd2f60SMing Lei 1952fc17b653SChristoph Hellwig return BLK_STS_OK; 1953b5dd2f60SMing Lei } 1954b5dd2f60SMing Lei 1955b5dd2f60SMing Lei static void loop_handle_cmd(struct loop_cmd *cmd) 1956b5dd2f60SMing Lei { 19571894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 19581894e916SJens Axboe const bool write = op_is_write(req_op(rq)); 19591894e916SJens Axboe struct loop_device *lo = rq->q->queuedata; 1960f4829a9bSChristoph Hellwig int ret = 0; 1961b5dd2f60SMing Lei 1962f4829a9bSChristoph Hellwig if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) { 1963f4829a9bSChristoph Hellwig ret = -EIO; 1964b5dd2f60SMing Lei goto failed; 1965f4829a9bSChristoph Hellwig } 1966b5dd2f60SMing Lei 19671894e916SJens Axboe ret = do_req_filebacked(lo, rq); 1968b5dd2f60SMing Lei failed: 1969bc07c10aSMing Lei /* complete non-aio request */ 1970fe2cb290SChristoph Hellwig if (!cmd->use_aio || ret) { 19718cd55087SEvan Green if (ret == -EOPNOTSUPP) 19728cd55087SEvan Green cmd->ret = ret; 19738cd55087SEvan Green else 1974fe2cb290SChristoph Hellwig cmd->ret = ret ? -EIO : 0; 19751894e916SJens Axboe blk_mq_complete_request(rq); 1976fe2cb290SChristoph Hellwig } 1977b5dd2f60SMing Lei } 1978b5dd2f60SMing Lei 1979e03a3d7aSMing Lei static void loop_queue_work(struct kthread_work *work) 1980b5dd2f60SMing Lei { 1981b5dd2f60SMing Lei struct loop_cmd *cmd = 1982e03a3d7aSMing Lei container_of(work, struct loop_cmd, work); 1983b5dd2f60SMing Lei 1984b5dd2f60SMing Lei loop_handle_cmd(cmd); 1985b5dd2f60SMing Lei } 1986b5dd2f60SMing Lei 1987d6296d39SChristoph Hellwig static int loop_init_request(struct blk_mq_tag_set *set, struct request *rq, 1988d6296d39SChristoph Hellwig unsigned int hctx_idx, unsigned int numa_node) 1989b5dd2f60SMing Lei { 1990b5dd2f60SMing Lei struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 1991b5dd2f60SMing Lei 19923989144fSPetr Mladek kthread_init_work(&cmd->work, loop_queue_work); 1993b5dd2f60SMing Lei return 0; 1994b5dd2f60SMing Lei } 1995b5dd2f60SMing Lei 1996f363b089SEric Biggers static const struct blk_mq_ops loop_mq_ops = { 1997b5dd2f60SMing Lei .queue_rq = loop_queue_rq, 1998b5dd2f60SMing Lei .init_request = loop_init_request, 1999fe2cb290SChristoph Hellwig .complete = lo_complete_rq, 2000b5dd2f60SMing Lei }; 2001b5dd2f60SMing Lei 200234dd82afSKay Sievers static int loop_add(struct loop_device **l, int i) 20031da177e4SLinus Torvalds { 200473285082SKen Chen struct loop_device *lo; 200573285082SKen Chen struct gendisk *disk; 200634dd82afSKay Sievers int err; 20071da177e4SLinus Torvalds 200834dd82afSKay Sievers err = -ENOMEM; 200968d740d7SSilva Paulo lo = kzalloc(sizeof(*lo), GFP_KERNEL); 201068d740d7SSilva Paulo if (!lo) 201173285082SKen Chen goto out; 201234dd82afSKay Sievers 2013ef7e7c82SMikulas Patocka lo->lo_state = Lo_unbound; 2014ef7e7c82SMikulas Patocka 2015c718aa65STejun Heo /* allocate id, if @id >= 0, we're requesting that specific id */ 201634dd82afSKay Sievers if (i >= 0) { 2017c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL); 2018c718aa65STejun Heo if (err == -ENOSPC) 201934dd82afSKay Sievers err = -EEXIST; 202034dd82afSKay Sievers } else { 2021c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL); 202234dd82afSKay Sievers } 202334dd82afSKay Sievers if (err < 0) 202434dd82afSKay Sievers goto out_free_dev; 2025c718aa65STejun Heo i = err; 20261da177e4SLinus Torvalds 2027183cfb57SWei Yongjun err = -ENOMEM; 2028b5dd2f60SMing Lei lo->tag_set.ops = &loop_mq_ops; 2029b5dd2f60SMing Lei lo->tag_set.nr_hw_queues = 1; 2030b5dd2f60SMing Lei lo->tag_set.queue_depth = 128; 2031b5dd2f60SMing Lei lo->tag_set.numa_node = NUMA_NO_NODE; 2032b5dd2f60SMing Lei lo->tag_set.cmd_size = sizeof(struct loop_cmd); 203356d18f62SMing Lei lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 2034b5dd2f60SMing Lei lo->tag_set.driver_data = lo; 2035b5dd2f60SMing Lei 2036b5dd2f60SMing Lei err = blk_mq_alloc_tag_set(&lo->tag_set); 2037b5dd2f60SMing Lei if (err) 20383ec981e3SMikulas Patocka goto out_free_idr; 203973285082SKen Chen 2040b5dd2f60SMing Lei lo->lo_queue = blk_mq_init_queue(&lo->tag_set); 204138a3499fSChengguang Xu if (IS_ERR(lo->lo_queue)) { 2042b5dd2f60SMing Lei err = PTR_ERR(lo->lo_queue); 2043b5dd2f60SMing Lei goto out_cleanup_tags; 2044b5dd2f60SMing Lei } 2045ef7e7c82SMikulas Patocka lo->lo_queue->queuedata = lo; 2046ef7e7c82SMikulas Patocka 204754bb0adeSShaohua Li blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS); 204840326d8aSShaohua Li 20495b5e20f4SMing Lei /* 205040326d8aSShaohua Li * By default, we do buffer IO, so it doesn't make sense to enable 205140326d8aSShaohua Li * merge because the I/O submitted to backing file is handled page by 205240326d8aSShaohua Li * page. For directio mode, merge does help to dispatch bigger request 205340326d8aSShaohua Li * to underlayer disk. We will enable merge once directio is enabled. 20545b5e20f4SMing Lei */ 20558b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 20565b5e20f4SMing Lei 20577a649737SMinfei Huang err = -ENOMEM; 2058476a4813SLaurent Vivier disk = lo->lo_disk = alloc_disk(1 << part_shift); 205973285082SKen Chen if (!disk) 206073285082SKen Chen goto out_free_queue; 206173285082SKen Chen 2062e03c8dd1SKay Sievers /* 2063e03c8dd1SKay Sievers * Disable partition scanning by default. The in-kernel partition 2064e03c8dd1SKay Sievers * scanning can be requested individually per-device during its 2065e03c8dd1SKay Sievers * setup. Userspace can always add and remove partitions from all 2066e03c8dd1SKay Sievers * devices. The needed partition minors are allocated from the 2067e03c8dd1SKay Sievers * extended minor space, the main loop device numbers will continue 2068e03c8dd1SKay Sievers * to match the loop minors, regardless of the number of partitions 2069e03c8dd1SKay Sievers * used. 2070e03c8dd1SKay Sievers * 2071e03c8dd1SKay Sievers * If max_part is given, partition scanning is globally enabled for 2072e03c8dd1SKay Sievers * all loop devices. The minors for the main loop devices will be 2073e03c8dd1SKay Sievers * multiples of max_part. 2074e03c8dd1SKay Sievers * 2075e03c8dd1SKay Sievers * Note: Global-for-all-devices, set-only-at-init, read-only module 2076e03c8dd1SKay Sievers * parameteters like 'max_loop' and 'max_part' make things needlessly 2077e03c8dd1SKay Sievers * complicated, are too static, inflexible and may surprise 2078e03c8dd1SKay Sievers * userspace tools. Parameters like this in general should be avoided. 2079e03c8dd1SKay Sievers */ 2080e03c8dd1SKay Sievers if (!part_shift) 2081e03c8dd1SKay Sievers disk->flags |= GENHD_FL_NO_PART_SCAN; 2082e03c8dd1SKay Sievers disk->flags |= GENHD_FL_EXT_DEVT; 2083f8933667SMing Lei atomic_set(&lo->lo_refcnt, 0); 20841da177e4SLinus Torvalds lo->lo_number = i; 20851da177e4SLinus Torvalds spin_lock_init(&lo->lo_lock); 20861da177e4SLinus Torvalds disk->major = LOOP_MAJOR; 2087476a4813SLaurent Vivier disk->first_minor = i << part_shift; 20881da177e4SLinus Torvalds disk->fops = &lo_fops; 20891da177e4SLinus Torvalds disk->private_data = lo; 20901da177e4SLinus Torvalds disk->queue = lo->lo_queue; 209173285082SKen Chen sprintf(disk->disk_name, "loop%d", i); 209234dd82afSKay Sievers add_disk(disk); 209334dd82afSKay Sievers *l = lo; 209434dd82afSKay Sievers return lo->lo_number; 209573285082SKen Chen 209673285082SKen Chen out_free_queue: 209773285082SKen Chen blk_cleanup_queue(lo->lo_queue); 2098b5dd2f60SMing Lei out_cleanup_tags: 2099b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set); 21003ec981e3SMikulas Patocka out_free_idr: 21013ec981e3SMikulas Patocka idr_remove(&loop_index_idr, i); 210273285082SKen Chen out_free_dev: 210373285082SKen Chen kfree(lo); 210473285082SKen Chen out: 210534dd82afSKay Sievers return err; 21061da177e4SLinus Torvalds } 21071da177e4SLinus Torvalds 210834dd82afSKay Sievers static void loop_remove(struct loop_device *lo) 210973285082SKen Chen { 21106cd18e71SNeilBrown del_gendisk(lo->lo_disk); 21110fa8ebddSBart Van Assche blk_cleanup_queue(lo->lo_queue); 2112b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set); 211373285082SKen Chen put_disk(lo->lo_disk); 211473285082SKen Chen kfree(lo); 211573285082SKen Chen } 211673285082SKen Chen 2117770fe30aSKay Sievers static int find_free_cb(int id, void *ptr, void *data) 2118770fe30aSKay Sievers { 2119770fe30aSKay Sievers struct loop_device *lo = ptr; 2120770fe30aSKay Sievers struct loop_device **l = data; 2121770fe30aSKay Sievers 2122770fe30aSKay Sievers if (lo->lo_state == Lo_unbound) { 2123770fe30aSKay Sievers *l = lo; 2124770fe30aSKay Sievers return 1; 2125770fe30aSKay Sievers } 2126770fe30aSKay Sievers return 0; 2127770fe30aSKay Sievers } 2128770fe30aSKay Sievers 212934dd82afSKay Sievers static int loop_lookup(struct loop_device **l, int i) 2130a47653fcSKen Chen { 2131a47653fcSKen Chen struct loop_device *lo; 213234dd82afSKay Sievers int ret = -ENODEV; 2133a47653fcSKen Chen 2134770fe30aSKay Sievers if (i < 0) { 2135770fe30aSKay Sievers int err; 2136770fe30aSKay Sievers 2137770fe30aSKay Sievers err = idr_for_each(&loop_index_idr, &find_free_cb, &lo); 2138770fe30aSKay Sievers if (err == 1) { 2139770fe30aSKay Sievers *l = lo; 2140770fe30aSKay Sievers ret = lo->lo_number; 2141770fe30aSKay Sievers } 2142770fe30aSKay Sievers goto out; 2143a47653fcSKen Chen } 2144a47653fcSKen Chen 2145770fe30aSKay Sievers /* lookup and return a specific i */ 214634dd82afSKay Sievers lo = idr_find(&loop_index_idr, i); 2147a47653fcSKen Chen if (lo) { 214834dd82afSKay Sievers *l = lo; 214934dd82afSKay Sievers ret = lo->lo_number; 2150a47653fcSKen Chen } 2151770fe30aSKay Sievers out: 215234dd82afSKay Sievers return ret; 2153a47653fcSKen Chen } 2154a47653fcSKen Chen 215573285082SKen Chen static struct kobject *loop_probe(dev_t dev, int *part, void *data) 215673285082SKen Chen { 2157705962ccSAl Viro struct loop_device *lo; 215807002e99SAl Viro struct kobject *kobj; 215934dd82afSKay Sievers int err; 216073285082SKen Chen 21610a42e99bSJan Kara mutex_lock(&loop_ctl_mutex); 216234dd82afSKay Sievers err = loop_lookup(&lo, MINOR(dev) >> part_shift); 216334dd82afSKay Sievers if (err < 0) 216434dd82afSKay Sievers err = loop_add(&lo, MINOR(dev) >> part_shift); 216534dd82afSKay Sievers if (err < 0) 2166a207f593SMikulas Patocka kobj = NULL; 216734dd82afSKay Sievers else 21683079c22eSJan Kara kobj = get_disk_and_module(lo->lo_disk); 21690a42e99bSJan Kara mutex_unlock(&loop_ctl_mutex); 217073285082SKen Chen 217173285082SKen Chen *part = 0; 217207002e99SAl Viro return kobj; 217373285082SKen Chen } 217473285082SKen Chen 2175770fe30aSKay Sievers static long loop_control_ioctl(struct file *file, unsigned int cmd, 2176770fe30aSKay Sievers unsigned long parm) 2177770fe30aSKay Sievers { 2178770fe30aSKay Sievers struct loop_device *lo; 21790a42e99bSJan Kara int ret; 2180770fe30aSKay Sievers 21810a42e99bSJan Kara ret = mutex_lock_killable(&loop_ctl_mutex); 21820a42e99bSJan Kara if (ret) 21830a42e99bSJan Kara return ret; 21840a42e99bSJan Kara 21850a42e99bSJan Kara ret = -ENOSYS; 2186770fe30aSKay Sievers switch (cmd) { 2187770fe30aSKay Sievers case LOOP_CTL_ADD: 2188770fe30aSKay Sievers ret = loop_lookup(&lo, parm); 2189770fe30aSKay Sievers if (ret >= 0) { 2190770fe30aSKay Sievers ret = -EEXIST; 2191770fe30aSKay Sievers break; 2192770fe30aSKay Sievers } 2193770fe30aSKay Sievers ret = loop_add(&lo, parm); 2194770fe30aSKay Sievers break; 2195770fe30aSKay Sievers case LOOP_CTL_REMOVE: 2196770fe30aSKay Sievers ret = loop_lookup(&lo, parm); 2197770fe30aSKay Sievers if (ret < 0) 2198770fe30aSKay Sievers break; 2199770fe30aSKay Sievers if (lo->lo_state != Lo_unbound) { 2200770fe30aSKay Sievers ret = -EBUSY; 2201770fe30aSKay Sievers break; 2202770fe30aSKay Sievers } 2203f8933667SMing Lei if (atomic_read(&lo->lo_refcnt) > 0) { 2204770fe30aSKay Sievers ret = -EBUSY; 2205770fe30aSKay Sievers break; 2206770fe30aSKay Sievers } 2207770fe30aSKay Sievers lo->lo_disk->private_data = NULL; 2208770fe30aSKay Sievers idr_remove(&loop_index_idr, lo->lo_number); 2209770fe30aSKay Sievers loop_remove(lo); 2210770fe30aSKay Sievers break; 2211770fe30aSKay Sievers case LOOP_CTL_GET_FREE: 2212770fe30aSKay Sievers ret = loop_lookup(&lo, -1); 2213770fe30aSKay Sievers if (ret >= 0) 2214770fe30aSKay Sievers break; 2215770fe30aSKay Sievers ret = loop_add(&lo, -1); 2216770fe30aSKay Sievers } 22170a42e99bSJan Kara mutex_unlock(&loop_ctl_mutex); 2218770fe30aSKay Sievers 2219770fe30aSKay Sievers return ret; 2220770fe30aSKay Sievers } 2221770fe30aSKay Sievers 2222770fe30aSKay Sievers static const struct file_operations loop_ctl_fops = { 2223770fe30aSKay Sievers .open = nonseekable_open, 2224770fe30aSKay Sievers .unlocked_ioctl = loop_control_ioctl, 2225770fe30aSKay Sievers .compat_ioctl = loop_control_ioctl, 2226770fe30aSKay Sievers .owner = THIS_MODULE, 2227770fe30aSKay Sievers .llseek = noop_llseek, 2228770fe30aSKay Sievers }; 2229770fe30aSKay Sievers 2230770fe30aSKay Sievers static struct miscdevice loop_misc = { 2231770fe30aSKay Sievers .minor = LOOP_CTRL_MINOR, 2232770fe30aSKay Sievers .name = "loop-control", 2233770fe30aSKay Sievers .fops = &loop_ctl_fops, 2234770fe30aSKay Sievers }; 2235770fe30aSKay Sievers 2236770fe30aSKay Sievers MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR); 2237770fe30aSKay Sievers MODULE_ALIAS("devname:loop-control"); 2238770fe30aSKay Sievers 223973285082SKen Chen static int __init loop_init(void) 224073285082SKen Chen { 2241a47653fcSKen Chen int i, nr; 2242a47653fcSKen Chen unsigned long range; 224334dd82afSKay Sievers struct loop_device *lo; 2244770fe30aSKay Sievers int err; 2245a47653fcSKen Chen 2246476a4813SLaurent Vivier part_shift = 0; 2247ac04fee0SNamhyung Kim if (max_part > 0) { 2248476a4813SLaurent Vivier part_shift = fls(max_part); 2249476a4813SLaurent Vivier 2250ac04fee0SNamhyung Kim /* 2251ac04fee0SNamhyung Kim * Adjust max_part according to part_shift as it is exported 2252ac04fee0SNamhyung Kim * to user space so that user can decide correct minor number 2253ac04fee0SNamhyung Kim * if [s]he want to create more devices. 2254ac04fee0SNamhyung Kim * 2255ac04fee0SNamhyung Kim * Note that -1 is required because partition 0 is reserved 2256ac04fee0SNamhyung Kim * for the whole disk. 2257ac04fee0SNamhyung Kim */ 2258ac04fee0SNamhyung Kim max_part = (1UL << part_shift) - 1; 2259ac04fee0SNamhyung Kim } 2260ac04fee0SNamhyung Kim 2261b1a66504SGuo Chao if ((1UL << part_shift) > DISK_MAX_PARTS) { 2262b1a66504SGuo Chao err = -EINVAL; 2263a8c1d064SAnton Volkov goto err_out; 2264b1a66504SGuo Chao } 226578f4bb36SNamhyung Kim 2266b1a66504SGuo Chao if (max_loop > 1UL << (MINORBITS - part_shift)) { 2267b1a66504SGuo Chao err = -EINVAL; 2268a8c1d064SAnton Volkov goto err_out; 2269b1a66504SGuo Chao } 227073285082SKen Chen 2271d134b00bSKay Sievers /* 2272d134b00bSKay Sievers * If max_loop is specified, create that many devices upfront. 2273d134b00bSKay Sievers * This also becomes a hard limit. If max_loop is not specified, 2274d134b00bSKay Sievers * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module 2275d134b00bSKay Sievers * init time. Loop devices can be requested on-demand with the 2276d134b00bSKay Sievers * /dev/loop-control interface, or be instantiated by accessing 2277d134b00bSKay Sievers * a 'dead' device node. 2278d134b00bSKay Sievers */ 227973285082SKen Chen if (max_loop) { 2280a47653fcSKen Chen nr = max_loop; 2281a1c15c59SNamhyung Kim range = max_loop << part_shift; 2282a47653fcSKen Chen } else { 2283d134b00bSKay Sievers nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT; 2284a1c15c59SNamhyung Kim range = 1UL << MINORBITS; 228573285082SKen Chen } 2286a47653fcSKen Chen 2287a8c1d064SAnton Volkov err = misc_register(&loop_misc); 2288a8c1d064SAnton Volkov if (err < 0) 2289a8c1d064SAnton Volkov goto err_out; 2290a8c1d064SAnton Volkov 2291a8c1d064SAnton Volkov 2292b1a66504SGuo Chao if (register_blkdev(LOOP_MAJOR, "loop")) { 2293b1a66504SGuo Chao err = -EIO; 2294b1a66504SGuo Chao goto misc_out; 2295b1a66504SGuo Chao } 2296a47653fcSKen Chen 2297a47653fcSKen Chen blk_register_region(MKDEV(LOOP_MAJOR, 0), range, 2298a47653fcSKen Chen THIS_MODULE, loop_probe, NULL, NULL); 2299a47653fcSKen Chen 2300d134b00bSKay Sievers /* pre-create number of devices given by config or max_loop */ 23010a42e99bSJan Kara mutex_lock(&loop_ctl_mutex); 230234dd82afSKay Sievers for (i = 0; i < nr; i++) 230334dd82afSKay Sievers loop_add(&lo, i); 23040a42e99bSJan Kara mutex_unlock(&loop_ctl_mutex); 230534dd82afSKay Sievers 230673285082SKen Chen printk(KERN_INFO "loop: module loaded\n"); 23071da177e4SLinus Torvalds return 0; 2308b1a66504SGuo Chao 2309b1a66504SGuo Chao misc_out: 2310b1a66504SGuo Chao misc_deregister(&loop_misc); 2311a8c1d064SAnton Volkov err_out: 2312b1a66504SGuo Chao return err; 231334dd82afSKay Sievers } 2314a47653fcSKen Chen 231534dd82afSKay Sievers static int loop_exit_cb(int id, void *ptr, void *data) 231634dd82afSKay Sievers { 231734dd82afSKay Sievers struct loop_device *lo = ptr; 2318a47653fcSKen Chen 231934dd82afSKay Sievers loop_remove(lo); 232034dd82afSKay Sievers return 0; 23211da177e4SLinus Torvalds } 23221da177e4SLinus Torvalds 232373285082SKen Chen static void __exit loop_exit(void) 23241da177e4SLinus Torvalds { 2325a47653fcSKen Chen unsigned long range; 23261da177e4SLinus Torvalds 2327a1c15c59SNamhyung Kim range = max_loop ? max_loop << part_shift : 1UL << MINORBITS; 2328a47653fcSKen Chen 232934dd82afSKay Sievers idr_for_each(&loop_index_idr, &loop_exit_cb, NULL); 233034dd82afSKay Sievers idr_destroy(&loop_index_idr); 233173285082SKen Chen 2332a47653fcSKen Chen blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range); 233300d59405SAkinobu Mita unregister_blkdev(LOOP_MAJOR, "loop"); 2334770fe30aSKay Sievers 2335770fe30aSKay Sievers misc_deregister(&loop_misc); 23361da177e4SLinus Torvalds } 23371da177e4SLinus Torvalds 23381da177e4SLinus Torvalds module_init(loop_init); 23391da177e4SLinus Torvalds module_exit(loop_exit); 23401da177e4SLinus Torvalds 23411da177e4SLinus Torvalds #ifndef MODULE 23421da177e4SLinus Torvalds static int __init max_loop_setup(char *str) 23431da177e4SLinus Torvalds { 23441da177e4SLinus Torvalds max_loop = simple_strtol(str, NULL, 0); 23451da177e4SLinus Torvalds return 1; 23461da177e4SLinus Torvalds } 23471da177e4SLinus Torvalds 23481da177e4SLinus Torvalds __setup("max_loop=", max_loop_setup); 23491da177e4SLinus Torvalds #endif 2350