11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/drivers/block/loop.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Written by Theodore Ts'o, 3/29/93 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Copyright 1993 by Theodore Ts'o. Redistribution of this file is 71da177e4SLinus Torvalds * permitted under the GNU General Public License. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993 101da177e4SLinus Torvalds * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994 131da177e4SLinus Torvalds * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997 161da177e4SLinus Torvalds * 171da177e4SLinus Torvalds * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998 201da177e4SLinus Torvalds * 211da177e4SLinus Torvalds * Loadable modules and other fixes by AK, 1998 221da177e4SLinus Torvalds * 231da177e4SLinus Torvalds * Make real block number available to downstream transfer functions, enables 241da177e4SLinus Torvalds * CBC (and relatives) mode encryption requiring unique IVs per data block. 251da177e4SLinus Torvalds * Reed H. Petty, rhp@draper.net 261da177e4SLinus Torvalds * 271da177e4SLinus Torvalds * Maximum number of loop devices now dynamic via max_loop module parameter. 281da177e4SLinus Torvalds * Russell Kroll <rkroll@exploits.org> 19990701 291da177e4SLinus Torvalds * 301da177e4SLinus Torvalds * Maximum number of loop devices when compiled-in now selectable by passing 311da177e4SLinus Torvalds * max_loop=<1-255> to the kernel on boot. 3296de0e25SJan Engelhardt * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999 331da177e4SLinus Torvalds * 341da177e4SLinus Torvalds * Completely rewrite request handling to be make_request_fn style and 351da177e4SLinus Torvalds * non blocking, pushing work to a helper thread. Lots of fixes from 361da177e4SLinus Torvalds * Al Viro too. 371da177e4SLinus Torvalds * Jens Axboe <axboe@suse.de>, Nov 2000 381da177e4SLinus Torvalds * 391da177e4SLinus Torvalds * Support up to 256 loop devices 401da177e4SLinus Torvalds * Heinz Mauelshagen <mge@sistina.com>, Feb 2002 411da177e4SLinus Torvalds * 421da177e4SLinus Torvalds * Support for falling back on the write file operation when the address space 434e02ed4bSNick Piggin * operations write_begin is not available on the backing filesystem. 441da177e4SLinus Torvalds * Anton Altaparmakov, 16 Feb 2005 451da177e4SLinus Torvalds * 461da177e4SLinus Torvalds * Still To Fix: 471da177e4SLinus Torvalds * - Advisory locking is ignored here. 481da177e4SLinus Torvalds * - Should use an own CAP_* category instead of CAP_SYS_ADMIN 491da177e4SLinus Torvalds * 501da177e4SLinus Torvalds */ 511da177e4SLinus Torvalds 521da177e4SLinus Torvalds #include <linux/module.h> 531da177e4SLinus Torvalds #include <linux/moduleparam.h> 541da177e4SLinus Torvalds #include <linux/sched.h> 551da177e4SLinus Torvalds #include <linux/fs.h> 561da177e4SLinus Torvalds #include <linux/file.h> 571da177e4SLinus Torvalds #include <linux/stat.h> 581da177e4SLinus Torvalds #include <linux/errno.h> 591da177e4SLinus Torvalds #include <linux/major.h> 601da177e4SLinus Torvalds #include <linux/wait.h> 611da177e4SLinus Torvalds #include <linux/blkdev.h> 621da177e4SLinus Torvalds #include <linux/blkpg.h> 631da177e4SLinus Torvalds #include <linux/init.h> 641da177e4SLinus Torvalds #include <linux/swap.h> 651da177e4SLinus Torvalds #include <linux/slab.h> 66863d5b82SDavid Howells #include <linux/compat.h> 671da177e4SLinus Torvalds #include <linux/suspend.h> 6883144186SRafael J. Wysocki #include <linux/freezer.h> 692a48fc0aSArnd Bergmann #include <linux/mutex.h> 701da177e4SLinus Torvalds #include <linux/writeback.h> 711da177e4SLinus Torvalds #include <linux/completion.h> 721da177e4SLinus Torvalds #include <linux/highmem.h> 736c997918SSerge E. Hallyn #include <linux/kthread.h> 74d6b29d7cSJens Axboe #include <linux/splice.h> 75ee862730SMilan Broz #include <linux/sysfs.h> 76770fe30aSKay Sievers #include <linux/miscdevice.h> 77dfaa2ef6SLukas Czerner #include <linux/falloc.h> 78283e7e5dSAl Viro #include <linux/uio.h> 7983a87611SAl Viro #include "loop.h" 801da177e4SLinus Torvalds 817c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 821da177e4SLinus Torvalds 8334dd82afSKay Sievers static DEFINE_IDR(loop_index_idr); 8434dd82afSKay Sievers static DEFINE_MUTEX(loop_index_mutex); 851da177e4SLinus Torvalds 86476a4813SLaurent Vivier static int max_part; 87476a4813SLaurent Vivier static int part_shift; 88476a4813SLaurent Vivier 891da177e4SLinus Torvalds static int transfer_xor(struct loop_device *lo, int cmd, 901da177e4SLinus Torvalds struct page *raw_page, unsigned raw_off, 911da177e4SLinus Torvalds struct page *loop_page, unsigned loop_off, 921da177e4SLinus Torvalds int size, sector_t real_block) 931da177e4SLinus Torvalds { 94cfd8005cSCong Wang char *raw_buf = kmap_atomic(raw_page) + raw_off; 95cfd8005cSCong Wang char *loop_buf = kmap_atomic(loop_page) + loop_off; 961da177e4SLinus Torvalds char *in, *out, *key; 971da177e4SLinus Torvalds int i, keysize; 981da177e4SLinus Torvalds 991da177e4SLinus Torvalds if (cmd == READ) { 1001da177e4SLinus Torvalds in = raw_buf; 1011da177e4SLinus Torvalds out = loop_buf; 1021da177e4SLinus Torvalds } else { 1031da177e4SLinus Torvalds in = loop_buf; 1041da177e4SLinus Torvalds out = raw_buf; 1051da177e4SLinus Torvalds } 1061da177e4SLinus Torvalds 1071da177e4SLinus Torvalds key = lo->lo_encrypt_key; 1081da177e4SLinus Torvalds keysize = lo->lo_encrypt_key_size; 1091da177e4SLinus Torvalds for (i = 0; i < size; i++) 1101da177e4SLinus Torvalds *out++ = *in++ ^ key[(i & 511) % keysize]; 1111da177e4SLinus Torvalds 112cfd8005cSCong Wang kunmap_atomic(loop_buf); 113cfd8005cSCong Wang kunmap_atomic(raw_buf); 1141da177e4SLinus Torvalds cond_resched(); 1151da177e4SLinus Torvalds return 0; 1161da177e4SLinus Torvalds } 1171da177e4SLinus Torvalds 1181da177e4SLinus Torvalds static int xor_init(struct loop_device *lo, const struct loop_info64 *info) 1191da177e4SLinus Torvalds { 1201da177e4SLinus Torvalds if (unlikely(info->lo_encrypt_key_size <= 0)) 1211da177e4SLinus Torvalds return -EINVAL; 1221da177e4SLinus Torvalds return 0; 1231da177e4SLinus Torvalds } 1241da177e4SLinus Torvalds 1251da177e4SLinus Torvalds static struct loop_func_table none_funcs = { 1261da177e4SLinus Torvalds .number = LO_CRYPT_NONE, 1271da177e4SLinus Torvalds }; 1281da177e4SLinus Torvalds 1291da177e4SLinus Torvalds static struct loop_func_table xor_funcs = { 1301da177e4SLinus Torvalds .number = LO_CRYPT_XOR, 1311da177e4SLinus Torvalds .transfer = transfer_xor, 1321da177e4SLinus Torvalds .init = xor_init 1331da177e4SLinus Torvalds }; 1341da177e4SLinus Torvalds 1351da177e4SLinus Torvalds /* xfer_funcs[0] is special - its release function is never called */ 1361da177e4SLinus Torvalds static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = { 1371da177e4SLinus Torvalds &none_funcs, 1381da177e4SLinus Torvalds &xor_funcs 1391da177e4SLinus Torvalds }; 1401da177e4SLinus Torvalds 1417035b5dfSDmitry Monakhov static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file) 1421da177e4SLinus Torvalds { 143b7a1da69SGuo Chao loff_t loopsize; 1441da177e4SLinus Torvalds 1451da177e4SLinus Torvalds /* Compute loopsize in bytes */ 146b7a1da69SGuo Chao loopsize = i_size_read(file->f_mapping->host); 147b7a1da69SGuo Chao if (offset > 0) 148b7a1da69SGuo Chao loopsize -= offset; 149b7a1da69SGuo Chao /* offset is beyond i_size, weird but possible */ 1507035b5dfSDmitry Monakhov if (loopsize < 0) 1517035b5dfSDmitry Monakhov return 0; 1521da177e4SLinus Torvalds 1537035b5dfSDmitry Monakhov if (sizelimit > 0 && sizelimit < loopsize) 1547035b5dfSDmitry Monakhov loopsize = sizelimit; 1551da177e4SLinus Torvalds /* 1561da177e4SLinus Torvalds * Unfortunately, if we want to do I/O on the device, 1571da177e4SLinus Torvalds * the number of 512-byte sectors has to fit into a sector_t. 1581da177e4SLinus Torvalds */ 1591da177e4SLinus Torvalds return loopsize >> 9; 1601da177e4SLinus Torvalds } 1611da177e4SLinus Torvalds 1627035b5dfSDmitry Monakhov static loff_t get_loop_size(struct loop_device *lo, struct file *file) 1631da177e4SLinus Torvalds { 1647035b5dfSDmitry Monakhov return get_size(lo->lo_offset, lo->lo_sizelimit, file); 1657035b5dfSDmitry Monakhov } 1667035b5dfSDmitry Monakhov 1672e5ab5f3SMing Lei static void __loop_update_dio(struct loop_device *lo, bool dio) 1682e5ab5f3SMing Lei { 1692e5ab5f3SMing Lei struct file *file = lo->lo_backing_file; 1702e5ab5f3SMing Lei struct address_space *mapping = file->f_mapping; 1712e5ab5f3SMing Lei struct inode *inode = mapping->host; 1722e5ab5f3SMing Lei unsigned short sb_bsize = 0; 1732e5ab5f3SMing Lei unsigned dio_align = 0; 1742e5ab5f3SMing Lei bool use_dio; 1752e5ab5f3SMing Lei 1762e5ab5f3SMing Lei if (inode->i_sb->s_bdev) { 1772e5ab5f3SMing Lei sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 1782e5ab5f3SMing Lei dio_align = sb_bsize - 1; 1792e5ab5f3SMing Lei } 1802e5ab5f3SMing Lei 1812e5ab5f3SMing Lei /* 1822e5ab5f3SMing Lei * We support direct I/O only if lo_offset is aligned with the 1832e5ab5f3SMing Lei * logical I/O size of backing device, and the logical block 1842e5ab5f3SMing Lei * size of loop is bigger than the backing device's and the loop 1852e5ab5f3SMing Lei * needn't transform transfer. 1862e5ab5f3SMing Lei * 1872e5ab5f3SMing Lei * TODO: the above condition may be loosed in the future, and 1882e5ab5f3SMing Lei * direct I/O may be switched runtime at that time because most 18989d790abSMasahiro Yamada * of requests in sane applications should be PAGE_SIZE aligned 1902e5ab5f3SMing Lei */ 1912e5ab5f3SMing Lei if (dio) { 1922e5ab5f3SMing Lei if (queue_logical_block_size(lo->lo_queue) >= sb_bsize && 1932e5ab5f3SMing Lei !(lo->lo_offset & dio_align) && 1942e5ab5f3SMing Lei mapping->a_ops->direct_IO && 1952e5ab5f3SMing Lei !lo->transfer) 1962e5ab5f3SMing Lei use_dio = true; 1972e5ab5f3SMing Lei else 1982e5ab5f3SMing Lei use_dio = false; 1992e5ab5f3SMing Lei } else { 2002e5ab5f3SMing Lei use_dio = false; 2012e5ab5f3SMing Lei } 2022e5ab5f3SMing Lei 2032e5ab5f3SMing Lei if (lo->use_dio == use_dio) 2042e5ab5f3SMing Lei return; 2052e5ab5f3SMing Lei 2062e5ab5f3SMing Lei /* flush dirty pages before changing direct IO */ 2072e5ab5f3SMing Lei vfs_fsync(file, 0); 2082e5ab5f3SMing Lei 2092e5ab5f3SMing Lei /* 2102e5ab5f3SMing Lei * The flag of LO_FLAGS_DIRECT_IO is handled similarly with 2112e5ab5f3SMing Lei * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup 2122e5ab5f3SMing Lei * will get updated by ioctl(LOOP_GET_STATUS) 2132e5ab5f3SMing Lei */ 2142e5ab5f3SMing Lei blk_mq_freeze_queue(lo->lo_queue); 2152e5ab5f3SMing Lei lo->use_dio = use_dio; 21640326d8aSShaohua Li if (use_dio) { 2178b904b5bSBart Van Assche blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2182e5ab5f3SMing Lei lo->lo_flags |= LO_FLAGS_DIRECT_IO; 21940326d8aSShaohua Li } else { 2208b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2212e5ab5f3SMing Lei lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; 22240326d8aSShaohua Li } 2232e5ab5f3SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 2242e5ab5f3SMing Lei } 2252e5ab5f3SMing Lei 2267035b5dfSDmitry Monakhov static int 2271e6ec9eaSOmar Sandoval figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit) 2287035b5dfSDmitry Monakhov { 2297035b5dfSDmitry Monakhov loff_t size = get_size(offset, sizelimit, lo->lo_backing_file); 2301da177e4SLinus Torvalds sector_t x = (sector_t)size; 2317b0576a3SGuo Chao struct block_device *bdev = lo->lo_device; 2321da177e4SLinus Torvalds 2331da177e4SLinus Torvalds if (unlikely((loff_t)x != size)) 2341da177e4SLinus Torvalds return -EFBIG; 2357035b5dfSDmitry Monakhov if (lo->lo_offset != offset) 2367035b5dfSDmitry Monakhov lo->lo_offset = offset; 2377035b5dfSDmitry Monakhov if (lo->lo_sizelimit != sizelimit) 2387035b5dfSDmitry Monakhov lo->lo_sizelimit = sizelimit; 23973285082SKen Chen set_capacity(lo->lo_disk, x); 2407b0576a3SGuo Chao bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9); 2417b0576a3SGuo Chao /* let user-space know about the new size */ 2427b0576a3SGuo Chao kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 2431da177e4SLinus Torvalds return 0; 2441da177e4SLinus Torvalds } 2451da177e4SLinus Torvalds 2461da177e4SLinus Torvalds static inline int 2471da177e4SLinus Torvalds lo_do_transfer(struct loop_device *lo, int cmd, 2481da177e4SLinus Torvalds struct page *rpage, unsigned roffs, 2491da177e4SLinus Torvalds struct page *lpage, unsigned loffs, 2501da177e4SLinus Torvalds int size, sector_t rblock) 2511da177e4SLinus Torvalds { 252aa4d8616SChristoph Hellwig int ret; 253aa4d8616SChristoph Hellwig 254aa4d8616SChristoph Hellwig ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); 255aa4d8616SChristoph Hellwig if (likely(!ret)) 2561da177e4SLinus Torvalds return 0; 2571da177e4SLinus Torvalds 258aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR 259aa4d8616SChristoph Hellwig "loop: Transfer error at byte offset %llu, length %i.\n", 260aa4d8616SChristoph Hellwig (unsigned long long)rblock << 9, size); 261aa4d8616SChristoph Hellwig return ret; 2621da177e4SLinus Torvalds } 2631da177e4SLinus Torvalds 264aa4d8616SChristoph Hellwig static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos) 2651da177e4SLinus Torvalds { 266aa4d8616SChristoph Hellwig struct iov_iter i; 2671da177e4SLinus Torvalds ssize_t bw; 268283e7e5dSAl Viro 2691d037577SRoss Zwisler iov_iter_bvec(&i, ITER_BVEC | WRITE, bvec, 1, bvec->bv_len); 2701da177e4SLinus Torvalds 27103d95eb2SAl Viro file_start_write(file); 272abbb6589SChristoph Hellwig bw = vfs_iter_write(file, &i, ppos, 0); 27303d95eb2SAl Viro file_end_write(file); 274aa4d8616SChristoph Hellwig 275aa4d8616SChristoph Hellwig if (likely(bw == bvec->bv_len)) 2761da177e4SLinus Torvalds return 0; 277aa4d8616SChristoph Hellwig 278aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR 279aa4d8616SChristoph Hellwig "loop: Write error at byte offset %llu, length %i.\n", 280aa4d8616SChristoph Hellwig (unsigned long long)*ppos, bvec->bv_len); 2811da177e4SLinus Torvalds if (bw >= 0) 2821da177e4SLinus Torvalds bw = -EIO; 2831da177e4SLinus Torvalds return bw; 2841da177e4SLinus Torvalds } 2851da177e4SLinus Torvalds 286aa4d8616SChristoph Hellwig static int lo_write_simple(struct loop_device *lo, struct request *rq, 287aa4d8616SChristoph Hellwig loff_t pos) 2881da177e4SLinus Torvalds { 289aa4d8616SChristoph Hellwig struct bio_vec bvec; 290aa4d8616SChristoph Hellwig struct req_iterator iter; 291aa4d8616SChristoph Hellwig int ret = 0; 292aa4d8616SChristoph Hellwig 293aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) { 294aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos); 295aa4d8616SChristoph Hellwig if (ret < 0) 296aa4d8616SChristoph Hellwig break; 2971da177e4SLinus Torvalds cond_resched(); 2981da177e4SLinus Torvalds } 2991da177e4SLinus Torvalds 300aa4d8616SChristoph Hellwig return ret; 301aa4d8616SChristoph Hellwig } 302aa4d8616SChristoph Hellwig 303aa4d8616SChristoph Hellwig /* 304456be148SChristoph Hellwig * This is the slow, transforming version that needs to double buffer the 305456be148SChristoph Hellwig * data as it cannot do the transformations in place without having direct 306456be148SChristoph Hellwig * access to the destination pages of the backing file. 3071da177e4SLinus Torvalds */ 308aa4d8616SChristoph Hellwig static int lo_write_transfer(struct loop_device *lo, struct request *rq, 309aa4d8616SChristoph Hellwig loff_t pos) 3101da177e4SLinus Torvalds { 311aa4d8616SChristoph Hellwig struct bio_vec bvec, b; 31230112013SMing Lei struct req_iterator iter; 313aa4d8616SChristoph Hellwig struct page *page; 3147988613bSKent Overstreet int ret = 0; 3151da177e4SLinus Torvalds 316aa4d8616SChristoph Hellwig page = alloc_page(GFP_NOIO); 3171da177e4SLinus Torvalds if (unlikely(!page)) 318aa4d8616SChristoph Hellwig return -ENOMEM; 319456be148SChristoph Hellwig 32030112013SMing Lei rq_for_each_segment(bvec, rq, iter) { 321aa4d8616SChristoph Hellwig ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page, 322aa4d8616SChristoph Hellwig bvec.bv_offset, bvec.bv_len, pos >> 9); 323aa4d8616SChristoph Hellwig if (unlikely(ret)) 324aa4d8616SChristoph Hellwig break; 325aa4d8616SChristoph Hellwig 326aa4d8616SChristoph Hellwig b.bv_page = page; 327aa4d8616SChristoph Hellwig b.bv_offset = 0; 328aa4d8616SChristoph Hellwig b.bv_len = bvec.bv_len; 329aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &b, &pos); 3301da177e4SLinus Torvalds if (ret < 0) 3311da177e4SLinus Torvalds break; 3321da177e4SLinus Torvalds } 333aa4d8616SChristoph Hellwig 3341da177e4SLinus Torvalds __free_page(page); 3351da177e4SLinus Torvalds return ret; 3361da177e4SLinus Torvalds } 3371da177e4SLinus Torvalds 338aa4d8616SChristoph Hellwig static int lo_read_simple(struct loop_device *lo, struct request *rq, 339aa4d8616SChristoph Hellwig loff_t pos) 3401da177e4SLinus Torvalds { 3417988613bSKent Overstreet struct bio_vec bvec; 34230112013SMing Lei struct req_iterator iter; 343aa4d8616SChristoph Hellwig struct iov_iter i; 344aa4d8616SChristoph Hellwig ssize_t len; 3451da177e4SLinus Torvalds 34630112013SMing Lei rq_for_each_segment(bvec, rq, iter) { 347aa4d8616SChristoph Hellwig iov_iter_bvec(&i, ITER_BVEC, &bvec, 1, bvec.bv_len); 34818e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 349aa4d8616SChristoph Hellwig if (len < 0) 350aa4d8616SChristoph Hellwig return len; 351306df071SDave Young 352aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page); 353aa4d8616SChristoph Hellwig 354aa4d8616SChristoph Hellwig if (len != bvec.bv_len) { 35530112013SMing Lei struct bio *bio; 35630112013SMing Lei 35730112013SMing Lei __rq_for_each_bio(bio, rq) 358306df071SDave Young zero_fill_bio(bio); 3591da177e4SLinus Torvalds break; 360306df071SDave Young } 361aa4d8616SChristoph Hellwig cond_resched(); 3621da177e4SLinus Torvalds } 363aa4d8616SChristoph Hellwig 364306df071SDave Young return 0; 3651da177e4SLinus Torvalds } 3661da177e4SLinus Torvalds 367aa4d8616SChristoph Hellwig static int lo_read_transfer(struct loop_device *lo, struct request *rq, 368aa4d8616SChristoph Hellwig loff_t pos) 369aa4d8616SChristoph Hellwig { 370aa4d8616SChristoph Hellwig struct bio_vec bvec, b; 371aa4d8616SChristoph Hellwig struct req_iterator iter; 372aa4d8616SChristoph Hellwig struct iov_iter i; 373aa4d8616SChristoph Hellwig struct page *page; 374aa4d8616SChristoph Hellwig ssize_t len; 375aa4d8616SChristoph Hellwig int ret = 0; 376aa4d8616SChristoph Hellwig 377aa4d8616SChristoph Hellwig page = alloc_page(GFP_NOIO); 378aa4d8616SChristoph Hellwig if (unlikely(!page)) 379aa4d8616SChristoph Hellwig return -ENOMEM; 380aa4d8616SChristoph Hellwig 381aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) { 382aa4d8616SChristoph Hellwig loff_t offset = pos; 383aa4d8616SChristoph Hellwig 384aa4d8616SChristoph Hellwig b.bv_page = page; 385aa4d8616SChristoph Hellwig b.bv_offset = 0; 386aa4d8616SChristoph Hellwig b.bv_len = bvec.bv_len; 387aa4d8616SChristoph Hellwig 388aa4d8616SChristoph Hellwig iov_iter_bvec(&i, ITER_BVEC, &b, 1, b.bv_len); 38918e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 390aa4d8616SChristoph Hellwig if (len < 0) { 391aa4d8616SChristoph Hellwig ret = len; 392aa4d8616SChristoph Hellwig goto out_free_page; 393aa4d8616SChristoph Hellwig } 394aa4d8616SChristoph Hellwig 395aa4d8616SChristoph Hellwig ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page, 396aa4d8616SChristoph Hellwig bvec.bv_offset, len, offset >> 9); 397aa4d8616SChristoph Hellwig if (ret) 398aa4d8616SChristoph Hellwig goto out_free_page; 399aa4d8616SChristoph Hellwig 400aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page); 401aa4d8616SChristoph Hellwig 402aa4d8616SChristoph Hellwig if (len != bvec.bv_len) { 403aa4d8616SChristoph Hellwig struct bio *bio; 404aa4d8616SChristoph Hellwig 405aa4d8616SChristoph Hellwig __rq_for_each_bio(bio, rq) 406aa4d8616SChristoph Hellwig zero_fill_bio(bio); 407aa4d8616SChristoph Hellwig break; 408aa4d8616SChristoph Hellwig } 409aa4d8616SChristoph Hellwig } 410aa4d8616SChristoph Hellwig 411aa4d8616SChristoph Hellwig ret = 0; 412aa4d8616SChristoph Hellwig out_free_page: 413aa4d8616SChristoph Hellwig __free_page(page); 414aa4d8616SChristoph Hellwig return ret; 415aa4d8616SChristoph Hellwig } 416aa4d8616SChristoph Hellwig 417cf655d95SMing Lei static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos) 418cf655d95SMing Lei { 419cf655d95SMing Lei /* 420cf655d95SMing Lei * We use punch hole to reclaim the free space used by the 421cf655d95SMing Lei * image a.k.a. discard. However we do not support discard if 422cf655d95SMing Lei * encryption is enabled, because it may give an attacker 423cf655d95SMing Lei * useful information. 424cf655d95SMing Lei */ 425cf655d95SMing Lei struct file *file = lo->lo_backing_file; 426cf655d95SMing Lei int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; 427cf655d95SMing Lei int ret; 428cf655d95SMing Lei 429cf655d95SMing Lei if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) { 430cf655d95SMing Lei ret = -EOPNOTSUPP; 431cf655d95SMing Lei goto out; 432cf655d95SMing Lei } 433cf655d95SMing Lei 434cf655d95SMing Lei ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq)); 435cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP)) 436cf655d95SMing Lei ret = -EIO; 437cf655d95SMing Lei out: 438cf655d95SMing Lei return ret; 439cf655d95SMing Lei } 440cf655d95SMing Lei 441cf655d95SMing Lei static int lo_req_flush(struct loop_device *lo, struct request *rq) 442cf655d95SMing Lei { 443cf655d95SMing Lei struct file *file = lo->lo_backing_file; 444cf655d95SMing Lei int ret = vfs_fsync(file, 0); 445cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL)) 446cf655d95SMing Lei ret = -EIO; 447cf655d95SMing Lei 448cf655d95SMing Lei return ret; 449cf655d95SMing Lei } 450cf655d95SMing Lei 451fe2cb290SChristoph Hellwig static void lo_complete_rq(struct request *rq) 452bc07c10aSMing Lei { 453fe2cb290SChristoph Hellwig struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 454f9de14bcSJens Axboe blk_status_t ret = BLK_STS_OK; 455bc07c10aSMing Lei 456f9de14bcSJens Axboe if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) || 457f9de14bcSJens Axboe req_op(rq) != REQ_OP_READ) { 458f9de14bcSJens Axboe if (cmd->ret < 0) 459f9de14bcSJens Axboe ret = BLK_STS_IOERR; 460f9de14bcSJens Axboe goto end_io; 461bc07c10aSMing Lei } 462fe2cb290SChristoph Hellwig 463f9de14bcSJens Axboe /* 464f9de14bcSJens Axboe * Short READ - if we got some data, advance our request and 465f9de14bcSJens Axboe * retry it. If we got no data, end the rest with EIO. 466f9de14bcSJens Axboe */ 467f9de14bcSJens Axboe if (cmd->ret) { 468f9de14bcSJens Axboe blk_update_request(rq, BLK_STS_OK, cmd->ret); 469f9de14bcSJens Axboe cmd->ret = 0; 470f9de14bcSJens Axboe blk_mq_requeue_request(rq, true); 471f9de14bcSJens Axboe } else { 472f9de14bcSJens Axboe if (cmd->use_aio) { 473f9de14bcSJens Axboe struct bio *bio = rq->bio; 474f9de14bcSJens Axboe 475f9de14bcSJens Axboe while (bio) { 476f9de14bcSJens Axboe zero_fill_bio(bio); 477f9de14bcSJens Axboe bio = bio->bi_next; 478f9de14bcSJens Axboe } 479f9de14bcSJens Axboe } 480f9de14bcSJens Axboe ret = BLK_STS_IOERR; 481f9de14bcSJens Axboe end_io: 482f9de14bcSJens Axboe blk_mq_end_request(rq, ret); 483f9de14bcSJens Axboe } 484bc07c10aSMing Lei } 485bc07c10aSMing Lei 48692d77332SShaohua Li static void lo_rw_aio_do_completion(struct loop_cmd *cmd) 48792d77332SShaohua Li { 4881894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 4891894e916SJens Axboe 49092d77332SShaohua Li if (!atomic_dec_and_test(&cmd->ref)) 49192d77332SShaohua Li return; 49292d77332SShaohua Li kfree(cmd->bvec); 49392d77332SShaohua Li cmd->bvec = NULL; 4941894e916SJens Axboe blk_mq_complete_request(rq); 49592d77332SShaohua Li } 49692d77332SShaohua Li 497bc07c10aSMing Lei static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2) 498bc07c10aSMing Lei { 499bc07c10aSMing Lei struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb); 500bc07c10aSMing Lei 501d4478e92SShaohua Li if (cmd->css) 502d4478e92SShaohua Li css_put(cmd->css); 503fe2cb290SChristoph Hellwig cmd->ret = ret; 50492d77332SShaohua Li lo_rw_aio_do_completion(cmd); 505bc07c10aSMing Lei } 506bc07c10aSMing Lei 507bc07c10aSMing Lei static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, 508bc07c10aSMing Lei loff_t pos, bool rw) 509bc07c10aSMing Lei { 510bc07c10aSMing Lei struct iov_iter iter; 511bc07c10aSMing Lei struct bio_vec *bvec; 5121894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 51340326d8aSShaohua Li struct bio *bio = rq->bio; 514bc07c10aSMing Lei struct file *file = lo->lo_backing_file; 51540326d8aSShaohua Li unsigned int offset; 51640326d8aSShaohua Li int segments = 0; 517bc07c10aSMing Lei int ret; 518bc07c10aSMing Lei 51940326d8aSShaohua Li if (rq->bio != rq->biotail) { 52040326d8aSShaohua Li struct req_iterator iter; 52140326d8aSShaohua Li struct bio_vec tmp; 522bc07c10aSMing Lei 52340326d8aSShaohua Li __rq_for_each_bio(bio, rq) 52440326d8aSShaohua Li segments += bio_segments(bio); 52540326d8aSShaohua Li bvec = kmalloc(sizeof(struct bio_vec) * segments, GFP_NOIO); 52640326d8aSShaohua Li if (!bvec) 52740326d8aSShaohua Li return -EIO; 52840326d8aSShaohua Li cmd->bvec = bvec; 52940326d8aSShaohua Li 530a7297a6aSMing Lei /* 53140326d8aSShaohua Li * The bios of the request may be started from the middle of 53240326d8aSShaohua Li * the 'bvec' because of bio splitting, so we can't directly 53340326d8aSShaohua Li * copy bio->bi_iov_vec to new bvec. The rq_for_each_segment 53440326d8aSShaohua Li * API will take care of all details for us. 535a7297a6aSMing Lei */ 53640326d8aSShaohua Li rq_for_each_segment(tmp, rq, iter) { 53740326d8aSShaohua Li *bvec = tmp; 53840326d8aSShaohua Li bvec++; 53940326d8aSShaohua Li } 54040326d8aSShaohua Li bvec = cmd->bvec; 54140326d8aSShaohua Li offset = 0; 54240326d8aSShaohua Li } else { 54340326d8aSShaohua Li /* 54440326d8aSShaohua Li * Same here, this bio may be started from the middle of the 54540326d8aSShaohua Li * 'bvec' because of bio splitting, so offset from the bvec 54640326d8aSShaohua Li * must be passed to iov iterator 54740326d8aSShaohua Li */ 54840326d8aSShaohua Li offset = bio->bi_iter.bi_bvec_done; 54940326d8aSShaohua Li bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 55040326d8aSShaohua Li segments = bio_segments(bio); 55140326d8aSShaohua Li } 55292d77332SShaohua Li atomic_set(&cmd->ref, 2); 55340326d8aSShaohua Li 55440326d8aSShaohua Li iov_iter_bvec(&iter, ITER_BVEC | rw, bvec, 55540326d8aSShaohua Li segments, blk_rq_bytes(rq)); 55640326d8aSShaohua Li iter.iov_offset = offset; 557bc07c10aSMing Lei 558bc07c10aSMing Lei cmd->iocb.ki_pos = pos; 559bc07c10aSMing Lei cmd->iocb.ki_filp = file; 560bc07c10aSMing Lei cmd->iocb.ki_complete = lo_rw_aio_complete; 561bc07c10aSMing Lei cmd->iocb.ki_flags = IOCB_DIRECT; 562d4478e92SShaohua Li if (cmd->css) 563d4478e92SShaohua Li kthread_associate_blkcg(cmd->css); 564bc07c10aSMing Lei 565bc07c10aSMing Lei if (rw == WRITE) 566bb7462b6SMiklos Szeredi ret = call_write_iter(file, &cmd->iocb, &iter); 567bc07c10aSMing Lei else 568bb7462b6SMiklos Szeredi ret = call_read_iter(file, &cmd->iocb, &iter); 569bc07c10aSMing Lei 57092d77332SShaohua Li lo_rw_aio_do_completion(cmd); 571d4478e92SShaohua Li kthread_associate_blkcg(NULL); 57292d77332SShaohua Li 573bc07c10aSMing Lei if (ret != -EIOCBQUEUED) 574bc07c10aSMing Lei cmd->iocb.ki_complete(&cmd->iocb, ret, 0); 575bc07c10aSMing Lei return 0; 576bc07c10aSMing Lei } 577bc07c10aSMing Lei 578c1c87c2bSChristoph Hellwig static int do_req_filebacked(struct loop_device *lo, struct request *rq) 579bc07c10aSMing Lei { 580bc07c10aSMing Lei struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 581c1c87c2bSChristoph Hellwig loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; 582bc07c10aSMing Lei 583bc07c10aSMing Lei /* 584bc07c10aSMing Lei * lo_write_simple and lo_read_simple should have been covered 585bc07c10aSMing Lei * by io submit style function like lo_rw_aio(), one blocker 586bc07c10aSMing Lei * is that lo_read_simple() need to call flush_dcache_page after 587bc07c10aSMing Lei * the page is written from kernel, and it isn't easy to handle 588bc07c10aSMing Lei * this in io submit style function which submits all segments 589bc07c10aSMing Lei * of the req at one time. And direct read IO doesn't need to 590bc07c10aSMing Lei * run flush_dcache_page(). 591bc07c10aSMing Lei */ 592c1c87c2bSChristoph Hellwig switch (req_op(rq)) { 593c1c87c2bSChristoph Hellwig case REQ_OP_FLUSH: 594c1c87c2bSChristoph Hellwig return lo_req_flush(lo, rq); 595c1c87c2bSChristoph Hellwig case REQ_OP_DISCARD: 59619372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 597c1c87c2bSChristoph Hellwig return lo_discard(lo, rq, pos); 598c1c87c2bSChristoph Hellwig case REQ_OP_WRITE: 599c1c87c2bSChristoph Hellwig if (lo->transfer) 600c1c87c2bSChristoph Hellwig return lo_write_transfer(lo, rq, pos); 601c1c87c2bSChristoph Hellwig else if (cmd->use_aio) 602c1c87c2bSChristoph Hellwig return lo_rw_aio(lo, cmd, pos, WRITE); 603c1c87c2bSChristoph Hellwig else 604bc07c10aSMing Lei return lo_write_simple(lo, rq, pos); 605c1c87c2bSChristoph Hellwig case REQ_OP_READ: 606c1c87c2bSChristoph Hellwig if (lo->transfer) 607c1c87c2bSChristoph Hellwig return lo_read_transfer(lo, rq, pos); 608c1c87c2bSChristoph Hellwig else if (cmd->use_aio) 609c1c87c2bSChristoph Hellwig return lo_rw_aio(lo, cmd, pos, READ); 610bc07c10aSMing Lei else 611bc07c10aSMing Lei return lo_read_simple(lo, rq, pos); 612c1c87c2bSChristoph Hellwig default: 613c1c87c2bSChristoph Hellwig WARN_ON_ONCE(1); 614c1c87c2bSChristoph Hellwig return -EIO; 615c1c87c2bSChristoph Hellwig break; 616bc07c10aSMing Lei } 6171da177e4SLinus Torvalds } 6181da177e4SLinus Torvalds 6192e5ab5f3SMing Lei static inline void loop_update_dio(struct loop_device *lo) 6202e5ab5f3SMing Lei { 6212e5ab5f3SMing Lei __loop_update_dio(lo, io_is_direct(lo->lo_backing_file) | 6222e5ab5f3SMing Lei lo->use_dio); 6232e5ab5f3SMing Lei } 6242e5ab5f3SMing Lei 62506f0e9e6SMing Lei static void loop_reread_partitions(struct loop_device *lo, 62606f0e9e6SMing Lei struct block_device *bdev) 62706f0e9e6SMing Lei { 62806f0e9e6SMing Lei int rc; 62906f0e9e6SMing Lei 63006f0e9e6SMing Lei /* 63106f0e9e6SMing Lei * bd_mutex has been held already in release path, so don't 63206f0e9e6SMing Lei * acquire it if this function is called in such case. 63306f0e9e6SMing Lei * 63406f0e9e6SMing Lei * If the reread partition isn't from release path, lo_refcnt 63506f0e9e6SMing Lei * must be at least one and it can only become zero when the 63606f0e9e6SMing Lei * current holder is released. 63706f0e9e6SMing Lei */ 63806f0e9e6SMing Lei if (!atomic_read(&lo->lo_refcnt)) 63906f0e9e6SMing Lei rc = __blkdev_reread_part(bdev); 64006f0e9e6SMing Lei else 64106f0e9e6SMing Lei rc = blkdev_reread_part(bdev); 64206f0e9e6SMing Lei if (rc) 64306f0e9e6SMing Lei pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n", 64406f0e9e6SMing Lei __func__, lo->lo_number, lo->lo_file_name, rc); 64506f0e9e6SMing Lei } 64606f0e9e6SMing Lei 647*d2ac838eSTheodore Ts'o static inline int is_loop_device(struct file *file) 648*d2ac838eSTheodore Ts'o { 649*d2ac838eSTheodore Ts'o struct inode *i = file->f_mapping->host; 650*d2ac838eSTheodore Ts'o 651*d2ac838eSTheodore Ts'o return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR; 652*d2ac838eSTheodore Ts'o } 653*d2ac838eSTheodore Ts'o 654*d2ac838eSTheodore Ts'o static int loop_validate_file(struct file *file, struct block_device *bdev) 655*d2ac838eSTheodore Ts'o { 656*d2ac838eSTheodore Ts'o struct inode *inode = file->f_mapping->host; 657*d2ac838eSTheodore Ts'o struct file *f = file; 658*d2ac838eSTheodore Ts'o 659*d2ac838eSTheodore Ts'o /* Avoid recursion */ 660*d2ac838eSTheodore Ts'o while (is_loop_device(f)) { 661*d2ac838eSTheodore Ts'o struct loop_device *l; 662*d2ac838eSTheodore Ts'o 663*d2ac838eSTheodore Ts'o if (f->f_mapping->host->i_bdev == bdev) 664*d2ac838eSTheodore Ts'o return -EBADF; 665*d2ac838eSTheodore Ts'o 666*d2ac838eSTheodore Ts'o l = f->f_mapping->host->i_bdev->bd_disk->private_data; 667*d2ac838eSTheodore Ts'o if (l->lo_state == Lo_unbound) { 668*d2ac838eSTheodore Ts'o return -EINVAL; 669*d2ac838eSTheodore Ts'o } 670*d2ac838eSTheodore Ts'o f = l->lo_backing_file; 671*d2ac838eSTheodore Ts'o } 672*d2ac838eSTheodore Ts'o if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode)) 673*d2ac838eSTheodore Ts'o return -EINVAL; 674*d2ac838eSTheodore Ts'o return 0; 675*d2ac838eSTheodore Ts'o } 676*d2ac838eSTheodore Ts'o 6771da177e4SLinus Torvalds /* 6781da177e4SLinus Torvalds * loop_change_fd switched the backing store of a loopback device to 6791da177e4SLinus Torvalds * a new file. This is useful for operating system installers to free up 6801da177e4SLinus Torvalds * the original file and in High Availability environments to switch to 6811da177e4SLinus Torvalds * an alternative location for the content in case of server meltdown. 6821da177e4SLinus Torvalds * This can only work if the loop device is used read-only, and if the 6831da177e4SLinus Torvalds * new backing store is the same size and type as the old backing store. 6841da177e4SLinus Torvalds */ 685bb214884SAl Viro static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, 686bb214884SAl Viro unsigned int arg) 6871da177e4SLinus Torvalds { 6881da177e4SLinus Torvalds struct file *file, *old_file; 6891da177e4SLinus Torvalds struct inode *inode; 6901da177e4SLinus Torvalds int error; 6911da177e4SLinus Torvalds 6921da177e4SLinus Torvalds error = -ENXIO; 6931da177e4SLinus Torvalds if (lo->lo_state != Lo_bound) 6941da177e4SLinus Torvalds goto out; 6951da177e4SLinus Torvalds 6961da177e4SLinus Torvalds /* the loop device has to be read-only */ 6971da177e4SLinus Torvalds error = -EINVAL; 6981da177e4SLinus Torvalds if (!(lo->lo_flags & LO_FLAGS_READ_ONLY)) 6991da177e4SLinus Torvalds goto out; 7001da177e4SLinus Torvalds 7011da177e4SLinus Torvalds error = -EBADF; 7021da177e4SLinus Torvalds file = fget(arg); 7031da177e4SLinus Torvalds if (!file) 7041da177e4SLinus Torvalds goto out; 7051da177e4SLinus Torvalds 706*d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev); 707*d2ac838eSTheodore Ts'o if (error) 708*d2ac838eSTheodore Ts'o goto out_putf; 709*d2ac838eSTheodore Ts'o 7101da177e4SLinus Torvalds inode = file->f_mapping->host; 7111da177e4SLinus Torvalds old_file = lo->lo_backing_file; 7121da177e4SLinus Torvalds 7131da177e4SLinus Torvalds error = -EINVAL; 7141da177e4SLinus Torvalds 7151da177e4SLinus Torvalds /* size of the new backing store needs to be the same */ 7161da177e4SLinus Torvalds if (get_loop_size(lo, file) != get_loop_size(lo, old_file)) 7171da177e4SLinus Torvalds goto out_putf; 7181da177e4SLinus Torvalds 7191da177e4SLinus Torvalds /* and ... switch */ 72043cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 72143cade80SOmar Sandoval mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask); 72243cade80SOmar Sandoval lo->lo_backing_file = file; 72343cade80SOmar Sandoval lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping); 72443cade80SOmar Sandoval mapping_set_gfp_mask(file->f_mapping, 72543cade80SOmar Sandoval lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 72643cade80SOmar Sandoval loop_update_dio(lo); 72743cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 7281da177e4SLinus Torvalds 7291da177e4SLinus Torvalds fput(old_file); 730e03c8dd1SKay Sievers if (lo->lo_flags & LO_FLAGS_PARTSCAN) 73106f0e9e6SMing Lei loop_reread_partitions(lo, bdev); 7321da177e4SLinus Torvalds return 0; 7331da177e4SLinus Torvalds 7341da177e4SLinus Torvalds out_putf: 7351da177e4SLinus Torvalds fput(file); 7361da177e4SLinus Torvalds out: 7371da177e4SLinus Torvalds return error; 7381da177e4SLinus Torvalds } 7391da177e4SLinus Torvalds 740ee862730SMilan Broz /* loop sysfs attributes */ 741ee862730SMilan Broz 742ee862730SMilan Broz static ssize_t loop_attr_show(struct device *dev, char *page, 743ee862730SMilan Broz ssize_t (*callback)(struct loop_device *, char *)) 744ee862730SMilan Broz { 74534dd82afSKay Sievers struct gendisk *disk = dev_to_disk(dev); 74634dd82afSKay Sievers struct loop_device *lo = disk->private_data; 747ee862730SMilan Broz 74834dd82afSKay Sievers return callback(lo, page); 749ee862730SMilan Broz } 750ee862730SMilan Broz 751ee862730SMilan Broz #define LOOP_ATTR_RO(_name) \ 752ee862730SMilan Broz static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \ 753ee862730SMilan Broz static ssize_t loop_attr_do_show_##_name(struct device *d, \ 754ee862730SMilan Broz struct device_attribute *attr, char *b) \ 755ee862730SMilan Broz { \ 756ee862730SMilan Broz return loop_attr_show(d, b, loop_attr_##_name##_show); \ 757ee862730SMilan Broz } \ 758ee862730SMilan Broz static struct device_attribute loop_attr_##_name = \ 7595657a819SJoe Perches __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL); 760ee862730SMilan Broz 761ee862730SMilan Broz static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf) 762ee862730SMilan Broz { 763ee862730SMilan Broz ssize_t ret; 764ee862730SMilan Broz char *p = NULL; 765ee862730SMilan Broz 76605eb0f25SKay Sievers spin_lock_irq(&lo->lo_lock); 767ee862730SMilan Broz if (lo->lo_backing_file) 7689bf39ab2SMiklos Szeredi p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1); 76905eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock); 770ee862730SMilan Broz 771ee862730SMilan Broz if (IS_ERR_OR_NULL(p)) 772ee862730SMilan Broz ret = PTR_ERR(p); 773ee862730SMilan Broz else { 774ee862730SMilan Broz ret = strlen(p); 775ee862730SMilan Broz memmove(buf, p, ret); 776ee862730SMilan Broz buf[ret++] = '\n'; 777ee862730SMilan Broz buf[ret] = 0; 778ee862730SMilan Broz } 779ee862730SMilan Broz 780ee862730SMilan Broz return ret; 781ee862730SMilan Broz } 782ee862730SMilan Broz 783ee862730SMilan Broz static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf) 784ee862730SMilan Broz { 785ee862730SMilan Broz return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset); 786ee862730SMilan Broz } 787ee862730SMilan Broz 788ee862730SMilan Broz static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf) 789ee862730SMilan Broz { 790ee862730SMilan Broz return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit); 791ee862730SMilan Broz } 792ee862730SMilan Broz 793ee862730SMilan Broz static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf) 794ee862730SMilan Broz { 795ee862730SMilan Broz int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR); 796ee862730SMilan Broz 797ee862730SMilan Broz return sprintf(buf, "%s\n", autoclear ? "1" : "0"); 798ee862730SMilan Broz } 799ee862730SMilan Broz 800e03c8dd1SKay Sievers static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf) 801e03c8dd1SKay Sievers { 802e03c8dd1SKay Sievers int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN); 803e03c8dd1SKay Sievers 804e03c8dd1SKay Sievers return sprintf(buf, "%s\n", partscan ? "1" : "0"); 805e03c8dd1SKay Sievers } 806e03c8dd1SKay Sievers 8072e5ab5f3SMing Lei static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf) 8082e5ab5f3SMing Lei { 8092e5ab5f3SMing Lei int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO); 8102e5ab5f3SMing Lei 8112e5ab5f3SMing Lei return sprintf(buf, "%s\n", dio ? "1" : "0"); 8122e5ab5f3SMing Lei } 8132e5ab5f3SMing Lei 814ee862730SMilan Broz LOOP_ATTR_RO(backing_file); 815ee862730SMilan Broz LOOP_ATTR_RO(offset); 816ee862730SMilan Broz LOOP_ATTR_RO(sizelimit); 817ee862730SMilan Broz LOOP_ATTR_RO(autoclear); 818e03c8dd1SKay Sievers LOOP_ATTR_RO(partscan); 8192e5ab5f3SMing Lei LOOP_ATTR_RO(dio); 820ee862730SMilan Broz 821ee862730SMilan Broz static struct attribute *loop_attrs[] = { 822ee862730SMilan Broz &loop_attr_backing_file.attr, 823ee862730SMilan Broz &loop_attr_offset.attr, 824ee862730SMilan Broz &loop_attr_sizelimit.attr, 825ee862730SMilan Broz &loop_attr_autoclear.attr, 826e03c8dd1SKay Sievers &loop_attr_partscan.attr, 8272e5ab5f3SMing Lei &loop_attr_dio.attr, 828ee862730SMilan Broz NULL, 829ee862730SMilan Broz }; 830ee862730SMilan Broz 831ee862730SMilan Broz static struct attribute_group loop_attribute_group = { 832ee862730SMilan Broz .name = "loop", 833ee862730SMilan Broz .attrs= loop_attrs, 834ee862730SMilan Broz }; 835ee862730SMilan Broz 836d3349b6bSTetsuo Handa static void loop_sysfs_init(struct loop_device *lo) 837ee862730SMilan Broz { 838d3349b6bSTetsuo Handa lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj, 839ee862730SMilan Broz &loop_attribute_group); 840ee862730SMilan Broz } 841ee862730SMilan Broz 842ee862730SMilan Broz static void loop_sysfs_exit(struct loop_device *lo) 843ee862730SMilan Broz { 844d3349b6bSTetsuo Handa if (lo->sysfs_inited) 845ee862730SMilan Broz sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj, 846ee862730SMilan Broz &loop_attribute_group); 847ee862730SMilan Broz } 848ee862730SMilan Broz 849dfaa2ef6SLukas Czerner static void loop_config_discard(struct loop_device *lo) 850dfaa2ef6SLukas Czerner { 851dfaa2ef6SLukas Czerner struct file *file = lo->lo_backing_file; 852dfaa2ef6SLukas Czerner struct inode *inode = file->f_mapping->host; 853dfaa2ef6SLukas Czerner struct request_queue *q = lo->lo_queue; 854dfaa2ef6SLukas Czerner 855dfaa2ef6SLukas Czerner /* 856dfaa2ef6SLukas Czerner * We use punch hole to reclaim the free space used by the 85712a64d2fSOlaf Hering * image a.k.a. discard. However we do not support discard if 858dfaa2ef6SLukas Czerner * encryption is enabled, because it may give an attacker 859dfaa2ef6SLukas Czerner * useful information. 860dfaa2ef6SLukas Czerner */ 861dfaa2ef6SLukas Czerner if ((!file->f_op->fallocate) || 862dfaa2ef6SLukas Czerner lo->lo_encrypt_key_size) { 863dfaa2ef6SLukas Czerner q->limits.discard_granularity = 0; 864dfaa2ef6SLukas Czerner q->limits.discard_alignment = 0; 8652bb4cd5cSJens Axboe blk_queue_max_discard_sectors(q, 0); 86619372e27SChristoph Hellwig blk_queue_max_write_zeroes_sectors(q, 0); 8678b904b5bSBart Van Assche blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q); 868dfaa2ef6SLukas Czerner return; 869dfaa2ef6SLukas Czerner } 870dfaa2ef6SLukas Czerner 871dfaa2ef6SLukas Czerner q->limits.discard_granularity = inode->i_sb->s_blocksize; 872dfaf3c03SLukas Czerner q->limits.discard_alignment = 0; 873f2c6df7dSHannes Reinecke 8741e6ec9eaSOmar Sandoval blk_queue_max_discard_sectors(q, UINT_MAX >> 9); 8751e6ec9eaSOmar Sandoval blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9); 8768b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 877dfaa2ef6SLukas Czerner } 878dfaa2ef6SLukas Czerner 879e03a3d7aSMing Lei static void loop_unprepare_queue(struct loop_device *lo) 880e03a3d7aSMing Lei { 8813989144fSPetr Mladek kthread_flush_worker(&lo->worker); 882e03a3d7aSMing Lei kthread_stop(lo->worker_task); 883e03a3d7aSMing Lei } 884e03a3d7aSMing Lei 885b2ee7d46SNeilBrown static int loop_kthread_worker_fn(void *worker_ptr) 886b2ee7d46SNeilBrown { 887b2ee7d46SNeilBrown current->flags |= PF_LESS_THROTTLE; 888b2ee7d46SNeilBrown return kthread_worker_fn(worker_ptr); 889b2ee7d46SNeilBrown } 890b2ee7d46SNeilBrown 891e03a3d7aSMing Lei static int loop_prepare_queue(struct loop_device *lo) 892e03a3d7aSMing Lei { 8933989144fSPetr Mladek kthread_init_worker(&lo->worker); 894b2ee7d46SNeilBrown lo->worker_task = kthread_run(loop_kthread_worker_fn, 895e03a3d7aSMing Lei &lo->worker, "loop%d", lo->lo_number); 896e03a3d7aSMing Lei if (IS_ERR(lo->worker_task)) 897e03a3d7aSMing Lei return -ENOMEM; 898e03a3d7aSMing Lei set_user_nice(lo->worker_task, MIN_NICE); 899e03a3d7aSMing Lei return 0; 900e03a3d7aSMing Lei } 901e03a3d7aSMing Lei 902bb214884SAl Viro static int loop_set_fd(struct loop_device *lo, fmode_t mode, 9031da177e4SLinus Torvalds struct block_device *bdev, unsigned int arg) 9041da177e4SLinus Torvalds { 905*d2ac838eSTheodore Ts'o struct file *file; 9061da177e4SLinus Torvalds struct inode *inode; 9071da177e4SLinus Torvalds struct address_space *mapping; 9081da177e4SLinus Torvalds int lo_flags = 0; 9091da177e4SLinus Torvalds int error; 9101da177e4SLinus Torvalds loff_t size; 9111da177e4SLinus Torvalds 9121da177e4SLinus Torvalds /* This is safe, since we have a reference from open(). */ 9131da177e4SLinus Torvalds __module_get(THIS_MODULE); 9141da177e4SLinus Torvalds 9151da177e4SLinus Torvalds error = -EBADF; 9161da177e4SLinus Torvalds file = fget(arg); 9171da177e4SLinus Torvalds if (!file) 9181da177e4SLinus Torvalds goto out; 9191da177e4SLinus Torvalds 9201da177e4SLinus Torvalds error = -EBUSY; 9211da177e4SLinus Torvalds if (lo->lo_state != Lo_unbound) 9221da177e4SLinus Torvalds goto out_putf; 9231da177e4SLinus Torvalds 924*d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev); 925*d2ac838eSTheodore Ts'o if (error) 9261da177e4SLinus Torvalds goto out_putf; 9271da177e4SLinus Torvalds 9281da177e4SLinus Torvalds mapping = file->f_mapping; 9291da177e4SLinus Torvalds inode = mapping->host; 9301da177e4SLinus Torvalds 931456be148SChristoph Hellwig if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) || 932283e7e5dSAl Viro !file->f_op->write_iter) 9331da177e4SLinus Torvalds lo_flags |= LO_FLAGS_READ_ONLY; 9341da177e4SLinus Torvalds 9351da177e4SLinus Torvalds error = -EFBIG; 936456be148SChristoph Hellwig size = get_loop_size(lo, file); 937456be148SChristoph Hellwig if ((loff_t)(sector_t)size != size) 9381da177e4SLinus Torvalds goto out_putf; 939e03a3d7aSMing Lei error = loop_prepare_queue(lo); 940e03a3d7aSMing Lei if (error) 941f4aa4c7bSMing Lei goto out_putf; 9421da177e4SLinus Torvalds 943456be148SChristoph Hellwig error = 0; 9441da177e4SLinus Torvalds 9451da177e4SLinus Torvalds set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0); 9461da177e4SLinus Torvalds 9472e5ab5f3SMing Lei lo->use_dio = false; 9481da177e4SLinus Torvalds lo->lo_device = bdev; 9491da177e4SLinus Torvalds lo->lo_flags = lo_flags; 9501da177e4SLinus Torvalds lo->lo_backing_file = file; 951aa4d8616SChristoph Hellwig lo->transfer = NULL; 9521da177e4SLinus Torvalds lo->ioctl = NULL; 9531da177e4SLinus Torvalds lo->lo_sizelimit = 0; 9541da177e4SLinus Torvalds lo->old_gfp_mask = mapping_gfp_mask(mapping); 9551da177e4SLinus Torvalds mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 9561da177e4SLinus Torvalds 95768db1961SNikanth Karthikesan if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) 95821d0727fSJens Axboe blk_queue_write_cache(lo->lo_queue, true, false); 95968db1961SNikanth Karthikesan 9602e5ab5f3SMing Lei loop_update_dio(lo); 96173285082SKen Chen set_capacity(lo->lo_disk, size); 9621da177e4SLinus Torvalds bd_set_size(bdev, size << 9); 963ee862730SMilan Broz loop_sysfs_init(lo); 964c3473c63SDavid Zeuthen /* let user-space know about the new size */ 965c3473c63SDavid Zeuthen kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 9661da177e4SLinus Torvalds 9678a0740c4SOmar Sandoval set_blocksize(bdev, S_ISBLK(inode->i_mode) ? 9688a0740c4SOmar Sandoval block_size(inode->i_bdev) : PAGE_SIZE); 9691da177e4SLinus Torvalds 9706c997918SSerge E. Hallyn lo->lo_state = Lo_bound; 971e03c8dd1SKay Sievers if (part_shift) 972e03c8dd1SKay Sievers lo->lo_flags |= LO_FLAGS_PARTSCAN; 973e03c8dd1SKay Sievers if (lo->lo_flags & LO_FLAGS_PARTSCAN) 97406f0e9e6SMing Lei loop_reread_partitions(lo, bdev); 975c1681bf8SAnatol Pomozov 976c1681bf8SAnatol Pomozov /* Grab the block_device to prevent its destruction after we 977c1681bf8SAnatol Pomozov * put /dev/loopXX inode. Later in loop_clr_fd() we bdput(bdev). 978c1681bf8SAnatol Pomozov */ 979c1681bf8SAnatol Pomozov bdgrab(bdev); 9801da177e4SLinus Torvalds return 0; 9811da177e4SLinus Torvalds 9821da177e4SLinus Torvalds out_putf: 9831da177e4SLinus Torvalds fput(file); 9841da177e4SLinus Torvalds out: 9851da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */ 9861da177e4SLinus Torvalds module_put(THIS_MODULE); 9871da177e4SLinus Torvalds return error; 9881da177e4SLinus Torvalds } 9891da177e4SLinus Torvalds 9901da177e4SLinus Torvalds static int 9911da177e4SLinus Torvalds loop_release_xfer(struct loop_device *lo) 9921da177e4SLinus Torvalds { 9931da177e4SLinus Torvalds int err = 0; 9941da177e4SLinus Torvalds struct loop_func_table *xfer = lo->lo_encryption; 9951da177e4SLinus Torvalds 9961da177e4SLinus Torvalds if (xfer) { 9971da177e4SLinus Torvalds if (xfer->release) 9981da177e4SLinus Torvalds err = xfer->release(lo); 9991da177e4SLinus Torvalds lo->transfer = NULL; 10001da177e4SLinus Torvalds lo->lo_encryption = NULL; 10011da177e4SLinus Torvalds module_put(xfer->owner); 10021da177e4SLinus Torvalds } 10031da177e4SLinus Torvalds return err; 10041da177e4SLinus Torvalds } 10051da177e4SLinus Torvalds 10061da177e4SLinus Torvalds static int 10071da177e4SLinus Torvalds loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, 10081da177e4SLinus Torvalds const struct loop_info64 *i) 10091da177e4SLinus Torvalds { 10101da177e4SLinus Torvalds int err = 0; 10111da177e4SLinus Torvalds 10121da177e4SLinus Torvalds if (xfer) { 10131da177e4SLinus Torvalds struct module *owner = xfer->owner; 10141da177e4SLinus Torvalds 10151da177e4SLinus Torvalds if (!try_module_get(owner)) 10161da177e4SLinus Torvalds return -EINVAL; 10171da177e4SLinus Torvalds if (xfer->init) 10181da177e4SLinus Torvalds err = xfer->init(lo, i); 10191da177e4SLinus Torvalds if (err) 10201da177e4SLinus Torvalds module_put(owner); 10211da177e4SLinus Torvalds else 10221da177e4SLinus Torvalds lo->lo_encryption = xfer; 10231da177e4SLinus Torvalds } 10241da177e4SLinus Torvalds return err; 10251da177e4SLinus Torvalds } 10261da177e4SLinus Torvalds 10274c823cc3SAyan George static int loop_clr_fd(struct loop_device *lo) 10281da177e4SLinus Torvalds { 10291da177e4SLinus Torvalds struct file *filp = lo->lo_backing_file; 1030b4e3ca1aSAl Viro gfp_t gfp = lo->old_gfp_mask; 10314c823cc3SAyan George struct block_device *bdev = lo->lo_device; 10321da177e4SLinus Torvalds 10331da177e4SLinus Torvalds if (lo->lo_state != Lo_bound) 10341da177e4SLinus Torvalds return -ENXIO; 10351da177e4SLinus Torvalds 1036a1ecac3bSDave Chinner /* 1037a1ecac3bSDave Chinner * If we've explicitly asked to tear down the loop device, 1038a1ecac3bSDave Chinner * and it has an elevated reference count, set it for auto-teardown when 1039a1ecac3bSDave Chinner * the last reference goes away. This stops $!~#$@ udev from 1040a1ecac3bSDave Chinner * preventing teardown because it decided that it needs to run blkid on 1041a1ecac3bSDave Chinner * the loopback device whenever they appear. xfstests is notorious for 1042a1ecac3bSDave Chinner * failing tests because blkid via udev races with a losetup 1043a1ecac3bSDave Chinner * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d 1044a1ecac3bSDave Chinner * command to fail with EBUSY. 1045a1ecac3bSDave Chinner */ 1046f8933667SMing Lei if (atomic_read(&lo->lo_refcnt) > 1) { 1047a1ecac3bSDave Chinner lo->lo_flags |= LO_FLAGS_AUTOCLEAR; 1048a1ecac3bSDave Chinner mutex_unlock(&lo->lo_ctl_mutex); 1049a1ecac3bSDave Chinner return 0; 1050a1ecac3bSDave Chinner } 10511da177e4SLinus Torvalds 10521da177e4SLinus Torvalds if (filp == NULL) 10531da177e4SLinus Torvalds return -EINVAL; 10541da177e4SLinus Torvalds 1055f8933667SMing Lei /* freeze request queue during the transition */ 1056f8933667SMing Lei blk_mq_freeze_queue(lo->lo_queue); 1057f8933667SMing Lei 10581da177e4SLinus Torvalds spin_lock_irq(&lo->lo_lock); 10591da177e4SLinus Torvalds lo->lo_state = Lo_rundown; 10601da177e4SLinus Torvalds lo->lo_backing_file = NULL; 106105eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock); 10621da177e4SLinus Torvalds 10631da177e4SLinus Torvalds loop_release_xfer(lo); 10641da177e4SLinus Torvalds lo->transfer = NULL; 10651da177e4SLinus Torvalds lo->ioctl = NULL; 10661da177e4SLinus Torvalds lo->lo_device = NULL; 10671da177e4SLinus Torvalds lo->lo_encryption = NULL; 10681da177e4SLinus Torvalds lo->lo_offset = 0; 10691da177e4SLinus Torvalds lo->lo_sizelimit = 0; 10701da177e4SLinus Torvalds lo->lo_encrypt_key_size = 0; 10711da177e4SLinus Torvalds memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE); 10721da177e4SLinus Torvalds memset(lo->lo_crypt_name, 0, LO_NAME_SIZE); 10731da177e4SLinus Torvalds memset(lo->lo_file_name, 0, LO_NAME_SIZE); 107489e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, 512); 1075bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, 512); 1076bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, 512); 1077c1681bf8SAnatol Pomozov if (bdev) { 1078c1681bf8SAnatol Pomozov bdput(bdev); 1079f98393a6SPeter Zijlstra invalidate_bdev(bdev); 1080eedffa28SJeff Layton bdev->bd_inode->i_mapping->wb_err = 0; 1081c1681bf8SAnatol Pomozov } 108273285082SKen Chen set_capacity(lo->lo_disk, 0); 108351a0bb0cSMilan Broz loop_sysfs_exit(lo); 1084c3473c63SDavid Zeuthen if (bdev) { 10851da177e4SLinus Torvalds bd_set_size(bdev, 0); 1086c3473c63SDavid Zeuthen /* let user-space know about this change */ 1087c3473c63SDavid Zeuthen kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 1088c3473c63SDavid Zeuthen } 10891da177e4SLinus Torvalds mapping_set_gfp_mask(filp->f_mapping, gfp); 10901da177e4SLinus Torvalds lo->lo_state = Lo_unbound; 10911da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */ 10921da177e4SLinus Torvalds module_put(THIS_MODULE); 1093f8933667SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 1094f8933667SMing Lei 1095c2fccc1cSJens Axboe if (lo->lo_flags & LO_FLAGS_PARTSCAN && bdev) 109606f0e9e6SMing Lei loop_reread_partitions(lo, bdev); 1097e03c8dd1SKay Sievers lo->lo_flags = 0; 1098e03c8dd1SKay Sievers if (!part_shift) 1099e03c8dd1SKay Sievers lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN; 1100e03a3d7aSMing Lei loop_unprepare_queue(lo); 1101f028f3b2SNikanth Karthikesan mutex_unlock(&lo->lo_ctl_mutex); 1102f028f3b2SNikanth Karthikesan /* 1103f028f3b2SNikanth Karthikesan * Need not hold lo_ctl_mutex to fput backing file. 1104f028f3b2SNikanth Karthikesan * Calling fput holding lo_ctl_mutex triggers a circular 1105f028f3b2SNikanth Karthikesan * lock dependency possibility warning as fput can take 1106f028f3b2SNikanth Karthikesan * bd_mutex which is usually taken before lo_ctl_mutex. 1107f028f3b2SNikanth Karthikesan */ 1108f028f3b2SNikanth Karthikesan fput(filp); 11091da177e4SLinus Torvalds return 0; 11101da177e4SLinus Torvalds } 11111da177e4SLinus Torvalds 11121da177e4SLinus Torvalds static int 11131da177e4SLinus Torvalds loop_set_status(struct loop_device *lo, const struct loop_info64 *info) 11141da177e4SLinus Torvalds { 11151da177e4SLinus Torvalds int err; 11161da177e4SLinus Torvalds struct loop_func_table *xfer; 1117e4849737SEric W. Biederman kuid_t uid = current_uid(); 11181da177e4SLinus Torvalds 1119b0fafa81SDavid Howells if (lo->lo_encrypt_key_size && 1120e4849737SEric W. Biederman !uid_eq(lo->lo_key_owner, uid) && 11211da177e4SLinus Torvalds !capable(CAP_SYS_ADMIN)) 11221da177e4SLinus Torvalds return -EPERM; 11231da177e4SLinus Torvalds if (lo->lo_state != Lo_bound) 11241da177e4SLinus Torvalds return -ENXIO; 11251da177e4SLinus Torvalds if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) 11261da177e4SLinus Torvalds return -EINVAL; 11271da177e4SLinus Torvalds 1128ecdd0959SMing Lei /* I/O need to be drained during transfer transition */ 1129ecdd0959SMing Lei blk_mq_freeze_queue(lo->lo_queue); 1130ecdd0959SMing Lei 11311da177e4SLinus Torvalds err = loop_release_xfer(lo); 11321da177e4SLinus Torvalds if (err) 1133ecdd0959SMing Lei goto exit; 11341da177e4SLinus Torvalds 11351da177e4SLinus Torvalds if (info->lo_encrypt_type) { 11361da177e4SLinus Torvalds unsigned int type = info->lo_encrypt_type; 11371da177e4SLinus Torvalds 11381e047eaaSTetsuo Handa if (type >= MAX_LO_CRYPT) { 11391e047eaaSTetsuo Handa err = -EINVAL; 11401e047eaaSTetsuo Handa goto exit; 11411e047eaaSTetsuo Handa } 11421da177e4SLinus Torvalds xfer = xfer_funcs[type]; 11431e047eaaSTetsuo Handa if (xfer == NULL) { 11441e047eaaSTetsuo Handa err = -EINVAL; 11451e047eaaSTetsuo Handa goto exit; 11461e047eaaSTetsuo Handa } 11471da177e4SLinus Torvalds } else 11481da177e4SLinus Torvalds xfer = NULL; 11491da177e4SLinus Torvalds 11501da177e4SLinus Torvalds err = loop_init_xfer(lo, xfer, info); 11511da177e4SLinus Torvalds if (err) 1152ecdd0959SMing Lei goto exit; 11531da177e4SLinus Torvalds 11541da177e4SLinus Torvalds if (lo->lo_offset != info->lo_offset || 11551e6ec9eaSOmar Sandoval lo->lo_sizelimit != info->lo_sizelimit) { 11561e6ec9eaSOmar Sandoval if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) { 1157ecdd0959SMing Lei err = -EFBIG; 1158ecdd0959SMing Lei goto exit; 1159ecdd0959SMing Lei } 1160b040ad9cSArnd Bergmann } 1161541c742aSGuo Chao 1162dfaa2ef6SLukas Czerner loop_config_discard(lo); 11631da177e4SLinus Torvalds 11641da177e4SLinus Torvalds memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); 11651da177e4SLinus Torvalds memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); 11661da177e4SLinus Torvalds lo->lo_file_name[LO_NAME_SIZE-1] = 0; 11671da177e4SLinus Torvalds lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; 11681da177e4SLinus Torvalds 11691da177e4SLinus Torvalds if (!xfer) 11701da177e4SLinus Torvalds xfer = &none_funcs; 11711da177e4SLinus Torvalds lo->transfer = xfer->transfer; 11721da177e4SLinus Torvalds lo->ioctl = xfer->ioctl; 11731da177e4SLinus Torvalds 117496c58655SDavid Woodhouse if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) != 117596c58655SDavid Woodhouse (info->lo_flags & LO_FLAGS_AUTOCLEAR)) 117696c58655SDavid Woodhouse lo->lo_flags ^= LO_FLAGS_AUTOCLEAR; 117796c58655SDavid Woodhouse 11781da177e4SLinus Torvalds lo->lo_encrypt_key_size = info->lo_encrypt_key_size; 11791da177e4SLinus Torvalds lo->lo_init[0] = info->lo_init[0]; 11801da177e4SLinus Torvalds lo->lo_init[1] = info->lo_init[1]; 11811da177e4SLinus Torvalds if (info->lo_encrypt_key_size) { 11821da177e4SLinus Torvalds memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, 11831da177e4SLinus Torvalds info->lo_encrypt_key_size); 1184b0fafa81SDavid Howells lo->lo_key_owner = uid; 11851da177e4SLinus Torvalds } 11861da177e4SLinus Torvalds 11872e5ab5f3SMing Lei /* update dio if lo_offset or transfer is changed */ 11882e5ab5f3SMing Lei __loop_update_dio(lo, lo->use_dio); 11892e5ab5f3SMing Lei 1190ecdd0959SMing Lei exit: 1191ecdd0959SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 1192e02898b4SOmar Sandoval 1193e02898b4SOmar Sandoval if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) && 1194e02898b4SOmar Sandoval !(lo->lo_flags & LO_FLAGS_PARTSCAN)) { 1195e02898b4SOmar Sandoval lo->lo_flags |= LO_FLAGS_PARTSCAN; 1196e02898b4SOmar Sandoval lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; 1197e02898b4SOmar Sandoval loop_reread_partitions(lo, lo->lo_device); 1198e02898b4SOmar Sandoval } 1199e02898b4SOmar Sandoval 1200ecdd0959SMing Lei return err; 12011da177e4SLinus Torvalds } 12021da177e4SLinus Torvalds 12031da177e4SLinus Torvalds static int 12041da177e4SLinus Torvalds loop_get_status(struct loop_device *lo, struct loop_info64 *info) 12051da177e4SLinus Torvalds { 12062d1d4c1eSOmar Sandoval struct file *file; 12071da177e4SLinus Torvalds struct kstat stat; 12082d1d4c1eSOmar Sandoval int ret; 12091da177e4SLinus Torvalds 12102d1d4c1eSOmar Sandoval if (lo->lo_state != Lo_bound) { 12112d1d4c1eSOmar Sandoval mutex_unlock(&lo->lo_ctl_mutex); 12121da177e4SLinus Torvalds return -ENXIO; 12132d1d4c1eSOmar Sandoval } 12142d1d4c1eSOmar Sandoval 12151da177e4SLinus Torvalds memset(info, 0, sizeof(*info)); 12161da177e4SLinus Torvalds info->lo_number = lo->lo_number; 12171da177e4SLinus Torvalds info->lo_offset = lo->lo_offset; 12181da177e4SLinus Torvalds info->lo_sizelimit = lo->lo_sizelimit; 12191da177e4SLinus Torvalds info->lo_flags = lo->lo_flags; 12201da177e4SLinus Torvalds memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE); 12211da177e4SLinus Torvalds memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE); 12221da177e4SLinus Torvalds info->lo_encrypt_type = 12231da177e4SLinus Torvalds lo->lo_encryption ? lo->lo_encryption->number : 0; 12241da177e4SLinus Torvalds if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) { 12251da177e4SLinus Torvalds info->lo_encrypt_key_size = lo->lo_encrypt_key_size; 12261da177e4SLinus Torvalds memcpy(info->lo_encrypt_key, lo->lo_encrypt_key, 12271da177e4SLinus Torvalds lo->lo_encrypt_key_size); 12281da177e4SLinus Torvalds } 12292d1d4c1eSOmar Sandoval 12302d1d4c1eSOmar Sandoval /* Drop lo_ctl_mutex while we call into the filesystem. */ 12312d1d4c1eSOmar Sandoval file = get_file(lo->lo_backing_file); 12322d1d4c1eSOmar Sandoval mutex_unlock(&lo->lo_ctl_mutex); 12332d1d4c1eSOmar Sandoval ret = vfs_getattr(&file->f_path, &stat, STATX_INO, 12342d1d4c1eSOmar Sandoval AT_STATX_SYNC_AS_STAT); 12352d1d4c1eSOmar Sandoval if (!ret) { 12362d1d4c1eSOmar Sandoval info->lo_device = huge_encode_dev(stat.dev); 12372d1d4c1eSOmar Sandoval info->lo_inode = stat.ino; 12382d1d4c1eSOmar Sandoval info->lo_rdevice = huge_encode_dev(stat.rdev); 12392d1d4c1eSOmar Sandoval } 12402d1d4c1eSOmar Sandoval fput(file); 12412d1d4c1eSOmar Sandoval return ret; 12421da177e4SLinus Torvalds } 12431da177e4SLinus Torvalds 12441da177e4SLinus Torvalds static void 12451da177e4SLinus Torvalds loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64) 12461da177e4SLinus Torvalds { 12471da177e4SLinus Torvalds memset(info64, 0, sizeof(*info64)); 12481da177e4SLinus Torvalds info64->lo_number = info->lo_number; 12491da177e4SLinus Torvalds info64->lo_device = info->lo_device; 12501da177e4SLinus Torvalds info64->lo_inode = info->lo_inode; 12511da177e4SLinus Torvalds info64->lo_rdevice = info->lo_rdevice; 12521da177e4SLinus Torvalds info64->lo_offset = info->lo_offset; 12531da177e4SLinus Torvalds info64->lo_sizelimit = 0; 12541da177e4SLinus Torvalds info64->lo_encrypt_type = info->lo_encrypt_type; 12551da177e4SLinus Torvalds info64->lo_encrypt_key_size = info->lo_encrypt_key_size; 12561da177e4SLinus Torvalds info64->lo_flags = info->lo_flags; 12571da177e4SLinus Torvalds info64->lo_init[0] = info->lo_init[0]; 12581da177e4SLinus Torvalds info64->lo_init[1] = info->lo_init[1]; 12591da177e4SLinus Torvalds if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 12601da177e4SLinus Torvalds memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE); 12611da177e4SLinus Torvalds else 12621da177e4SLinus Torvalds memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); 12631da177e4SLinus Torvalds memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE); 12641da177e4SLinus Torvalds } 12651da177e4SLinus Torvalds 12661da177e4SLinus Torvalds static int 12671da177e4SLinus Torvalds loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info) 12681da177e4SLinus Torvalds { 12691da177e4SLinus Torvalds memset(info, 0, sizeof(*info)); 12701da177e4SLinus Torvalds info->lo_number = info64->lo_number; 12711da177e4SLinus Torvalds info->lo_device = info64->lo_device; 12721da177e4SLinus Torvalds info->lo_inode = info64->lo_inode; 12731da177e4SLinus Torvalds info->lo_rdevice = info64->lo_rdevice; 12741da177e4SLinus Torvalds info->lo_offset = info64->lo_offset; 12751da177e4SLinus Torvalds info->lo_encrypt_type = info64->lo_encrypt_type; 12761da177e4SLinus Torvalds info->lo_encrypt_key_size = info64->lo_encrypt_key_size; 12771da177e4SLinus Torvalds info->lo_flags = info64->lo_flags; 12781da177e4SLinus Torvalds info->lo_init[0] = info64->lo_init[0]; 12791da177e4SLinus Torvalds info->lo_init[1] = info64->lo_init[1]; 12801da177e4SLinus Torvalds if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 12811da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 12821da177e4SLinus Torvalds else 12831da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); 12841da177e4SLinus Torvalds memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 12851da177e4SLinus Torvalds 12861da177e4SLinus Torvalds /* error in case values were truncated */ 12871da177e4SLinus Torvalds if (info->lo_device != info64->lo_device || 12881da177e4SLinus Torvalds info->lo_rdevice != info64->lo_rdevice || 12891da177e4SLinus Torvalds info->lo_inode != info64->lo_inode || 12901da177e4SLinus Torvalds info->lo_offset != info64->lo_offset) 12911da177e4SLinus Torvalds return -EOVERFLOW; 12921da177e4SLinus Torvalds 12931da177e4SLinus Torvalds return 0; 12941da177e4SLinus Torvalds } 12951da177e4SLinus Torvalds 12961da177e4SLinus Torvalds static int 12971da177e4SLinus Torvalds loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg) 12981da177e4SLinus Torvalds { 12991da177e4SLinus Torvalds struct loop_info info; 13001da177e4SLinus Torvalds struct loop_info64 info64; 13011da177e4SLinus Torvalds 13021da177e4SLinus Torvalds if (copy_from_user(&info, arg, sizeof (struct loop_info))) 13031da177e4SLinus Torvalds return -EFAULT; 13041da177e4SLinus Torvalds loop_info64_from_old(&info, &info64); 13051da177e4SLinus Torvalds return loop_set_status(lo, &info64); 13061da177e4SLinus Torvalds } 13071da177e4SLinus Torvalds 13081da177e4SLinus Torvalds static int 13091da177e4SLinus Torvalds loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg) 13101da177e4SLinus Torvalds { 13111da177e4SLinus Torvalds struct loop_info64 info64; 13121da177e4SLinus Torvalds 13131da177e4SLinus Torvalds if (copy_from_user(&info64, arg, sizeof (struct loop_info64))) 13141da177e4SLinus Torvalds return -EFAULT; 13151da177e4SLinus Torvalds return loop_set_status(lo, &info64); 13161da177e4SLinus Torvalds } 13171da177e4SLinus Torvalds 13181da177e4SLinus Torvalds static int 13191da177e4SLinus Torvalds loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) { 13201da177e4SLinus Torvalds struct loop_info info; 13211da177e4SLinus Torvalds struct loop_info64 info64; 1322bdac616dSOmar Sandoval int err; 13231da177e4SLinus Torvalds 1324bdac616dSOmar Sandoval if (!arg) { 1325bdac616dSOmar Sandoval mutex_unlock(&lo->lo_ctl_mutex); 1326bdac616dSOmar Sandoval return -EINVAL; 1327bdac616dSOmar Sandoval } 13281da177e4SLinus Torvalds err = loop_get_status(lo, &info64); 13291da177e4SLinus Torvalds if (!err) 13301da177e4SLinus Torvalds err = loop_info64_to_old(&info64, &info); 13311da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info, sizeof(info))) 13321da177e4SLinus Torvalds err = -EFAULT; 13331da177e4SLinus Torvalds 13341da177e4SLinus Torvalds return err; 13351da177e4SLinus Torvalds } 13361da177e4SLinus Torvalds 13371da177e4SLinus Torvalds static int 13381da177e4SLinus Torvalds loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { 13391da177e4SLinus Torvalds struct loop_info64 info64; 1340bdac616dSOmar Sandoval int err; 13411da177e4SLinus Torvalds 1342bdac616dSOmar Sandoval if (!arg) { 1343bdac616dSOmar Sandoval mutex_unlock(&lo->lo_ctl_mutex); 1344bdac616dSOmar Sandoval return -EINVAL; 1345bdac616dSOmar Sandoval } 13461da177e4SLinus Torvalds err = loop_get_status(lo, &info64); 13471da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info64, sizeof(info64))) 13481da177e4SLinus Torvalds err = -EFAULT; 13491da177e4SLinus Torvalds 13501da177e4SLinus Torvalds return err; 13511da177e4SLinus Torvalds } 13521da177e4SLinus Torvalds 135351001b7dSHannes Reinecke static int loop_set_capacity(struct loop_device *lo) 135453d66608SJ. R. Okajima { 135553d66608SJ. R. Okajima if (unlikely(lo->lo_state != Lo_bound)) 13567b0576a3SGuo Chao return -ENXIO; 135753d66608SJ. R. Okajima 13581e6ec9eaSOmar Sandoval return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit); 135953d66608SJ. R. Okajima } 136053d66608SJ. R. Okajima 1361ab1cb278SMing Lei static int loop_set_dio(struct loop_device *lo, unsigned long arg) 1362ab1cb278SMing Lei { 1363ab1cb278SMing Lei int error = -ENXIO; 1364ab1cb278SMing Lei if (lo->lo_state != Lo_bound) 1365ab1cb278SMing Lei goto out; 1366ab1cb278SMing Lei 1367ab1cb278SMing Lei __loop_update_dio(lo, !!arg); 1368ab1cb278SMing Lei if (lo->use_dio == !!arg) 1369ab1cb278SMing Lei return 0; 1370ab1cb278SMing Lei error = -EINVAL; 1371ab1cb278SMing Lei out: 1372ab1cb278SMing Lei return error; 1373ab1cb278SMing Lei } 1374ab1cb278SMing Lei 137589e4fdecSOmar Sandoval static int loop_set_block_size(struct loop_device *lo, unsigned long arg) 137689e4fdecSOmar Sandoval { 137789e4fdecSOmar Sandoval if (lo->lo_state != Lo_bound) 137889e4fdecSOmar Sandoval return -ENXIO; 137989e4fdecSOmar Sandoval 138089e4fdecSOmar Sandoval if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg)) 138189e4fdecSOmar Sandoval return -EINVAL; 138289e4fdecSOmar Sandoval 138389e4fdecSOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 138489e4fdecSOmar Sandoval 138589e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, arg); 1386bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, arg); 1387bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, arg); 138889e4fdecSOmar Sandoval loop_update_dio(lo); 138989e4fdecSOmar Sandoval 139089e4fdecSOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 139189e4fdecSOmar Sandoval 139289e4fdecSOmar Sandoval return 0; 139389e4fdecSOmar Sandoval } 139489e4fdecSOmar Sandoval 1395bb214884SAl Viro static int lo_ioctl(struct block_device *bdev, fmode_t mode, 13961da177e4SLinus Torvalds unsigned int cmd, unsigned long arg) 13971da177e4SLinus Torvalds { 1398bb214884SAl Viro struct loop_device *lo = bdev->bd_disk->private_data; 13991da177e4SLinus Torvalds int err; 14001da177e4SLinus Torvalds 14013148ffbdSOmar Sandoval err = mutex_lock_killable_nested(&lo->lo_ctl_mutex, 1); 14023148ffbdSOmar Sandoval if (err) 14033148ffbdSOmar Sandoval goto out_unlocked; 14043148ffbdSOmar Sandoval 14051da177e4SLinus Torvalds switch (cmd) { 14061da177e4SLinus Torvalds case LOOP_SET_FD: 1407bb214884SAl Viro err = loop_set_fd(lo, mode, bdev, arg); 14081da177e4SLinus Torvalds break; 14091da177e4SLinus Torvalds case LOOP_CHANGE_FD: 1410bb214884SAl Viro err = loop_change_fd(lo, bdev, arg); 14111da177e4SLinus Torvalds break; 14121da177e4SLinus Torvalds case LOOP_CLR_FD: 1413f028f3b2SNikanth Karthikesan /* loop_clr_fd would have unlocked lo_ctl_mutex on success */ 14144c823cc3SAyan George err = loop_clr_fd(lo); 1415f028f3b2SNikanth Karthikesan if (!err) 1416f028f3b2SNikanth Karthikesan goto out_unlocked; 14171da177e4SLinus Torvalds break; 14181da177e4SLinus Torvalds case LOOP_SET_STATUS: 14197035b5dfSDmitry Monakhov err = -EPERM; 14207035b5dfSDmitry Monakhov if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) 14217035b5dfSDmitry Monakhov err = loop_set_status_old(lo, 14227035b5dfSDmitry Monakhov (struct loop_info __user *)arg); 14231da177e4SLinus Torvalds break; 14241da177e4SLinus Torvalds case LOOP_GET_STATUS: 14251da177e4SLinus Torvalds err = loop_get_status_old(lo, (struct loop_info __user *) arg); 14262d1d4c1eSOmar Sandoval /* loop_get_status() unlocks lo_ctl_mutex */ 14272d1d4c1eSOmar Sandoval goto out_unlocked; 14281da177e4SLinus Torvalds case LOOP_SET_STATUS64: 14297035b5dfSDmitry Monakhov err = -EPERM; 14307035b5dfSDmitry Monakhov if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) 14317035b5dfSDmitry Monakhov err = loop_set_status64(lo, 14327035b5dfSDmitry Monakhov (struct loop_info64 __user *) arg); 14331da177e4SLinus Torvalds break; 14341da177e4SLinus Torvalds case LOOP_GET_STATUS64: 14351da177e4SLinus Torvalds err = loop_get_status64(lo, (struct loop_info64 __user *) arg); 14362d1d4c1eSOmar Sandoval /* loop_get_status() unlocks lo_ctl_mutex */ 14372d1d4c1eSOmar Sandoval goto out_unlocked; 143853d66608SJ. R. Okajima case LOOP_SET_CAPACITY: 143953d66608SJ. R. Okajima err = -EPERM; 144053d66608SJ. R. Okajima if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) 144151001b7dSHannes Reinecke err = loop_set_capacity(lo); 144253d66608SJ. R. Okajima break; 1443ab1cb278SMing Lei case LOOP_SET_DIRECT_IO: 1444ab1cb278SMing Lei err = -EPERM; 1445ab1cb278SMing Lei if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) 1446ab1cb278SMing Lei err = loop_set_dio(lo, arg); 1447ab1cb278SMing Lei break; 144889e4fdecSOmar Sandoval case LOOP_SET_BLOCK_SIZE: 144989e4fdecSOmar Sandoval err = -EPERM; 145089e4fdecSOmar Sandoval if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) 145189e4fdecSOmar Sandoval err = loop_set_block_size(lo, arg); 145289e4fdecSOmar Sandoval break; 14531da177e4SLinus Torvalds default: 14541da177e4SLinus Torvalds err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL; 14551da177e4SLinus Torvalds } 1456f85221ddSIngo Molnar mutex_unlock(&lo->lo_ctl_mutex); 1457f028f3b2SNikanth Karthikesan 1458f028f3b2SNikanth Karthikesan out_unlocked: 14591da177e4SLinus Torvalds return err; 14601da177e4SLinus Torvalds } 14611da177e4SLinus Torvalds 1462863d5b82SDavid Howells #ifdef CONFIG_COMPAT 1463863d5b82SDavid Howells struct compat_loop_info { 1464863d5b82SDavid Howells compat_int_t lo_number; /* ioctl r/o */ 1465863d5b82SDavid Howells compat_dev_t lo_device; /* ioctl r/o */ 1466863d5b82SDavid Howells compat_ulong_t lo_inode; /* ioctl r/o */ 1467863d5b82SDavid Howells compat_dev_t lo_rdevice; /* ioctl r/o */ 1468863d5b82SDavid Howells compat_int_t lo_offset; 1469863d5b82SDavid Howells compat_int_t lo_encrypt_type; 1470863d5b82SDavid Howells compat_int_t lo_encrypt_key_size; /* ioctl w/o */ 1471863d5b82SDavid Howells compat_int_t lo_flags; /* ioctl r/o */ 1472863d5b82SDavid Howells char lo_name[LO_NAME_SIZE]; 1473863d5b82SDavid Howells unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ 1474863d5b82SDavid Howells compat_ulong_t lo_init[2]; 1475863d5b82SDavid Howells char reserved[4]; 1476863d5b82SDavid Howells }; 1477863d5b82SDavid Howells 1478863d5b82SDavid Howells /* 1479863d5b82SDavid Howells * Transfer 32-bit compatibility structure in userspace to 64-bit loop info 1480863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver 1481863d5b82SDavid Howells */ 1482863d5b82SDavid Howells static noinline int 1483ba674cfcSAl Viro loop_info64_from_compat(const struct compat_loop_info __user *arg, 1484863d5b82SDavid Howells struct loop_info64 *info64) 1485863d5b82SDavid Howells { 1486863d5b82SDavid Howells struct compat_loop_info info; 1487863d5b82SDavid Howells 1488863d5b82SDavid Howells if (copy_from_user(&info, arg, sizeof(info))) 1489863d5b82SDavid Howells return -EFAULT; 1490863d5b82SDavid Howells 1491863d5b82SDavid Howells memset(info64, 0, sizeof(*info64)); 1492863d5b82SDavid Howells info64->lo_number = info.lo_number; 1493863d5b82SDavid Howells info64->lo_device = info.lo_device; 1494863d5b82SDavid Howells info64->lo_inode = info.lo_inode; 1495863d5b82SDavid Howells info64->lo_rdevice = info.lo_rdevice; 1496863d5b82SDavid Howells info64->lo_offset = info.lo_offset; 1497863d5b82SDavid Howells info64->lo_sizelimit = 0; 1498863d5b82SDavid Howells info64->lo_encrypt_type = info.lo_encrypt_type; 1499863d5b82SDavid Howells info64->lo_encrypt_key_size = info.lo_encrypt_key_size; 1500863d5b82SDavid Howells info64->lo_flags = info.lo_flags; 1501863d5b82SDavid Howells info64->lo_init[0] = info.lo_init[0]; 1502863d5b82SDavid Howells info64->lo_init[1] = info.lo_init[1]; 1503863d5b82SDavid Howells if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1504863d5b82SDavid Howells memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE); 1505863d5b82SDavid Howells else 1506863d5b82SDavid Howells memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); 1507863d5b82SDavid Howells memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE); 1508863d5b82SDavid Howells return 0; 1509863d5b82SDavid Howells } 1510863d5b82SDavid Howells 1511863d5b82SDavid Howells /* 1512863d5b82SDavid Howells * Transfer 64-bit loop info to 32-bit compatibility structure in userspace 1513863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver 1514863d5b82SDavid Howells */ 1515863d5b82SDavid Howells static noinline int 1516863d5b82SDavid Howells loop_info64_to_compat(const struct loop_info64 *info64, 1517863d5b82SDavid Howells struct compat_loop_info __user *arg) 1518863d5b82SDavid Howells { 1519863d5b82SDavid Howells struct compat_loop_info info; 1520863d5b82SDavid Howells 1521863d5b82SDavid Howells memset(&info, 0, sizeof(info)); 1522863d5b82SDavid Howells info.lo_number = info64->lo_number; 1523863d5b82SDavid Howells info.lo_device = info64->lo_device; 1524863d5b82SDavid Howells info.lo_inode = info64->lo_inode; 1525863d5b82SDavid Howells info.lo_rdevice = info64->lo_rdevice; 1526863d5b82SDavid Howells info.lo_offset = info64->lo_offset; 1527863d5b82SDavid Howells info.lo_encrypt_type = info64->lo_encrypt_type; 1528863d5b82SDavid Howells info.lo_encrypt_key_size = info64->lo_encrypt_key_size; 1529863d5b82SDavid Howells info.lo_flags = info64->lo_flags; 1530863d5b82SDavid Howells info.lo_init[0] = info64->lo_init[0]; 1531863d5b82SDavid Howells info.lo_init[1] = info64->lo_init[1]; 1532863d5b82SDavid Howells if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1533863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 1534863d5b82SDavid Howells else 1535863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); 1536863d5b82SDavid Howells memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 1537863d5b82SDavid Howells 1538863d5b82SDavid Howells /* error in case values were truncated */ 1539863d5b82SDavid Howells if (info.lo_device != info64->lo_device || 1540863d5b82SDavid Howells info.lo_rdevice != info64->lo_rdevice || 1541863d5b82SDavid Howells info.lo_inode != info64->lo_inode || 1542863d5b82SDavid Howells info.lo_offset != info64->lo_offset || 1543863d5b82SDavid Howells info.lo_init[0] != info64->lo_init[0] || 1544863d5b82SDavid Howells info.lo_init[1] != info64->lo_init[1]) 1545863d5b82SDavid Howells return -EOVERFLOW; 1546863d5b82SDavid Howells 1547863d5b82SDavid Howells if (copy_to_user(arg, &info, sizeof(info))) 1548863d5b82SDavid Howells return -EFAULT; 1549863d5b82SDavid Howells return 0; 1550863d5b82SDavid Howells } 1551863d5b82SDavid Howells 1552863d5b82SDavid Howells static int 1553863d5b82SDavid Howells loop_set_status_compat(struct loop_device *lo, 1554863d5b82SDavid Howells const struct compat_loop_info __user *arg) 1555863d5b82SDavid Howells { 1556863d5b82SDavid Howells struct loop_info64 info64; 1557863d5b82SDavid Howells int ret; 1558863d5b82SDavid Howells 1559863d5b82SDavid Howells ret = loop_info64_from_compat(arg, &info64); 1560863d5b82SDavid Howells if (ret < 0) 1561863d5b82SDavid Howells return ret; 1562863d5b82SDavid Howells return loop_set_status(lo, &info64); 1563863d5b82SDavid Howells } 1564863d5b82SDavid Howells 1565863d5b82SDavid Howells static int 1566863d5b82SDavid Howells loop_get_status_compat(struct loop_device *lo, 1567863d5b82SDavid Howells struct compat_loop_info __user *arg) 1568863d5b82SDavid Howells { 1569863d5b82SDavid Howells struct loop_info64 info64; 1570bdac616dSOmar Sandoval int err; 1571863d5b82SDavid Howells 1572bdac616dSOmar Sandoval if (!arg) { 1573bdac616dSOmar Sandoval mutex_unlock(&lo->lo_ctl_mutex); 1574bdac616dSOmar Sandoval return -EINVAL; 1575bdac616dSOmar Sandoval } 1576863d5b82SDavid Howells err = loop_get_status(lo, &info64); 1577863d5b82SDavid Howells if (!err) 1578863d5b82SDavid Howells err = loop_info64_to_compat(&info64, arg); 1579863d5b82SDavid Howells return err; 1580863d5b82SDavid Howells } 1581863d5b82SDavid Howells 1582bb214884SAl Viro static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode, 1583bb214884SAl Viro unsigned int cmd, unsigned long arg) 1584863d5b82SDavid Howells { 1585bb214884SAl Viro struct loop_device *lo = bdev->bd_disk->private_data; 1586863d5b82SDavid Howells int err; 1587863d5b82SDavid Howells 1588863d5b82SDavid Howells switch(cmd) { 1589863d5b82SDavid Howells case LOOP_SET_STATUS: 15903148ffbdSOmar Sandoval err = mutex_lock_killable(&lo->lo_ctl_mutex); 15913148ffbdSOmar Sandoval if (!err) { 15923148ffbdSOmar Sandoval err = loop_set_status_compat(lo, 15933148ffbdSOmar Sandoval (const struct compat_loop_info __user *)arg); 1594863d5b82SDavid Howells mutex_unlock(&lo->lo_ctl_mutex); 15953148ffbdSOmar Sandoval } 1596863d5b82SDavid Howells break; 1597863d5b82SDavid Howells case LOOP_GET_STATUS: 15983148ffbdSOmar Sandoval err = mutex_lock_killable(&lo->lo_ctl_mutex); 15993148ffbdSOmar Sandoval if (!err) { 16003148ffbdSOmar Sandoval err = loop_get_status_compat(lo, 16013148ffbdSOmar Sandoval (struct compat_loop_info __user *)arg); 16022d1d4c1eSOmar Sandoval /* loop_get_status() unlocks lo_ctl_mutex */ 16033148ffbdSOmar Sandoval } 1604863d5b82SDavid Howells break; 160553d66608SJ. R. Okajima case LOOP_SET_CAPACITY: 1606863d5b82SDavid Howells case LOOP_CLR_FD: 1607863d5b82SDavid Howells case LOOP_GET_STATUS64: 1608863d5b82SDavid Howells case LOOP_SET_STATUS64: 1609863d5b82SDavid Howells arg = (unsigned long) compat_ptr(arg); 1610863d5b82SDavid Howells case LOOP_SET_FD: 1611863d5b82SDavid Howells case LOOP_CHANGE_FD: 1612bb214884SAl Viro err = lo_ioctl(bdev, mode, cmd, arg); 1613863d5b82SDavid Howells break; 1614863d5b82SDavid Howells default: 1615863d5b82SDavid Howells err = -ENOIOCTLCMD; 1616863d5b82SDavid Howells break; 1617863d5b82SDavid Howells } 1618863d5b82SDavid Howells return err; 1619863d5b82SDavid Howells } 1620863d5b82SDavid Howells #endif 1621863d5b82SDavid Howells 1622bb214884SAl Viro static int lo_open(struct block_device *bdev, fmode_t mode) 16231da177e4SLinus Torvalds { 1624770fe30aSKay Sievers struct loop_device *lo; 1625770fe30aSKay Sievers int err = 0; 1626770fe30aSKay Sievers 1627770fe30aSKay Sievers mutex_lock(&loop_index_mutex); 1628770fe30aSKay Sievers lo = bdev->bd_disk->private_data; 1629770fe30aSKay Sievers if (!lo) { 1630770fe30aSKay Sievers err = -ENXIO; 1631770fe30aSKay Sievers goto out; 1632770fe30aSKay Sievers } 16331da177e4SLinus Torvalds 1634f8933667SMing Lei atomic_inc(&lo->lo_refcnt); 1635770fe30aSKay Sievers out: 1636770fe30aSKay Sievers mutex_unlock(&loop_index_mutex); 1637770fe30aSKay Sievers return err; 16381da177e4SLinus Torvalds } 16391da177e4SLinus Torvalds 1640ae665016SLinus Torvalds static void __lo_release(struct loop_device *lo) 16411da177e4SLinus Torvalds { 1642ffcd7dcaSAlexander Beregalov int err; 16431da177e4SLinus Torvalds 1644f8933667SMing Lei if (atomic_dec_return(&lo->lo_refcnt)) 1645f8933667SMing Lei return; 1646f8933667SMing Lei 1647f85221ddSIngo Molnar mutex_lock(&lo->lo_ctl_mutex); 164814f27939SMilan Broz if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) { 164914f27939SMilan Broz /* 165014f27939SMilan Broz * In autoclear mode, stop the loop thread 165114f27939SMilan Broz * and remove configuration after last close. 165214f27939SMilan Broz */ 16534c823cc3SAyan George err = loop_clr_fd(lo); 1654ffcd7dcaSAlexander Beregalov if (!err) 1655db2a144bSAl Viro return; 165643cade80SOmar Sandoval } else if (lo->lo_state == Lo_bound) { 165714f27939SMilan Broz /* 165814f27939SMilan Broz * Otherwise keep thread (if running) and config, 165914f27939SMilan Broz * but flush possible ongoing bios in thread. 166014f27939SMilan Broz */ 166143cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 166243cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 166314f27939SMilan Broz } 166496c58655SDavid Woodhouse 1665f85221ddSIngo Molnar mutex_unlock(&lo->lo_ctl_mutex); 16661da177e4SLinus Torvalds } 16671da177e4SLinus Torvalds 1668ae665016SLinus Torvalds static void lo_release(struct gendisk *disk, fmode_t mode) 1669ae665016SLinus Torvalds { 1670ae665016SLinus Torvalds mutex_lock(&loop_index_mutex); 1671ae665016SLinus Torvalds __lo_release(disk->private_data); 1672ae665016SLinus Torvalds mutex_unlock(&loop_index_mutex); 1673ae665016SLinus Torvalds } 1674ae665016SLinus Torvalds 167583d5cde4SAlexey Dobriyan static const struct block_device_operations lo_fops = { 16761da177e4SLinus Torvalds .owner = THIS_MODULE, 1677bb214884SAl Viro .open = lo_open, 1678bb214884SAl Viro .release = lo_release, 1679bb214884SAl Viro .ioctl = lo_ioctl, 1680863d5b82SDavid Howells #ifdef CONFIG_COMPAT 1681bb214884SAl Viro .compat_ioctl = lo_compat_ioctl, 1682863d5b82SDavid Howells #endif 16831da177e4SLinus Torvalds }; 16841da177e4SLinus Torvalds 16851da177e4SLinus Torvalds /* 16861da177e4SLinus Torvalds * And now the modules code and kernel interface. 16871da177e4SLinus Torvalds */ 168873285082SKen Chen static int max_loop; 16895657a819SJoe Perches module_param(max_loop, int, 0444); 1690a47653fcSKen Chen MODULE_PARM_DESC(max_loop, "Maximum number of loop devices"); 16915657a819SJoe Perches module_param(max_part, int, 0444); 1692476a4813SLaurent Vivier MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device"); 16931da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 16941da177e4SLinus Torvalds MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR); 16951da177e4SLinus Torvalds 16961da177e4SLinus Torvalds int loop_register_transfer(struct loop_func_table *funcs) 16971da177e4SLinus Torvalds { 16981da177e4SLinus Torvalds unsigned int n = funcs->number; 16991da177e4SLinus Torvalds 17001da177e4SLinus Torvalds if (n >= MAX_LO_CRYPT || xfer_funcs[n]) 17011da177e4SLinus Torvalds return -EINVAL; 17021da177e4SLinus Torvalds xfer_funcs[n] = funcs; 17031da177e4SLinus Torvalds return 0; 17041da177e4SLinus Torvalds } 17051da177e4SLinus Torvalds 170634dd82afSKay Sievers static int unregister_transfer_cb(int id, void *ptr, void *data) 170734dd82afSKay Sievers { 170834dd82afSKay Sievers struct loop_device *lo = ptr; 170934dd82afSKay Sievers struct loop_func_table *xfer = data; 171034dd82afSKay Sievers 171134dd82afSKay Sievers mutex_lock(&lo->lo_ctl_mutex); 171234dd82afSKay Sievers if (lo->lo_encryption == xfer) 171334dd82afSKay Sievers loop_release_xfer(lo); 171434dd82afSKay Sievers mutex_unlock(&lo->lo_ctl_mutex); 171534dd82afSKay Sievers return 0; 171634dd82afSKay Sievers } 171734dd82afSKay Sievers 17181da177e4SLinus Torvalds int loop_unregister_transfer(int number) 17191da177e4SLinus Torvalds { 17201da177e4SLinus Torvalds unsigned int n = number; 17211da177e4SLinus Torvalds struct loop_func_table *xfer; 17221da177e4SLinus Torvalds 17231da177e4SLinus Torvalds if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL) 17241da177e4SLinus Torvalds return -EINVAL; 17251da177e4SLinus Torvalds 17261da177e4SLinus Torvalds xfer_funcs[n] = NULL; 172734dd82afSKay Sievers idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer); 17281da177e4SLinus Torvalds return 0; 17291da177e4SLinus Torvalds } 17301da177e4SLinus Torvalds 17311da177e4SLinus Torvalds EXPORT_SYMBOL(loop_register_transfer); 17321da177e4SLinus Torvalds EXPORT_SYMBOL(loop_unregister_transfer); 17331da177e4SLinus Torvalds 1734fc17b653SChristoph Hellwig static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, 1735b5dd2f60SMing Lei const struct blk_mq_queue_data *bd) 1736b5dd2f60SMing Lei { 17371894e916SJens Axboe struct request *rq = bd->rq; 17381894e916SJens Axboe struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 17391894e916SJens Axboe struct loop_device *lo = rq->q->queuedata; 1740b5dd2f60SMing Lei 17411894e916SJens Axboe blk_mq_start_request(rq); 1742b5dd2f60SMing Lei 1743f4aa4c7bSMing Lei if (lo->lo_state != Lo_bound) 1744fc17b653SChristoph Hellwig return BLK_STS_IOERR; 1745f4aa4c7bSMing Lei 17461894e916SJens Axboe switch (req_op(rq)) { 1747f0225cacSChristoph Hellwig case REQ_OP_FLUSH: 1748f0225cacSChristoph Hellwig case REQ_OP_DISCARD: 174919372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 1750bc07c10aSMing Lei cmd->use_aio = false; 1751f0225cacSChristoph Hellwig break; 1752f0225cacSChristoph Hellwig default: 1753f0225cacSChristoph Hellwig cmd->use_aio = lo->use_dio; 1754f0225cacSChristoph Hellwig break; 1755f0225cacSChristoph Hellwig } 1756bc07c10aSMing Lei 1757d4478e92SShaohua Li /* always use the first bio's css */ 17580b508bc9SShaohua Li #ifdef CONFIG_BLK_CGROUP 17591894e916SJens Axboe if (cmd->use_aio && rq->bio && rq->bio->bi_css) { 17601894e916SJens Axboe cmd->css = rq->bio->bi_css; 1761d4478e92SShaohua Li css_get(cmd->css); 1762d4478e92SShaohua Li } else 1763d4478e92SShaohua Li #endif 1764d4478e92SShaohua Li cmd->css = NULL; 17653989144fSPetr Mladek kthread_queue_work(&lo->worker, &cmd->work); 1766b5dd2f60SMing Lei 1767fc17b653SChristoph Hellwig return BLK_STS_OK; 1768b5dd2f60SMing Lei } 1769b5dd2f60SMing Lei 1770b5dd2f60SMing Lei static void loop_handle_cmd(struct loop_cmd *cmd) 1771b5dd2f60SMing Lei { 17721894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 17731894e916SJens Axboe const bool write = op_is_write(req_op(rq)); 17741894e916SJens Axboe struct loop_device *lo = rq->q->queuedata; 1775f4829a9bSChristoph Hellwig int ret = 0; 1776b5dd2f60SMing Lei 1777f4829a9bSChristoph Hellwig if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) { 1778f4829a9bSChristoph Hellwig ret = -EIO; 1779b5dd2f60SMing Lei goto failed; 1780f4829a9bSChristoph Hellwig } 1781b5dd2f60SMing Lei 17821894e916SJens Axboe ret = do_req_filebacked(lo, rq); 1783b5dd2f60SMing Lei failed: 1784bc07c10aSMing Lei /* complete non-aio request */ 1785fe2cb290SChristoph Hellwig if (!cmd->use_aio || ret) { 1786fe2cb290SChristoph Hellwig cmd->ret = ret ? -EIO : 0; 17871894e916SJens Axboe blk_mq_complete_request(rq); 1788fe2cb290SChristoph Hellwig } 1789b5dd2f60SMing Lei } 1790b5dd2f60SMing Lei 1791e03a3d7aSMing Lei static void loop_queue_work(struct kthread_work *work) 1792b5dd2f60SMing Lei { 1793b5dd2f60SMing Lei struct loop_cmd *cmd = 1794e03a3d7aSMing Lei container_of(work, struct loop_cmd, work); 1795b5dd2f60SMing Lei 1796b5dd2f60SMing Lei loop_handle_cmd(cmd); 1797b5dd2f60SMing Lei } 1798b5dd2f60SMing Lei 1799d6296d39SChristoph Hellwig static int loop_init_request(struct blk_mq_tag_set *set, struct request *rq, 1800d6296d39SChristoph Hellwig unsigned int hctx_idx, unsigned int numa_node) 1801b5dd2f60SMing Lei { 1802b5dd2f60SMing Lei struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 1803b5dd2f60SMing Lei 18043989144fSPetr Mladek kthread_init_work(&cmd->work, loop_queue_work); 1805b5dd2f60SMing Lei return 0; 1806b5dd2f60SMing Lei } 1807b5dd2f60SMing Lei 1808f363b089SEric Biggers static const struct blk_mq_ops loop_mq_ops = { 1809b5dd2f60SMing Lei .queue_rq = loop_queue_rq, 1810b5dd2f60SMing Lei .init_request = loop_init_request, 1811fe2cb290SChristoph Hellwig .complete = lo_complete_rq, 1812b5dd2f60SMing Lei }; 1813b5dd2f60SMing Lei 181434dd82afSKay Sievers static int loop_add(struct loop_device **l, int i) 18151da177e4SLinus Torvalds { 181673285082SKen Chen struct loop_device *lo; 181773285082SKen Chen struct gendisk *disk; 181834dd82afSKay Sievers int err; 18191da177e4SLinus Torvalds 182034dd82afSKay Sievers err = -ENOMEM; 182168d740d7SSilva Paulo lo = kzalloc(sizeof(*lo), GFP_KERNEL); 182268d740d7SSilva Paulo if (!lo) 182373285082SKen Chen goto out; 182434dd82afSKay Sievers 1825ef7e7c82SMikulas Patocka lo->lo_state = Lo_unbound; 1826ef7e7c82SMikulas Patocka 1827c718aa65STejun Heo /* allocate id, if @id >= 0, we're requesting that specific id */ 182834dd82afSKay Sievers if (i >= 0) { 1829c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL); 1830c718aa65STejun Heo if (err == -ENOSPC) 183134dd82afSKay Sievers err = -EEXIST; 183234dd82afSKay Sievers } else { 1833c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL); 183434dd82afSKay Sievers } 183534dd82afSKay Sievers if (err < 0) 183634dd82afSKay Sievers goto out_free_dev; 1837c718aa65STejun Heo i = err; 18381da177e4SLinus Torvalds 1839183cfb57SWei Yongjun err = -ENOMEM; 1840b5dd2f60SMing Lei lo->tag_set.ops = &loop_mq_ops; 1841b5dd2f60SMing Lei lo->tag_set.nr_hw_queues = 1; 1842b5dd2f60SMing Lei lo->tag_set.queue_depth = 128; 1843b5dd2f60SMing Lei lo->tag_set.numa_node = NUMA_NO_NODE; 1844b5dd2f60SMing Lei lo->tag_set.cmd_size = sizeof(struct loop_cmd); 1845b5dd2f60SMing Lei lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE; 1846b5dd2f60SMing Lei lo->tag_set.driver_data = lo; 1847b5dd2f60SMing Lei 1848b5dd2f60SMing Lei err = blk_mq_alloc_tag_set(&lo->tag_set); 1849b5dd2f60SMing Lei if (err) 18503ec981e3SMikulas Patocka goto out_free_idr; 185173285082SKen Chen 1852b5dd2f60SMing Lei lo->lo_queue = blk_mq_init_queue(&lo->tag_set); 1853b5dd2f60SMing Lei if (IS_ERR_OR_NULL(lo->lo_queue)) { 1854b5dd2f60SMing Lei err = PTR_ERR(lo->lo_queue); 1855b5dd2f60SMing Lei goto out_cleanup_tags; 1856b5dd2f60SMing Lei } 1857ef7e7c82SMikulas Patocka lo->lo_queue->queuedata = lo; 1858ef7e7c82SMikulas Patocka 185954bb0adeSShaohua Li blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS); 186040326d8aSShaohua Li 18615b5e20f4SMing Lei /* 186240326d8aSShaohua Li * By default, we do buffer IO, so it doesn't make sense to enable 186340326d8aSShaohua Li * merge because the I/O submitted to backing file is handled page by 186440326d8aSShaohua Li * page. For directio mode, merge does help to dispatch bigger request 186540326d8aSShaohua Li * to underlayer disk. We will enable merge once directio is enabled. 18665b5e20f4SMing Lei */ 18678b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 18685b5e20f4SMing Lei 18697a649737SMinfei Huang err = -ENOMEM; 1870476a4813SLaurent Vivier disk = lo->lo_disk = alloc_disk(1 << part_shift); 187173285082SKen Chen if (!disk) 187273285082SKen Chen goto out_free_queue; 187373285082SKen Chen 1874e03c8dd1SKay Sievers /* 1875e03c8dd1SKay Sievers * Disable partition scanning by default. The in-kernel partition 1876e03c8dd1SKay Sievers * scanning can be requested individually per-device during its 1877e03c8dd1SKay Sievers * setup. Userspace can always add and remove partitions from all 1878e03c8dd1SKay Sievers * devices. The needed partition minors are allocated from the 1879e03c8dd1SKay Sievers * extended minor space, the main loop device numbers will continue 1880e03c8dd1SKay Sievers * to match the loop minors, regardless of the number of partitions 1881e03c8dd1SKay Sievers * used. 1882e03c8dd1SKay Sievers * 1883e03c8dd1SKay Sievers * If max_part is given, partition scanning is globally enabled for 1884e03c8dd1SKay Sievers * all loop devices. The minors for the main loop devices will be 1885e03c8dd1SKay Sievers * multiples of max_part. 1886e03c8dd1SKay Sievers * 1887e03c8dd1SKay Sievers * Note: Global-for-all-devices, set-only-at-init, read-only module 1888e03c8dd1SKay Sievers * parameteters like 'max_loop' and 'max_part' make things needlessly 1889e03c8dd1SKay Sievers * complicated, are too static, inflexible and may surprise 1890e03c8dd1SKay Sievers * userspace tools. Parameters like this in general should be avoided. 1891e03c8dd1SKay Sievers */ 1892e03c8dd1SKay Sievers if (!part_shift) 1893e03c8dd1SKay Sievers disk->flags |= GENHD_FL_NO_PART_SCAN; 1894e03c8dd1SKay Sievers disk->flags |= GENHD_FL_EXT_DEVT; 1895f85221ddSIngo Molnar mutex_init(&lo->lo_ctl_mutex); 1896f8933667SMing Lei atomic_set(&lo->lo_refcnt, 0); 18971da177e4SLinus Torvalds lo->lo_number = i; 18981da177e4SLinus Torvalds spin_lock_init(&lo->lo_lock); 18991da177e4SLinus Torvalds disk->major = LOOP_MAJOR; 1900476a4813SLaurent Vivier disk->first_minor = i << part_shift; 19011da177e4SLinus Torvalds disk->fops = &lo_fops; 19021da177e4SLinus Torvalds disk->private_data = lo; 19031da177e4SLinus Torvalds disk->queue = lo->lo_queue; 190473285082SKen Chen sprintf(disk->disk_name, "loop%d", i); 190534dd82afSKay Sievers add_disk(disk); 190634dd82afSKay Sievers *l = lo; 190734dd82afSKay Sievers return lo->lo_number; 190873285082SKen Chen 190973285082SKen Chen out_free_queue: 191073285082SKen Chen blk_cleanup_queue(lo->lo_queue); 1911b5dd2f60SMing Lei out_cleanup_tags: 1912b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set); 19133ec981e3SMikulas Patocka out_free_idr: 19143ec981e3SMikulas Patocka idr_remove(&loop_index_idr, i); 191573285082SKen Chen out_free_dev: 191673285082SKen Chen kfree(lo); 191773285082SKen Chen out: 191834dd82afSKay Sievers return err; 19191da177e4SLinus Torvalds } 19201da177e4SLinus Torvalds 192134dd82afSKay Sievers static void loop_remove(struct loop_device *lo) 192273285082SKen Chen { 19236cd18e71SNeilBrown del_gendisk(lo->lo_disk); 19240fa8ebddSBart Van Assche blk_cleanup_queue(lo->lo_queue); 1925b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set); 192673285082SKen Chen put_disk(lo->lo_disk); 192773285082SKen Chen kfree(lo); 192873285082SKen Chen } 192973285082SKen Chen 1930770fe30aSKay Sievers static int find_free_cb(int id, void *ptr, void *data) 1931770fe30aSKay Sievers { 1932770fe30aSKay Sievers struct loop_device *lo = ptr; 1933770fe30aSKay Sievers struct loop_device **l = data; 1934770fe30aSKay Sievers 1935770fe30aSKay Sievers if (lo->lo_state == Lo_unbound) { 1936770fe30aSKay Sievers *l = lo; 1937770fe30aSKay Sievers return 1; 1938770fe30aSKay Sievers } 1939770fe30aSKay Sievers return 0; 1940770fe30aSKay Sievers } 1941770fe30aSKay Sievers 194234dd82afSKay Sievers static int loop_lookup(struct loop_device **l, int i) 1943a47653fcSKen Chen { 1944a47653fcSKen Chen struct loop_device *lo; 194534dd82afSKay Sievers int ret = -ENODEV; 1946a47653fcSKen Chen 1947770fe30aSKay Sievers if (i < 0) { 1948770fe30aSKay Sievers int err; 1949770fe30aSKay Sievers 1950770fe30aSKay Sievers err = idr_for_each(&loop_index_idr, &find_free_cb, &lo); 1951770fe30aSKay Sievers if (err == 1) { 1952770fe30aSKay Sievers *l = lo; 1953770fe30aSKay Sievers ret = lo->lo_number; 1954770fe30aSKay Sievers } 1955770fe30aSKay Sievers goto out; 1956a47653fcSKen Chen } 1957a47653fcSKen Chen 1958770fe30aSKay Sievers /* lookup and return a specific i */ 195934dd82afSKay Sievers lo = idr_find(&loop_index_idr, i); 1960a47653fcSKen Chen if (lo) { 196134dd82afSKay Sievers *l = lo; 196234dd82afSKay Sievers ret = lo->lo_number; 1963a47653fcSKen Chen } 1964770fe30aSKay Sievers out: 196534dd82afSKay Sievers return ret; 1966a47653fcSKen Chen } 1967a47653fcSKen Chen 196873285082SKen Chen static struct kobject *loop_probe(dev_t dev, int *part, void *data) 196973285082SKen Chen { 1970705962ccSAl Viro struct loop_device *lo; 197107002e99SAl Viro struct kobject *kobj; 197234dd82afSKay Sievers int err; 197373285082SKen Chen 197434dd82afSKay Sievers mutex_lock(&loop_index_mutex); 197534dd82afSKay Sievers err = loop_lookup(&lo, MINOR(dev) >> part_shift); 197634dd82afSKay Sievers if (err < 0) 197734dd82afSKay Sievers err = loop_add(&lo, MINOR(dev) >> part_shift); 197834dd82afSKay Sievers if (err < 0) 1979a207f593SMikulas Patocka kobj = NULL; 198034dd82afSKay Sievers else 19813079c22eSJan Kara kobj = get_disk_and_module(lo->lo_disk); 198234dd82afSKay Sievers mutex_unlock(&loop_index_mutex); 198373285082SKen Chen 198473285082SKen Chen *part = 0; 198507002e99SAl Viro return kobj; 198673285082SKen Chen } 198773285082SKen Chen 1988770fe30aSKay Sievers static long loop_control_ioctl(struct file *file, unsigned int cmd, 1989770fe30aSKay Sievers unsigned long parm) 1990770fe30aSKay Sievers { 1991770fe30aSKay Sievers struct loop_device *lo; 1992770fe30aSKay Sievers int ret = -ENOSYS; 1993770fe30aSKay Sievers 1994770fe30aSKay Sievers mutex_lock(&loop_index_mutex); 1995770fe30aSKay Sievers switch (cmd) { 1996770fe30aSKay Sievers case LOOP_CTL_ADD: 1997770fe30aSKay Sievers ret = loop_lookup(&lo, parm); 1998770fe30aSKay Sievers if (ret >= 0) { 1999770fe30aSKay Sievers ret = -EEXIST; 2000770fe30aSKay Sievers break; 2001770fe30aSKay Sievers } 2002770fe30aSKay Sievers ret = loop_add(&lo, parm); 2003770fe30aSKay Sievers break; 2004770fe30aSKay Sievers case LOOP_CTL_REMOVE: 2005770fe30aSKay Sievers ret = loop_lookup(&lo, parm); 2006770fe30aSKay Sievers if (ret < 0) 2007770fe30aSKay Sievers break; 20083148ffbdSOmar Sandoval ret = mutex_lock_killable(&lo->lo_ctl_mutex); 20093148ffbdSOmar Sandoval if (ret) 20103148ffbdSOmar Sandoval break; 2011770fe30aSKay Sievers if (lo->lo_state != Lo_unbound) { 2012770fe30aSKay Sievers ret = -EBUSY; 2013770fe30aSKay Sievers mutex_unlock(&lo->lo_ctl_mutex); 2014770fe30aSKay Sievers break; 2015770fe30aSKay Sievers } 2016f8933667SMing Lei if (atomic_read(&lo->lo_refcnt) > 0) { 2017770fe30aSKay Sievers ret = -EBUSY; 2018770fe30aSKay Sievers mutex_unlock(&lo->lo_ctl_mutex); 2019770fe30aSKay Sievers break; 2020770fe30aSKay Sievers } 2021770fe30aSKay Sievers lo->lo_disk->private_data = NULL; 2022770fe30aSKay Sievers mutex_unlock(&lo->lo_ctl_mutex); 2023770fe30aSKay Sievers idr_remove(&loop_index_idr, lo->lo_number); 2024770fe30aSKay Sievers loop_remove(lo); 2025770fe30aSKay Sievers break; 2026770fe30aSKay Sievers case LOOP_CTL_GET_FREE: 2027770fe30aSKay Sievers ret = loop_lookup(&lo, -1); 2028770fe30aSKay Sievers if (ret >= 0) 2029770fe30aSKay Sievers break; 2030770fe30aSKay Sievers ret = loop_add(&lo, -1); 2031770fe30aSKay Sievers } 2032770fe30aSKay Sievers mutex_unlock(&loop_index_mutex); 2033770fe30aSKay Sievers 2034770fe30aSKay Sievers return ret; 2035770fe30aSKay Sievers } 2036770fe30aSKay Sievers 2037770fe30aSKay Sievers static const struct file_operations loop_ctl_fops = { 2038770fe30aSKay Sievers .open = nonseekable_open, 2039770fe30aSKay Sievers .unlocked_ioctl = loop_control_ioctl, 2040770fe30aSKay Sievers .compat_ioctl = loop_control_ioctl, 2041770fe30aSKay Sievers .owner = THIS_MODULE, 2042770fe30aSKay Sievers .llseek = noop_llseek, 2043770fe30aSKay Sievers }; 2044770fe30aSKay Sievers 2045770fe30aSKay Sievers static struct miscdevice loop_misc = { 2046770fe30aSKay Sievers .minor = LOOP_CTRL_MINOR, 2047770fe30aSKay Sievers .name = "loop-control", 2048770fe30aSKay Sievers .fops = &loop_ctl_fops, 2049770fe30aSKay Sievers }; 2050770fe30aSKay Sievers 2051770fe30aSKay Sievers MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR); 2052770fe30aSKay Sievers MODULE_ALIAS("devname:loop-control"); 2053770fe30aSKay Sievers 205473285082SKen Chen static int __init loop_init(void) 205573285082SKen Chen { 2056a47653fcSKen Chen int i, nr; 2057a47653fcSKen Chen unsigned long range; 205834dd82afSKay Sievers struct loop_device *lo; 2059770fe30aSKay Sievers int err; 2060a47653fcSKen Chen 2061476a4813SLaurent Vivier part_shift = 0; 2062ac04fee0SNamhyung Kim if (max_part > 0) { 2063476a4813SLaurent Vivier part_shift = fls(max_part); 2064476a4813SLaurent Vivier 2065ac04fee0SNamhyung Kim /* 2066ac04fee0SNamhyung Kim * Adjust max_part according to part_shift as it is exported 2067ac04fee0SNamhyung Kim * to user space so that user can decide correct minor number 2068ac04fee0SNamhyung Kim * if [s]he want to create more devices. 2069ac04fee0SNamhyung Kim * 2070ac04fee0SNamhyung Kim * Note that -1 is required because partition 0 is reserved 2071ac04fee0SNamhyung Kim * for the whole disk. 2072ac04fee0SNamhyung Kim */ 2073ac04fee0SNamhyung Kim max_part = (1UL << part_shift) - 1; 2074ac04fee0SNamhyung Kim } 2075ac04fee0SNamhyung Kim 2076b1a66504SGuo Chao if ((1UL << part_shift) > DISK_MAX_PARTS) { 2077b1a66504SGuo Chao err = -EINVAL; 2078a8c1d064SAnton Volkov goto err_out; 2079b1a66504SGuo Chao } 208078f4bb36SNamhyung Kim 2081b1a66504SGuo Chao if (max_loop > 1UL << (MINORBITS - part_shift)) { 2082b1a66504SGuo Chao err = -EINVAL; 2083a8c1d064SAnton Volkov goto err_out; 2084b1a66504SGuo Chao } 208573285082SKen Chen 2086d134b00bSKay Sievers /* 2087d134b00bSKay Sievers * If max_loop is specified, create that many devices upfront. 2088d134b00bSKay Sievers * This also becomes a hard limit. If max_loop is not specified, 2089d134b00bSKay Sievers * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module 2090d134b00bSKay Sievers * init time. Loop devices can be requested on-demand with the 2091d134b00bSKay Sievers * /dev/loop-control interface, or be instantiated by accessing 2092d134b00bSKay Sievers * a 'dead' device node. 2093d134b00bSKay Sievers */ 209473285082SKen Chen if (max_loop) { 2095a47653fcSKen Chen nr = max_loop; 2096a1c15c59SNamhyung Kim range = max_loop << part_shift; 2097a47653fcSKen Chen } else { 2098d134b00bSKay Sievers nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT; 2099a1c15c59SNamhyung Kim range = 1UL << MINORBITS; 210073285082SKen Chen } 2101a47653fcSKen Chen 2102a8c1d064SAnton Volkov err = misc_register(&loop_misc); 2103a8c1d064SAnton Volkov if (err < 0) 2104a8c1d064SAnton Volkov goto err_out; 2105a8c1d064SAnton Volkov 2106a8c1d064SAnton Volkov 2107b1a66504SGuo Chao if (register_blkdev(LOOP_MAJOR, "loop")) { 2108b1a66504SGuo Chao err = -EIO; 2109b1a66504SGuo Chao goto misc_out; 2110b1a66504SGuo Chao } 2111a47653fcSKen Chen 2112a47653fcSKen Chen blk_register_region(MKDEV(LOOP_MAJOR, 0), range, 2113a47653fcSKen Chen THIS_MODULE, loop_probe, NULL, NULL); 2114a47653fcSKen Chen 2115d134b00bSKay Sievers /* pre-create number of devices given by config or max_loop */ 211634dd82afSKay Sievers mutex_lock(&loop_index_mutex); 211734dd82afSKay Sievers for (i = 0; i < nr; i++) 211834dd82afSKay Sievers loop_add(&lo, i); 211934dd82afSKay Sievers mutex_unlock(&loop_index_mutex); 212034dd82afSKay Sievers 212173285082SKen Chen printk(KERN_INFO "loop: module loaded\n"); 21221da177e4SLinus Torvalds return 0; 2123b1a66504SGuo Chao 2124b1a66504SGuo Chao misc_out: 2125b1a66504SGuo Chao misc_deregister(&loop_misc); 2126a8c1d064SAnton Volkov err_out: 2127b1a66504SGuo Chao return err; 212834dd82afSKay Sievers } 2129a47653fcSKen Chen 213034dd82afSKay Sievers static int loop_exit_cb(int id, void *ptr, void *data) 213134dd82afSKay Sievers { 213234dd82afSKay Sievers struct loop_device *lo = ptr; 2133a47653fcSKen Chen 213434dd82afSKay Sievers loop_remove(lo); 213534dd82afSKay Sievers return 0; 21361da177e4SLinus Torvalds } 21371da177e4SLinus Torvalds 213873285082SKen Chen static void __exit loop_exit(void) 21391da177e4SLinus Torvalds { 2140a47653fcSKen Chen unsigned long range; 21411da177e4SLinus Torvalds 2142a1c15c59SNamhyung Kim range = max_loop ? max_loop << part_shift : 1UL << MINORBITS; 2143a47653fcSKen Chen 214434dd82afSKay Sievers idr_for_each(&loop_index_idr, &loop_exit_cb, NULL); 214534dd82afSKay Sievers idr_destroy(&loop_index_idr); 214673285082SKen Chen 2147a47653fcSKen Chen blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range); 214800d59405SAkinobu Mita unregister_blkdev(LOOP_MAJOR, "loop"); 2149770fe30aSKay Sievers 2150770fe30aSKay Sievers misc_deregister(&loop_misc); 21511da177e4SLinus Torvalds } 21521da177e4SLinus Torvalds 21531da177e4SLinus Torvalds module_init(loop_init); 21541da177e4SLinus Torvalds module_exit(loop_exit); 21551da177e4SLinus Torvalds 21561da177e4SLinus Torvalds #ifndef MODULE 21571da177e4SLinus Torvalds static int __init max_loop_setup(char *str) 21581da177e4SLinus Torvalds { 21591da177e4SLinus Torvalds max_loop = simple_strtol(str, NULL, 0); 21601da177e4SLinus Torvalds return 1; 21611da177e4SLinus Torvalds } 21621da177e4SLinus Torvalds 21631da177e4SLinus Torvalds __setup("max_loop=", max_loop_setup); 21641da177e4SLinus Torvalds #endif 2165