11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/drivers/block/loop.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Written by Theodore Ts'o, 3/29/93 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Copyright 1993 by Theodore Ts'o. Redistribution of this file is 71da177e4SLinus Torvalds * permitted under the GNU General Public License. 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993 101da177e4SLinus Torvalds * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996 111da177e4SLinus Torvalds * 121da177e4SLinus Torvalds * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994 131da177e4SLinus Torvalds * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996 141da177e4SLinus Torvalds * 151da177e4SLinus Torvalds * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997 161da177e4SLinus Torvalds * 171da177e4SLinus Torvalds * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998 181da177e4SLinus Torvalds * 191da177e4SLinus Torvalds * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998 201da177e4SLinus Torvalds * 211da177e4SLinus Torvalds * Loadable modules and other fixes by AK, 1998 221da177e4SLinus Torvalds * 231da177e4SLinus Torvalds * Make real block number available to downstream transfer functions, enables 241da177e4SLinus Torvalds * CBC (and relatives) mode encryption requiring unique IVs per data block. 251da177e4SLinus Torvalds * Reed H. Petty, rhp@draper.net 261da177e4SLinus Torvalds * 271da177e4SLinus Torvalds * Maximum number of loop devices now dynamic via max_loop module parameter. 281da177e4SLinus Torvalds * Russell Kroll <rkroll@exploits.org> 19990701 291da177e4SLinus Torvalds * 301da177e4SLinus Torvalds * Maximum number of loop devices when compiled-in now selectable by passing 311da177e4SLinus Torvalds * max_loop=<1-255> to the kernel on boot. 3296de0e25SJan Engelhardt * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999 331da177e4SLinus Torvalds * 341da177e4SLinus Torvalds * Completely rewrite request handling to be make_request_fn style and 351da177e4SLinus Torvalds * non blocking, pushing work to a helper thread. Lots of fixes from 361da177e4SLinus Torvalds * Al Viro too. 371da177e4SLinus Torvalds * Jens Axboe <axboe@suse.de>, Nov 2000 381da177e4SLinus Torvalds * 391da177e4SLinus Torvalds * Support up to 256 loop devices 401da177e4SLinus Torvalds * Heinz Mauelshagen <mge@sistina.com>, Feb 2002 411da177e4SLinus Torvalds * 421da177e4SLinus Torvalds * Support for falling back on the write file operation when the address space 434e02ed4bSNick Piggin * operations write_begin is not available on the backing filesystem. 441da177e4SLinus Torvalds * Anton Altaparmakov, 16 Feb 2005 451da177e4SLinus Torvalds * 461da177e4SLinus Torvalds * Still To Fix: 471da177e4SLinus Torvalds * - Advisory locking is ignored here. 481da177e4SLinus Torvalds * - Should use an own CAP_* category instead of CAP_SYS_ADMIN 491da177e4SLinus Torvalds * 501da177e4SLinus Torvalds */ 511da177e4SLinus Torvalds 521da177e4SLinus Torvalds #include <linux/module.h> 531da177e4SLinus Torvalds #include <linux/moduleparam.h> 541da177e4SLinus Torvalds #include <linux/sched.h> 551da177e4SLinus Torvalds #include <linux/fs.h> 564ee60ec1SMatthew Wilcox (Oracle) #include <linux/pagemap.h> 571da177e4SLinus Torvalds #include <linux/file.h> 581da177e4SLinus Torvalds #include <linux/stat.h> 591da177e4SLinus Torvalds #include <linux/errno.h> 601da177e4SLinus Torvalds #include <linux/major.h> 611da177e4SLinus Torvalds #include <linux/wait.h> 621da177e4SLinus Torvalds #include <linux/blkdev.h> 631da177e4SLinus Torvalds #include <linux/blkpg.h> 641da177e4SLinus Torvalds #include <linux/init.h> 651da177e4SLinus Torvalds #include <linux/swap.h> 661da177e4SLinus Torvalds #include <linux/slab.h> 67863d5b82SDavid Howells #include <linux/compat.h> 681da177e4SLinus Torvalds #include <linux/suspend.h> 6983144186SRafael J. Wysocki #include <linux/freezer.h> 702a48fc0aSArnd Bergmann #include <linux/mutex.h> 711da177e4SLinus Torvalds #include <linux/writeback.h> 721da177e4SLinus Torvalds #include <linux/completion.h> 731da177e4SLinus Torvalds #include <linux/highmem.h> 74d6b29d7cSJens Axboe #include <linux/splice.h> 75ee862730SMilan Broz #include <linux/sysfs.h> 76770fe30aSKay Sievers #include <linux/miscdevice.h> 77dfaa2ef6SLukas Czerner #include <linux/falloc.h> 78283e7e5dSAl Viro #include <linux/uio.h> 79d9a08a9eSAdam Manzanares #include <linux/ioprio.h> 80db6638d7SDennis Zhou #include <linux/blk-cgroup.h> 81c74d40e8SDan Schatzberg #include <linux/sched/mm.h> 82d9a08a9eSAdam Manzanares 8383a87611SAl Viro #include "loop.h" 841da177e4SLinus Torvalds 857c0f6ba6SLinus Torvalds #include <linux/uaccess.h> 861da177e4SLinus Torvalds 8787579e9bSDan Schatzberg #define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ) 8887579e9bSDan Schatzberg 8934dd82afSKay Sievers static DEFINE_IDR(loop_index_idr); 90310ca162STetsuo Handa static DEFINE_MUTEX(loop_ctl_mutex); 913ce6e1f6STetsuo Handa static DEFINE_MUTEX(loop_validate_mutex); 923ce6e1f6STetsuo Handa 933ce6e1f6STetsuo Handa /** 943ce6e1f6STetsuo Handa * loop_global_lock_killable() - take locks for safe loop_validate_file() test 953ce6e1f6STetsuo Handa * 963ce6e1f6STetsuo Handa * @lo: struct loop_device 973ce6e1f6STetsuo Handa * @global: true if @lo is about to bind another "struct loop_device", false otherwise 983ce6e1f6STetsuo Handa * 993ce6e1f6STetsuo Handa * Returns 0 on success, -EINTR otherwise. 1003ce6e1f6STetsuo Handa * 1013ce6e1f6STetsuo Handa * Since loop_validate_file() traverses on other "struct loop_device" if 1023ce6e1f6STetsuo Handa * is_loop_device() is true, we need a global lock for serializing concurrent 1033ce6e1f6STetsuo Handa * loop_configure()/loop_change_fd()/__loop_clr_fd() calls. 1043ce6e1f6STetsuo Handa */ 1053ce6e1f6STetsuo Handa static int loop_global_lock_killable(struct loop_device *lo, bool global) 1063ce6e1f6STetsuo Handa { 1073ce6e1f6STetsuo Handa int err; 1083ce6e1f6STetsuo Handa 1093ce6e1f6STetsuo Handa if (global) { 1103ce6e1f6STetsuo Handa err = mutex_lock_killable(&loop_validate_mutex); 1113ce6e1f6STetsuo Handa if (err) 1123ce6e1f6STetsuo Handa return err; 1133ce6e1f6STetsuo Handa } 1143ce6e1f6STetsuo Handa err = mutex_lock_killable(&lo->lo_mutex); 1153ce6e1f6STetsuo Handa if (err && global) 1163ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex); 1173ce6e1f6STetsuo Handa return err; 1183ce6e1f6STetsuo Handa } 1193ce6e1f6STetsuo Handa 1203ce6e1f6STetsuo Handa /** 1213ce6e1f6STetsuo Handa * loop_global_unlock() - release locks taken by loop_global_lock_killable() 1223ce6e1f6STetsuo Handa * 1233ce6e1f6STetsuo Handa * @lo: struct loop_device 1243ce6e1f6STetsuo Handa * @global: true if @lo was about to bind another "struct loop_device", false otherwise 1253ce6e1f6STetsuo Handa */ 1263ce6e1f6STetsuo Handa static void loop_global_unlock(struct loop_device *lo, bool global) 1273ce6e1f6STetsuo Handa { 1283ce6e1f6STetsuo Handa mutex_unlock(&lo->lo_mutex); 1293ce6e1f6STetsuo Handa if (global) 1303ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex); 1313ce6e1f6STetsuo Handa } 1321da177e4SLinus Torvalds 133476a4813SLaurent Vivier static int max_part; 134476a4813SLaurent Vivier static int part_shift; 135476a4813SLaurent Vivier 1361da177e4SLinus Torvalds static int transfer_xor(struct loop_device *lo, int cmd, 1371da177e4SLinus Torvalds struct page *raw_page, unsigned raw_off, 1381da177e4SLinus Torvalds struct page *loop_page, unsigned loop_off, 1391da177e4SLinus Torvalds int size, sector_t real_block) 1401da177e4SLinus Torvalds { 141cfd8005cSCong Wang char *raw_buf = kmap_atomic(raw_page) + raw_off; 142cfd8005cSCong Wang char *loop_buf = kmap_atomic(loop_page) + loop_off; 1431da177e4SLinus Torvalds char *in, *out, *key; 1441da177e4SLinus Torvalds int i, keysize; 1451da177e4SLinus Torvalds 1461da177e4SLinus Torvalds if (cmd == READ) { 1471da177e4SLinus Torvalds in = raw_buf; 1481da177e4SLinus Torvalds out = loop_buf; 1491da177e4SLinus Torvalds } else { 1501da177e4SLinus Torvalds in = loop_buf; 1511da177e4SLinus Torvalds out = raw_buf; 1521da177e4SLinus Torvalds } 1531da177e4SLinus Torvalds 1541da177e4SLinus Torvalds key = lo->lo_encrypt_key; 1551da177e4SLinus Torvalds keysize = lo->lo_encrypt_key_size; 1561da177e4SLinus Torvalds for (i = 0; i < size; i++) 1571da177e4SLinus Torvalds *out++ = *in++ ^ key[(i & 511) % keysize]; 1581da177e4SLinus Torvalds 159cfd8005cSCong Wang kunmap_atomic(loop_buf); 160cfd8005cSCong Wang kunmap_atomic(raw_buf); 1611da177e4SLinus Torvalds cond_resched(); 1621da177e4SLinus Torvalds return 0; 1631da177e4SLinus Torvalds } 1641da177e4SLinus Torvalds 1651da177e4SLinus Torvalds static int xor_init(struct loop_device *lo, const struct loop_info64 *info) 1661da177e4SLinus Torvalds { 1671da177e4SLinus Torvalds if (unlikely(info->lo_encrypt_key_size <= 0)) 1681da177e4SLinus Torvalds return -EINVAL; 1691da177e4SLinus Torvalds return 0; 1701da177e4SLinus Torvalds } 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds static struct loop_func_table none_funcs = { 1731da177e4SLinus Torvalds .number = LO_CRYPT_NONE, 1741da177e4SLinus Torvalds }; 1751da177e4SLinus Torvalds 1761da177e4SLinus Torvalds static struct loop_func_table xor_funcs = { 1771da177e4SLinus Torvalds .number = LO_CRYPT_XOR, 1781da177e4SLinus Torvalds .transfer = transfer_xor, 1791da177e4SLinus Torvalds .init = xor_init 1801da177e4SLinus Torvalds }; 1811da177e4SLinus Torvalds 1821da177e4SLinus Torvalds /* xfer_funcs[0] is special - its release function is never called */ 1831da177e4SLinus Torvalds static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = { 1841da177e4SLinus Torvalds &none_funcs, 1851da177e4SLinus Torvalds &xor_funcs 1861da177e4SLinus Torvalds }; 1871da177e4SLinus Torvalds 1887035b5dfSDmitry Monakhov static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file) 1891da177e4SLinus Torvalds { 190b7a1da69SGuo Chao loff_t loopsize; 1911da177e4SLinus Torvalds 1921da177e4SLinus Torvalds /* Compute loopsize in bytes */ 193b7a1da69SGuo Chao loopsize = i_size_read(file->f_mapping->host); 194b7a1da69SGuo Chao if (offset > 0) 195b7a1da69SGuo Chao loopsize -= offset; 196b7a1da69SGuo Chao /* offset is beyond i_size, weird but possible */ 1977035b5dfSDmitry Monakhov if (loopsize < 0) 1987035b5dfSDmitry Monakhov return 0; 1991da177e4SLinus Torvalds 2007035b5dfSDmitry Monakhov if (sizelimit > 0 && sizelimit < loopsize) 2017035b5dfSDmitry Monakhov loopsize = sizelimit; 2021da177e4SLinus Torvalds /* 2031da177e4SLinus Torvalds * Unfortunately, if we want to do I/O on the device, 2041da177e4SLinus Torvalds * the number of 512-byte sectors has to fit into a sector_t. 2051da177e4SLinus Torvalds */ 2061da177e4SLinus Torvalds return loopsize >> 9; 2071da177e4SLinus Torvalds } 2081da177e4SLinus Torvalds 2097035b5dfSDmitry Monakhov static loff_t get_loop_size(struct loop_device *lo, struct file *file) 2101da177e4SLinus Torvalds { 2117035b5dfSDmitry Monakhov return get_size(lo->lo_offset, lo->lo_sizelimit, file); 2127035b5dfSDmitry Monakhov } 2137035b5dfSDmitry Monakhov 2142e5ab5f3SMing Lei static void __loop_update_dio(struct loop_device *lo, bool dio) 2152e5ab5f3SMing Lei { 2162e5ab5f3SMing Lei struct file *file = lo->lo_backing_file; 2172e5ab5f3SMing Lei struct address_space *mapping = file->f_mapping; 2182e5ab5f3SMing Lei struct inode *inode = mapping->host; 2192e5ab5f3SMing Lei unsigned short sb_bsize = 0; 2202e5ab5f3SMing Lei unsigned dio_align = 0; 2212e5ab5f3SMing Lei bool use_dio; 2222e5ab5f3SMing Lei 2232e5ab5f3SMing Lei if (inode->i_sb->s_bdev) { 2242e5ab5f3SMing Lei sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 2252e5ab5f3SMing Lei dio_align = sb_bsize - 1; 2262e5ab5f3SMing Lei } 2272e5ab5f3SMing Lei 2282e5ab5f3SMing Lei /* 2292e5ab5f3SMing Lei * We support direct I/O only if lo_offset is aligned with the 2302e5ab5f3SMing Lei * logical I/O size of backing device, and the logical block 2312e5ab5f3SMing Lei * size of loop is bigger than the backing device's and the loop 2322e5ab5f3SMing Lei * needn't transform transfer. 2332e5ab5f3SMing Lei * 2342e5ab5f3SMing Lei * TODO: the above condition may be loosed in the future, and 2352e5ab5f3SMing Lei * direct I/O may be switched runtime at that time because most 23689d790abSMasahiro Yamada * of requests in sane applications should be PAGE_SIZE aligned 2372e5ab5f3SMing Lei */ 2382e5ab5f3SMing Lei if (dio) { 2392e5ab5f3SMing Lei if (queue_logical_block_size(lo->lo_queue) >= sb_bsize && 2402e5ab5f3SMing Lei !(lo->lo_offset & dio_align) && 2412e5ab5f3SMing Lei mapping->a_ops->direct_IO && 2422e5ab5f3SMing Lei !lo->transfer) 2432e5ab5f3SMing Lei use_dio = true; 2442e5ab5f3SMing Lei else 2452e5ab5f3SMing Lei use_dio = false; 2462e5ab5f3SMing Lei } else { 2472e5ab5f3SMing Lei use_dio = false; 2482e5ab5f3SMing Lei } 2492e5ab5f3SMing Lei 2502e5ab5f3SMing Lei if (lo->use_dio == use_dio) 2512e5ab5f3SMing Lei return; 2522e5ab5f3SMing Lei 2532e5ab5f3SMing Lei /* flush dirty pages before changing direct IO */ 2542e5ab5f3SMing Lei vfs_fsync(file, 0); 2552e5ab5f3SMing Lei 2562e5ab5f3SMing Lei /* 2572e5ab5f3SMing Lei * The flag of LO_FLAGS_DIRECT_IO is handled similarly with 2582e5ab5f3SMing Lei * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup 2592e5ab5f3SMing Lei * will get updated by ioctl(LOOP_GET_STATUS) 2602e5ab5f3SMing Lei */ 2610fbcf579SMartijn Coenen if (lo->lo_state == Lo_bound) 2622e5ab5f3SMing Lei blk_mq_freeze_queue(lo->lo_queue); 2632e5ab5f3SMing Lei lo->use_dio = use_dio; 26440326d8aSShaohua Li if (use_dio) { 2658b904b5bSBart Van Assche blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2662e5ab5f3SMing Lei lo->lo_flags |= LO_FLAGS_DIRECT_IO; 26740326d8aSShaohua Li } else { 2688b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2692e5ab5f3SMing Lei lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; 27040326d8aSShaohua Li } 2710fbcf579SMartijn Coenen if (lo->lo_state == Lo_bound) 2722e5ab5f3SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 2732e5ab5f3SMing Lei } 2742e5ab5f3SMing Lei 2755795b6f5SMartijn Coenen /** 2765795b6f5SMartijn Coenen * loop_set_size() - sets device size and notifies userspace 2775795b6f5SMartijn Coenen * @lo: struct loop_device to set the size for 2785795b6f5SMartijn Coenen * @size: new size of the loop device 2795795b6f5SMartijn Coenen * 2805795b6f5SMartijn Coenen * Callers must validate that the size passed into this function fits into 2815795b6f5SMartijn Coenen * a sector_t, eg using loop_validate_size() 2825795b6f5SMartijn Coenen */ 2835795b6f5SMartijn Coenen static void loop_set_size(struct loop_device *lo, loff_t size) 2845795b6f5SMartijn Coenen { 285449f4ec9SChristoph Hellwig if (!set_capacity_and_notify(lo->lo_disk, size)) 2863b4f85d0SChristoph Hellwig kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE); 2871da177e4SLinus Torvalds } 2881da177e4SLinus Torvalds 2891da177e4SLinus Torvalds static inline int 2901da177e4SLinus Torvalds lo_do_transfer(struct loop_device *lo, int cmd, 2911da177e4SLinus Torvalds struct page *rpage, unsigned roffs, 2921da177e4SLinus Torvalds struct page *lpage, unsigned loffs, 2931da177e4SLinus Torvalds int size, sector_t rblock) 2941da177e4SLinus Torvalds { 295aa4d8616SChristoph Hellwig int ret; 296aa4d8616SChristoph Hellwig 297aa4d8616SChristoph Hellwig ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); 298aa4d8616SChristoph Hellwig if (likely(!ret)) 2991da177e4SLinus Torvalds return 0; 3001da177e4SLinus Torvalds 301aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR 302aa4d8616SChristoph Hellwig "loop: Transfer error at byte offset %llu, length %i.\n", 303aa4d8616SChristoph Hellwig (unsigned long long)rblock << 9, size); 304aa4d8616SChristoph Hellwig return ret; 3051da177e4SLinus Torvalds } 3061da177e4SLinus Torvalds 307aa4d8616SChristoph Hellwig static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos) 3081da177e4SLinus Torvalds { 309aa4d8616SChristoph Hellwig struct iov_iter i; 3101da177e4SLinus Torvalds ssize_t bw; 311283e7e5dSAl Viro 312b6207430SChristoph Hellwig iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len); 3131da177e4SLinus Torvalds 31403d95eb2SAl Viro file_start_write(file); 315abbb6589SChristoph Hellwig bw = vfs_iter_write(file, &i, ppos, 0); 31603d95eb2SAl Viro file_end_write(file); 317aa4d8616SChristoph Hellwig 318aa4d8616SChristoph Hellwig if (likely(bw == bvec->bv_len)) 3191da177e4SLinus Torvalds return 0; 320aa4d8616SChristoph Hellwig 321aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR 322aa4d8616SChristoph Hellwig "loop: Write error at byte offset %llu, length %i.\n", 323aa4d8616SChristoph Hellwig (unsigned long long)*ppos, bvec->bv_len); 3241da177e4SLinus Torvalds if (bw >= 0) 3251da177e4SLinus Torvalds bw = -EIO; 3261da177e4SLinus Torvalds return bw; 3271da177e4SLinus Torvalds } 3281da177e4SLinus Torvalds 329aa4d8616SChristoph Hellwig static int lo_write_simple(struct loop_device *lo, struct request *rq, 330aa4d8616SChristoph Hellwig loff_t pos) 3311da177e4SLinus Torvalds { 332aa4d8616SChristoph Hellwig struct bio_vec bvec; 333aa4d8616SChristoph Hellwig struct req_iterator iter; 334aa4d8616SChristoph Hellwig int ret = 0; 335aa4d8616SChristoph Hellwig 336aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) { 337aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos); 338aa4d8616SChristoph Hellwig if (ret < 0) 339aa4d8616SChristoph Hellwig break; 3401da177e4SLinus Torvalds cond_resched(); 3411da177e4SLinus Torvalds } 3421da177e4SLinus Torvalds 343aa4d8616SChristoph Hellwig return ret; 344aa4d8616SChristoph Hellwig } 345aa4d8616SChristoph Hellwig 346aa4d8616SChristoph Hellwig /* 347456be148SChristoph Hellwig * This is the slow, transforming version that needs to double buffer the 348456be148SChristoph Hellwig * data as it cannot do the transformations in place without having direct 349456be148SChristoph Hellwig * access to the destination pages of the backing file. 3501da177e4SLinus Torvalds */ 351aa4d8616SChristoph Hellwig static int lo_write_transfer(struct loop_device *lo, struct request *rq, 352aa4d8616SChristoph Hellwig loff_t pos) 3531da177e4SLinus Torvalds { 354aa4d8616SChristoph Hellwig struct bio_vec bvec, b; 35530112013SMing Lei struct req_iterator iter; 356aa4d8616SChristoph Hellwig struct page *page; 3577988613bSKent Overstreet int ret = 0; 3581da177e4SLinus Torvalds 359aa4d8616SChristoph Hellwig page = alloc_page(GFP_NOIO); 3601da177e4SLinus Torvalds if (unlikely(!page)) 361aa4d8616SChristoph Hellwig return -ENOMEM; 362456be148SChristoph Hellwig 36330112013SMing Lei rq_for_each_segment(bvec, rq, iter) { 364aa4d8616SChristoph Hellwig ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page, 365aa4d8616SChristoph Hellwig bvec.bv_offset, bvec.bv_len, pos >> 9); 366aa4d8616SChristoph Hellwig if (unlikely(ret)) 367aa4d8616SChristoph Hellwig break; 368aa4d8616SChristoph Hellwig 369aa4d8616SChristoph Hellwig b.bv_page = page; 370aa4d8616SChristoph Hellwig b.bv_offset = 0; 371aa4d8616SChristoph Hellwig b.bv_len = bvec.bv_len; 372aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &b, &pos); 3731da177e4SLinus Torvalds if (ret < 0) 3741da177e4SLinus Torvalds break; 3751da177e4SLinus Torvalds } 376aa4d8616SChristoph Hellwig 3771da177e4SLinus Torvalds __free_page(page); 3781da177e4SLinus Torvalds return ret; 3791da177e4SLinus Torvalds } 3801da177e4SLinus Torvalds 381aa4d8616SChristoph Hellwig static int lo_read_simple(struct loop_device *lo, struct request *rq, 382aa4d8616SChristoph Hellwig loff_t pos) 3831da177e4SLinus Torvalds { 3847988613bSKent Overstreet struct bio_vec bvec; 38530112013SMing Lei struct req_iterator iter; 386aa4d8616SChristoph Hellwig struct iov_iter i; 387aa4d8616SChristoph Hellwig ssize_t len; 3881da177e4SLinus Torvalds 38930112013SMing Lei rq_for_each_segment(bvec, rq, iter) { 390b6207430SChristoph Hellwig iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len); 39118e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 392aa4d8616SChristoph Hellwig if (len < 0) 393aa4d8616SChristoph Hellwig return len; 394306df071SDave Young 395aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page); 396aa4d8616SChristoph Hellwig 397aa4d8616SChristoph Hellwig if (len != bvec.bv_len) { 39830112013SMing Lei struct bio *bio; 39930112013SMing Lei 40030112013SMing Lei __rq_for_each_bio(bio, rq) 401306df071SDave Young zero_fill_bio(bio); 4021da177e4SLinus Torvalds break; 403306df071SDave Young } 404aa4d8616SChristoph Hellwig cond_resched(); 4051da177e4SLinus Torvalds } 406aa4d8616SChristoph Hellwig 407306df071SDave Young return 0; 4081da177e4SLinus Torvalds } 4091da177e4SLinus Torvalds 410aa4d8616SChristoph Hellwig static int lo_read_transfer(struct loop_device *lo, struct request *rq, 411aa4d8616SChristoph Hellwig loff_t pos) 412aa4d8616SChristoph Hellwig { 413aa4d8616SChristoph Hellwig struct bio_vec bvec, b; 414aa4d8616SChristoph Hellwig struct req_iterator iter; 415aa4d8616SChristoph Hellwig struct iov_iter i; 416aa4d8616SChristoph Hellwig struct page *page; 417aa4d8616SChristoph Hellwig ssize_t len; 418aa4d8616SChristoph Hellwig int ret = 0; 419aa4d8616SChristoph Hellwig 420aa4d8616SChristoph Hellwig page = alloc_page(GFP_NOIO); 421aa4d8616SChristoph Hellwig if (unlikely(!page)) 422aa4d8616SChristoph Hellwig return -ENOMEM; 423aa4d8616SChristoph Hellwig 424aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) { 425aa4d8616SChristoph Hellwig loff_t offset = pos; 426aa4d8616SChristoph Hellwig 427aa4d8616SChristoph Hellwig b.bv_page = page; 428aa4d8616SChristoph Hellwig b.bv_offset = 0; 429aa4d8616SChristoph Hellwig b.bv_len = bvec.bv_len; 430aa4d8616SChristoph Hellwig 431b6207430SChristoph Hellwig iov_iter_bvec(&i, READ, &b, 1, b.bv_len); 43218e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 433aa4d8616SChristoph Hellwig if (len < 0) { 434aa4d8616SChristoph Hellwig ret = len; 435aa4d8616SChristoph Hellwig goto out_free_page; 436aa4d8616SChristoph Hellwig } 437aa4d8616SChristoph Hellwig 438aa4d8616SChristoph Hellwig ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page, 439aa4d8616SChristoph Hellwig bvec.bv_offset, len, offset >> 9); 440aa4d8616SChristoph Hellwig if (ret) 441aa4d8616SChristoph Hellwig goto out_free_page; 442aa4d8616SChristoph Hellwig 443aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page); 444aa4d8616SChristoph Hellwig 445aa4d8616SChristoph Hellwig if (len != bvec.bv_len) { 446aa4d8616SChristoph Hellwig struct bio *bio; 447aa4d8616SChristoph Hellwig 448aa4d8616SChristoph Hellwig __rq_for_each_bio(bio, rq) 449aa4d8616SChristoph Hellwig zero_fill_bio(bio); 450aa4d8616SChristoph Hellwig break; 451aa4d8616SChristoph Hellwig } 452aa4d8616SChristoph Hellwig } 453aa4d8616SChristoph Hellwig 454aa4d8616SChristoph Hellwig ret = 0; 455aa4d8616SChristoph Hellwig out_free_page: 456aa4d8616SChristoph Hellwig __free_page(page); 457aa4d8616SChristoph Hellwig return ret; 458aa4d8616SChristoph Hellwig } 459aa4d8616SChristoph Hellwig 460efcfec57SDarrick J. Wong static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos, 461efcfec57SDarrick J. Wong int mode) 462cf655d95SMing Lei { 463cf655d95SMing Lei /* 464efcfec57SDarrick J. Wong * We use fallocate to manipulate the space mappings used by the image 465efcfec57SDarrick J. Wong * a.k.a. discard/zerorange. However we do not support this if 466efcfec57SDarrick J. Wong * encryption is enabled, because it may give an attacker useful 467efcfec57SDarrick J. Wong * information. 468cf655d95SMing Lei */ 469cf655d95SMing Lei struct file *file = lo->lo_backing_file; 470c52abf56SEvan Green struct request_queue *q = lo->lo_queue; 471cf655d95SMing Lei int ret; 472cf655d95SMing Lei 473efcfec57SDarrick J. Wong mode |= FALLOC_FL_KEEP_SIZE; 474efcfec57SDarrick J. Wong 475c52abf56SEvan Green if (!blk_queue_discard(q)) { 476cf655d95SMing Lei ret = -EOPNOTSUPP; 477cf655d95SMing Lei goto out; 478cf655d95SMing Lei } 479cf655d95SMing Lei 480cf655d95SMing Lei ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq)); 481cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP)) 482cf655d95SMing Lei ret = -EIO; 483cf655d95SMing Lei out: 484cf655d95SMing Lei return ret; 485cf655d95SMing Lei } 486cf655d95SMing Lei 487cf655d95SMing Lei static int lo_req_flush(struct loop_device *lo, struct request *rq) 488cf655d95SMing Lei { 489cf655d95SMing Lei struct file *file = lo->lo_backing_file; 490cf655d95SMing Lei int ret = vfs_fsync(file, 0); 491cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL)) 492cf655d95SMing Lei ret = -EIO; 493cf655d95SMing Lei 494cf655d95SMing Lei return ret; 495cf655d95SMing Lei } 496cf655d95SMing Lei 497fe2cb290SChristoph Hellwig static void lo_complete_rq(struct request *rq) 498bc07c10aSMing Lei { 499fe2cb290SChristoph Hellwig struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 500f9de14bcSJens Axboe blk_status_t ret = BLK_STS_OK; 501bc07c10aSMing Lei 502f9de14bcSJens Axboe if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) || 503f9de14bcSJens Axboe req_op(rq) != REQ_OP_READ) { 504f9de14bcSJens Axboe if (cmd->ret < 0) 5058cd55087SEvan Green ret = errno_to_blk_status(cmd->ret); 506f9de14bcSJens Axboe goto end_io; 507bc07c10aSMing Lei } 508fe2cb290SChristoph Hellwig 509f9de14bcSJens Axboe /* 510f9de14bcSJens Axboe * Short READ - if we got some data, advance our request and 511f9de14bcSJens Axboe * retry it. If we got no data, end the rest with EIO. 512f9de14bcSJens Axboe */ 513f9de14bcSJens Axboe if (cmd->ret) { 514f9de14bcSJens Axboe blk_update_request(rq, BLK_STS_OK, cmd->ret); 515f9de14bcSJens Axboe cmd->ret = 0; 516f9de14bcSJens Axboe blk_mq_requeue_request(rq, true); 517f9de14bcSJens Axboe } else { 518f9de14bcSJens Axboe if (cmd->use_aio) { 519f9de14bcSJens Axboe struct bio *bio = rq->bio; 520f9de14bcSJens Axboe 521f9de14bcSJens Axboe while (bio) { 522f9de14bcSJens Axboe zero_fill_bio(bio); 523f9de14bcSJens Axboe bio = bio->bi_next; 524f9de14bcSJens Axboe } 525f9de14bcSJens Axboe } 526f9de14bcSJens Axboe ret = BLK_STS_IOERR; 527f9de14bcSJens Axboe end_io: 528f9de14bcSJens Axboe blk_mq_end_request(rq, ret); 529f9de14bcSJens Axboe } 530bc07c10aSMing Lei } 531bc07c10aSMing Lei 53292d77332SShaohua Li static void lo_rw_aio_do_completion(struct loop_cmd *cmd) 53392d77332SShaohua Li { 5341894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 5351894e916SJens Axboe 53692d77332SShaohua Li if (!atomic_dec_and_test(&cmd->ref)) 53792d77332SShaohua Li return; 53892d77332SShaohua Li kfree(cmd->bvec); 53992d77332SShaohua Li cmd->bvec = NULL; 54015f73f5bSChristoph Hellwig if (likely(!blk_should_fake_timeout(rq->q))) 5411894e916SJens Axboe blk_mq_complete_request(rq); 54292d77332SShaohua Li } 54392d77332SShaohua Li 544bc07c10aSMing Lei static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2) 545bc07c10aSMing Lei { 546bc07c10aSMing Lei struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb); 547bc07c10aSMing Lei 548fe2cb290SChristoph Hellwig cmd->ret = ret; 54992d77332SShaohua Li lo_rw_aio_do_completion(cmd); 550bc07c10aSMing Lei } 551bc07c10aSMing Lei 552bc07c10aSMing Lei static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, 553bc07c10aSMing Lei loff_t pos, bool rw) 554bc07c10aSMing Lei { 555bc07c10aSMing Lei struct iov_iter iter; 55686af5952SMing Lei struct req_iterator rq_iter; 557bc07c10aSMing Lei struct bio_vec *bvec; 5581894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 55940326d8aSShaohua Li struct bio *bio = rq->bio; 560bc07c10aSMing Lei struct file *file = lo->lo_backing_file; 56186af5952SMing Lei struct bio_vec tmp; 56240326d8aSShaohua Li unsigned int offset; 56386af5952SMing Lei int nr_bvec = 0; 564bc07c10aSMing Lei int ret; 565bc07c10aSMing Lei 56686af5952SMing Lei rq_for_each_bvec(tmp, rq, rq_iter) 56786af5952SMing Lei nr_bvec++; 568bc07c10aSMing Lei 56986af5952SMing Lei if (rq->bio != rq->biotail) { 57086af5952SMing Lei 57186af5952SMing Lei bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec), 5726da2ec56SKees Cook GFP_NOIO); 57340326d8aSShaohua Li if (!bvec) 57440326d8aSShaohua Li return -EIO; 57540326d8aSShaohua Li cmd->bvec = bvec; 57640326d8aSShaohua Li 577a7297a6aSMing Lei /* 57840326d8aSShaohua Li * The bios of the request may be started from the middle of 57940326d8aSShaohua Li * the 'bvec' because of bio splitting, so we can't directly 58086af5952SMing Lei * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec 58140326d8aSShaohua Li * API will take care of all details for us. 582a7297a6aSMing Lei */ 58386af5952SMing Lei rq_for_each_bvec(tmp, rq, rq_iter) { 58440326d8aSShaohua Li *bvec = tmp; 58540326d8aSShaohua Li bvec++; 58640326d8aSShaohua Li } 58740326d8aSShaohua Li bvec = cmd->bvec; 58840326d8aSShaohua Li offset = 0; 58940326d8aSShaohua Li } else { 59040326d8aSShaohua Li /* 59140326d8aSShaohua Li * Same here, this bio may be started from the middle of the 59240326d8aSShaohua Li * 'bvec' because of bio splitting, so offset from the bvec 59340326d8aSShaohua Li * must be passed to iov iterator 59440326d8aSShaohua Li */ 59540326d8aSShaohua Li offset = bio->bi_iter.bi_bvec_done; 59640326d8aSShaohua Li bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 59740326d8aSShaohua Li } 59892d77332SShaohua Li atomic_set(&cmd->ref, 2); 59940326d8aSShaohua Li 600b6207430SChristoph Hellwig iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq)); 60140326d8aSShaohua Li iter.iov_offset = offset; 602bc07c10aSMing Lei 603bc07c10aSMing Lei cmd->iocb.ki_pos = pos; 604bc07c10aSMing Lei cmd->iocb.ki_filp = file; 605bc07c10aSMing Lei cmd->iocb.ki_complete = lo_rw_aio_complete; 606bc07c10aSMing Lei cmd->iocb.ki_flags = IOCB_DIRECT; 607d9a08a9eSAdam Manzanares cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); 608bc07c10aSMing Lei 609bc07c10aSMing Lei if (rw == WRITE) 610bb7462b6SMiklos Szeredi ret = call_write_iter(file, &cmd->iocb, &iter); 611bc07c10aSMing Lei else 612bb7462b6SMiklos Szeredi ret = call_read_iter(file, &cmd->iocb, &iter); 613bc07c10aSMing Lei 61492d77332SShaohua Li lo_rw_aio_do_completion(cmd); 61592d77332SShaohua Li 616bc07c10aSMing Lei if (ret != -EIOCBQUEUED) 617bc07c10aSMing Lei cmd->iocb.ki_complete(&cmd->iocb, ret, 0); 618bc07c10aSMing Lei return 0; 619bc07c10aSMing Lei } 620bc07c10aSMing Lei 621c1c87c2bSChristoph Hellwig static int do_req_filebacked(struct loop_device *lo, struct request *rq) 622bc07c10aSMing Lei { 623bc07c10aSMing Lei struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 624c1c87c2bSChristoph Hellwig loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; 625bc07c10aSMing Lei 626bc07c10aSMing Lei /* 627bc07c10aSMing Lei * lo_write_simple and lo_read_simple should have been covered 628bc07c10aSMing Lei * by io submit style function like lo_rw_aio(), one blocker 629bc07c10aSMing Lei * is that lo_read_simple() need to call flush_dcache_page after 630bc07c10aSMing Lei * the page is written from kernel, and it isn't easy to handle 631bc07c10aSMing Lei * this in io submit style function which submits all segments 632bc07c10aSMing Lei * of the req at one time. And direct read IO doesn't need to 633bc07c10aSMing Lei * run flush_dcache_page(). 634bc07c10aSMing Lei */ 635c1c87c2bSChristoph Hellwig switch (req_op(rq)) { 636c1c87c2bSChristoph Hellwig case REQ_OP_FLUSH: 637c1c87c2bSChristoph Hellwig return lo_req_flush(lo, rq); 63819372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 639efcfec57SDarrick J. Wong /* 640efcfec57SDarrick J. Wong * If the caller doesn't want deallocation, call zeroout to 641efcfec57SDarrick J. Wong * write zeroes the range. Otherwise, punch them out. 642efcfec57SDarrick J. Wong */ 643efcfec57SDarrick J. Wong return lo_fallocate(lo, rq, pos, 644efcfec57SDarrick J. Wong (rq->cmd_flags & REQ_NOUNMAP) ? 645efcfec57SDarrick J. Wong FALLOC_FL_ZERO_RANGE : 646efcfec57SDarrick J. Wong FALLOC_FL_PUNCH_HOLE); 647efcfec57SDarrick J. Wong case REQ_OP_DISCARD: 648efcfec57SDarrick J. Wong return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE); 649c1c87c2bSChristoph Hellwig case REQ_OP_WRITE: 650c1c87c2bSChristoph Hellwig if (lo->transfer) 651c1c87c2bSChristoph Hellwig return lo_write_transfer(lo, rq, pos); 652c1c87c2bSChristoph Hellwig else if (cmd->use_aio) 653c1c87c2bSChristoph Hellwig return lo_rw_aio(lo, cmd, pos, WRITE); 654c1c87c2bSChristoph Hellwig else 655bc07c10aSMing Lei return lo_write_simple(lo, rq, pos); 656c1c87c2bSChristoph Hellwig case REQ_OP_READ: 657c1c87c2bSChristoph Hellwig if (lo->transfer) 658c1c87c2bSChristoph Hellwig return lo_read_transfer(lo, rq, pos); 659c1c87c2bSChristoph Hellwig else if (cmd->use_aio) 660c1c87c2bSChristoph Hellwig return lo_rw_aio(lo, cmd, pos, READ); 661bc07c10aSMing Lei else 662bc07c10aSMing Lei return lo_read_simple(lo, rq, pos); 663c1c87c2bSChristoph Hellwig default: 664c1c87c2bSChristoph Hellwig WARN_ON_ONCE(1); 665c1c87c2bSChristoph Hellwig return -EIO; 666bc07c10aSMing Lei } 6671da177e4SLinus Torvalds } 6681da177e4SLinus Torvalds 6692e5ab5f3SMing Lei static inline void loop_update_dio(struct loop_device *lo) 6702e5ab5f3SMing Lei { 671efbe3c24SIra Weiny __loop_update_dio(lo, (lo->lo_backing_file->f_flags & O_DIRECT) | 6722e5ab5f3SMing Lei lo->use_dio); 6732e5ab5f3SMing Lei } 6742e5ab5f3SMing Lei 6750384264eSChristoph Hellwig static void loop_reread_partitions(struct loop_device *lo) 67606f0e9e6SMing Lei { 67706f0e9e6SMing Lei int rc; 67806f0e9e6SMing Lei 6790384264eSChristoph Hellwig mutex_lock(&lo->lo_disk->open_mutex); 6800384264eSChristoph Hellwig rc = bdev_disk_changed(lo->lo_disk, false); 6810384264eSChristoph Hellwig mutex_unlock(&lo->lo_disk->open_mutex); 68206f0e9e6SMing Lei if (rc) 68306f0e9e6SMing Lei pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n", 68406f0e9e6SMing Lei __func__, lo->lo_number, lo->lo_file_name, rc); 68506f0e9e6SMing Lei } 68606f0e9e6SMing Lei 687d2ac838eSTheodore Ts'o static inline int is_loop_device(struct file *file) 688d2ac838eSTheodore Ts'o { 689d2ac838eSTheodore Ts'o struct inode *i = file->f_mapping->host; 690d2ac838eSTheodore Ts'o 6916f24784fSAl Viro return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; 692d2ac838eSTheodore Ts'o } 693d2ac838eSTheodore Ts'o 694d2ac838eSTheodore Ts'o static int loop_validate_file(struct file *file, struct block_device *bdev) 695d2ac838eSTheodore Ts'o { 696d2ac838eSTheodore Ts'o struct inode *inode = file->f_mapping->host; 697d2ac838eSTheodore Ts'o struct file *f = file; 698d2ac838eSTheodore Ts'o 699d2ac838eSTheodore Ts'o /* Avoid recursion */ 700d2ac838eSTheodore Ts'o while (is_loop_device(f)) { 701d2ac838eSTheodore Ts'o struct loop_device *l; 702d2ac838eSTheodore Ts'o 7033ce6e1f6STetsuo Handa lockdep_assert_held(&loop_validate_mutex); 7044e7b5671SChristoph Hellwig if (f->f_mapping->host->i_rdev == bdev->bd_dev) 705d2ac838eSTheodore Ts'o return -EBADF; 706d2ac838eSTheodore Ts'o 7074e7b5671SChristoph Hellwig l = I_BDEV(f->f_mapping->host)->bd_disk->private_data; 7083ce6e1f6STetsuo Handa if (l->lo_state != Lo_bound) 709d2ac838eSTheodore Ts'o return -EINVAL; 7103ce6e1f6STetsuo Handa /* Order wrt setting lo->lo_backing_file in loop_configure(). */ 7113ce6e1f6STetsuo Handa rmb(); 712d2ac838eSTheodore Ts'o f = l->lo_backing_file; 713d2ac838eSTheodore Ts'o } 714d2ac838eSTheodore Ts'o if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode)) 715d2ac838eSTheodore Ts'o return -EINVAL; 716d2ac838eSTheodore Ts'o return 0; 717d2ac838eSTheodore Ts'o } 718d2ac838eSTheodore Ts'o 7191da177e4SLinus Torvalds /* 7201da177e4SLinus Torvalds * loop_change_fd switched the backing store of a loopback device to 7211da177e4SLinus Torvalds * a new file. This is useful for operating system installers to free up 7221da177e4SLinus Torvalds * the original file and in High Availability environments to switch to 7231da177e4SLinus Torvalds * an alternative location for the content in case of server meltdown. 7241da177e4SLinus Torvalds * This can only work if the loop device is used read-only, and if the 7251da177e4SLinus Torvalds * new backing store is the same size and type as the old backing store. 7261da177e4SLinus Torvalds */ 727bb214884SAl Viro static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, 728bb214884SAl Viro unsigned int arg) 7291da177e4SLinus Torvalds { 7303ce6e1f6STetsuo Handa struct file *file = fget(arg); 7313ce6e1f6STetsuo Handa struct file *old_file; 7321da177e4SLinus Torvalds int error; 73385b0a54aSJan Kara bool partscan; 7343ce6e1f6STetsuo Handa bool is_loop; 7351da177e4SLinus Torvalds 7363ce6e1f6STetsuo Handa if (!file) 7373ce6e1f6STetsuo Handa return -EBADF; 7383ce6e1f6STetsuo Handa is_loop = is_loop_device(file); 7393ce6e1f6STetsuo Handa error = loop_global_lock_killable(lo, is_loop); 740c3710770SJan Kara if (error) 7413ce6e1f6STetsuo Handa goto out_putf; 7421da177e4SLinus Torvalds error = -ENXIO; 7431da177e4SLinus Torvalds if (lo->lo_state != Lo_bound) 7441dded9acSJan Kara goto out_err; 7451da177e4SLinus Torvalds 7461da177e4SLinus Torvalds /* the loop device has to be read-only */ 7471da177e4SLinus Torvalds error = -EINVAL; 7481da177e4SLinus Torvalds if (!(lo->lo_flags & LO_FLAGS_READ_ONLY)) 7491dded9acSJan Kara goto out_err; 7501da177e4SLinus Torvalds 751d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev); 752d2ac838eSTheodore Ts'o if (error) 7531dded9acSJan Kara goto out_err; 754d2ac838eSTheodore Ts'o 7551da177e4SLinus Torvalds old_file = lo->lo_backing_file; 7561da177e4SLinus Torvalds 7571da177e4SLinus Torvalds error = -EINVAL; 7581da177e4SLinus Torvalds 7591da177e4SLinus Torvalds /* size of the new backing store needs to be the same */ 7601da177e4SLinus Torvalds if (get_loop_size(lo, file) != get_loop_size(lo, old_file)) 7611dded9acSJan Kara goto out_err; 7621da177e4SLinus Torvalds 7631da177e4SLinus Torvalds /* and ... switch */ 7649f65c489SMatteo Croce disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE); 76543cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 76643cade80SOmar Sandoval mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask); 76743cade80SOmar Sandoval lo->lo_backing_file = file; 76843cade80SOmar Sandoval lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping); 76943cade80SOmar Sandoval mapping_set_gfp_mask(file->f_mapping, 77043cade80SOmar Sandoval lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 77143cade80SOmar Sandoval loop_update_dio(lo); 77243cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 77385b0a54aSJan Kara partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 7743ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop); 7753ce6e1f6STetsuo Handa 7763ce6e1f6STetsuo Handa /* 7773ce6e1f6STetsuo Handa * Flush loop_validate_file() before fput(), for l->lo_backing_file 7783ce6e1f6STetsuo Handa * might be pointing at old_file which might be the last reference. 7793ce6e1f6STetsuo Handa */ 7803ce6e1f6STetsuo Handa if (!is_loop) { 7813ce6e1f6STetsuo Handa mutex_lock(&loop_validate_mutex); 7823ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex); 7833ce6e1f6STetsuo Handa } 7841dded9acSJan Kara /* 7856cc8e743SPavel Tatashin * We must drop file reference outside of lo_mutex as dropping 786a8698707SChristoph Hellwig * the file ref can take open_mutex which creates circular locking 7871dded9acSJan Kara * dependency. 7881dded9acSJan Kara */ 7891dded9acSJan Kara fput(old_file); 79085b0a54aSJan Kara if (partscan) 7910384264eSChristoph Hellwig loop_reread_partitions(lo); 7921da177e4SLinus Torvalds return 0; 7931da177e4SLinus Torvalds 7941dded9acSJan Kara out_err: 7953ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop); 7963ce6e1f6STetsuo Handa out_putf: 7971dded9acSJan Kara fput(file); 7981da177e4SLinus Torvalds return error; 7991da177e4SLinus Torvalds } 8001da177e4SLinus Torvalds 801ee862730SMilan Broz /* loop sysfs attributes */ 802ee862730SMilan Broz 803ee862730SMilan Broz static ssize_t loop_attr_show(struct device *dev, char *page, 804ee862730SMilan Broz ssize_t (*callback)(struct loop_device *, char *)) 805ee862730SMilan Broz { 80634dd82afSKay Sievers struct gendisk *disk = dev_to_disk(dev); 80734dd82afSKay Sievers struct loop_device *lo = disk->private_data; 808ee862730SMilan Broz 80934dd82afSKay Sievers return callback(lo, page); 810ee862730SMilan Broz } 811ee862730SMilan Broz 812ee862730SMilan Broz #define LOOP_ATTR_RO(_name) \ 813ee862730SMilan Broz static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \ 814ee862730SMilan Broz static ssize_t loop_attr_do_show_##_name(struct device *d, \ 815ee862730SMilan Broz struct device_attribute *attr, char *b) \ 816ee862730SMilan Broz { \ 817ee862730SMilan Broz return loop_attr_show(d, b, loop_attr_##_name##_show); \ 818ee862730SMilan Broz } \ 819ee862730SMilan Broz static struct device_attribute loop_attr_##_name = \ 8205657a819SJoe Perches __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL); 821ee862730SMilan Broz 822ee862730SMilan Broz static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf) 823ee862730SMilan Broz { 824ee862730SMilan Broz ssize_t ret; 825ee862730SMilan Broz char *p = NULL; 826ee862730SMilan Broz 82705eb0f25SKay Sievers spin_lock_irq(&lo->lo_lock); 828ee862730SMilan Broz if (lo->lo_backing_file) 8299bf39ab2SMiklos Szeredi p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1); 83005eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock); 831ee862730SMilan Broz 832ee862730SMilan Broz if (IS_ERR_OR_NULL(p)) 833ee862730SMilan Broz ret = PTR_ERR(p); 834ee862730SMilan Broz else { 835ee862730SMilan Broz ret = strlen(p); 836ee862730SMilan Broz memmove(buf, p, ret); 837ee862730SMilan Broz buf[ret++] = '\n'; 838ee862730SMilan Broz buf[ret] = 0; 839ee862730SMilan Broz } 840ee862730SMilan Broz 841ee862730SMilan Broz return ret; 842ee862730SMilan Broz } 843ee862730SMilan Broz 844ee862730SMilan Broz static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf) 845ee862730SMilan Broz { 846ee862730SMilan Broz return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset); 847ee862730SMilan Broz } 848ee862730SMilan Broz 849ee862730SMilan Broz static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf) 850ee862730SMilan Broz { 851ee862730SMilan Broz return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit); 852ee862730SMilan Broz } 853ee862730SMilan Broz 854ee862730SMilan Broz static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf) 855ee862730SMilan Broz { 856ee862730SMilan Broz int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR); 857ee862730SMilan Broz 858ee862730SMilan Broz return sprintf(buf, "%s\n", autoclear ? "1" : "0"); 859ee862730SMilan Broz } 860ee862730SMilan Broz 861e03c8dd1SKay Sievers static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf) 862e03c8dd1SKay Sievers { 863e03c8dd1SKay Sievers int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN); 864e03c8dd1SKay Sievers 865e03c8dd1SKay Sievers return sprintf(buf, "%s\n", partscan ? "1" : "0"); 866e03c8dd1SKay Sievers } 867e03c8dd1SKay Sievers 8682e5ab5f3SMing Lei static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf) 8692e5ab5f3SMing Lei { 8702e5ab5f3SMing Lei int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO); 8712e5ab5f3SMing Lei 8722e5ab5f3SMing Lei return sprintf(buf, "%s\n", dio ? "1" : "0"); 8732e5ab5f3SMing Lei } 8742e5ab5f3SMing Lei 875ee862730SMilan Broz LOOP_ATTR_RO(backing_file); 876ee862730SMilan Broz LOOP_ATTR_RO(offset); 877ee862730SMilan Broz LOOP_ATTR_RO(sizelimit); 878ee862730SMilan Broz LOOP_ATTR_RO(autoclear); 879e03c8dd1SKay Sievers LOOP_ATTR_RO(partscan); 8802e5ab5f3SMing Lei LOOP_ATTR_RO(dio); 881ee862730SMilan Broz 882ee862730SMilan Broz static struct attribute *loop_attrs[] = { 883ee862730SMilan Broz &loop_attr_backing_file.attr, 884ee862730SMilan Broz &loop_attr_offset.attr, 885ee862730SMilan Broz &loop_attr_sizelimit.attr, 886ee862730SMilan Broz &loop_attr_autoclear.attr, 887e03c8dd1SKay Sievers &loop_attr_partscan.attr, 8882e5ab5f3SMing Lei &loop_attr_dio.attr, 889ee862730SMilan Broz NULL, 890ee862730SMilan Broz }; 891ee862730SMilan Broz 892ee862730SMilan Broz static struct attribute_group loop_attribute_group = { 893ee862730SMilan Broz .name = "loop", 894ee862730SMilan Broz .attrs= loop_attrs, 895ee862730SMilan Broz }; 896ee862730SMilan Broz 897d3349b6bSTetsuo Handa static void loop_sysfs_init(struct loop_device *lo) 898ee862730SMilan Broz { 899d3349b6bSTetsuo Handa lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj, 900ee862730SMilan Broz &loop_attribute_group); 901ee862730SMilan Broz } 902ee862730SMilan Broz 903ee862730SMilan Broz static void loop_sysfs_exit(struct loop_device *lo) 904ee862730SMilan Broz { 905d3349b6bSTetsuo Handa if (lo->sysfs_inited) 906ee862730SMilan Broz sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj, 907ee862730SMilan Broz &loop_attribute_group); 908ee862730SMilan Broz } 909ee862730SMilan Broz 910dfaa2ef6SLukas Czerner static void loop_config_discard(struct loop_device *lo) 911dfaa2ef6SLukas Czerner { 912dfaa2ef6SLukas Czerner struct file *file = lo->lo_backing_file; 913dfaa2ef6SLukas Czerner struct inode *inode = file->f_mapping->host; 914dfaa2ef6SLukas Czerner struct request_queue *q = lo->lo_queue; 915bcb21c8cSMing Lei u32 granularity, max_discard_sectors; 916dfaa2ef6SLukas Czerner 917dfaa2ef6SLukas Czerner /* 918c52abf56SEvan Green * If the backing device is a block device, mirror its zeroing 919c52abf56SEvan Green * capability. Set the discard sectors to the block device's zeroing 920c52abf56SEvan Green * capabilities because loop discards result in blkdev_issue_zeroout(), 921c52abf56SEvan Green * not blkdev_issue_discard(). This maintains consistent behavior with 922c52abf56SEvan Green * file-backed loop devices: discarded regions read back as zero. 923c52abf56SEvan Green */ 924c52abf56SEvan Green if (S_ISBLK(inode->i_mode) && !lo->lo_encrypt_key_size) { 9254e7b5671SChristoph Hellwig struct request_queue *backingq = bdev_get_queue(I_BDEV(inode)); 926c52abf56SEvan Green 927bcb21c8cSMing Lei max_discard_sectors = backingq->limits.max_write_zeroes_sectors; 928bcb21c8cSMing Lei granularity = backingq->limits.discard_granularity ?: 929bcb21c8cSMing Lei queue_physical_block_size(backingq); 930c52abf56SEvan Green 931c52abf56SEvan Green /* 932dfaa2ef6SLukas Czerner * We use punch hole to reclaim the free space used by the 93312a64d2fSOlaf Hering * image a.k.a. discard. However we do not support discard if 934dfaa2ef6SLukas Czerner * encryption is enabled, because it may give an attacker 935dfaa2ef6SLukas Czerner * useful information. 936dfaa2ef6SLukas Czerner */ 937c52abf56SEvan Green } else if (!file->f_op->fallocate || lo->lo_encrypt_key_size) { 938bcb21c8cSMing Lei max_discard_sectors = 0; 939bcb21c8cSMing Lei granularity = 0; 940dfaa2ef6SLukas Czerner 941c52abf56SEvan Green } else { 942bcb21c8cSMing Lei max_discard_sectors = UINT_MAX >> 9; 943bcb21c8cSMing Lei granularity = inode->i_sb->s_blocksize; 944c52abf56SEvan Green } 945c52abf56SEvan Green 946bcb21c8cSMing Lei if (max_discard_sectors) { 947bcb21c8cSMing Lei q->limits.discard_granularity = granularity; 948bcb21c8cSMing Lei blk_queue_max_discard_sectors(q, max_discard_sectors); 949bcb21c8cSMing Lei blk_queue_max_write_zeroes_sectors(q, max_discard_sectors); 9508b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 951bcb21c8cSMing Lei } else { 952bcb21c8cSMing Lei q->limits.discard_granularity = 0; 953bcb21c8cSMing Lei blk_queue_max_discard_sectors(q, 0); 954bcb21c8cSMing Lei blk_queue_max_write_zeroes_sectors(q, 0); 955c52abf56SEvan Green blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q); 956dfaa2ef6SLukas Czerner } 957bcb21c8cSMing Lei q->limits.discard_alignment = 0; 958bcb21c8cSMing Lei } 959dfaa2ef6SLukas Czerner 96087579e9bSDan Schatzberg struct loop_worker { 96187579e9bSDan Schatzberg struct rb_node rb_node; 96287579e9bSDan Schatzberg struct work_struct work; 96387579e9bSDan Schatzberg struct list_head cmd_list; 96487579e9bSDan Schatzberg struct list_head idle_list; 96587579e9bSDan Schatzberg struct loop_device *lo; 966c74d40e8SDan Schatzberg struct cgroup_subsys_state *blkcg_css; 96787579e9bSDan Schatzberg unsigned long last_ran_at; 96887579e9bSDan Schatzberg }; 969e03a3d7aSMing Lei 97087579e9bSDan Schatzberg static void loop_workfn(struct work_struct *work); 97187579e9bSDan Schatzberg static void loop_rootcg_workfn(struct work_struct *work); 97287579e9bSDan Schatzberg static void loop_free_idle_workers(struct timer_list *timer); 973b2ee7d46SNeilBrown 97487579e9bSDan Schatzberg #ifdef CONFIG_BLK_CGROUP 97587579e9bSDan Schatzberg static inline int queue_on_root_worker(struct cgroup_subsys_state *css) 976e03a3d7aSMing Lei { 97787579e9bSDan Schatzberg return !css || css == blkcg_root_css; 97887579e9bSDan Schatzberg } 97987579e9bSDan Schatzberg #else 98087579e9bSDan Schatzberg static inline int queue_on_root_worker(struct cgroup_subsys_state *css) 98187579e9bSDan Schatzberg { 98287579e9bSDan Schatzberg return !css; 98387579e9bSDan Schatzberg } 98487579e9bSDan Schatzberg #endif 98587579e9bSDan Schatzberg 98687579e9bSDan Schatzberg static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd) 98787579e9bSDan Schatzberg { 98887579e9bSDan Schatzberg struct rb_node **node = &(lo->worker_tree.rb_node), *parent = NULL; 98987579e9bSDan Schatzberg struct loop_worker *cur_worker, *worker = NULL; 99087579e9bSDan Schatzberg struct work_struct *work; 99187579e9bSDan Schatzberg struct list_head *cmd_list; 99287579e9bSDan Schatzberg 99387579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock); 99487579e9bSDan Schatzberg 995c74d40e8SDan Schatzberg if (queue_on_root_worker(cmd->blkcg_css)) 99687579e9bSDan Schatzberg goto queue_work; 99787579e9bSDan Schatzberg 99887579e9bSDan Schatzberg node = &lo->worker_tree.rb_node; 99987579e9bSDan Schatzberg 100087579e9bSDan Schatzberg while (*node) { 100187579e9bSDan Schatzberg parent = *node; 100287579e9bSDan Schatzberg cur_worker = container_of(*node, struct loop_worker, rb_node); 1003c74d40e8SDan Schatzberg if (cur_worker->blkcg_css == cmd->blkcg_css) { 100487579e9bSDan Schatzberg worker = cur_worker; 100587579e9bSDan Schatzberg break; 1006c74d40e8SDan Schatzberg } else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) { 100787579e9bSDan Schatzberg node = &(*node)->rb_left; 100887579e9bSDan Schatzberg } else { 100987579e9bSDan Schatzberg node = &(*node)->rb_right; 101087579e9bSDan Schatzberg } 101187579e9bSDan Schatzberg } 101287579e9bSDan Schatzberg if (worker) 101387579e9bSDan Schatzberg goto queue_work; 101487579e9bSDan Schatzberg 101587579e9bSDan Schatzberg worker = kzalloc(sizeof(struct loop_worker), GFP_NOWAIT | __GFP_NOWARN); 101687579e9bSDan Schatzberg /* 101787579e9bSDan Schatzberg * In the event we cannot allocate a worker, just queue on the 1018c74d40e8SDan Schatzberg * rootcg worker and issue the I/O as the rootcg 101987579e9bSDan Schatzberg */ 1020c74d40e8SDan Schatzberg if (!worker) { 1021c74d40e8SDan Schatzberg cmd->blkcg_css = NULL; 1022c74d40e8SDan Schatzberg if (cmd->memcg_css) 1023c74d40e8SDan Schatzberg css_put(cmd->memcg_css); 1024c74d40e8SDan Schatzberg cmd->memcg_css = NULL; 102587579e9bSDan Schatzberg goto queue_work; 1026c74d40e8SDan Schatzberg } 102787579e9bSDan Schatzberg 1028c74d40e8SDan Schatzberg worker->blkcg_css = cmd->blkcg_css; 1029c74d40e8SDan Schatzberg css_get(worker->blkcg_css); 103087579e9bSDan Schatzberg INIT_WORK(&worker->work, loop_workfn); 103187579e9bSDan Schatzberg INIT_LIST_HEAD(&worker->cmd_list); 103287579e9bSDan Schatzberg INIT_LIST_HEAD(&worker->idle_list); 103387579e9bSDan Schatzberg worker->lo = lo; 103487579e9bSDan Schatzberg rb_link_node(&worker->rb_node, parent, node); 103587579e9bSDan Schatzberg rb_insert_color(&worker->rb_node, &lo->worker_tree); 103687579e9bSDan Schatzberg queue_work: 103787579e9bSDan Schatzberg if (worker) { 103887579e9bSDan Schatzberg /* 103987579e9bSDan Schatzberg * We need to remove from the idle list here while 104087579e9bSDan Schatzberg * holding the lock so that the idle timer doesn't 104187579e9bSDan Schatzberg * free the worker 104287579e9bSDan Schatzberg */ 104387579e9bSDan Schatzberg if (!list_empty(&worker->idle_list)) 104487579e9bSDan Schatzberg list_del_init(&worker->idle_list); 104587579e9bSDan Schatzberg work = &worker->work; 104687579e9bSDan Schatzberg cmd_list = &worker->cmd_list; 104787579e9bSDan Schatzberg } else { 104887579e9bSDan Schatzberg work = &lo->rootcg_work; 104987579e9bSDan Schatzberg cmd_list = &lo->rootcg_cmd_list; 105087579e9bSDan Schatzberg } 105187579e9bSDan Schatzberg list_add_tail(&cmd->list_entry, cmd_list); 105287579e9bSDan Schatzberg queue_work(lo->workqueue, work); 105387579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock); 1054e03a3d7aSMing Lei } 1055e03a3d7aSMing Lei 105656a85fd8SHolger Hoffstätte static void loop_update_rotational(struct loop_device *lo) 105756a85fd8SHolger Hoffstätte { 105856a85fd8SHolger Hoffstätte struct file *file = lo->lo_backing_file; 105956a85fd8SHolger Hoffstätte struct inode *file_inode = file->f_mapping->host; 106056a85fd8SHolger Hoffstätte struct block_device *file_bdev = file_inode->i_sb->s_bdev; 106156a85fd8SHolger Hoffstätte struct request_queue *q = lo->lo_queue; 106256a85fd8SHolger Hoffstätte bool nonrot = true; 106356a85fd8SHolger Hoffstätte 106456a85fd8SHolger Hoffstätte /* not all filesystems (e.g. tmpfs) have a sb->s_bdev */ 106556a85fd8SHolger Hoffstätte if (file_bdev) 106656a85fd8SHolger Hoffstätte nonrot = blk_queue_nonrot(bdev_get_queue(file_bdev)); 106756a85fd8SHolger Hoffstätte 106856a85fd8SHolger Hoffstätte if (nonrot) 106956a85fd8SHolger Hoffstätte blk_queue_flag_set(QUEUE_FLAG_NONROT, q); 107056a85fd8SHolger Hoffstätte else 107156a85fd8SHolger Hoffstätte blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); 107256a85fd8SHolger Hoffstätte } 107356a85fd8SHolger Hoffstätte 10741da177e4SLinus Torvalds static int 10751da177e4SLinus Torvalds loop_release_xfer(struct loop_device *lo) 10761da177e4SLinus Torvalds { 10771da177e4SLinus Torvalds int err = 0; 10781da177e4SLinus Torvalds struct loop_func_table *xfer = lo->lo_encryption; 10791da177e4SLinus Torvalds 10801da177e4SLinus Torvalds if (xfer) { 10811da177e4SLinus Torvalds if (xfer->release) 10821da177e4SLinus Torvalds err = xfer->release(lo); 10831da177e4SLinus Torvalds lo->transfer = NULL; 10841da177e4SLinus Torvalds lo->lo_encryption = NULL; 10851da177e4SLinus Torvalds module_put(xfer->owner); 10861da177e4SLinus Torvalds } 10871da177e4SLinus Torvalds return err; 10881da177e4SLinus Torvalds } 10891da177e4SLinus Torvalds 10901da177e4SLinus Torvalds static int 10911da177e4SLinus Torvalds loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, 10921da177e4SLinus Torvalds const struct loop_info64 *i) 10931da177e4SLinus Torvalds { 10941da177e4SLinus Torvalds int err = 0; 10951da177e4SLinus Torvalds 10961da177e4SLinus Torvalds if (xfer) { 10971da177e4SLinus Torvalds struct module *owner = xfer->owner; 10981da177e4SLinus Torvalds 10991da177e4SLinus Torvalds if (!try_module_get(owner)) 11001da177e4SLinus Torvalds return -EINVAL; 11011da177e4SLinus Torvalds if (xfer->init) 11021da177e4SLinus Torvalds err = xfer->init(lo, i); 11031da177e4SLinus Torvalds if (err) 11041da177e4SLinus Torvalds module_put(owner); 11051da177e4SLinus Torvalds else 11061da177e4SLinus Torvalds lo->lo_encryption = xfer; 11071da177e4SLinus Torvalds } 11081da177e4SLinus Torvalds return err; 11091da177e4SLinus Torvalds } 11101da177e4SLinus Torvalds 111162ab466cSMartijn Coenen /** 111262ab466cSMartijn Coenen * loop_set_status_from_info - configure device from loop_info 111362ab466cSMartijn Coenen * @lo: struct loop_device to configure 111462ab466cSMartijn Coenen * @info: struct loop_info64 to configure the device with 111562ab466cSMartijn Coenen * 111662ab466cSMartijn Coenen * Configures the loop device parameters according to the passed 111762ab466cSMartijn Coenen * in loop_info64 configuration. 111862ab466cSMartijn Coenen */ 111962ab466cSMartijn Coenen static int 112062ab466cSMartijn Coenen loop_set_status_from_info(struct loop_device *lo, 112162ab466cSMartijn Coenen const struct loop_info64 *info) 112262ab466cSMartijn Coenen { 112362ab466cSMartijn Coenen int err; 112462ab466cSMartijn Coenen struct loop_func_table *xfer; 112562ab466cSMartijn Coenen kuid_t uid = current_uid(); 112662ab466cSMartijn Coenen 112762ab466cSMartijn Coenen if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) 112862ab466cSMartijn Coenen return -EINVAL; 112962ab466cSMartijn Coenen 113062ab466cSMartijn Coenen err = loop_release_xfer(lo); 113162ab466cSMartijn Coenen if (err) 113262ab466cSMartijn Coenen return err; 113362ab466cSMartijn Coenen 113462ab466cSMartijn Coenen if (info->lo_encrypt_type) { 113562ab466cSMartijn Coenen unsigned int type = info->lo_encrypt_type; 113662ab466cSMartijn Coenen 113762ab466cSMartijn Coenen if (type >= MAX_LO_CRYPT) 113862ab466cSMartijn Coenen return -EINVAL; 113962ab466cSMartijn Coenen xfer = xfer_funcs[type]; 114062ab466cSMartijn Coenen if (xfer == NULL) 114162ab466cSMartijn Coenen return -EINVAL; 114262ab466cSMartijn Coenen } else 114362ab466cSMartijn Coenen xfer = NULL; 114462ab466cSMartijn Coenen 114562ab466cSMartijn Coenen err = loop_init_xfer(lo, xfer, info); 114662ab466cSMartijn Coenen if (err) 114762ab466cSMartijn Coenen return err; 114862ab466cSMartijn Coenen 114962ab466cSMartijn Coenen lo->lo_offset = info->lo_offset; 115062ab466cSMartijn Coenen lo->lo_sizelimit = info->lo_sizelimit; 115162ab466cSMartijn Coenen memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); 115262ab466cSMartijn Coenen memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); 115362ab466cSMartijn Coenen lo->lo_file_name[LO_NAME_SIZE-1] = 0; 115462ab466cSMartijn Coenen lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; 115562ab466cSMartijn Coenen 115662ab466cSMartijn Coenen if (!xfer) 115762ab466cSMartijn Coenen xfer = &none_funcs; 115862ab466cSMartijn Coenen lo->transfer = xfer->transfer; 115962ab466cSMartijn Coenen lo->ioctl = xfer->ioctl; 116062ab466cSMartijn Coenen 1161faf1d254SMartijn Coenen lo->lo_flags = info->lo_flags; 116262ab466cSMartijn Coenen 116362ab466cSMartijn Coenen lo->lo_encrypt_key_size = info->lo_encrypt_key_size; 116462ab466cSMartijn Coenen lo->lo_init[0] = info->lo_init[0]; 116562ab466cSMartijn Coenen lo->lo_init[1] = info->lo_init[1]; 116662ab466cSMartijn Coenen if (info->lo_encrypt_key_size) { 116762ab466cSMartijn Coenen memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, 116862ab466cSMartijn Coenen info->lo_encrypt_key_size); 116962ab466cSMartijn Coenen lo->lo_key_owner = uid; 117062ab466cSMartijn Coenen } 117162ab466cSMartijn Coenen 117262ab466cSMartijn Coenen return 0; 117362ab466cSMartijn Coenen } 117462ab466cSMartijn Coenen 11753448914eSMartijn Coenen static int loop_configure(struct loop_device *lo, fmode_t mode, 11763448914eSMartijn Coenen struct block_device *bdev, 11773448914eSMartijn Coenen const struct loop_config *config) 11781da177e4SLinus Torvalds { 11793ce6e1f6STetsuo Handa struct file *file = fget(config->fd); 11801da177e4SLinus Torvalds struct inode *inode; 11811da177e4SLinus Torvalds struct address_space *mapping; 11821da177e4SLinus Torvalds int error; 11831da177e4SLinus Torvalds loff_t size; 11841da177e4SLinus Torvalds bool partscan; 11853448914eSMartijn Coenen unsigned short bsize; 11863ce6e1f6STetsuo Handa bool is_loop; 11873ce6e1f6STetsuo Handa 11883ce6e1f6STetsuo Handa if (!file) 11893ce6e1f6STetsuo Handa return -EBADF; 11903ce6e1f6STetsuo Handa is_loop = is_loop_device(file); 11911da177e4SLinus Torvalds 11921da177e4SLinus Torvalds /* This is safe, since we have a reference from open(). */ 11931da177e4SLinus Torvalds __module_get(THIS_MODULE); 11941da177e4SLinus Torvalds 11951da177e4SLinus Torvalds /* 11961da177e4SLinus Torvalds * If we don't hold exclusive handle for the device, upgrade to it 11971da177e4SLinus Torvalds * here to avoid changing device under exclusive owner. 11981da177e4SLinus Torvalds */ 11991da177e4SLinus Torvalds if (!(mode & FMODE_EXCL)) { 120037c3fc9aSChristoph Hellwig error = bd_prepare_to_claim(bdev, loop_configure); 1201ecbe6bc0SChristoph Hellwig if (error) 12021da177e4SLinus Torvalds goto out_putf; 12031da177e4SLinus Torvalds } 12041da177e4SLinus Torvalds 12053ce6e1f6STetsuo Handa error = loop_global_lock_killable(lo, is_loop); 12061da177e4SLinus Torvalds if (error) 12071da177e4SLinus Torvalds goto out_bdev; 12081da177e4SLinus Torvalds 12091da177e4SLinus Torvalds error = -EBUSY; 12101da177e4SLinus Torvalds if (lo->lo_state != Lo_unbound) 12111da177e4SLinus Torvalds goto out_unlock; 12121da177e4SLinus Torvalds 12131da177e4SLinus Torvalds error = loop_validate_file(file, bdev); 12141da177e4SLinus Torvalds if (error) 12151da177e4SLinus Torvalds goto out_unlock; 12161da177e4SLinus Torvalds 12171da177e4SLinus Torvalds mapping = file->f_mapping; 12181da177e4SLinus Torvalds inode = mapping->host; 12191da177e4SLinus Torvalds 12203448914eSMartijn Coenen if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) { 12213448914eSMartijn Coenen error = -EINVAL; 12223448914eSMartijn Coenen goto out_unlock; 12233448914eSMartijn Coenen } 12243448914eSMartijn Coenen 12253448914eSMartijn Coenen if (config->block_size) { 1226*af3c570fSXie Yongji error = blk_validate_block_size(config->block_size); 12273448914eSMartijn Coenen if (error) 12283448914eSMartijn Coenen goto out_unlock; 12293448914eSMartijn Coenen } 12303448914eSMartijn Coenen 12313448914eSMartijn Coenen error = loop_set_status_from_info(lo, &config->info); 12323448914eSMartijn Coenen if (error) 12333448914eSMartijn Coenen goto out_unlock; 12343448914eSMartijn Coenen 12351da177e4SLinus Torvalds if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) || 12361da177e4SLinus Torvalds !file->f_op->write_iter) 12373448914eSMartijn Coenen lo->lo_flags |= LO_FLAGS_READ_ONLY; 1238083a6a50SMartijn Coenen 123987579e9bSDan Schatzberg lo->workqueue = alloc_workqueue("loop%d", 124087579e9bSDan Schatzberg WQ_UNBOUND | WQ_FREEZABLE, 124187579e9bSDan Schatzberg 0, 124287579e9bSDan Schatzberg lo->lo_number); 124387579e9bSDan Schatzberg if (!lo->workqueue) { 124487579e9bSDan Schatzberg error = -ENOMEM; 12451da177e4SLinus Torvalds goto out_unlock; 124687579e9bSDan Schatzberg } 12471da177e4SLinus Torvalds 12489f65c489SMatteo Croce disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE); 12497a2f0ce1SChristoph Hellwig set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0); 12501da177e4SLinus Torvalds 125187579e9bSDan Schatzberg INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn); 125287579e9bSDan Schatzberg INIT_LIST_HEAD(&lo->rootcg_cmd_list); 125387579e9bSDan Schatzberg INIT_LIST_HEAD(&lo->idle_worker_list); 125487579e9bSDan Schatzberg lo->worker_tree = RB_ROOT; 125587579e9bSDan Schatzberg timer_setup(&lo->timer, loop_free_idle_workers, 125687579e9bSDan Schatzberg TIMER_DEFERRABLE); 12573448914eSMartijn Coenen lo->use_dio = lo->lo_flags & LO_FLAGS_DIRECT_IO; 12581da177e4SLinus Torvalds lo->lo_device = bdev; 12591da177e4SLinus Torvalds lo->lo_backing_file = file; 12601da177e4SLinus Torvalds lo->old_gfp_mask = mapping_gfp_mask(mapping); 12611da177e4SLinus Torvalds mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 12621da177e4SLinus Torvalds 12633448914eSMartijn Coenen if (!(lo->lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) 12641da177e4SLinus Torvalds blk_queue_write_cache(lo->lo_queue, true, false); 12651da177e4SLinus Torvalds 12663448914eSMartijn Coenen if (config->block_size) 12673448914eSMartijn Coenen bsize = config->block_size; 126896ed320dSLinus Torvalds else if ((lo->lo_backing_file->f_flags & O_DIRECT) && inode->i_sb->s_bdev) 12691da177e4SLinus Torvalds /* In case of direct I/O, match underlying block size */ 12703448914eSMartijn Coenen bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 12713448914eSMartijn Coenen else 12723448914eSMartijn Coenen bsize = 512; 12731da177e4SLinus Torvalds 12741da177e4SLinus Torvalds blk_queue_logical_block_size(lo->lo_queue, bsize); 12751da177e4SLinus Torvalds blk_queue_physical_block_size(lo->lo_queue, bsize); 12761da177e4SLinus Torvalds blk_queue_io_min(lo->lo_queue, bsize); 12771da177e4SLinus Torvalds 12782b9ac22bSKristian Klausen loop_config_discard(lo); 12791da177e4SLinus Torvalds loop_update_rotational(lo); 12801da177e4SLinus Torvalds loop_update_dio(lo); 12811da177e4SLinus Torvalds loop_sysfs_init(lo); 128279e5dc59SMartijn Coenen 128379e5dc59SMartijn Coenen size = get_loop_size(lo, file); 12845795b6f5SMartijn Coenen loop_set_size(lo, size); 12851da177e4SLinus Torvalds 12863ce6e1f6STetsuo Handa /* Order wrt reading lo_state in loop_validate_file(). */ 12873ce6e1f6STetsuo Handa wmb(); 12883ce6e1f6STetsuo Handa 12891da177e4SLinus Torvalds lo->lo_state = Lo_bound; 12901da177e4SLinus Torvalds if (part_shift) 12911da177e4SLinus Torvalds lo->lo_flags |= LO_FLAGS_PARTSCAN; 12921da177e4SLinus Torvalds partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 1293fe6a8fc5SLennart Poettering if (partscan) 1294fe6a8fc5SLennart Poettering lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; 12951da177e4SLinus Torvalds 12963ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop); 12971da177e4SLinus Torvalds if (partscan) 12980384264eSChristoph Hellwig loop_reread_partitions(lo); 129937c3fc9aSChristoph Hellwig if (!(mode & FMODE_EXCL)) 130037c3fc9aSChristoph Hellwig bd_abort_claiming(bdev, loop_configure); 13011da177e4SLinus Torvalds return 0; 13021da177e4SLinus Torvalds 13031da177e4SLinus Torvalds out_unlock: 13043ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop); 13051da177e4SLinus Torvalds out_bdev: 130637c3fc9aSChristoph Hellwig if (!(mode & FMODE_EXCL)) 130737c3fc9aSChristoph Hellwig bd_abort_claiming(bdev, loop_configure); 13081da177e4SLinus Torvalds out_putf: 13091da177e4SLinus Torvalds fput(file); 13101da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */ 13111da177e4SLinus Torvalds module_put(THIS_MODULE); 13121da177e4SLinus Torvalds return error; 13131da177e4SLinus Torvalds } 13141da177e4SLinus Torvalds 13150da03cabSJan Kara static int __loop_clr_fd(struct loop_device *lo, bool release) 13161da177e4SLinus Torvalds { 13177ccd0791SJan Kara struct file *filp = NULL; 1318b4e3ca1aSAl Viro gfp_t gfp = lo->old_gfp_mask; 13197ccd0791SJan Kara int err = 0; 13200da03cabSJan Kara bool partscan = false; 13210da03cabSJan Kara int lo_number; 132287579e9bSDan Schatzberg struct loop_worker *pos, *worker; 13231da177e4SLinus Torvalds 13243ce6e1f6STetsuo Handa /* 13253ce6e1f6STetsuo Handa * Flush loop_configure() and loop_change_fd(). It is acceptable for 13263ce6e1f6STetsuo Handa * loop_validate_file() to succeed, for actual clear operation has not 13273ce6e1f6STetsuo Handa * started yet. 13283ce6e1f6STetsuo Handa */ 13293ce6e1f6STetsuo Handa mutex_lock(&loop_validate_mutex); 13303ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex); 13313ce6e1f6STetsuo Handa /* 13323ce6e1f6STetsuo Handa * loop_validate_file() now fails because l->lo_state != Lo_bound 13333ce6e1f6STetsuo Handa * became visible. 13343ce6e1f6STetsuo Handa */ 13353ce6e1f6STetsuo Handa 13366cc8e743SPavel Tatashin mutex_lock(&lo->lo_mutex); 13377ccd0791SJan Kara if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) { 13387ccd0791SJan Kara err = -ENXIO; 13397ccd0791SJan Kara goto out_unlock; 13407ccd0791SJan Kara } 13411da177e4SLinus Torvalds 13427ccd0791SJan Kara filp = lo->lo_backing_file; 13437ccd0791SJan Kara if (filp == NULL) { 13447ccd0791SJan Kara err = -EINVAL; 13457ccd0791SJan Kara goto out_unlock; 13467ccd0791SJan Kara } 13471da177e4SLinus Torvalds 13484ceddce5SMauricio Faria de Oliveira if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags)) 13494ceddce5SMauricio Faria de Oliveira blk_queue_write_cache(lo->lo_queue, false, false); 13504ceddce5SMauricio Faria de Oliveira 1351f8933667SMing Lei /* freeze request queue during the transition */ 1352f8933667SMing Lei blk_mq_freeze_queue(lo->lo_queue); 1353f8933667SMing Lei 135487579e9bSDan Schatzberg destroy_workqueue(lo->workqueue); 135587579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock); 135687579e9bSDan Schatzberg list_for_each_entry_safe(worker, pos, &lo->idle_worker_list, 135787579e9bSDan Schatzberg idle_list) { 135887579e9bSDan Schatzberg list_del(&worker->idle_list); 135987579e9bSDan Schatzberg rb_erase(&worker->rb_node, &lo->worker_tree); 1360c74d40e8SDan Schatzberg css_put(worker->blkcg_css); 136187579e9bSDan Schatzberg kfree(worker); 136287579e9bSDan Schatzberg } 136387579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock); 136487579e9bSDan Schatzberg del_timer_sync(&lo->timer); 136587579e9bSDan Schatzberg 13661da177e4SLinus Torvalds spin_lock_irq(&lo->lo_lock); 13671da177e4SLinus Torvalds lo->lo_backing_file = NULL; 136805eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock); 13691da177e4SLinus Torvalds 13701da177e4SLinus Torvalds loop_release_xfer(lo); 13711da177e4SLinus Torvalds lo->transfer = NULL; 13721da177e4SLinus Torvalds lo->ioctl = NULL; 13731da177e4SLinus Torvalds lo->lo_device = NULL; 13741da177e4SLinus Torvalds lo->lo_encryption = NULL; 13751da177e4SLinus Torvalds lo->lo_offset = 0; 13761da177e4SLinus Torvalds lo->lo_sizelimit = 0; 13771da177e4SLinus Torvalds lo->lo_encrypt_key_size = 0; 13781da177e4SLinus Torvalds memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE); 13791da177e4SLinus Torvalds memset(lo->lo_crypt_name, 0, LO_NAME_SIZE); 13801da177e4SLinus Torvalds memset(lo->lo_file_name, 0, LO_NAME_SIZE); 138189e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, 512); 1382bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, 512); 1383bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, 512); 1384e515be8fSXie Yongji invalidate_disk(lo->lo_disk); 138551a0bb0cSMilan Broz loop_sysfs_exit(lo); 1386c3473c63SDavid Zeuthen /* let user-space know about this change */ 138719f553dbSXie Yongji kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE); 13881da177e4SLinus Torvalds mapping_set_gfp_mask(filp->f_mapping, gfp); 13891da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */ 13901da177e4SLinus Torvalds module_put(THIS_MODULE); 1391f8933667SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 1392f8933667SMing Lei 139319f553dbSXie Yongji partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 13940da03cabSJan Kara lo_number = lo->lo_number; 13959f65c489SMatteo Croce disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE); 13960da03cabSJan Kara out_unlock: 13976cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 13980da03cabSJan Kara if (partscan) { 1399d57f3374SJan Kara /* 1400a8698707SChristoph Hellwig * open_mutex has been held already in release path, so don't 1401d57f3374SJan Kara * acquire it if this function is called in such case. 1402d57f3374SJan Kara * 1403d57f3374SJan Kara * If the reread partition isn't from release path, lo_refcnt 1404d57f3374SJan Kara * must be at least one and it can only become zero when the 1405d57f3374SJan Kara * current holder is released. 1406d57f3374SJan Kara */ 1407f0b870dfSChristoph Hellwig if (!release) 14080384264eSChristoph Hellwig mutex_lock(&lo->lo_disk->open_mutex); 14090384264eSChristoph Hellwig err = bdev_disk_changed(lo->lo_disk, false); 1410f0b870dfSChristoph Hellwig if (!release) 14110384264eSChristoph Hellwig mutex_unlock(&lo->lo_disk->open_mutex); 141240853d6fSDongli Zhang if (err) 1413d57f3374SJan Kara pr_warn("%s: partition scan of loop%d failed (rc=%d)\n", 14140da03cabSJan Kara __func__, lo_number, err); 1415d57f3374SJan Kara /* Device is gone, no point in returning error */ 1416d57f3374SJan Kara err = 0; 1417d57f3374SJan Kara } 1418758a58d0SDongli Zhang 1419758a58d0SDongli Zhang /* 1420758a58d0SDongli Zhang * lo->lo_state is set to Lo_unbound here after above partscan has 1421758a58d0SDongli Zhang * finished. 1422758a58d0SDongli Zhang * 1423758a58d0SDongli Zhang * There cannot be anybody else entering __loop_clr_fd() as 1424758a58d0SDongli Zhang * lo->lo_backing_file is already cleared and Lo_rundown state 1425758a58d0SDongli Zhang * protects us from all the other places trying to change the 'lo' 1426758a58d0SDongli Zhang * device. 1427758a58d0SDongli Zhang */ 14286cc8e743SPavel Tatashin mutex_lock(&lo->lo_mutex); 1429758a58d0SDongli Zhang lo->lo_flags = 0; 1430758a58d0SDongli Zhang if (!part_shift) 1431758a58d0SDongli Zhang lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN; 1432758a58d0SDongli Zhang lo->lo_state = Lo_unbound; 14336cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 1434758a58d0SDongli Zhang 1435f028f3b2SNikanth Karthikesan /* 14366cc8e743SPavel Tatashin * Need not hold lo_mutex to fput backing file. Calling fput holding 14376cc8e743SPavel Tatashin * lo_mutex triggers a circular lock dependency possibility warning as 1438a8698707SChristoph Hellwig * fput can take open_mutex which is usually taken before lo_mutex. 1439f028f3b2SNikanth Karthikesan */ 14407ccd0791SJan Kara if (filp) 1441f028f3b2SNikanth Karthikesan fput(filp); 14427ccd0791SJan Kara return err; 14431da177e4SLinus Torvalds } 14441da177e4SLinus Torvalds 1445a2505b79SJan Kara static int loop_clr_fd(struct loop_device *lo) 1446a2505b79SJan Kara { 14477ccd0791SJan Kara int err; 14487ccd0791SJan Kara 14496cc8e743SPavel Tatashin err = mutex_lock_killable(&lo->lo_mutex); 14507ccd0791SJan Kara if (err) 14517ccd0791SJan Kara return err; 14527ccd0791SJan Kara if (lo->lo_state != Lo_bound) { 14536cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 1454a2505b79SJan Kara return -ENXIO; 14557ccd0791SJan Kara } 1456a2505b79SJan Kara /* 1457a2505b79SJan Kara * If we've explicitly asked to tear down the loop device, 1458a2505b79SJan Kara * and it has an elevated reference count, set it for auto-teardown when 1459a2505b79SJan Kara * the last reference goes away. This stops $!~#$@ udev from 1460a2505b79SJan Kara * preventing teardown because it decided that it needs to run blkid on 1461a2505b79SJan Kara * the loopback device whenever they appear. xfstests is notorious for 1462a2505b79SJan Kara * failing tests because blkid via udev races with a losetup 1463a2505b79SJan Kara * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d 1464a2505b79SJan Kara * command to fail with EBUSY. 1465a2505b79SJan Kara */ 1466a2505b79SJan Kara if (atomic_read(&lo->lo_refcnt) > 1) { 1467a2505b79SJan Kara lo->lo_flags |= LO_FLAGS_AUTOCLEAR; 14686cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 1469a2505b79SJan Kara return 0; 1470a2505b79SJan Kara } 1471a2505b79SJan Kara lo->lo_state = Lo_rundown; 14726cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 1473a2505b79SJan Kara 14740da03cabSJan Kara return __loop_clr_fd(lo, false); 1475a2505b79SJan Kara } 1476a2505b79SJan Kara 14771da177e4SLinus Torvalds static int 14781da177e4SLinus Torvalds loop_set_status(struct loop_device *lo, const struct loop_info64 *info) 14791da177e4SLinus Torvalds { 14801da177e4SLinus Torvalds int err; 14810c3796c2SMartijn Coenen kuid_t uid = current_uid(); 1482faf1d254SMartijn Coenen int prev_lo_flags; 148385b0a54aSJan Kara bool partscan = false; 14840c3796c2SMartijn Coenen bool size_changed = false; 14851da177e4SLinus Torvalds 14866cc8e743SPavel Tatashin err = mutex_lock_killable(&lo->lo_mutex); 1487550df5fdSJan Kara if (err) 1488550df5fdSJan Kara return err; 1489b0fafa81SDavid Howells if (lo->lo_encrypt_key_size && 1490e4849737SEric W. Biederman !uid_eq(lo->lo_key_owner, uid) && 1491550df5fdSJan Kara !capable(CAP_SYS_ADMIN)) { 1492550df5fdSJan Kara err = -EPERM; 1493550df5fdSJan Kara goto out_unlock; 1494550df5fdSJan Kara } 1495550df5fdSJan Kara if (lo->lo_state != Lo_bound) { 1496550df5fdSJan Kara err = -ENXIO; 1497550df5fdSJan Kara goto out_unlock; 1498550df5fdSJan Kara } 14991da177e4SLinus Torvalds 15005db470e2SJaegeuk Kim if (lo->lo_offset != info->lo_offset || 15015db470e2SJaegeuk Kim lo->lo_sizelimit != info->lo_sizelimit) { 15020c3796c2SMartijn Coenen size_changed = true; 15035db470e2SJaegeuk Kim sync_blockdev(lo->lo_device); 1504f4bd34b1SZheng Bin invalidate_bdev(lo->lo_device); 15055db470e2SJaegeuk Kim } 15065db470e2SJaegeuk Kim 1507ecdd0959SMing Lei /* I/O need to be drained during transfer transition */ 1508ecdd0959SMing Lei blk_mq_freeze_queue(lo->lo_queue); 1509ecdd0959SMing Lei 15100c3796c2SMartijn Coenen if (size_changed && lo->lo_device->bd_inode->i_mapping->nrpages) { 1511f4bd34b1SZheng Bin /* If any pages were dirtied after invalidate_bdev(), try again */ 15125db470e2SJaegeuk Kim err = -EAGAIN; 15135db470e2SJaegeuk Kim pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", 15145db470e2SJaegeuk Kim __func__, lo->lo_number, lo->lo_file_name, 15155db470e2SJaegeuk Kim lo->lo_device->bd_inode->i_mapping->nrpages); 15165db470e2SJaegeuk Kim goto out_unfreeze; 15175db470e2SJaegeuk Kim } 15180c3796c2SMartijn Coenen 1519faf1d254SMartijn Coenen prev_lo_flags = lo->lo_flags; 1520faf1d254SMartijn Coenen 15210c3796c2SMartijn Coenen err = loop_set_status_from_info(lo, info); 15220c3796c2SMartijn Coenen if (err) 1523550df5fdSJan Kara goto out_unfreeze; 15240c3796c2SMartijn Coenen 1525faf1d254SMartijn Coenen /* Mask out flags that can't be set using LOOP_SET_STATUS. */ 15266ac92fb5SMartijn Coenen lo->lo_flags &= LOOP_SET_STATUS_SETTABLE_FLAGS; 1527faf1d254SMartijn Coenen /* For those flags, use the previous values instead */ 1528faf1d254SMartijn Coenen lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS; 1529faf1d254SMartijn Coenen /* For flags that can't be cleared, use previous values too */ 1530faf1d254SMartijn Coenen lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_CLEARABLE_FLAGS; 1531faf1d254SMartijn Coenen 1532b0bd158dSMartijn Coenen if (size_changed) { 1533b0bd158dSMartijn Coenen loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit, 1534b0bd158dSMartijn Coenen lo->lo_backing_file); 1535b0bd158dSMartijn Coenen loop_set_size(lo, new_size); 1536b040ad9cSArnd Bergmann } 1537541c742aSGuo Chao 1538dfaa2ef6SLukas Czerner loop_config_discard(lo); 15391da177e4SLinus Torvalds 15402e5ab5f3SMing Lei /* update dio if lo_offset or transfer is changed */ 15412e5ab5f3SMing Lei __loop_update_dio(lo, lo->use_dio); 15422e5ab5f3SMing Lei 1543550df5fdSJan Kara out_unfreeze: 1544ecdd0959SMing Lei blk_mq_unfreeze_queue(lo->lo_queue); 1545e02898b4SOmar Sandoval 1546faf1d254SMartijn Coenen if (!err && (lo->lo_flags & LO_FLAGS_PARTSCAN) && 1547faf1d254SMartijn Coenen !(prev_lo_flags & LO_FLAGS_PARTSCAN)) { 1548e02898b4SOmar Sandoval lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; 154985b0a54aSJan Kara partscan = true; 1550e02898b4SOmar Sandoval } 1551550df5fdSJan Kara out_unlock: 15526cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 155385b0a54aSJan Kara if (partscan) 15540384264eSChristoph Hellwig loop_reread_partitions(lo); 1555e02898b4SOmar Sandoval 1556ecdd0959SMing Lei return err; 15571da177e4SLinus Torvalds } 15581da177e4SLinus Torvalds 15591da177e4SLinus Torvalds static int 15601da177e4SLinus Torvalds loop_get_status(struct loop_device *lo, struct loop_info64 *info) 15611da177e4SLinus Torvalds { 1562b1ab5fa3STetsuo Handa struct path path; 15631da177e4SLinus Torvalds struct kstat stat; 15642d1d4c1eSOmar Sandoval int ret; 15651da177e4SLinus Torvalds 15666cc8e743SPavel Tatashin ret = mutex_lock_killable(&lo->lo_mutex); 15674a5ce9baSJan Kara if (ret) 15684a5ce9baSJan Kara return ret; 15692d1d4c1eSOmar Sandoval if (lo->lo_state != Lo_bound) { 15706cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 15711da177e4SLinus Torvalds return -ENXIO; 15722d1d4c1eSOmar Sandoval } 15732d1d4c1eSOmar Sandoval 15741da177e4SLinus Torvalds memset(info, 0, sizeof(*info)); 15751da177e4SLinus Torvalds info->lo_number = lo->lo_number; 15761da177e4SLinus Torvalds info->lo_offset = lo->lo_offset; 15771da177e4SLinus Torvalds info->lo_sizelimit = lo->lo_sizelimit; 15781da177e4SLinus Torvalds info->lo_flags = lo->lo_flags; 15791da177e4SLinus Torvalds memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE); 15801da177e4SLinus Torvalds memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE); 15811da177e4SLinus Torvalds info->lo_encrypt_type = 15821da177e4SLinus Torvalds lo->lo_encryption ? lo->lo_encryption->number : 0; 15831da177e4SLinus Torvalds if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) { 15841da177e4SLinus Torvalds info->lo_encrypt_key_size = lo->lo_encrypt_key_size; 15851da177e4SLinus Torvalds memcpy(info->lo_encrypt_key, lo->lo_encrypt_key, 15861da177e4SLinus Torvalds lo->lo_encrypt_key_size); 15871da177e4SLinus Torvalds } 15882d1d4c1eSOmar Sandoval 15896cc8e743SPavel Tatashin /* Drop lo_mutex while we call into the filesystem. */ 1590b1ab5fa3STetsuo Handa path = lo->lo_backing_file->f_path; 1591b1ab5fa3STetsuo Handa path_get(&path); 15926cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 1593b1ab5fa3STetsuo Handa ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT); 15942d1d4c1eSOmar Sandoval if (!ret) { 15952d1d4c1eSOmar Sandoval info->lo_device = huge_encode_dev(stat.dev); 15962d1d4c1eSOmar Sandoval info->lo_inode = stat.ino; 15972d1d4c1eSOmar Sandoval info->lo_rdevice = huge_encode_dev(stat.rdev); 15982d1d4c1eSOmar Sandoval } 1599b1ab5fa3STetsuo Handa path_put(&path); 16002d1d4c1eSOmar Sandoval return ret; 16011da177e4SLinus Torvalds } 16021da177e4SLinus Torvalds 16031da177e4SLinus Torvalds static void 16041da177e4SLinus Torvalds loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64) 16051da177e4SLinus Torvalds { 16061da177e4SLinus Torvalds memset(info64, 0, sizeof(*info64)); 16071da177e4SLinus Torvalds info64->lo_number = info->lo_number; 16081da177e4SLinus Torvalds info64->lo_device = info->lo_device; 16091da177e4SLinus Torvalds info64->lo_inode = info->lo_inode; 16101da177e4SLinus Torvalds info64->lo_rdevice = info->lo_rdevice; 16111da177e4SLinus Torvalds info64->lo_offset = info->lo_offset; 16121da177e4SLinus Torvalds info64->lo_sizelimit = 0; 16131da177e4SLinus Torvalds info64->lo_encrypt_type = info->lo_encrypt_type; 16141da177e4SLinus Torvalds info64->lo_encrypt_key_size = info->lo_encrypt_key_size; 16151da177e4SLinus Torvalds info64->lo_flags = info->lo_flags; 16161da177e4SLinus Torvalds info64->lo_init[0] = info->lo_init[0]; 16171da177e4SLinus Torvalds info64->lo_init[1] = info->lo_init[1]; 16181da177e4SLinus Torvalds if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 16191da177e4SLinus Torvalds memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE); 16201da177e4SLinus Torvalds else 16211da177e4SLinus Torvalds memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); 16221da177e4SLinus Torvalds memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE); 16231da177e4SLinus Torvalds } 16241da177e4SLinus Torvalds 16251da177e4SLinus Torvalds static int 16261da177e4SLinus Torvalds loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info) 16271da177e4SLinus Torvalds { 16281da177e4SLinus Torvalds memset(info, 0, sizeof(*info)); 16291da177e4SLinus Torvalds info->lo_number = info64->lo_number; 16301da177e4SLinus Torvalds info->lo_device = info64->lo_device; 16311da177e4SLinus Torvalds info->lo_inode = info64->lo_inode; 16321da177e4SLinus Torvalds info->lo_rdevice = info64->lo_rdevice; 16331da177e4SLinus Torvalds info->lo_offset = info64->lo_offset; 16341da177e4SLinus Torvalds info->lo_encrypt_type = info64->lo_encrypt_type; 16351da177e4SLinus Torvalds info->lo_encrypt_key_size = info64->lo_encrypt_key_size; 16361da177e4SLinus Torvalds info->lo_flags = info64->lo_flags; 16371da177e4SLinus Torvalds info->lo_init[0] = info64->lo_init[0]; 16381da177e4SLinus Torvalds info->lo_init[1] = info64->lo_init[1]; 16391da177e4SLinus Torvalds if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 16401da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 16411da177e4SLinus Torvalds else 16421da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); 16431da177e4SLinus Torvalds memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 16441da177e4SLinus Torvalds 16451da177e4SLinus Torvalds /* error in case values were truncated */ 16461da177e4SLinus Torvalds if (info->lo_device != info64->lo_device || 16471da177e4SLinus Torvalds info->lo_rdevice != info64->lo_rdevice || 16481da177e4SLinus Torvalds info->lo_inode != info64->lo_inode || 16491da177e4SLinus Torvalds info->lo_offset != info64->lo_offset) 16501da177e4SLinus Torvalds return -EOVERFLOW; 16511da177e4SLinus Torvalds 16521da177e4SLinus Torvalds return 0; 16531da177e4SLinus Torvalds } 16541da177e4SLinus Torvalds 16551da177e4SLinus Torvalds static int 16561da177e4SLinus Torvalds loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg) 16571da177e4SLinus Torvalds { 16581da177e4SLinus Torvalds struct loop_info info; 16591da177e4SLinus Torvalds struct loop_info64 info64; 16601da177e4SLinus Torvalds 16611da177e4SLinus Torvalds if (copy_from_user(&info, arg, sizeof (struct loop_info))) 16621da177e4SLinus Torvalds return -EFAULT; 16631da177e4SLinus Torvalds loop_info64_from_old(&info, &info64); 16641da177e4SLinus Torvalds return loop_set_status(lo, &info64); 16651da177e4SLinus Torvalds } 16661da177e4SLinus Torvalds 16671da177e4SLinus Torvalds static int 16681da177e4SLinus Torvalds loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg) 16691da177e4SLinus Torvalds { 16701da177e4SLinus Torvalds struct loop_info64 info64; 16711da177e4SLinus Torvalds 16721da177e4SLinus Torvalds if (copy_from_user(&info64, arg, sizeof (struct loop_info64))) 16731da177e4SLinus Torvalds return -EFAULT; 16741da177e4SLinus Torvalds return loop_set_status(lo, &info64); 16751da177e4SLinus Torvalds } 16761da177e4SLinus Torvalds 16771da177e4SLinus Torvalds static int 16781da177e4SLinus Torvalds loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) { 16791da177e4SLinus Torvalds struct loop_info info; 16801da177e4SLinus Torvalds struct loop_info64 info64; 1681bdac616dSOmar Sandoval int err; 16821da177e4SLinus Torvalds 16834a5ce9baSJan Kara if (!arg) 1684bdac616dSOmar Sandoval return -EINVAL; 16851da177e4SLinus Torvalds err = loop_get_status(lo, &info64); 16861da177e4SLinus Torvalds if (!err) 16871da177e4SLinus Torvalds err = loop_info64_to_old(&info64, &info); 16881da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info, sizeof(info))) 16891da177e4SLinus Torvalds err = -EFAULT; 16901da177e4SLinus Torvalds 16911da177e4SLinus Torvalds return err; 16921da177e4SLinus Torvalds } 16931da177e4SLinus Torvalds 16941da177e4SLinus Torvalds static int 16951da177e4SLinus Torvalds loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { 16961da177e4SLinus Torvalds struct loop_info64 info64; 1697bdac616dSOmar Sandoval int err; 16981da177e4SLinus Torvalds 16994a5ce9baSJan Kara if (!arg) 1700bdac616dSOmar Sandoval return -EINVAL; 17011da177e4SLinus Torvalds err = loop_get_status(lo, &info64); 17021da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info64, sizeof(info64))) 17031da177e4SLinus Torvalds err = -EFAULT; 17041da177e4SLinus Torvalds 17051da177e4SLinus Torvalds return err; 17061da177e4SLinus Torvalds } 17071da177e4SLinus Torvalds 170851001b7dSHannes Reinecke static int loop_set_capacity(struct loop_device *lo) 170953d66608SJ. R. Okajima { 17100a6ed1b5SMartijn Coenen loff_t size; 17110a6ed1b5SMartijn Coenen 171253d66608SJ. R. Okajima if (unlikely(lo->lo_state != Lo_bound)) 17137b0576a3SGuo Chao return -ENXIO; 171453d66608SJ. R. Okajima 17150a6ed1b5SMartijn Coenen size = get_loop_size(lo, lo->lo_backing_file); 17160a6ed1b5SMartijn Coenen loop_set_size(lo, size); 1717083a6a50SMartijn Coenen 1718083a6a50SMartijn Coenen return 0; 171953d66608SJ. R. Okajima } 172053d66608SJ. R. Okajima 1721ab1cb278SMing Lei static int loop_set_dio(struct loop_device *lo, unsigned long arg) 1722ab1cb278SMing Lei { 1723ab1cb278SMing Lei int error = -ENXIO; 1724ab1cb278SMing Lei if (lo->lo_state != Lo_bound) 1725ab1cb278SMing Lei goto out; 1726ab1cb278SMing Lei 1727ab1cb278SMing Lei __loop_update_dio(lo, !!arg); 1728ab1cb278SMing Lei if (lo->use_dio == !!arg) 1729ab1cb278SMing Lei return 0; 1730ab1cb278SMing Lei error = -EINVAL; 1731ab1cb278SMing Lei out: 1732ab1cb278SMing Lei return error; 1733ab1cb278SMing Lei } 1734ab1cb278SMing Lei 173589e4fdecSOmar Sandoval static int loop_set_block_size(struct loop_device *lo, unsigned long arg) 173689e4fdecSOmar Sandoval { 17375db470e2SJaegeuk Kim int err = 0; 17385db470e2SJaegeuk Kim 173989e4fdecSOmar Sandoval if (lo->lo_state != Lo_bound) 174089e4fdecSOmar Sandoval return -ENXIO; 174189e4fdecSOmar Sandoval 1742*af3c570fSXie Yongji err = blk_validate_block_size(arg); 17433448914eSMartijn Coenen if (err) 17443448914eSMartijn Coenen return err; 174589e4fdecSOmar Sandoval 17467e81f99aSMartijn Coenen if (lo->lo_queue->limits.logical_block_size == arg) 17477e81f99aSMartijn Coenen return 0; 17487e81f99aSMartijn Coenen 17495db470e2SJaegeuk Kim sync_blockdev(lo->lo_device); 1750f4bd34b1SZheng Bin invalidate_bdev(lo->lo_device); 17515db470e2SJaegeuk Kim 175289e4fdecSOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 175389e4fdecSOmar Sandoval 1754f4bd34b1SZheng Bin /* invalidate_bdev should have truncated all the pages */ 17557e81f99aSMartijn Coenen if (lo->lo_device->bd_inode->i_mapping->nrpages) { 17565db470e2SJaegeuk Kim err = -EAGAIN; 17575db470e2SJaegeuk Kim pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", 17585db470e2SJaegeuk Kim __func__, lo->lo_number, lo->lo_file_name, 17595db470e2SJaegeuk Kim lo->lo_device->bd_inode->i_mapping->nrpages); 17605db470e2SJaegeuk Kim goto out_unfreeze; 17615db470e2SJaegeuk Kim } 17625db470e2SJaegeuk Kim 176389e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, arg); 1764bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, arg); 1765bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, arg); 176689e4fdecSOmar Sandoval loop_update_dio(lo); 17675db470e2SJaegeuk Kim out_unfreeze: 176889e4fdecSOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 176989e4fdecSOmar Sandoval 17705db470e2SJaegeuk Kim return err; 177189e4fdecSOmar Sandoval } 177289e4fdecSOmar Sandoval 1773a1316544SJan Kara static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd, 1774a1316544SJan Kara unsigned long arg) 17751da177e4SLinus Torvalds { 17761da177e4SLinus Torvalds int err; 17771da177e4SLinus Torvalds 17786cc8e743SPavel Tatashin err = mutex_lock_killable(&lo->lo_mutex); 17793148ffbdSOmar Sandoval if (err) 1780a1316544SJan Kara return err; 17811da177e4SLinus Torvalds switch (cmd) { 178253d66608SJ. R. Okajima case LOOP_SET_CAPACITY: 178351001b7dSHannes Reinecke err = loop_set_capacity(lo); 178453d66608SJ. R. Okajima break; 1785ab1cb278SMing Lei case LOOP_SET_DIRECT_IO: 1786ab1cb278SMing Lei err = loop_set_dio(lo, arg); 1787ab1cb278SMing Lei break; 178889e4fdecSOmar Sandoval case LOOP_SET_BLOCK_SIZE: 178989e4fdecSOmar Sandoval err = loop_set_block_size(lo, arg); 179089e4fdecSOmar Sandoval break; 17911da177e4SLinus Torvalds default: 17921da177e4SLinus Torvalds err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL; 17931da177e4SLinus Torvalds } 17946cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 1795a1316544SJan Kara return err; 1796a1316544SJan Kara } 1797f028f3b2SNikanth Karthikesan 1798a1316544SJan Kara static int lo_ioctl(struct block_device *bdev, fmode_t mode, 1799a1316544SJan Kara unsigned int cmd, unsigned long arg) 1800a1316544SJan Kara { 1801a1316544SJan Kara struct loop_device *lo = bdev->bd_disk->private_data; 1802571fae6eSMartijn Coenen void __user *argp = (void __user *) arg; 1803a1316544SJan Kara int err; 1804a1316544SJan Kara 1805a1316544SJan Kara switch (cmd) { 18063448914eSMartijn Coenen case LOOP_SET_FD: { 18073448914eSMartijn Coenen /* 18083448914eSMartijn Coenen * Legacy case - pass in a zeroed out struct loop_config with 18093448914eSMartijn Coenen * only the file descriptor set , which corresponds with the 18103448914eSMartijn Coenen * default parameters we'd have used otherwise. 18113448914eSMartijn Coenen */ 18123448914eSMartijn Coenen struct loop_config config; 18133448914eSMartijn Coenen 18143448914eSMartijn Coenen memset(&config, 0, sizeof(config)); 18153448914eSMartijn Coenen config.fd = arg; 18163448914eSMartijn Coenen 18173448914eSMartijn Coenen return loop_configure(lo, mode, bdev, &config); 18183448914eSMartijn Coenen } 18193448914eSMartijn Coenen case LOOP_CONFIGURE: { 18203448914eSMartijn Coenen struct loop_config config; 18213448914eSMartijn Coenen 18223448914eSMartijn Coenen if (copy_from_user(&config, argp, sizeof(config))) 18233448914eSMartijn Coenen return -EFAULT; 18243448914eSMartijn Coenen 18253448914eSMartijn Coenen return loop_configure(lo, mode, bdev, &config); 18263448914eSMartijn Coenen } 1827a1316544SJan Kara case LOOP_CHANGE_FD: 1828c3710770SJan Kara return loop_change_fd(lo, bdev, arg); 1829a1316544SJan Kara case LOOP_CLR_FD: 18307ccd0791SJan Kara return loop_clr_fd(lo); 1831a1316544SJan Kara case LOOP_SET_STATUS: 1832a1316544SJan Kara err = -EPERM; 1833a1316544SJan Kara if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { 1834571fae6eSMartijn Coenen err = loop_set_status_old(lo, argp); 1835a1316544SJan Kara } 1836a1316544SJan Kara break; 1837a1316544SJan Kara case LOOP_GET_STATUS: 1838571fae6eSMartijn Coenen return loop_get_status_old(lo, argp); 1839a1316544SJan Kara case LOOP_SET_STATUS64: 1840a1316544SJan Kara err = -EPERM; 1841a1316544SJan Kara if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { 1842571fae6eSMartijn Coenen err = loop_set_status64(lo, argp); 1843a1316544SJan Kara } 1844a1316544SJan Kara break; 1845a1316544SJan Kara case LOOP_GET_STATUS64: 1846571fae6eSMartijn Coenen return loop_get_status64(lo, argp); 1847a1316544SJan Kara case LOOP_SET_CAPACITY: 1848a1316544SJan Kara case LOOP_SET_DIRECT_IO: 1849a1316544SJan Kara case LOOP_SET_BLOCK_SIZE: 1850a1316544SJan Kara if (!(mode & FMODE_WRITE) && !capable(CAP_SYS_ADMIN)) 1851a1316544SJan Kara return -EPERM; 1852df561f66SGustavo A. R. Silva fallthrough; 1853a1316544SJan Kara default: 1854a1316544SJan Kara err = lo_simple_ioctl(lo, cmd, arg); 1855a1316544SJan Kara break; 1856a1316544SJan Kara } 1857a1316544SJan Kara 18581da177e4SLinus Torvalds return err; 18591da177e4SLinus Torvalds } 18601da177e4SLinus Torvalds 1861863d5b82SDavid Howells #ifdef CONFIG_COMPAT 1862863d5b82SDavid Howells struct compat_loop_info { 1863863d5b82SDavid Howells compat_int_t lo_number; /* ioctl r/o */ 1864863d5b82SDavid Howells compat_dev_t lo_device; /* ioctl r/o */ 1865863d5b82SDavid Howells compat_ulong_t lo_inode; /* ioctl r/o */ 1866863d5b82SDavid Howells compat_dev_t lo_rdevice; /* ioctl r/o */ 1867863d5b82SDavid Howells compat_int_t lo_offset; 1868863d5b82SDavid Howells compat_int_t lo_encrypt_type; 1869863d5b82SDavid Howells compat_int_t lo_encrypt_key_size; /* ioctl w/o */ 1870863d5b82SDavid Howells compat_int_t lo_flags; /* ioctl r/o */ 1871863d5b82SDavid Howells char lo_name[LO_NAME_SIZE]; 1872863d5b82SDavid Howells unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ 1873863d5b82SDavid Howells compat_ulong_t lo_init[2]; 1874863d5b82SDavid Howells char reserved[4]; 1875863d5b82SDavid Howells }; 1876863d5b82SDavid Howells 1877863d5b82SDavid Howells /* 1878863d5b82SDavid Howells * Transfer 32-bit compatibility structure in userspace to 64-bit loop info 1879863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver 1880863d5b82SDavid Howells */ 1881863d5b82SDavid Howells static noinline int 1882ba674cfcSAl Viro loop_info64_from_compat(const struct compat_loop_info __user *arg, 1883863d5b82SDavid Howells struct loop_info64 *info64) 1884863d5b82SDavid Howells { 1885863d5b82SDavid Howells struct compat_loop_info info; 1886863d5b82SDavid Howells 1887863d5b82SDavid Howells if (copy_from_user(&info, arg, sizeof(info))) 1888863d5b82SDavid Howells return -EFAULT; 1889863d5b82SDavid Howells 1890863d5b82SDavid Howells memset(info64, 0, sizeof(*info64)); 1891863d5b82SDavid Howells info64->lo_number = info.lo_number; 1892863d5b82SDavid Howells info64->lo_device = info.lo_device; 1893863d5b82SDavid Howells info64->lo_inode = info.lo_inode; 1894863d5b82SDavid Howells info64->lo_rdevice = info.lo_rdevice; 1895863d5b82SDavid Howells info64->lo_offset = info.lo_offset; 1896863d5b82SDavid Howells info64->lo_sizelimit = 0; 1897863d5b82SDavid Howells info64->lo_encrypt_type = info.lo_encrypt_type; 1898863d5b82SDavid Howells info64->lo_encrypt_key_size = info.lo_encrypt_key_size; 1899863d5b82SDavid Howells info64->lo_flags = info.lo_flags; 1900863d5b82SDavid Howells info64->lo_init[0] = info.lo_init[0]; 1901863d5b82SDavid Howells info64->lo_init[1] = info.lo_init[1]; 1902863d5b82SDavid Howells if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1903863d5b82SDavid Howells memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE); 1904863d5b82SDavid Howells else 1905863d5b82SDavid Howells memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); 1906863d5b82SDavid Howells memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE); 1907863d5b82SDavid Howells return 0; 1908863d5b82SDavid Howells } 1909863d5b82SDavid Howells 1910863d5b82SDavid Howells /* 1911863d5b82SDavid Howells * Transfer 64-bit loop info to 32-bit compatibility structure in userspace 1912863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver 1913863d5b82SDavid Howells */ 1914863d5b82SDavid Howells static noinline int 1915863d5b82SDavid Howells loop_info64_to_compat(const struct loop_info64 *info64, 1916863d5b82SDavid Howells struct compat_loop_info __user *arg) 1917863d5b82SDavid Howells { 1918863d5b82SDavid Howells struct compat_loop_info info; 1919863d5b82SDavid Howells 1920863d5b82SDavid Howells memset(&info, 0, sizeof(info)); 1921863d5b82SDavid Howells info.lo_number = info64->lo_number; 1922863d5b82SDavid Howells info.lo_device = info64->lo_device; 1923863d5b82SDavid Howells info.lo_inode = info64->lo_inode; 1924863d5b82SDavid Howells info.lo_rdevice = info64->lo_rdevice; 1925863d5b82SDavid Howells info.lo_offset = info64->lo_offset; 1926863d5b82SDavid Howells info.lo_encrypt_type = info64->lo_encrypt_type; 1927863d5b82SDavid Howells info.lo_encrypt_key_size = info64->lo_encrypt_key_size; 1928863d5b82SDavid Howells info.lo_flags = info64->lo_flags; 1929863d5b82SDavid Howells info.lo_init[0] = info64->lo_init[0]; 1930863d5b82SDavid Howells info.lo_init[1] = info64->lo_init[1]; 1931863d5b82SDavid Howells if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1932863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 1933863d5b82SDavid Howells else 1934863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); 1935863d5b82SDavid Howells memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 1936863d5b82SDavid Howells 1937863d5b82SDavid Howells /* error in case values were truncated */ 1938863d5b82SDavid Howells if (info.lo_device != info64->lo_device || 1939863d5b82SDavid Howells info.lo_rdevice != info64->lo_rdevice || 1940863d5b82SDavid Howells info.lo_inode != info64->lo_inode || 1941863d5b82SDavid Howells info.lo_offset != info64->lo_offset || 1942863d5b82SDavid Howells info.lo_init[0] != info64->lo_init[0] || 1943863d5b82SDavid Howells info.lo_init[1] != info64->lo_init[1]) 1944863d5b82SDavid Howells return -EOVERFLOW; 1945863d5b82SDavid Howells 1946863d5b82SDavid Howells if (copy_to_user(arg, &info, sizeof(info))) 1947863d5b82SDavid Howells return -EFAULT; 1948863d5b82SDavid Howells return 0; 1949863d5b82SDavid Howells } 1950863d5b82SDavid Howells 1951863d5b82SDavid Howells static int 1952863d5b82SDavid Howells loop_set_status_compat(struct loop_device *lo, 1953863d5b82SDavid Howells const struct compat_loop_info __user *arg) 1954863d5b82SDavid Howells { 1955863d5b82SDavid Howells struct loop_info64 info64; 1956863d5b82SDavid Howells int ret; 1957863d5b82SDavid Howells 1958863d5b82SDavid Howells ret = loop_info64_from_compat(arg, &info64); 1959863d5b82SDavid Howells if (ret < 0) 1960863d5b82SDavid Howells return ret; 1961863d5b82SDavid Howells return loop_set_status(lo, &info64); 1962863d5b82SDavid Howells } 1963863d5b82SDavid Howells 1964863d5b82SDavid Howells static int 1965863d5b82SDavid Howells loop_get_status_compat(struct loop_device *lo, 1966863d5b82SDavid Howells struct compat_loop_info __user *arg) 1967863d5b82SDavid Howells { 1968863d5b82SDavid Howells struct loop_info64 info64; 1969bdac616dSOmar Sandoval int err; 1970863d5b82SDavid Howells 19714a5ce9baSJan Kara if (!arg) 1972bdac616dSOmar Sandoval return -EINVAL; 1973863d5b82SDavid Howells err = loop_get_status(lo, &info64); 1974863d5b82SDavid Howells if (!err) 1975863d5b82SDavid Howells err = loop_info64_to_compat(&info64, arg); 1976863d5b82SDavid Howells return err; 1977863d5b82SDavid Howells } 1978863d5b82SDavid Howells 1979bb214884SAl Viro static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode, 1980bb214884SAl Viro unsigned int cmd, unsigned long arg) 1981863d5b82SDavid Howells { 1982bb214884SAl Viro struct loop_device *lo = bdev->bd_disk->private_data; 1983863d5b82SDavid Howells int err; 1984863d5b82SDavid Howells 1985863d5b82SDavid Howells switch(cmd) { 1986863d5b82SDavid Howells case LOOP_SET_STATUS: 19873148ffbdSOmar Sandoval err = loop_set_status_compat(lo, 19883148ffbdSOmar Sandoval (const struct compat_loop_info __user *)arg); 1989863d5b82SDavid Howells break; 1990863d5b82SDavid Howells case LOOP_GET_STATUS: 19913148ffbdSOmar Sandoval err = loop_get_status_compat(lo, 19923148ffbdSOmar Sandoval (struct compat_loop_info __user *)arg); 1993863d5b82SDavid Howells break; 199453d66608SJ. R. Okajima case LOOP_SET_CAPACITY: 1995863d5b82SDavid Howells case LOOP_CLR_FD: 1996863d5b82SDavid Howells case LOOP_GET_STATUS64: 1997863d5b82SDavid Howells case LOOP_SET_STATUS64: 19983448914eSMartijn Coenen case LOOP_CONFIGURE: 1999863d5b82SDavid Howells arg = (unsigned long) compat_ptr(arg); 2000df561f66SGustavo A. R. Silva fallthrough; 2001863d5b82SDavid Howells case LOOP_SET_FD: 2002863d5b82SDavid Howells case LOOP_CHANGE_FD: 20039fea4b39SEvan Green case LOOP_SET_BLOCK_SIZE: 2004fdbe4eeeSAlessio Balsini case LOOP_SET_DIRECT_IO: 2005bb214884SAl Viro err = lo_ioctl(bdev, mode, cmd, arg); 2006863d5b82SDavid Howells break; 2007863d5b82SDavid Howells default: 2008863d5b82SDavid Howells err = -ENOIOCTLCMD; 2009863d5b82SDavid Howells break; 2010863d5b82SDavid Howells } 2011863d5b82SDavid Howells return err; 2012863d5b82SDavid Howells } 2013863d5b82SDavid Howells #endif 2014863d5b82SDavid Howells 2015bb214884SAl Viro static int lo_open(struct block_device *bdev, fmode_t mode) 20161da177e4SLinus Torvalds { 2017990e7811SChristoph Hellwig struct loop_device *lo = bdev->bd_disk->private_data; 20180a42e99bSJan Kara int err; 2019770fe30aSKay Sievers 20206cc8e743SPavel Tatashin err = mutex_lock_killable(&lo->lo_mutex); 20216cc8e743SPavel Tatashin if (err) 2022770fe30aSKay Sievers return err; 2023990e7811SChristoph Hellwig if (lo->lo_state == Lo_deleting) 2024990e7811SChristoph Hellwig err = -ENXIO; 2025990e7811SChristoph Hellwig else 20266cc8e743SPavel Tatashin atomic_inc(&lo->lo_refcnt); 20276cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 2028990e7811SChristoph Hellwig return err; 20291da177e4SLinus Torvalds } 20301da177e4SLinus Torvalds 2031967d1dc1SJan Kara static void lo_release(struct gendisk *disk, fmode_t mode) 20321da177e4SLinus Torvalds { 20336cc8e743SPavel Tatashin struct loop_device *lo = disk->private_data; 20341da177e4SLinus Torvalds 20356cc8e743SPavel Tatashin mutex_lock(&lo->lo_mutex); 2036f8933667SMing Lei if (atomic_dec_return(&lo->lo_refcnt)) 20370a42e99bSJan Kara goto out_unlock; 2038f8933667SMing Lei 203914f27939SMilan Broz if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) { 2040a2505b79SJan Kara if (lo->lo_state != Lo_bound) 2041a2505b79SJan Kara goto out_unlock; 2042a2505b79SJan Kara lo->lo_state = Lo_rundown; 20436cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 204414f27939SMilan Broz /* 204514f27939SMilan Broz * In autoclear mode, stop the loop thread 204614f27939SMilan Broz * and remove configuration after last close. 204714f27939SMilan Broz */ 20480da03cabSJan Kara __loop_clr_fd(lo, true); 20490a42e99bSJan Kara return; 205043cade80SOmar Sandoval } else if (lo->lo_state == Lo_bound) { 205114f27939SMilan Broz /* 205214f27939SMilan Broz * Otherwise keep thread (if running) and config, 205314f27939SMilan Broz * but flush possible ongoing bios in thread. 205414f27939SMilan Broz */ 205543cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue); 205643cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue); 205714f27939SMilan Broz } 205896c58655SDavid Woodhouse 20590a42e99bSJan Kara out_unlock: 20606cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 2061ae665016SLinus Torvalds } 2062ae665016SLinus Torvalds 206383d5cde4SAlexey Dobriyan static const struct block_device_operations lo_fops = { 20641da177e4SLinus Torvalds .owner = THIS_MODULE, 2065bb214884SAl Viro .open = lo_open, 2066bb214884SAl Viro .release = lo_release, 2067bb214884SAl Viro .ioctl = lo_ioctl, 2068863d5b82SDavid Howells #ifdef CONFIG_COMPAT 2069bb214884SAl Viro .compat_ioctl = lo_compat_ioctl, 2070863d5b82SDavid Howells #endif 20711da177e4SLinus Torvalds }; 20721da177e4SLinus Torvalds 20731da177e4SLinus Torvalds /* 20741da177e4SLinus Torvalds * And now the modules code and kernel interface. 20751da177e4SLinus Torvalds */ 207673285082SKen Chen static int max_loop; 20775657a819SJoe Perches module_param(max_loop, int, 0444); 2078a47653fcSKen Chen MODULE_PARM_DESC(max_loop, "Maximum number of loop devices"); 20795657a819SJoe Perches module_param(max_part, int, 0444); 2080476a4813SLaurent Vivier MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device"); 20811da177e4SLinus Torvalds MODULE_LICENSE("GPL"); 20821da177e4SLinus Torvalds MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR); 20831da177e4SLinus Torvalds 20841da177e4SLinus Torvalds int loop_register_transfer(struct loop_func_table *funcs) 20851da177e4SLinus Torvalds { 20861da177e4SLinus Torvalds unsigned int n = funcs->number; 20871da177e4SLinus Torvalds 20881da177e4SLinus Torvalds if (n >= MAX_LO_CRYPT || xfer_funcs[n]) 20891da177e4SLinus Torvalds return -EINVAL; 20901da177e4SLinus Torvalds xfer_funcs[n] = funcs; 20911da177e4SLinus Torvalds return 0; 20921da177e4SLinus Torvalds } 20931da177e4SLinus Torvalds 20941da177e4SLinus Torvalds int loop_unregister_transfer(int number) 20951da177e4SLinus Torvalds { 20961da177e4SLinus Torvalds unsigned int n = number; 20971da177e4SLinus Torvalds struct loop_func_table *xfer; 20981da177e4SLinus Torvalds 20991da177e4SLinus Torvalds if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL) 21001da177e4SLinus Torvalds return -EINVAL; 21011c500ad7STetsuo Handa /* 21021c500ad7STetsuo Handa * This function is called from only cleanup_cryptoloop(). 21031c500ad7STetsuo Handa * Given that each loop device that has a transfer enabled holds a 21041c500ad7STetsuo Handa * reference to the module implementing it we should never get here 21051c500ad7STetsuo Handa * with a transfer that is set (unless forced module unloading is 21061c500ad7STetsuo Handa * requested). Thus, check module's refcount and warn if this is 21071c500ad7STetsuo Handa * not a clean unloading. 21081c500ad7STetsuo Handa */ 21091c500ad7STetsuo Handa #ifdef CONFIG_MODULE_UNLOAD 21101c500ad7STetsuo Handa if (xfer->owner && module_refcount(xfer->owner) != -1) 21111c500ad7STetsuo Handa pr_err("Danger! Unregistering an in use transfer function.\n"); 21121c500ad7STetsuo Handa #endif 21131da177e4SLinus Torvalds 21141da177e4SLinus Torvalds xfer_funcs[n] = NULL; 21151da177e4SLinus Torvalds return 0; 21161da177e4SLinus Torvalds } 21171da177e4SLinus Torvalds 21181da177e4SLinus Torvalds EXPORT_SYMBOL(loop_register_transfer); 21191da177e4SLinus Torvalds EXPORT_SYMBOL(loop_unregister_transfer); 21201da177e4SLinus Torvalds 2121fc17b653SChristoph Hellwig static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, 2122b5dd2f60SMing Lei const struct blk_mq_queue_data *bd) 2123b5dd2f60SMing Lei { 21241894e916SJens Axboe struct request *rq = bd->rq; 21251894e916SJens Axboe struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 21261894e916SJens Axboe struct loop_device *lo = rq->q->queuedata; 2127b5dd2f60SMing Lei 21281894e916SJens Axboe blk_mq_start_request(rq); 2129b5dd2f60SMing Lei 2130f4aa4c7bSMing Lei if (lo->lo_state != Lo_bound) 2131fc17b653SChristoph Hellwig return BLK_STS_IOERR; 2132f4aa4c7bSMing Lei 21331894e916SJens Axboe switch (req_op(rq)) { 2134f0225cacSChristoph Hellwig case REQ_OP_FLUSH: 2135f0225cacSChristoph Hellwig case REQ_OP_DISCARD: 213619372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES: 2137bc07c10aSMing Lei cmd->use_aio = false; 2138f0225cacSChristoph Hellwig break; 2139f0225cacSChristoph Hellwig default: 2140f0225cacSChristoph Hellwig cmd->use_aio = lo->use_dio; 2141f0225cacSChristoph Hellwig break; 2142f0225cacSChristoph Hellwig } 2143bc07c10aSMing Lei 2144d4478e92SShaohua Li /* always use the first bio's css */ 2145c74d40e8SDan Schatzberg cmd->blkcg_css = NULL; 2146c74d40e8SDan Schatzberg cmd->memcg_css = NULL; 21470b508bc9SShaohua Li #ifdef CONFIG_BLK_CGROUP 2148c74d40e8SDan Schatzberg if (rq->bio && rq->bio->bi_blkg) { 2149c74d40e8SDan Schatzberg cmd->blkcg_css = &bio_blkcg(rq->bio)->css; 2150c74d40e8SDan Schatzberg #ifdef CONFIG_MEMCG 2151c74d40e8SDan Schatzberg cmd->memcg_css = 2152c74d40e8SDan Schatzberg cgroup_get_e_css(cmd->blkcg_css->cgroup, 2153c74d40e8SDan Schatzberg &memory_cgrp_subsys); 2154d4478e92SShaohua Li #endif 2155c74d40e8SDan Schatzberg } 2156c74d40e8SDan Schatzberg #endif 215787579e9bSDan Schatzberg loop_queue_work(lo, cmd); 2158b5dd2f60SMing Lei 2159fc17b653SChristoph Hellwig return BLK_STS_OK; 2160b5dd2f60SMing Lei } 2161b5dd2f60SMing Lei 2162b5dd2f60SMing Lei static void loop_handle_cmd(struct loop_cmd *cmd) 2163b5dd2f60SMing Lei { 21641894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd); 21651894e916SJens Axboe const bool write = op_is_write(req_op(rq)); 21661894e916SJens Axboe struct loop_device *lo = rq->q->queuedata; 2167f4829a9bSChristoph Hellwig int ret = 0; 2168c74d40e8SDan Schatzberg struct mem_cgroup *old_memcg = NULL; 2169b5dd2f60SMing Lei 2170f4829a9bSChristoph Hellwig if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) { 2171f4829a9bSChristoph Hellwig ret = -EIO; 2172b5dd2f60SMing Lei goto failed; 2173f4829a9bSChristoph Hellwig } 2174b5dd2f60SMing Lei 2175c74d40e8SDan Schatzberg if (cmd->blkcg_css) 2176c74d40e8SDan Schatzberg kthread_associate_blkcg(cmd->blkcg_css); 2177c74d40e8SDan Schatzberg if (cmd->memcg_css) 2178c74d40e8SDan Schatzberg old_memcg = set_active_memcg( 2179c74d40e8SDan Schatzberg mem_cgroup_from_css(cmd->memcg_css)); 2180c74d40e8SDan Schatzberg 21811894e916SJens Axboe ret = do_req_filebacked(lo, rq); 2182c74d40e8SDan Schatzberg 2183c74d40e8SDan Schatzberg if (cmd->blkcg_css) 2184c74d40e8SDan Schatzberg kthread_associate_blkcg(NULL); 2185c74d40e8SDan Schatzberg 2186c74d40e8SDan Schatzberg if (cmd->memcg_css) { 2187c74d40e8SDan Schatzberg set_active_memcg(old_memcg); 2188c74d40e8SDan Schatzberg css_put(cmd->memcg_css); 2189c74d40e8SDan Schatzberg } 2190b5dd2f60SMing Lei failed: 2191bc07c10aSMing Lei /* complete non-aio request */ 2192fe2cb290SChristoph Hellwig if (!cmd->use_aio || ret) { 21938cd55087SEvan Green if (ret == -EOPNOTSUPP) 21948cd55087SEvan Green cmd->ret = ret; 21958cd55087SEvan Green else 2196fe2cb290SChristoph Hellwig cmd->ret = ret ? -EIO : 0; 219715f73f5bSChristoph Hellwig if (likely(!blk_should_fake_timeout(rq->q))) 21981894e916SJens Axboe blk_mq_complete_request(rq); 2199fe2cb290SChristoph Hellwig } 2200b5dd2f60SMing Lei } 2201b5dd2f60SMing Lei 220287579e9bSDan Schatzberg static void loop_set_timer(struct loop_device *lo) 2203b5dd2f60SMing Lei { 220487579e9bSDan Schatzberg timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT); 2205b5dd2f60SMing Lei } 2206b5dd2f60SMing Lei 220787579e9bSDan Schatzberg static void loop_process_work(struct loop_worker *worker, 220887579e9bSDan Schatzberg struct list_head *cmd_list, struct loop_device *lo) 2209b5dd2f60SMing Lei { 221087579e9bSDan Schatzberg int orig_flags = current->flags; 221187579e9bSDan Schatzberg struct loop_cmd *cmd; 2212b5dd2f60SMing Lei 221387579e9bSDan Schatzberg current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; 221487579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock); 221587579e9bSDan Schatzberg while (!list_empty(cmd_list)) { 221687579e9bSDan Schatzberg cmd = container_of( 221787579e9bSDan Schatzberg cmd_list->next, struct loop_cmd, list_entry); 221887579e9bSDan Schatzberg list_del(cmd_list->next); 221987579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock); 222087579e9bSDan Schatzberg 222187579e9bSDan Schatzberg loop_handle_cmd(cmd); 222287579e9bSDan Schatzberg cond_resched(); 222387579e9bSDan Schatzberg 222487579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock); 222587579e9bSDan Schatzberg } 222687579e9bSDan Schatzberg 222787579e9bSDan Schatzberg /* 222887579e9bSDan Schatzberg * We only add to the idle list if there are no pending cmds 222987579e9bSDan Schatzberg * *and* the worker will not run again which ensures that it 223087579e9bSDan Schatzberg * is safe to free any worker on the idle list 223187579e9bSDan Schatzberg */ 223287579e9bSDan Schatzberg if (worker && !work_pending(&worker->work)) { 223387579e9bSDan Schatzberg worker->last_ran_at = jiffies; 223487579e9bSDan Schatzberg list_add_tail(&worker->idle_list, &lo->idle_worker_list); 223587579e9bSDan Schatzberg loop_set_timer(lo); 223687579e9bSDan Schatzberg } 223787579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock); 223887579e9bSDan Schatzberg current->flags = orig_flags; 223987579e9bSDan Schatzberg } 224087579e9bSDan Schatzberg 224187579e9bSDan Schatzberg static void loop_workfn(struct work_struct *work) 224287579e9bSDan Schatzberg { 224387579e9bSDan Schatzberg struct loop_worker *worker = 224487579e9bSDan Schatzberg container_of(work, struct loop_worker, work); 224587579e9bSDan Schatzberg loop_process_work(worker, &worker->cmd_list, worker->lo); 224687579e9bSDan Schatzberg } 224787579e9bSDan Schatzberg 224887579e9bSDan Schatzberg static void loop_rootcg_workfn(struct work_struct *work) 224987579e9bSDan Schatzberg { 225087579e9bSDan Schatzberg struct loop_device *lo = 225187579e9bSDan Schatzberg container_of(work, struct loop_device, rootcg_work); 225287579e9bSDan Schatzberg loop_process_work(NULL, &lo->rootcg_cmd_list, lo); 225387579e9bSDan Schatzberg } 225487579e9bSDan Schatzberg 225587579e9bSDan Schatzberg static void loop_free_idle_workers(struct timer_list *timer) 225687579e9bSDan Schatzberg { 225787579e9bSDan Schatzberg struct loop_device *lo = container_of(timer, struct loop_device, timer); 225887579e9bSDan Schatzberg struct loop_worker *pos, *worker; 225987579e9bSDan Schatzberg 226087579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock); 226187579e9bSDan Schatzberg list_for_each_entry_safe(worker, pos, &lo->idle_worker_list, 226287579e9bSDan Schatzberg idle_list) { 226387579e9bSDan Schatzberg if (time_is_after_jiffies(worker->last_ran_at + 226487579e9bSDan Schatzberg LOOP_IDLE_WORKER_TIMEOUT)) 226587579e9bSDan Schatzberg break; 226687579e9bSDan Schatzberg list_del(&worker->idle_list); 226787579e9bSDan Schatzberg rb_erase(&worker->rb_node, &lo->worker_tree); 2268c74d40e8SDan Schatzberg css_put(worker->blkcg_css); 226987579e9bSDan Schatzberg kfree(worker); 227087579e9bSDan Schatzberg } 227187579e9bSDan Schatzberg if (!list_empty(&lo->idle_worker_list)) 227287579e9bSDan Schatzberg loop_set_timer(lo); 227387579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock); 2274b5dd2f60SMing Lei } 2275b5dd2f60SMing Lei 2276f363b089SEric Biggers static const struct blk_mq_ops loop_mq_ops = { 2277b5dd2f60SMing Lei .queue_rq = loop_queue_rq, 2278fe2cb290SChristoph Hellwig .complete = lo_complete_rq, 2279b5dd2f60SMing Lei }; 2280b5dd2f60SMing Lei 2281d6da83d0SChristoph Hellwig static int loop_add(int i) 22821da177e4SLinus Torvalds { 228373285082SKen Chen struct loop_device *lo; 228473285082SKen Chen struct gendisk *disk; 228534dd82afSKay Sievers int err; 22861da177e4SLinus Torvalds 228734dd82afSKay Sievers err = -ENOMEM; 228868d740d7SSilva Paulo lo = kzalloc(sizeof(*lo), GFP_KERNEL); 228968d740d7SSilva Paulo if (!lo) 229073285082SKen Chen goto out; 2291ef7e7c82SMikulas Patocka lo->lo_state = Lo_unbound; 2292ef7e7c82SMikulas Patocka 229318d1f200SChristoph Hellwig err = mutex_lock_killable(&loop_ctl_mutex); 229418d1f200SChristoph Hellwig if (err) 229518d1f200SChristoph Hellwig goto out_free_dev; 229618d1f200SChristoph Hellwig 2297c718aa65STejun Heo /* allocate id, if @id >= 0, we're requesting that specific id */ 229834dd82afSKay Sievers if (i >= 0) { 2299c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL); 2300c718aa65STejun Heo if (err == -ENOSPC) 230134dd82afSKay Sievers err = -EEXIST; 230234dd82afSKay Sievers } else { 2303c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL); 230434dd82afSKay Sievers } 23051c500ad7STetsuo Handa mutex_unlock(&loop_ctl_mutex); 230634dd82afSKay Sievers if (err < 0) 23071c500ad7STetsuo Handa goto out_free_dev; 2308c718aa65STejun Heo i = err; 23091da177e4SLinus Torvalds 2310183cfb57SWei Yongjun err = -ENOMEM; 2311b5dd2f60SMing Lei lo->tag_set.ops = &loop_mq_ops; 2312b5dd2f60SMing Lei lo->tag_set.nr_hw_queues = 1; 2313b5dd2f60SMing Lei lo->tag_set.queue_depth = 128; 2314b5dd2f60SMing Lei lo->tag_set.numa_node = NUMA_NO_NODE; 2315b5dd2f60SMing Lei lo->tag_set.cmd_size = sizeof(struct loop_cmd); 23162112f5c1SBart Van Assche lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING | 23172112f5c1SBart Van Assche BLK_MQ_F_NO_SCHED_BY_DEFAULT; 2318b5dd2f60SMing Lei lo->tag_set.driver_data = lo; 2319b5dd2f60SMing Lei 2320b5dd2f60SMing Lei err = blk_mq_alloc_tag_set(&lo->tag_set); 2321b5dd2f60SMing Lei if (err) 23223ec981e3SMikulas Patocka goto out_free_idr; 232373285082SKen Chen 23241c99502fSChristoph Hellwig disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, lo); 23251c99502fSChristoph Hellwig if (IS_ERR(disk)) { 23261c99502fSChristoph Hellwig err = PTR_ERR(disk); 2327b5dd2f60SMing Lei goto out_cleanup_tags; 2328b5dd2f60SMing Lei } 23291c99502fSChristoph Hellwig lo->lo_queue = lo->lo_disk->queue; 2330ef7e7c82SMikulas Patocka 233154bb0adeSShaohua Li blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS); 233240326d8aSShaohua Li 23335b5e20f4SMing Lei /* 233440326d8aSShaohua Li * By default, we do buffer IO, so it doesn't make sense to enable 233540326d8aSShaohua Li * merge because the I/O submitted to backing file is handled page by 233640326d8aSShaohua Li * page. For directio mode, merge does help to dispatch bigger request 233740326d8aSShaohua Li * to underlayer disk. We will enable merge once directio is enabled. 23385b5e20f4SMing Lei */ 23398b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 23405b5e20f4SMing Lei 2341e03c8dd1SKay Sievers /* 2342e03c8dd1SKay Sievers * Disable partition scanning by default. The in-kernel partition 2343e03c8dd1SKay Sievers * scanning can be requested individually per-device during its 2344e03c8dd1SKay Sievers * setup. Userspace can always add and remove partitions from all 2345e03c8dd1SKay Sievers * devices. The needed partition minors are allocated from the 2346e03c8dd1SKay Sievers * extended minor space, the main loop device numbers will continue 2347e03c8dd1SKay Sievers * to match the loop minors, regardless of the number of partitions 2348e03c8dd1SKay Sievers * used. 2349e03c8dd1SKay Sievers * 2350e03c8dd1SKay Sievers * If max_part is given, partition scanning is globally enabled for 2351e03c8dd1SKay Sievers * all loop devices. The minors for the main loop devices will be 2352e03c8dd1SKay Sievers * multiples of max_part. 2353e03c8dd1SKay Sievers * 2354e03c8dd1SKay Sievers * Note: Global-for-all-devices, set-only-at-init, read-only module 2355e03c8dd1SKay Sievers * parameteters like 'max_loop' and 'max_part' make things needlessly 2356e03c8dd1SKay Sievers * complicated, are too static, inflexible and may surprise 2357e03c8dd1SKay Sievers * userspace tools. Parameters like this in general should be avoided. 2358e03c8dd1SKay Sievers */ 2359e03c8dd1SKay Sievers if (!part_shift) 2360e03c8dd1SKay Sievers disk->flags |= GENHD_FL_NO_PART_SCAN; 2361e03c8dd1SKay Sievers disk->flags |= GENHD_FL_EXT_DEVT; 2362f8933667SMing Lei atomic_set(&lo->lo_refcnt, 0); 23636cc8e743SPavel Tatashin mutex_init(&lo->lo_mutex); 23641da177e4SLinus Torvalds lo->lo_number = i; 23651da177e4SLinus Torvalds spin_lock_init(&lo->lo_lock); 236687579e9bSDan Schatzberg spin_lock_init(&lo->lo_work_lock); 23671da177e4SLinus Torvalds disk->major = LOOP_MAJOR; 2368476a4813SLaurent Vivier disk->first_minor = i << part_shift; 23691c99502fSChristoph Hellwig disk->minors = 1 << part_shift; 23701da177e4SLinus Torvalds disk->fops = &lo_fops; 23711da177e4SLinus Torvalds disk->private_data = lo; 23721da177e4SLinus Torvalds disk->queue = lo->lo_queue; 23739f65c489SMatteo Croce disk->events = DISK_EVENT_MEDIA_CHANGE; 23749f65c489SMatteo Croce disk->event_flags = DISK_EVENT_FLAG_UEVENT; 237573285082SKen Chen sprintf(disk->disk_name, "loop%d", i); 23761c500ad7STetsuo Handa /* Make this loop device reachable from pathname. */ 237734dd82afSKay Sievers add_disk(disk); 23781c500ad7STetsuo Handa /* Show this loop device. */ 23791c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex); 23801c500ad7STetsuo Handa lo->idr_visible = true; 238118d1f200SChristoph Hellwig mutex_unlock(&loop_ctl_mutex); 238218d1f200SChristoph Hellwig return i; 238373285082SKen Chen 2384b5dd2f60SMing Lei out_cleanup_tags: 2385b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set); 23863ec981e3SMikulas Patocka out_free_idr: 23871c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex); 23883ec981e3SMikulas Patocka idr_remove(&loop_index_idr, i); 238918d1f200SChristoph Hellwig mutex_unlock(&loop_ctl_mutex); 239073285082SKen Chen out_free_dev: 239173285082SKen Chen kfree(lo); 239273285082SKen Chen out: 239334dd82afSKay Sievers return err; 23941da177e4SLinus Torvalds } 23951da177e4SLinus Torvalds 239634dd82afSKay Sievers static void loop_remove(struct loop_device *lo) 239773285082SKen Chen { 23981c500ad7STetsuo Handa /* Make this loop device unreachable from pathname. */ 23996cd18e71SNeilBrown del_gendisk(lo->lo_disk); 24001c99502fSChristoph Hellwig blk_cleanup_disk(lo->lo_disk); 2401b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set); 24021c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex); 24031c500ad7STetsuo Handa idr_remove(&loop_index_idr, lo->lo_number); 24041c500ad7STetsuo Handa mutex_unlock(&loop_ctl_mutex); 24051c500ad7STetsuo Handa /* There is no route which can find this loop device. */ 24066cc8e743SPavel Tatashin mutex_destroy(&lo->lo_mutex); 240773285082SKen Chen kfree(lo); 240873285082SKen Chen } 240973285082SKen Chen 24108410d38cSChristoph Hellwig static void loop_probe(dev_t dev) 241173285082SKen Chen { 24128410d38cSChristoph Hellwig int idx = MINOR(dev) >> part_shift; 24138410d38cSChristoph Hellwig 24148410d38cSChristoph Hellwig if (max_loop && idx >= max_loop) 24158410d38cSChristoph Hellwig return; 2416d6da83d0SChristoph Hellwig loop_add(idx); 2417f9d10764SChristoph Hellwig } 2418f9d10764SChristoph Hellwig 2419f9d10764SChristoph Hellwig static int loop_control_remove(int idx) 2420770fe30aSKay Sievers { 2421770fe30aSKay Sievers struct loop_device *lo; 24220a42e99bSJan Kara int ret; 2423e5d66a10SChristoph Hellwig 2424e5d66a10SChristoph Hellwig if (idx < 0) { 2425e5d66a10SChristoph Hellwig pr_warn("deleting an unspecified loop device is not supported.\n"); 2426e5d66a10SChristoph Hellwig return -EINVAL; 2427e5d66a10SChristoph Hellwig } 2428770fe30aSKay Sievers 24291c500ad7STetsuo Handa /* Hide this loop device for serialization. */ 24300a42e99bSJan Kara ret = mutex_lock_killable(&loop_ctl_mutex); 24310a42e99bSJan Kara if (ret) 24320a42e99bSJan Kara return ret; 2433b9848081SChristoph Hellwig lo = idr_find(&loop_index_idr, idx); 24341c500ad7STetsuo Handa if (!lo || !lo->idr_visible) 2435b9848081SChristoph Hellwig ret = -ENODEV; 24361c500ad7STetsuo Handa else 24371c500ad7STetsuo Handa lo->idr_visible = false; 24381c500ad7STetsuo Handa mutex_unlock(&loop_ctl_mutex); 24391c500ad7STetsuo Handa if (ret) 24401c500ad7STetsuo Handa return ret; 2441f9d10764SChristoph Hellwig 24421c500ad7STetsuo Handa /* Check whether this loop device can be removed. */ 24436cc8e743SPavel Tatashin ret = mutex_lock_killable(&lo->lo_mutex); 24446cc8e743SPavel Tatashin if (ret) 24451c500ad7STetsuo Handa goto mark_visible; 2446f9d10764SChristoph Hellwig if (lo->lo_state != Lo_unbound || 2447f9d10764SChristoph Hellwig atomic_read(&lo->lo_refcnt) > 0) { 24486cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 2449770fe30aSKay Sievers ret = -EBUSY; 24501c500ad7STetsuo Handa goto mark_visible; 2451770fe30aSKay Sievers } 24521c500ad7STetsuo Handa /* Mark this loop device no longer open()-able. */ 2453990e7811SChristoph Hellwig lo->lo_state = Lo_deleting; 24546cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex); 2455f9d10764SChristoph Hellwig 2456770fe30aSKay Sievers loop_remove(lo); 24571c500ad7STetsuo Handa return 0; 24581c500ad7STetsuo Handa 24591c500ad7STetsuo Handa mark_visible: 24601c500ad7STetsuo Handa /* Show this loop device again. */ 24611c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex); 24621c500ad7STetsuo Handa lo->idr_visible = true; 2463f9d10764SChristoph Hellwig mutex_unlock(&loop_ctl_mutex); 2464f9d10764SChristoph Hellwig return ret; 2465770fe30aSKay Sievers } 2466f9d10764SChristoph Hellwig 2467f9d10764SChristoph Hellwig static int loop_control_get_free(int idx) 2468f9d10764SChristoph Hellwig { 2469f9d10764SChristoph Hellwig struct loop_device *lo; 2470b9848081SChristoph Hellwig int id, ret; 2471f9d10764SChristoph Hellwig 2472f9d10764SChristoph Hellwig ret = mutex_lock_killable(&loop_ctl_mutex); 2473f9d10764SChristoph Hellwig if (ret) 2474f9d10764SChristoph Hellwig return ret; 2475b9848081SChristoph Hellwig idr_for_each_entry(&loop_index_idr, lo, id) { 24761c500ad7STetsuo Handa /* Hitting a race results in creating a new loop device which is harmless. */ 24771c500ad7STetsuo Handa if (lo->idr_visible && data_race(lo->lo_state) == Lo_unbound) 2478b9848081SChristoph Hellwig goto found; 2479b9848081SChristoph Hellwig } 24800a42e99bSJan Kara mutex_unlock(&loop_ctl_mutex); 248118d1f200SChristoph Hellwig return loop_add(-1); 2482b9848081SChristoph Hellwig found: 2483b9848081SChristoph Hellwig mutex_unlock(&loop_ctl_mutex); 2484b9848081SChristoph Hellwig return id; 2485770fe30aSKay Sievers } 2486770fe30aSKay Sievers 2487f9d10764SChristoph Hellwig static long loop_control_ioctl(struct file *file, unsigned int cmd, 2488f9d10764SChristoph Hellwig unsigned long parm) 2489f9d10764SChristoph Hellwig { 2490f9d10764SChristoph Hellwig switch (cmd) { 2491f9d10764SChristoph Hellwig case LOOP_CTL_ADD: 249218d1f200SChristoph Hellwig return loop_add(parm); 2493f9d10764SChristoph Hellwig case LOOP_CTL_REMOVE: 2494f9d10764SChristoph Hellwig return loop_control_remove(parm); 2495f9d10764SChristoph Hellwig case LOOP_CTL_GET_FREE: 2496f9d10764SChristoph Hellwig return loop_control_get_free(parm); 2497f9d10764SChristoph Hellwig default: 2498f9d10764SChristoph Hellwig return -ENOSYS; 2499f9d10764SChristoph Hellwig } 2500f9d10764SChristoph Hellwig } 2501f9d10764SChristoph Hellwig 2502770fe30aSKay Sievers static const struct file_operations loop_ctl_fops = { 2503770fe30aSKay Sievers .open = nonseekable_open, 2504770fe30aSKay Sievers .unlocked_ioctl = loop_control_ioctl, 2505770fe30aSKay Sievers .compat_ioctl = loop_control_ioctl, 2506770fe30aSKay Sievers .owner = THIS_MODULE, 2507770fe30aSKay Sievers .llseek = noop_llseek, 2508770fe30aSKay Sievers }; 2509770fe30aSKay Sievers 2510770fe30aSKay Sievers static struct miscdevice loop_misc = { 2511770fe30aSKay Sievers .minor = LOOP_CTRL_MINOR, 2512770fe30aSKay Sievers .name = "loop-control", 2513770fe30aSKay Sievers .fops = &loop_ctl_fops, 2514770fe30aSKay Sievers }; 2515770fe30aSKay Sievers 2516770fe30aSKay Sievers MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR); 2517770fe30aSKay Sievers MODULE_ALIAS("devname:loop-control"); 2518770fe30aSKay Sievers 251973285082SKen Chen static int __init loop_init(void) 252073285082SKen Chen { 2521a47653fcSKen Chen int i, nr; 2522770fe30aSKay Sievers int err; 2523a47653fcSKen Chen 2524476a4813SLaurent Vivier part_shift = 0; 2525ac04fee0SNamhyung Kim if (max_part > 0) { 2526476a4813SLaurent Vivier part_shift = fls(max_part); 2527476a4813SLaurent Vivier 2528ac04fee0SNamhyung Kim /* 2529ac04fee0SNamhyung Kim * Adjust max_part according to part_shift as it is exported 2530ac04fee0SNamhyung Kim * to user space so that user can decide correct minor number 2531ac04fee0SNamhyung Kim * if [s]he want to create more devices. 2532ac04fee0SNamhyung Kim * 2533ac04fee0SNamhyung Kim * Note that -1 is required because partition 0 is reserved 2534ac04fee0SNamhyung Kim * for the whole disk. 2535ac04fee0SNamhyung Kim */ 2536ac04fee0SNamhyung Kim max_part = (1UL << part_shift) - 1; 2537ac04fee0SNamhyung Kim } 2538ac04fee0SNamhyung Kim 2539b1a66504SGuo Chao if ((1UL << part_shift) > DISK_MAX_PARTS) { 2540b1a66504SGuo Chao err = -EINVAL; 2541a8c1d064SAnton Volkov goto err_out; 2542b1a66504SGuo Chao } 254378f4bb36SNamhyung Kim 2544b1a66504SGuo Chao if (max_loop > 1UL << (MINORBITS - part_shift)) { 2545b1a66504SGuo Chao err = -EINVAL; 2546a8c1d064SAnton Volkov goto err_out; 2547b1a66504SGuo Chao } 254873285082SKen Chen 2549d134b00bSKay Sievers /* 2550d134b00bSKay Sievers * If max_loop is specified, create that many devices upfront. 2551d134b00bSKay Sievers * This also becomes a hard limit. If max_loop is not specified, 2552d134b00bSKay Sievers * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module 2553d134b00bSKay Sievers * init time. Loop devices can be requested on-demand with the 2554d134b00bSKay Sievers * /dev/loop-control interface, or be instantiated by accessing 2555d134b00bSKay Sievers * a 'dead' device node. 2556d134b00bSKay Sievers */ 2557aeb2b0b1SLukas Bulwahn if (max_loop) 2558a47653fcSKen Chen nr = max_loop; 2559aeb2b0b1SLukas Bulwahn else 2560d134b00bSKay Sievers nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT; 2561a47653fcSKen Chen 2562a8c1d064SAnton Volkov err = misc_register(&loop_misc); 2563a8c1d064SAnton Volkov if (err < 0) 2564a8c1d064SAnton Volkov goto err_out; 2565a8c1d064SAnton Volkov 2566a8c1d064SAnton Volkov 25678410d38cSChristoph Hellwig if (__register_blkdev(LOOP_MAJOR, "loop", loop_probe)) { 2568b1a66504SGuo Chao err = -EIO; 2569b1a66504SGuo Chao goto misc_out; 2570b1a66504SGuo Chao } 2571a47653fcSKen Chen 2572d134b00bSKay Sievers /* pre-create number of devices given by config or max_loop */ 257334dd82afSKay Sievers for (i = 0; i < nr; i++) 2574d6da83d0SChristoph Hellwig loop_add(i); 257534dd82afSKay Sievers 257673285082SKen Chen printk(KERN_INFO "loop: module loaded\n"); 25771da177e4SLinus Torvalds return 0; 2578b1a66504SGuo Chao 2579b1a66504SGuo Chao misc_out: 2580b1a66504SGuo Chao misc_deregister(&loop_misc); 2581a8c1d064SAnton Volkov err_out: 2582b1a66504SGuo Chao return err; 258334dd82afSKay Sievers } 2584a47653fcSKen Chen 258573285082SKen Chen static void __exit loop_exit(void) 25861da177e4SLinus Torvalds { 25878e60947dSChristoph Hellwig struct loop_device *lo; 25888e60947dSChristoph Hellwig int id; 25898e60947dSChristoph Hellwig 25908b52d8beSChristoph Hellwig unregister_blkdev(LOOP_MAJOR, "loop"); 25918b52d8beSChristoph Hellwig misc_deregister(&loop_misc); 2592200f9337SLuis Chamberlain 25931c500ad7STetsuo Handa /* 25941c500ad7STetsuo Handa * There is no need to use loop_ctl_mutex here, for nobody else can 25951c500ad7STetsuo Handa * access loop_index_idr when this module is unloading (unless forced 25961c500ad7STetsuo Handa * module unloading is requested). If this is not a clean unloading, 25971c500ad7STetsuo Handa * we have no means to avoid kernel crash. 25981c500ad7STetsuo Handa */ 25998e60947dSChristoph Hellwig idr_for_each_entry(&loop_index_idr, lo, id) 26008e60947dSChristoph Hellwig loop_remove(lo); 2601bd5c39edSChristoph Hellwig 2602bd5c39edSChristoph Hellwig idr_destroy(&loop_index_idr); 26031da177e4SLinus Torvalds } 26041da177e4SLinus Torvalds 26051da177e4SLinus Torvalds module_init(loop_init); 26061da177e4SLinus Torvalds module_exit(loop_exit); 26071da177e4SLinus Torvalds 26081da177e4SLinus Torvalds #ifndef MODULE 26091da177e4SLinus Torvalds static int __init max_loop_setup(char *str) 26101da177e4SLinus Torvalds { 26111da177e4SLinus Torvalds max_loop = simple_strtol(str, NULL, 0); 26121da177e4SLinus Torvalds return 1; 26131da177e4SLinus Torvalds } 26141da177e4SLinus Torvalds 26151da177e4SLinus Torvalds __setup("max_loop=", max_loop_setup); 26161da177e4SLinus Torvalds #endif 2617