1f21e6e18SChristoph Hellwig // SPDX-License-Identifier: GPL-2.0-only
21da177e4SLinus Torvalds /*
3f21e6e18SChristoph Hellwig * Copyright 1993 by Theodore Ts'o.
41da177e4SLinus Torvalds */
51da177e4SLinus Torvalds #include <linux/module.h>
61da177e4SLinus Torvalds #include <linux/moduleparam.h>
71da177e4SLinus Torvalds #include <linux/sched.h>
81da177e4SLinus Torvalds #include <linux/fs.h>
94ee60ec1SMatthew Wilcox (Oracle) #include <linux/pagemap.h>
101da177e4SLinus Torvalds #include <linux/file.h>
111da177e4SLinus Torvalds #include <linux/stat.h>
121da177e4SLinus Torvalds #include <linux/errno.h>
131da177e4SLinus Torvalds #include <linux/major.h>
141da177e4SLinus Torvalds #include <linux/wait.h>
151da177e4SLinus Torvalds #include <linux/blkpg.h>
161da177e4SLinus Torvalds #include <linux/init.h>
171da177e4SLinus Torvalds #include <linux/swap.h>
181da177e4SLinus Torvalds #include <linux/slab.h>
19863d5b82SDavid Howells #include <linux/compat.h>
201da177e4SLinus Torvalds #include <linux/suspend.h>
2183144186SRafael J. Wysocki #include <linux/freezer.h>
222a48fc0aSArnd Bergmann #include <linux/mutex.h>
231da177e4SLinus Torvalds #include <linux/writeback.h>
241da177e4SLinus Torvalds #include <linux/completion.h>
251da177e4SLinus Torvalds #include <linux/highmem.h>
26d6b29d7cSJens Axboe #include <linux/splice.h>
27ee862730SMilan Broz #include <linux/sysfs.h>
28770fe30aSKay Sievers #include <linux/miscdevice.h>
29dfaa2ef6SLukas Czerner #include <linux/falloc.h>
30283e7e5dSAl Viro #include <linux/uio.h>
31d9a08a9eSAdam Manzanares #include <linux/ioprio.h>
32db6638d7SDennis Zhou #include <linux/blk-cgroup.h>
33c74d40e8SDan Schatzberg #include <linux/sched/mm.h>
3406582bc8SMing Lei #include <linux/statfs.h>
357c0f6ba6SLinus Torvalds #include <linux/uaccess.h>
36754d9679SChristoph Hellwig #include <linux/blk-mq.h>
37754d9679SChristoph Hellwig #include <linux/spinlock.h>
38754d9679SChristoph Hellwig #include <uapi/linux/loop.h>
39754d9679SChristoph Hellwig
40754d9679SChristoph Hellwig /* Possible states of device */
41754d9679SChristoph Hellwig enum {
42754d9679SChristoph Hellwig Lo_unbound,
43754d9679SChristoph Hellwig Lo_bound,
44754d9679SChristoph Hellwig Lo_rundown,
45754d9679SChristoph Hellwig Lo_deleting,
46754d9679SChristoph Hellwig };
47754d9679SChristoph Hellwig
48754d9679SChristoph Hellwig struct loop_func_table;
49754d9679SChristoph Hellwig
50754d9679SChristoph Hellwig struct loop_device {
51754d9679SChristoph Hellwig int lo_number;
52754d9679SChristoph Hellwig loff_t lo_offset;
53754d9679SChristoph Hellwig loff_t lo_sizelimit;
54754d9679SChristoph Hellwig int lo_flags;
55754d9679SChristoph Hellwig char lo_file_name[LO_NAME_SIZE];
56754d9679SChristoph Hellwig
57754d9679SChristoph Hellwig struct file * lo_backing_file;
58754d9679SChristoph Hellwig struct block_device *lo_device;
59754d9679SChristoph Hellwig
60754d9679SChristoph Hellwig gfp_t old_gfp_mask;
61754d9679SChristoph Hellwig
62754d9679SChristoph Hellwig spinlock_t lo_lock;
63754d9679SChristoph Hellwig int lo_state;
64754d9679SChristoph Hellwig spinlock_t lo_work_lock;
65754d9679SChristoph Hellwig struct workqueue_struct *workqueue;
66754d9679SChristoph Hellwig struct work_struct rootcg_work;
67754d9679SChristoph Hellwig struct list_head rootcg_cmd_list;
68754d9679SChristoph Hellwig struct list_head idle_worker_list;
69754d9679SChristoph Hellwig struct rb_root worker_tree;
70754d9679SChristoph Hellwig struct timer_list timer;
71754d9679SChristoph Hellwig bool use_dio;
72754d9679SChristoph Hellwig bool sysfs_inited;
73754d9679SChristoph Hellwig
74754d9679SChristoph Hellwig struct request_queue *lo_queue;
75754d9679SChristoph Hellwig struct blk_mq_tag_set tag_set;
76754d9679SChristoph Hellwig struct gendisk *lo_disk;
77754d9679SChristoph Hellwig struct mutex lo_mutex;
78754d9679SChristoph Hellwig bool idr_visible;
79754d9679SChristoph Hellwig };
80754d9679SChristoph Hellwig
81754d9679SChristoph Hellwig struct loop_cmd {
82754d9679SChristoph Hellwig struct list_head list_entry;
83754d9679SChristoph Hellwig bool use_aio; /* use AIO interface to handle I/O */
84754d9679SChristoph Hellwig atomic_t ref; /* only for aio */
85754d9679SChristoph Hellwig long ret;
86754d9679SChristoph Hellwig struct kiocb iocb;
87754d9679SChristoph Hellwig struct bio_vec *bvec;
88754d9679SChristoph Hellwig struct cgroup_subsys_state *blkcg_css;
89754d9679SChristoph Hellwig struct cgroup_subsys_state *memcg_css;
90754d9679SChristoph Hellwig };
911da177e4SLinus Torvalds
9287579e9bSDan Schatzberg #define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ)
93e152a05fSBart Van Assche #define LOOP_DEFAULT_HW_Q_DEPTH 128
9487579e9bSDan Schatzberg
9534dd82afSKay Sievers static DEFINE_IDR(loop_index_idr);
96310ca162STetsuo Handa static DEFINE_MUTEX(loop_ctl_mutex);
973ce6e1f6STetsuo Handa static DEFINE_MUTEX(loop_validate_mutex);
983ce6e1f6STetsuo Handa
993ce6e1f6STetsuo Handa /**
1003ce6e1f6STetsuo Handa * loop_global_lock_killable() - take locks for safe loop_validate_file() test
1013ce6e1f6STetsuo Handa *
1023ce6e1f6STetsuo Handa * @lo: struct loop_device
1033ce6e1f6STetsuo Handa * @global: true if @lo is about to bind another "struct loop_device", false otherwise
1043ce6e1f6STetsuo Handa *
1053ce6e1f6STetsuo Handa * Returns 0 on success, -EINTR otherwise.
1063ce6e1f6STetsuo Handa *
1073ce6e1f6STetsuo Handa * Since loop_validate_file() traverses on other "struct loop_device" if
1083ce6e1f6STetsuo Handa * is_loop_device() is true, we need a global lock for serializing concurrent
1093ce6e1f6STetsuo Handa * loop_configure()/loop_change_fd()/__loop_clr_fd() calls.
1103ce6e1f6STetsuo Handa */
loop_global_lock_killable(struct loop_device * lo,bool global)1113ce6e1f6STetsuo Handa static int loop_global_lock_killable(struct loop_device *lo, bool global)
1123ce6e1f6STetsuo Handa {
1133ce6e1f6STetsuo Handa int err;
1143ce6e1f6STetsuo Handa
1153ce6e1f6STetsuo Handa if (global) {
1163ce6e1f6STetsuo Handa err = mutex_lock_killable(&loop_validate_mutex);
1173ce6e1f6STetsuo Handa if (err)
1183ce6e1f6STetsuo Handa return err;
1193ce6e1f6STetsuo Handa }
1203ce6e1f6STetsuo Handa err = mutex_lock_killable(&lo->lo_mutex);
1213ce6e1f6STetsuo Handa if (err && global)
1223ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex);
1233ce6e1f6STetsuo Handa return err;
1243ce6e1f6STetsuo Handa }
1253ce6e1f6STetsuo Handa
1263ce6e1f6STetsuo Handa /**
1273ce6e1f6STetsuo Handa * loop_global_unlock() - release locks taken by loop_global_lock_killable()
1283ce6e1f6STetsuo Handa *
1293ce6e1f6STetsuo Handa * @lo: struct loop_device
1303ce6e1f6STetsuo Handa * @global: true if @lo was about to bind another "struct loop_device", false otherwise
1313ce6e1f6STetsuo Handa */
loop_global_unlock(struct loop_device * lo,bool global)1323ce6e1f6STetsuo Handa static void loop_global_unlock(struct loop_device *lo, bool global)
1333ce6e1f6STetsuo Handa {
1343ce6e1f6STetsuo Handa mutex_unlock(&lo->lo_mutex);
1353ce6e1f6STetsuo Handa if (global)
1363ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex);
1373ce6e1f6STetsuo Handa }
1381da177e4SLinus Torvalds
139476a4813SLaurent Vivier static int max_part;
140476a4813SLaurent Vivier static int part_shift;
141476a4813SLaurent Vivier
get_size(loff_t offset,loff_t sizelimit,struct file * file)1427035b5dfSDmitry Monakhov static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
1431da177e4SLinus Torvalds {
144b7a1da69SGuo Chao loff_t loopsize;
1451da177e4SLinus Torvalds
1461da177e4SLinus Torvalds /* Compute loopsize in bytes */
147b7a1da69SGuo Chao loopsize = i_size_read(file->f_mapping->host);
148b7a1da69SGuo Chao if (offset > 0)
149b7a1da69SGuo Chao loopsize -= offset;
150b7a1da69SGuo Chao /* offset is beyond i_size, weird but possible */
1517035b5dfSDmitry Monakhov if (loopsize < 0)
1527035b5dfSDmitry Monakhov return 0;
1531da177e4SLinus Torvalds
1547035b5dfSDmitry Monakhov if (sizelimit > 0 && sizelimit < loopsize)
1557035b5dfSDmitry Monakhov loopsize = sizelimit;
1561da177e4SLinus Torvalds /*
1571da177e4SLinus Torvalds * Unfortunately, if we want to do I/O on the device,
1581da177e4SLinus Torvalds * the number of 512-byte sectors has to fit into a sector_t.
1591da177e4SLinus Torvalds */
1601da177e4SLinus Torvalds return loopsize >> 9;
1611da177e4SLinus Torvalds }
1621da177e4SLinus Torvalds
get_loop_size(struct loop_device * lo,struct file * file)1637035b5dfSDmitry Monakhov static loff_t get_loop_size(struct loop_device *lo, struct file *file)
1641da177e4SLinus Torvalds {
1657035b5dfSDmitry Monakhov return get_size(lo->lo_offset, lo->lo_sizelimit, file);
1667035b5dfSDmitry Monakhov }
1677035b5dfSDmitry Monakhov
168*96e84339SChristoph Hellwig /*
169*96e84339SChristoph Hellwig * We support direct I/O only if lo_offset is aligned with the logical I/O size
170*96e84339SChristoph Hellwig * of backing device, and the logical block size of loop is bigger than that of
171*96e84339SChristoph Hellwig * the backing device.
172*96e84339SChristoph Hellwig */
lo_bdev_can_use_dio(struct loop_device * lo,struct block_device * backing_bdev)173*96e84339SChristoph Hellwig static bool lo_bdev_can_use_dio(struct loop_device *lo,
174*96e84339SChristoph Hellwig struct block_device *backing_bdev)
175*96e84339SChristoph Hellwig {
176*96e84339SChristoph Hellwig unsigned short sb_bsize = bdev_logical_block_size(backing_bdev);
177*96e84339SChristoph Hellwig
178*96e84339SChristoph Hellwig if (queue_logical_block_size(lo->lo_queue) < sb_bsize)
179*96e84339SChristoph Hellwig return false;
180*96e84339SChristoph Hellwig if (lo->lo_offset & (sb_bsize - 1))
181*96e84339SChristoph Hellwig return false;
182*96e84339SChristoph Hellwig return true;
183*96e84339SChristoph Hellwig }
184*96e84339SChristoph Hellwig
__loop_update_dio(struct loop_device * lo,bool dio)1852e5ab5f3SMing Lei static void __loop_update_dio(struct loop_device *lo, bool dio)
1862e5ab5f3SMing Lei {
1872e5ab5f3SMing Lei struct file *file = lo->lo_backing_file;
188*96e84339SChristoph Hellwig struct inode *inode = file->f_mapping->host;
189*96e84339SChristoph Hellwig struct block_device *backing_bdev = NULL;
1902e5ab5f3SMing Lei bool use_dio;
1912e5ab5f3SMing Lei
192*96e84339SChristoph Hellwig if (S_ISBLK(inode->i_mode))
193*96e84339SChristoph Hellwig backing_bdev = I_BDEV(inode);
194*96e84339SChristoph Hellwig else if (inode->i_sb->s_bdev)
195*96e84339SChristoph Hellwig backing_bdev = inode->i_sb->s_bdev;
1962e5ab5f3SMing Lei
197*96e84339SChristoph Hellwig use_dio = dio && (file->f_mode & FMODE_CAN_ODIRECT) &&
198*96e84339SChristoph Hellwig (!backing_bdev || lo_bdev_can_use_dio(lo, backing_bdev));
1992e5ab5f3SMing Lei
2002e5ab5f3SMing Lei if (lo->use_dio == use_dio)
2012e5ab5f3SMing Lei return;
2022e5ab5f3SMing Lei
2032e5ab5f3SMing Lei /* flush dirty pages before changing direct IO */
2042e5ab5f3SMing Lei vfs_fsync(file, 0);
2052e5ab5f3SMing Lei
2062e5ab5f3SMing Lei /*
2072e5ab5f3SMing Lei * The flag of LO_FLAGS_DIRECT_IO is handled similarly with
2082e5ab5f3SMing Lei * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
2092e5ab5f3SMing Lei * will get updated by ioctl(LOOP_GET_STATUS)
2102e5ab5f3SMing Lei */
2110fbcf579SMartijn Coenen if (lo->lo_state == Lo_bound)
2122e5ab5f3SMing Lei blk_mq_freeze_queue(lo->lo_queue);
2132e5ab5f3SMing Lei lo->use_dio = use_dio;
21440326d8aSShaohua Li if (use_dio) {
2158b904b5bSBart Van Assche blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue);
2162e5ab5f3SMing Lei lo->lo_flags |= LO_FLAGS_DIRECT_IO;
21740326d8aSShaohua Li } else {
2188b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
2192e5ab5f3SMing Lei lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
22040326d8aSShaohua Li }
2210fbcf579SMartijn Coenen if (lo->lo_state == Lo_bound)
2222e5ab5f3SMing Lei blk_mq_unfreeze_queue(lo->lo_queue);
2232e5ab5f3SMing Lei }
2242e5ab5f3SMing Lei
2255795b6f5SMartijn Coenen /**
2265795b6f5SMartijn Coenen * loop_set_size() - sets device size and notifies userspace
2275795b6f5SMartijn Coenen * @lo: struct loop_device to set the size for
2285795b6f5SMartijn Coenen * @size: new size of the loop device
2295795b6f5SMartijn Coenen *
2305795b6f5SMartijn Coenen * Callers must validate that the size passed into this function fits into
2315795b6f5SMartijn Coenen * a sector_t, eg using loop_validate_size()
2325795b6f5SMartijn Coenen */
loop_set_size(struct loop_device * lo,loff_t size)2335795b6f5SMartijn Coenen static void loop_set_size(struct loop_device *lo, loff_t size)
2345795b6f5SMartijn Coenen {
235449f4ec9SChristoph Hellwig if (!set_capacity_and_notify(lo->lo_disk, size))
2363b4f85d0SChristoph Hellwig kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
2371da177e4SLinus Torvalds }
2381da177e4SLinus Torvalds
lo_write_bvec(struct file * file,struct bio_vec * bvec,loff_t * ppos)239aa4d8616SChristoph Hellwig static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
2401da177e4SLinus Torvalds {
241aa4d8616SChristoph Hellwig struct iov_iter i;
2421da177e4SLinus Torvalds ssize_t bw;
243283e7e5dSAl Viro
244de4eda9dSAl Viro iov_iter_bvec(&i, ITER_SOURCE, bvec, 1, bvec->bv_len);
2451da177e4SLinus Torvalds
24603d95eb2SAl Viro file_start_write(file);
247abbb6589SChristoph Hellwig bw = vfs_iter_write(file, &i, ppos, 0);
24803d95eb2SAl Viro file_end_write(file);
249aa4d8616SChristoph Hellwig
250aa4d8616SChristoph Hellwig if (likely(bw == bvec->bv_len))
2511da177e4SLinus Torvalds return 0;
252aa4d8616SChristoph Hellwig
253aa4d8616SChristoph Hellwig printk_ratelimited(KERN_ERR
254aa4d8616SChristoph Hellwig "loop: Write error at byte offset %llu, length %i.\n",
255aa4d8616SChristoph Hellwig (unsigned long long)*ppos, bvec->bv_len);
2561da177e4SLinus Torvalds if (bw >= 0)
2571da177e4SLinus Torvalds bw = -EIO;
2581da177e4SLinus Torvalds return bw;
2591da177e4SLinus Torvalds }
2601da177e4SLinus Torvalds
lo_write_simple(struct loop_device * lo,struct request * rq,loff_t pos)261aa4d8616SChristoph Hellwig static int lo_write_simple(struct loop_device *lo, struct request *rq,
262aa4d8616SChristoph Hellwig loff_t pos)
2631da177e4SLinus Torvalds {
264aa4d8616SChristoph Hellwig struct bio_vec bvec;
265aa4d8616SChristoph Hellwig struct req_iterator iter;
266aa4d8616SChristoph Hellwig int ret = 0;
267aa4d8616SChristoph Hellwig
268aa4d8616SChristoph Hellwig rq_for_each_segment(bvec, rq, iter) {
269aa4d8616SChristoph Hellwig ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
270aa4d8616SChristoph Hellwig if (ret < 0)
271aa4d8616SChristoph Hellwig break;
2721da177e4SLinus Torvalds cond_resched();
2731da177e4SLinus Torvalds }
2741da177e4SLinus Torvalds
275aa4d8616SChristoph Hellwig return ret;
276aa4d8616SChristoph Hellwig }
277aa4d8616SChristoph Hellwig
lo_read_simple(struct loop_device * lo,struct request * rq,loff_t pos)278aa4d8616SChristoph Hellwig static int lo_read_simple(struct loop_device *lo, struct request *rq,
279aa4d8616SChristoph Hellwig loff_t pos)
2801da177e4SLinus Torvalds {
2817988613bSKent Overstreet struct bio_vec bvec;
28230112013SMing Lei struct req_iterator iter;
283aa4d8616SChristoph Hellwig struct iov_iter i;
284aa4d8616SChristoph Hellwig ssize_t len;
2851da177e4SLinus Torvalds
28630112013SMing Lei rq_for_each_segment(bvec, rq, iter) {
287de4eda9dSAl Viro iov_iter_bvec(&i, ITER_DEST, &bvec, 1, bvec.bv_len);
28818e9710eSChristoph Hellwig len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
289aa4d8616SChristoph Hellwig if (len < 0)
290aa4d8616SChristoph Hellwig return len;
291306df071SDave Young
292aa4d8616SChristoph Hellwig flush_dcache_page(bvec.bv_page);
293aa4d8616SChristoph Hellwig
294aa4d8616SChristoph Hellwig if (len != bvec.bv_len) {
29530112013SMing Lei struct bio *bio;
29630112013SMing Lei
29730112013SMing Lei __rq_for_each_bio(bio, rq)
298306df071SDave Young zero_fill_bio(bio);
2991da177e4SLinus Torvalds break;
300306df071SDave Young }
301aa4d8616SChristoph Hellwig cond_resched();
3021da177e4SLinus Torvalds }
303aa4d8616SChristoph Hellwig
304306df071SDave Young return 0;
3051da177e4SLinus Torvalds }
3061da177e4SLinus Torvalds
lo_fallocate(struct loop_device * lo,struct request * rq,loff_t pos,int mode)307efcfec57SDarrick J. Wong static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos,
308efcfec57SDarrick J. Wong int mode)
309cf655d95SMing Lei {
310cf655d95SMing Lei /*
311efcfec57SDarrick J. Wong * We use fallocate to manipulate the space mappings used by the image
31247e96246SChristoph Hellwig * a.k.a. discard/zerorange.
313cf655d95SMing Lei */
314cf655d95SMing Lei struct file *file = lo->lo_backing_file;
315cf655d95SMing Lei int ret;
316cf655d95SMing Lei
317efcfec57SDarrick J. Wong mode |= FALLOC_FL_KEEP_SIZE;
318efcfec57SDarrick J. Wong
31970200574SChristoph Hellwig if (!bdev_max_discard_sectors(lo->lo_device))
32070200574SChristoph Hellwig return -EOPNOTSUPP;
321cf655d95SMing Lei
322cf655d95SMing Lei ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
323cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
32470200574SChristoph Hellwig return -EIO;
325cf655d95SMing Lei return ret;
326cf655d95SMing Lei }
327cf655d95SMing Lei
lo_req_flush(struct loop_device * lo,struct request * rq)328cf655d95SMing Lei static int lo_req_flush(struct loop_device *lo, struct request *rq)
329cf655d95SMing Lei {
3309c64e38cSChaitanya Kulkarni int ret = vfs_fsync(lo->lo_backing_file, 0);
331cf655d95SMing Lei if (unlikely(ret && ret != -EINVAL))
332cf655d95SMing Lei ret = -EIO;
333cf655d95SMing Lei
334cf655d95SMing Lei return ret;
335cf655d95SMing Lei }
336cf655d95SMing Lei
lo_complete_rq(struct request * rq)337fe2cb290SChristoph Hellwig static void lo_complete_rq(struct request *rq)
338bc07c10aSMing Lei {
339fe2cb290SChristoph Hellwig struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
340f9de14bcSJens Axboe blk_status_t ret = BLK_STS_OK;
341bc07c10aSMing Lei
342f9de14bcSJens Axboe if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
343f9de14bcSJens Axboe req_op(rq) != REQ_OP_READ) {
344f9de14bcSJens Axboe if (cmd->ret < 0)
3458cd55087SEvan Green ret = errno_to_blk_status(cmd->ret);
346f9de14bcSJens Axboe goto end_io;
347bc07c10aSMing Lei }
348fe2cb290SChristoph Hellwig
349f9de14bcSJens Axboe /*
350f9de14bcSJens Axboe * Short READ - if we got some data, advance our request and
351f9de14bcSJens Axboe * retry it. If we got no data, end the rest with EIO.
352f9de14bcSJens Axboe */
353f9de14bcSJens Axboe if (cmd->ret) {
354f9de14bcSJens Axboe blk_update_request(rq, BLK_STS_OK, cmd->ret);
355f9de14bcSJens Axboe cmd->ret = 0;
356f9de14bcSJens Axboe blk_mq_requeue_request(rq, true);
357f9de14bcSJens Axboe } else {
358f9de14bcSJens Axboe if (cmd->use_aio) {
359f9de14bcSJens Axboe struct bio *bio = rq->bio;
360f9de14bcSJens Axboe
361f9de14bcSJens Axboe while (bio) {
362f9de14bcSJens Axboe zero_fill_bio(bio);
363f9de14bcSJens Axboe bio = bio->bi_next;
364f9de14bcSJens Axboe }
365f9de14bcSJens Axboe }
366f9de14bcSJens Axboe ret = BLK_STS_IOERR;
367f9de14bcSJens Axboe end_io:
368f9de14bcSJens Axboe blk_mq_end_request(rq, ret);
369f9de14bcSJens Axboe }
370bc07c10aSMing Lei }
371bc07c10aSMing Lei
lo_rw_aio_do_completion(struct loop_cmd * cmd)37292d77332SShaohua Li static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
37392d77332SShaohua Li {
3741894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd);
3751894e916SJens Axboe
37692d77332SShaohua Li if (!atomic_dec_and_test(&cmd->ref))
37792d77332SShaohua Li return;
37892d77332SShaohua Li kfree(cmd->bvec);
37992d77332SShaohua Li cmd->bvec = NULL;
38015f73f5bSChristoph Hellwig if (likely(!blk_should_fake_timeout(rq->q)))
3811894e916SJens Axboe blk_mq_complete_request(rq);
38292d77332SShaohua Li }
38392d77332SShaohua Li
lo_rw_aio_complete(struct kiocb * iocb,long ret)3846b19b766SJens Axboe static void lo_rw_aio_complete(struct kiocb *iocb, long ret)
385bc07c10aSMing Lei {
386bc07c10aSMing Lei struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
387bc07c10aSMing Lei
388fe2cb290SChristoph Hellwig cmd->ret = ret;
38992d77332SShaohua Li lo_rw_aio_do_completion(cmd);
390bc07c10aSMing Lei }
391bc07c10aSMing Lei
lo_rw_aio(struct loop_device * lo,struct loop_cmd * cmd,loff_t pos,int rw)392bc07c10aSMing Lei static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
393de4eda9dSAl Viro loff_t pos, int rw)
394bc07c10aSMing Lei {
395bc07c10aSMing Lei struct iov_iter iter;
39686af5952SMing Lei struct req_iterator rq_iter;
397bc07c10aSMing Lei struct bio_vec *bvec;
3981894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd);
39940326d8aSShaohua Li struct bio *bio = rq->bio;
400bc07c10aSMing Lei struct file *file = lo->lo_backing_file;
40186af5952SMing Lei struct bio_vec tmp;
40240326d8aSShaohua Li unsigned int offset;
40386af5952SMing Lei int nr_bvec = 0;
404bc07c10aSMing Lei int ret;
405bc07c10aSMing Lei
40686af5952SMing Lei rq_for_each_bvec(tmp, rq, rq_iter)
40786af5952SMing Lei nr_bvec++;
408bc07c10aSMing Lei
40986af5952SMing Lei if (rq->bio != rq->biotail) {
41086af5952SMing Lei
41186af5952SMing Lei bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
4126da2ec56SKees Cook GFP_NOIO);
41340326d8aSShaohua Li if (!bvec)
41440326d8aSShaohua Li return -EIO;
41540326d8aSShaohua Li cmd->bvec = bvec;
41640326d8aSShaohua Li
417a7297a6aSMing Lei /*
41840326d8aSShaohua Li * The bios of the request may be started from the middle of
41940326d8aSShaohua Li * the 'bvec' because of bio splitting, so we can't directly
42086af5952SMing Lei * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec
42140326d8aSShaohua Li * API will take care of all details for us.
422a7297a6aSMing Lei */
42386af5952SMing Lei rq_for_each_bvec(tmp, rq, rq_iter) {
42440326d8aSShaohua Li *bvec = tmp;
42540326d8aSShaohua Li bvec++;
42640326d8aSShaohua Li }
42740326d8aSShaohua Li bvec = cmd->bvec;
42840326d8aSShaohua Li offset = 0;
42940326d8aSShaohua Li } else {
43040326d8aSShaohua Li /*
43140326d8aSShaohua Li * Same here, this bio may be started from the middle of the
43240326d8aSShaohua Li * 'bvec' because of bio splitting, so offset from the bvec
43340326d8aSShaohua Li * must be passed to iov iterator
43440326d8aSShaohua Li */
43540326d8aSShaohua Li offset = bio->bi_iter.bi_bvec_done;
43640326d8aSShaohua Li bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
43740326d8aSShaohua Li }
43892d77332SShaohua Li atomic_set(&cmd->ref, 2);
43940326d8aSShaohua Li
440b6207430SChristoph Hellwig iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq));
44140326d8aSShaohua Li iter.iov_offset = offset;
442bc07c10aSMing Lei
443bc07c10aSMing Lei cmd->iocb.ki_pos = pos;
444bc07c10aSMing Lei cmd->iocb.ki_filp = file;
445bc07c10aSMing Lei cmd->iocb.ki_complete = lo_rw_aio_complete;
446bc07c10aSMing Lei cmd->iocb.ki_flags = IOCB_DIRECT;
447d9a08a9eSAdam Manzanares cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
448bc07c10aSMing Lei
449de4eda9dSAl Viro if (rw == ITER_SOURCE)
450bb7462b6SMiklos Szeredi ret = call_write_iter(file, &cmd->iocb, &iter);
451bc07c10aSMing Lei else
452bb7462b6SMiklos Szeredi ret = call_read_iter(file, &cmd->iocb, &iter);
453bc07c10aSMing Lei
45492d77332SShaohua Li lo_rw_aio_do_completion(cmd);
45592d77332SShaohua Li
456bc07c10aSMing Lei if (ret != -EIOCBQUEUED)
4576b19b766SJens Axboe lo_rw_aio_complete(&cmd->iocb, ret);
458bc07c10aSMing Lei return 0;
459bc07c10aSMing Lei }
460bc07c10aSMing Lei
do_req_filebacked(struct loop_device * lo,struct request * rq)461c1c87c2bSChristoph Hellwig static int do_req_filebacked(struct loop_device *lo, struct request *rq)
462bc07c10aSMing Lei {
463bc07c10aSMing Lei struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
464c1c87c2bSChristoph Hellwig loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
465bc07c10aSMing Lei
466bc07c10aSMing Lei /*
467bc07c10aSMing Lei * lo_write_simple and lo_read_simple should have been covered
468bc07c10aSMing Lei * by io submit style function like lo_rw_aio(), one blocker
469bc07c10aSMing Lei * is that lo_read_simple() need to call flush_dcache_page after
470bc07c10aSMing Lei * the page is written from kernel, and it isn't easy to handle
471bc07c10aSMing Lei * this in io submit style function which submits all segments
472bc07c10aSMing Lei * of the req at one time. And direct read IO doesn't need to
473bc07c10aSMing Lei * run flush_dcache_page().
474bc07c10aSMing Lei */
475c1c87c2bSChristoph Hellwig switch (req_op(rq)) {
476c1c87c2bSChristoph Hellwig case REQ_OP_FLUSH:
477c1c87c2bSChristoph Hellwig return lo_req_flush(lo, rq);
47819372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES:
479efcfec57SDarrick J. Wong /*
480efcfec57SDarrick J. Wong * If the caller doesn't want deallocation, call zeroout to
481efcfec57SDarrick J. Wong * write zeroes the range. Otherwise, punch them out.
482efcfec57SDarrick J. Wong */
483efcfec57SDarrick J. Wong return lo_fallocate(lo, rq, pos,
484efcfec57SDarrick J. Wong (rq->cmd_flags & REQ_NOUNMAP) ?
485efcfec57SDarrick J. Wong FALLOC_FL_ZERO_RANGE :
486efcfec57SDarrick J. Wong FALLOC_FL_PUNCH_HOLE);
487efcfec57SDarrick J. Wong case REQ_OP_DISCARD:
488efcfec57SDarrick J. Wong return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE);
489c1c87c2bSChristoph Hellwig case REQ_OP_WRITE:
49047e96246SChristoph Hellwig if (cmd->use_aio)
491de4eda9dSAl Viro return lo_rw_aio(lo, cmd, pos, ITER_SOURCE);
492c1c87c2bSChristoph Hellwig else
493bc07c10aSMing Lei return lo_write_simple(lo, rq, pos);
494c1c87c2bSChristoph Hellwig case REQ_OP_READ:
49547e96246SChristoph Hellwig if (cmd->use_aio)
496de4eda9dSAl Viro return lo_rw_aio(lo, cmd, pos, ITER_DEST);
497bc07c10aSMing Lei else
498bc07c10aSMing Lei return lo_read_simple(lo, rq, pos);
499c1c87c2bSChristoph Hellwig default:
500c1c87c2bSChristoph Hellwig WARN_ON_ONCE(1);
501c1c87c2bSChristoph Hellwig return -EIO;
502bc07c10aSMing Lei }
5031da177e4SLinus Torvalds }
5041da177e4SLinus Torvalds
loop_update_dio(struct loop_device * lo)5052e5ab5f3SMing Lei static inline void loop_update_dio(struct loop_device *lo)
5062e5ab5f3SMing Lei {
507efbe3c24SIra Weiny __loop_update_dio(lo, (lo->lo_backing_file->f_flags & O_DIRECT) |
5082e5ab5f3SMing Lei lo->use_dio);
5092e5ab5f3SMing Lei }
5102e5ab5f3SMing Lei
loop_reread_partitions(struct loop_device * lo)5110384264eSChristoph Hellwig static void loop_reread_partitions(struct loop_device *lo)
51206f0e9e6SMing Lei {
51306f0e9e6SMing Lei int rc;
51406f0e9e6SMing Lei
5150384264eSChristoph Hellwig mutex_lock(&lo->lo_disk->open_mutex);
5160384264eSChristoph Hellwig rc = bdev_disk_changed(lo->lo_disk, false);
5170384264eSChristoph Hellwig mutex_unlock(&lo->lo_disk->open_mutex);
51806f0e9e6SMing Lei if (rc)
51906f0e9e6SMing Lei pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
52006f0e9e6SMing Lei __func__, lo->lo_number, lo->lo_file_name, rc);
52106f0e9e6SMing Lei }
52206f0e9e6SMing Lei
is_loop_device(struct file * file)523d2ac838eSTheodore Ts'o static inline int is_loop_device(struct file *file)
524d2ac838eSTheodore Ts'o {
525d2ac838eSTheodore Ts'o struct inode *i = file->f_mapping->host;
526d2ac838eSTheodore Ts'o
5276f24784fSAl Viro return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR;
528d2ac838eSTheodore Ts'o }
529d2ac838eSTheodore Ts'o
loop_validate_file(struct file * file,struct block_device * bdev)530d2ac838eSTheodore Ts'o static int loop_validate_file(struct file *file, struct block_device *bdev)
531d2ac838eSTheodore Ts'o {
532d2ac838eSTheodore Ts'o struct inode *inode = file->f_mapping->host;
533d2ac838eSTheodore Ts'o struct file *f = file;
534d2ac838eSTheodore Ts'o
535d2ac838eSTheodore Ts'o /* Avoid recursion */
536d2ac838eSTheodore Ts'o while (is_loop_device(f)) {
537d2ac838eSTheodore Ts'o struct loop_device *l;
538d2ac838eSTheodore Ts'o
5393ce6e1f6STetsuo Handa lockdep_assert_held(&loop_validate_mutex);
5404e7b5671SChristoph Hellwig if (f->f_mapping->host->i_rdev == bdev->bd_dev)
541d2ac838eSTheodore Ts'o return -EBADF;
542d2ac838eSTheodore Ts'o
5434e7b5671SChristoph Hellwig l = I_BDEV(f->f_mapping->host)->bd_disk->private_data;
5443ce6e1f6STetsuo Handa if (l->lo_state != Lo_bound)
545d2ac838eSTheodore Ts'o return -EINVAL;
5463ce6e1f6STetsuo Handa /* Order wrt setting lo->lo_backing_file in loop_configure(). */
5473ce6e1f6STetsuo Handa rmb();
548d2ac838eSTheodore Ts'o f = l->lo_backing_file;
549d2ac838eSTheodore Ts'o }
550d2ac838eSTheodore Ts'o if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
551d2ac838eSTheodore Ts'o return -EINVAL;
552d2ac838eSTheodore Ts'o return 0;
553d2ac838eSTheodore Ts'o }
554d2ac838eSTheodore Ts'o
5551da177e4SLinus Torvalds /*
5561da177e4SLinus Torvalds * loop_change_fd switched the backing store of a loopback device to
5571da177e4SLinus Torvalds * a new file. This is useful for operating system installers to free up
5581da177e4SLinus Torvalds * the original file and in High Availability environments to switch to
5591da177e4SLinus Torvalds * an alternative location for the content in case of server meltdown.
5601da177e4SLinus Torvalds * This can only work if the loop device is used read-only, and if the
5611da177e4SLinus Torvalds * new backing store is the same size and type as the old backing store.
5621da177e4SLinus Torvalds */
loop_change_fd(struct loop_device * lo,struct block_device * bdev,unsigned int arg)563bb214884SAl Viro static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
564bb214884SAl Viro unsigned int arg)
5651da177e4SLinus Torvalds {
5663ce6e1f6STetsuo Handa struct file *file = fget(arg);
5673ce6e1f6STetsuo Handa struct file *old_file;
5681da177e4SLinus Torvalds int error;
56985b0a54aSJan Kara bool partscan;
5703ce6e1f6STetsuo Handa bool is_loop;
5711da177e4SLinus Torvalds
5723ce6e1f6STetsuo Handa if (!file)
5733ce6e1f6STetsuo Handa return -EBADF;
574498ef5c7SChristoph Hellwig
575498ef5c7SChristoph Hellwig /* suppress uevents while reconfiguring the device */
576498ef5c7SChristoph Hellwig dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
577498ef5c7SChristoph Hellwig
5783ce6e1f6STetsuo Handa is_loop = is_loop_device(file);
5793ce6e1f6STetsuo Handa error = loop_global_lock_killable(lo, is_loop);
580c3710770SJan Kara if (error)
5813ce6e1f6STetsuo Handa goto out_putf;
5821da177e4SLinus Torvalds error = -ENXIO;
5831da177e4SLinus Torvalds if (lo->lo_state != Lo_bound)
5841dded9acSJan Kara goto out_err;
5851da177e4SLinus Torvalds
5861da177e4SLinus Torvalds /* the loop device has to be read-only */
5871da177e4SLinus Torvalds error = -EINVAL;
5881da177e4SLinus Torvalds if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
5891dded9acSJan Kara goto out_err;
5901da177e4SLinus Torvalds
591d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev);
592d2ac838eSTheodore Ts'o if (error)
5931dded9acSJan Kara goto out_err;
594d2ac838eSTheodore Ts'o
5951da177e4SLinus Torvalds old_file = lo->lo_backing_file;
5961da177e4SLinus Torvalds
5971da177e4SLinus Torvalds error = -EINVAL;
5981da177e4SLinus Torvalds
5991da177e4SLinus Torvalds /* size of the new backing store needs to be the same */
6001da177e4SLinus Torvalds if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
6011dded9acSJan Kara goto out_err;
6021da177e4SLinus Torvalds
6031da177e4SLinus Torvalds /* and ... switch */
604ab6860f6SChristoph Hellwig disk_force_media_change(lo->lo_disk);
60543cade80SOmar Sandoval blk_mq_freeze_queue(lo->lo_queue);
60643cade80SOmar Sandoval mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
60743cade80SOmar Sandoval lo->lo_backing_file = file;
60843cade80SOmar Sandoval lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
60943cade80SOmar Sandoval mapping_set_gfp_mask(file->f_mapping,
61043cade80SOmar Sandoval lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
61143cade80SOmar Sandoval loop_update_dio(lo);
61243cade80SOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue);
61385b0a54aSJan Kara partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
6143ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop);
6153ce6e1f6STetsuo Handa
6163ce6e1f6STetsuo Handa /*
6173ce6e1f6STetsuo Handa * Flush loop_validate_file() before fput(), for l->lo_backing_file
6183ce6e1f6STetsuo Handa * might be pointing at old_file which might be the last reference.
6193ce6e1f6STetsuo Handa */
6203ce6e1f6STetsuo Handa if (!is_loop) {
6213ce6e1f6STetsuo Handa mutex_lock(&loop_validate_mutex);
6223ce6e1f6STetsuo Handa mutex_unlock(&loop_validate_mutex);
6233ce6e1f6STetsuo Handa }
6241dded9acSJan Kara /*
6256cc8e743SPavel Tatashin * We must drop file reference outside of lo_mutex as dropping
626a8698707SChristoph Hellwig * the file ref can take open_mutex which creates circular locking
6271dded9acSJan Kara * dependency.
6281dded9acSJan Kara */
6291dded9acSJan Kara fput(old_file);
63085b0a54aSJan Kara if (partscan)
6310384264eSChristoph Hellwig loop_reread_partitions(lo);
632498ef5c7SChristoph Hellwig
633498ef5c7SChristoph Hellwig error = 0;
634498ef5c7SChristoph Hellwig done:
635498ef5c7SChristoph Hellwig /* enable and uncork uevent now that we are done */
636498ef5c7SChristoph Hellwig dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
637498ef5c7SChristoph Hellwig return error;
6381da177e4SLinus Torvalds
6391dded9acSJan Kara out_err:
6403ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop);
6413ce6e1f6STetsuo Handa out_putf:
6421dded9acSJan Kara fput(file);
643498ef5c7SChristoph Hellwig goto done;
6441da177e4SLinus Torvalds }
6451da177e4SLinus Torvalds
646ee862730SMilan Broz /* loop sysfs attributes */
647ee862730SMilan Broz
loop_attr_show(struct device * dev,char * page,ssize_t (* callback)(struct loop_device *,char *))648ee862730SMilan Broz static ssize_t loop_attr_show(struct device *dev, char *page,
649ee862730SMilan Broz ssize_t (*callback)(struct loop_device *, char *))
650ee862730SMilan Broz {
65134dd82afSKay Sievers struct gendisk *disk = dev_to_disk(dev);
65234dd82afSKay Sievers struct loop_device *lo = disk->private_data;
653ee862730SMilan Broz
65434dd82afSKay Sievers return callback(lo, page);
655ee862730SMilan Broz }
656ee862730SMilan Broz
657ee862730SMilan Broz #define LOOP_ATTR_RO(_name) \
658ee862730SMilan Broz static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \
659ee862730SMilan Broz static ssize_t loop_attr_do_show_##_name(struct device *d, \
660ee862730SMilan Broz struct device_attribute *attr, char *b) \
661ee862730SMilan Broz { \
662ee862730SMilan Broz return loop_attr_show(d, b, loop_attr_##_name##_show); \
663ee862730SMilan Broz } \
664ee862730SMilan Broz static struct device_attribute loop_attr_##_name = \
6655657a819SJoe Perches __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
666ee862730SMilan Broz
loop_attr_backing_file_show(struct loop_device * lo,char * buf)667ee862730SMilan Broz static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
668ee862730SMilan Broz {
669ee862730SMilan Broz ssize_t ret;
670ee862730SMilan Broz char *p = NULL;
671ee862730SMilan Broz
67205eb0f25SKay Sievers spin_lock_irq(&lo->lo_lock);
673ee862730SMilan Broz if (lo->lo_backing_file)
6749bf39ab2SMiklos Szeredi p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
67505eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock);
676ee862730SMilan Broz
677ee862730SMilan Broz if (IS_ERR_OR_NULL(p))
678ee862730SMilan Broz ret = PTR_ERR(p);
679ee862730SMilan Broz else {
680ee862730SMilan Broz ret = strlen(p);
681ee862730SMilan Broz memmove(buf, p, ret);
682ee862730SMilan Broz buf[ret++] = '\n';
683ee862730SMilan Broz buf[ret] = 0;
684ee862730SMilan Broz }
685ee862730SMilan Broz
686ee862730SMilan Broz return ret;
687ee862730SMilan Broz }
688ee862730SMilan Broz
loop_attr_offset_show(struct loop_device * lo,char * buf)689ee862730SMilan Broz static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
690ee862730SMilan Broz {
691b27824d3SChaitanya Kulkarni return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset);
692ee862730SMilan Broz }
693ee862730SMilan Broz
loop_attr_sizelimit_show(struct loop_device * lo,char * buf)694ee862730SMilan Broz static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
695ee862730SMilan Broz {
696b27824d3SChaitanya Kulkarni return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
697ee862730SMilan Broz }
698ee862730SMilan Broz
loop_attr_autoclear_show(struct loop_device * lo,char * buf)699ee862730SMilan Broz static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
700ee862730SMilan Broz {
701ee862730SMilan Broz int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
702ee862730SMilan Broz
703b27824d3SChaitanya Kulkarni return sysfs_emit(buf, "%s\n", autoclear ? "1" : "0");
704ee862730SMilan Broz }
705ee862730SMilan Broz
loop_attr_partscan_show(struct loop_device * lo,char * buf)706e03c8dd1SKay Sievers static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
707e03c8dd1SKay Sievers {
708e03c8dd1SKay Sievers int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
709e03c8dd1SKay Sievers
710b27824d3SChaitanya Kulkarni return sysfs_emit(buf, "%s\n", partscan ? "1" : "0");
711e03c8dd1SKay Sievers }
712e03c8dd1SKay Sievers
loop_attr_dio_show(struct loop_device * lo,char * buf)7132e5ab5f3SMing Lei static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
7142e5ab5f3SMing Lei {
7152e5ab5f3SMing Lei int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
7162e5ab5f3SMing Lei
717b27824d3SChaitanya Kulkarni return sysfs_emit(buf, "%s\n", dio ? "1" : "0");
7182e5ab5f3SMing Lei }
7192e5ab5f3SMing Lei
720ee862730SMilan Broz LOOP_ATTR_RO(backing_file);
721ee862730SMilan Broz LOOP_ATTR_RO(offset);
722ee862730SMilan Broz LOOP_ATTR_RO(sizelimit);
723ee862730SMilan Broz LOOP_ATTR_RO(autoclear);
724e03c8dd1SKay Sievers LOOP_ATTR_RO(partscan);
7252e5ab5f3SMing Lei LOOP_ATTR_RO(dio);
726ee862730SMilan Broz
727ee862730SMilan Broz static struct attribute *loop_attrs[] = {
728ee862730SMilan Broz &loop_attr_backing_file.attr,
729ee862730SMilan Broz &loop_attr_offset.attr,
730ee862730SMilan Broz &loop_attr_sizelimit.attr,
731ee862730SMilan Broz &loop_attr_autoclear.attr,
732e03c8dd1SKay Sievers &loop_attr_partscan.attr,
7332e5ab5f3SMing Lei &loop_attr_dio.attr,
734ee862730SMilan Broz NULL,
735ee862730SMilan Broz };
736ee862730SMilan Broz
737ee862730SMilan Broz static struct attribute_group loop_attribute_group = {
738ee862730SMilan Broz .name = "loop",
739ee862730SMilan Broz .attrs= loop_attrs,
740ee862730SMilan Broz };
741ee862730SMilan Broz
loop_sysfs_init(struct loop_device * lo)742d3349b6bSTetsuo Handa static void loop_sysfs_init(struct loop_device *lo)
743ee862730SMilan Broz {
744d3349b6bSTetsuo Handa lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
745ee862730SMilan Broz &loop_attribute_group);
746ee862730SMilan Broz }
747ee862730SMilan Broz
loop_sysfs_exit(struct loop_device * lo)748ee862730SMilan Broz static void loop_sysfs_exit(struct loop_device *lo)
749ee862730SMilan Broz {
750d3349b6bSTetsuo Handa if (lo->sysfs_inited)
751ee862730SMilan Broz sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
752ee862730SMilan Broz &loop_attribute_group);
753ee862730SMilan Broz }
754ee862730SMilan Broz
loop_config_discard(struct loop_device * lo)755dfaa2ef6SLukas Czerner static void loop_config_discard(struct loop_device *lo)
756dfaa2ef6SLukas Czerner {
757dfaa2ef6SLukas Czerner struct file *file = lo->lo_backing_file;
758dfaa2ef6SLukas Czerner struct inode *inode = file->f_mapping->host;
759dfaa2ef6SLukas Czerner struct request_queue *q = lo->lo_queue;
760bcb21c8cSMing Lei u32 granularity, max_discard_sectors;
761dfaa2ef6SLukas Czerner
762dfaa2ef6SLukas Czerner /*
763c52abf56SEvan Green * If the backing device is a block device, mirror its zeroing
764c52abf56SEvan Green * capability. Set the discard sectors to the block device's zeroing
765c52abf56SEvan Green * capabilities because loop discards result in blkdev_issue_zeroout(),
766c52abf56SEvan Green * not blkdev_issue_discard(). This maintains consistent behavior with
767c52abf56SEvan Green * file-backed loop devices: discarded regions read back as zero.
768c52abf56SEvan Green */
76947e96246SChristoph Hellwig if (S_ISBLK(inode->i_mode)) {
7704e7b5671SChristoph Hellwig struct request_queue *backingq = bdev_get_queue(I_BDEV(inode));
771c52abf56SEvan Green
772bcb21c8cSMing Lei max_discard_sectors = backingq->limits.max_write_zeroes_sectors;
7737b47ef52SChristoph Hellwig granularity = bdev_discard_granularity(I_BDEV(inode)) ?:
774bcb21c8cSMing Lei queue_physical_block_size(backingq);
775c52abf56SEvan Green
776c52abf56SEvan Green /*
777dfaa2ef6SLukas Czerner * We use punch hole to reclaim the free space used by the
77847e96246SChristoph Hellwig * image a.k.a. discard.
779dfaa2ef6SLukas Czerner */
78047e96246SChristoph Hellwig } else if (!file->f_op->fallocate) {
781bcb21c8cSMing Lei max_discard_sectors = 0;
782bcb21c8cSMing Lei granularity = 0;
783dfaa2ef6SLukas Czerner
784c52abf56SEvan Green } else {
78506582bc8SMing Lei struct kstatfs sbuf;
78606582bc8SMing Lei
787bcb21c8cSMing Lei max_discard_sectors = UINT_MAX >> 9;
78806582bc8SMing Lei if (!vfs_statfs(&file->f_path, &sbuf))
78906582bc8SMing Lei granularity = sbuf.f_bsize;
79006582bc8SMing Lei else
79106582bc8SMing Lei max_discard_sectors = 0;
792c52abf56SEvan Green }
793c52abf56SEvan Green
794bcb21c8cSMing Lei if (max_discard_sectors) {
795bcb21c8cSMing Lei q->limits.discard_granularity = granularity;
796bcb21c8cSMing Lei blk_queue_max_discard_sectors(q, max_discard_sectors);
797bcb21c8cSMing Lei blk_queue_max_write_zeroes_sectors(q, max_discard_sectors);
798bcb21c8cSMing Lei } else {
799bcb21c8cSMing Lei q->limits.discard_granularity = 0;
800bcb21c8cSMing Lei blk_queue_max_discard_sectors(q, 0);
801bcb21c8cSMing Lei blk_queue_max_write_zeroes_sectors(q, 0);
802dfaa2ef6SLukas Czerner }
803bcb21c8cSMing Lei }
804dfaa2ef6SLukas Czerner
80587579e9bSDan Schatzberg struct loop_worker {
80687579e9bSDan Schatzberg struct rb_node rb_node;
80787579e9bSDan Schatzberg struct work_struct work;
80887579e9bSDan Schatzberg struct list_head cmd_list;
80987579e9bSDan Schatzberg struct list_head idle_list;
81087579e9bSDan Schatzberg struct loop_device *lo;
811c74d40e8SDan Schatzberg struct cgroup_subsys_state *blkcg_css;
81287579e9bSDan Schatzberg unsigned long last_ran_at;
81387579e9bSDan Schatzberg };
814e03a3d7aSMing Lei
81587579e9bSDan Schatzberg static void loop_workfn(struct work_struct *work);
816b2ee7d46SNeilBrown
81787579e9bSDan Schatzberg #ifdef CONFIG_BLK_CGROUP
queue_on_root_worker(struct cgroup_subsys_state * css)81887579e9bSDan Schatzberg static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
819e03a3d7aSMing Lei {
82087579e9bSDan Schatzberg return !css || css == blkcg_root_css;
82187579e9bSDan Schatzberg }
82287579e9bSDan Schatzberg #else
queue_on_root_worker(struct cgroup_subsys_state * css)82387579e9bSDan Schatzberg static inline int queue_on_root_worker(struct cgroup_subsys_state *css)
82487579e9bSDan Schatzberg {
82587579e9bSDan Schatzberg return !css;
82687579e9bSDan Schatzberg }
82787579e9bSDan Schatzberg #endif
82887579e9bSDan Schatzberg
loop_queue_work(struct loop_device * lo,struct loop_cmd * cmd)82987579e9bSDan Schatzberg static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd)
83087579e9bSDan Schatzberg {
831413ec805SColin Ian King struct rb_node **node, *parent = NULL;
83287579e9bSDan Schatzberg struct loop_worker *cur_worker, *worker = NULL;
83387579e9bSDan Schatzberg struct work_struct *work;
83487579e9bSDan Schatzberg struct list_head *cmd_list;
83587579e9bSDan Schatzberg
83687579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock);
83787579e9bSDan Schatzberg
838c74d40e8SDan Schatzberg if (queue_on_root_worker(cmd->blkcg_css))
83987579e9bSDan Schatzberg goto queue_work;
84087579e9bSDan Schatzberg
84187579e9bSDan Schatzberg node = &lo->worker_tree.rb_node;
84287579e9bSDan Schatzberg
84387579e9bSDan Schatzberg while (*node) {
84487579e9bSDan Schatzberg parent = *node;
84587579e9bSDan Schatzberg cur_worker = container_of(*node, struct loop_worker, rb_node);
846c74d40e8SDan Schatzberg if (cur_worker->blkcg_css == cmd->blkcg_css) {
84787579e9bSDan Schatzberg worker = cur_worker;
84887579e9bSDan Schatzberg break;
849c74d40e8SDan Schatzberg } else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) {
85087579e9bSDan Schatzberg node = &(*node)->rb_left;
85187579e9bSDan Schatzberg } else {
85287579e9bSDan Schatzberg node = &(*node)->rb_right;
85387579e9bSDan Schatzberg }
85487579e9bSDan Schatzberg }
85587579e9bSDan Schatzberg if (worker)
85687579e9bSDan Schatzberg goto queue_work;
85787579e9bSDan Schatzberg
85887579e9bSDan Schatzberg worker = kzalloc(sizeof(struct loop_worker), GFP_NOWAIT | __GFP_NOWARN);
85987579e9bSDan Schatzberg /*
86087579e9bSDan Schatzberg * In the event we cannot allocate a worker, just queue on the
861c74d40e8SDan Schatzberg * rootcg worker and issue the I/O as the rootcg
86287579e9bSDan Schatzberg */
863c74d40e8SDan Schatzberg if (!worker) {
864c74d40e8SDan Schatzberg cmd->blkcg_css = NULL;
865c74d40e8SDan Schatzberg if (cmd->memcg_css)
866c74d40e8SDan Schatzberg css_put(cmd->memcg_css);
867c74d40e8SDan Schatzberg cmd->memcg_css = NULL;
86887579e9bSDan Schatzberg goto queue_work;
869c74d40e8SDan Schatzberg }
87087579e9bSDan Schatzberg
871c74d40e8SDan Schatzberg worker->blkcg_css = cmd->blkcg_css;
872c74d40e8SDan Schatzberg css_get(worker->blkcg_css);
87387579e9bSDan Schatzberg INIT_WORK(&worker->work, loop_workfn);
87487579e9bSDan Schatzberg INIT_LIST_HEAD(&worker->cmd_list);
87587579e9bSDan Schatzberg INIT_LIST_HEAD(&worker->idle_list);
87687579e9bSDan Schatzberg worker->lo = lo;
87787579e9bSDan Schatzberg rb_link_node(&worker->rb_node, parent, node);
87887579e9bSDan Schatzberg rb_insert_color(&worker->rb_node, &lo->worker_tree);
87987579e9bSDan Schatzberg queue_work:
88087579e9bSDan Schatzberg if (worker) {
88187579e9bSDan Schatzberg /*
88287579e9bSDan Schatzberg * We need to remove from the idle list here while
88387579e9bSDan Schatzberg * holding the lock so that the idle timer doesn't
88487579e9bSDan Schatzberg * free the worker
88587579e9bSDan Schatzberg */
88687579e9bSDan Schatzberg if (!list_empty(&worker->idle_list))
88787579e9bSDan Schatzberg list_del_init(&worker->idle_list);
88887579e9bSDan Schatzberg work = &worker->work;
88987579e9bSDan Schatzberg cmd_list = &worker->cmd_list;
89087579e9bSDan Schatzberg } else {
89187579e9bSDan Schatzberg work = &lo->rootcg_work;
89287579e9bSDan Schatzberg cmd_list = &lo->rootcg_cmd_list;
89387579e9bSDan Schatzberg }
89487579e9bSDan Schatzberg list_add_tail(&cmd->list_entry, cmd_list);
89587579e9bSDan Schatzberg queue_work(lo->workqueue, work);
89687579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock);
897e03a3d7aSMing Lei }
898e03a3d7aSMing Lei
loop_set_timer(struct loop_device * lo)8992cf429b5SChristoph Hellwig static void loop_set_timer(struct loop_device *lo)
9002cf429b5SChristoph Hellwig {
9012cf429b5SChristoph Hellwig timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT);
9022cf429b5SChristoph Hellwig }
9032cf429b5SChristoph Hellwig
loop_free_idle_workers(struct loop_device * lo,bool delete_all)9042cf429b5SChristoph Hellwig static void loop_free_idle_workers(struct loop_device *lo, bool delete_all)
9052cf429b5SChristoph Hellwig {
9062cf429b5SChristoph Hellwig struct loop_worker *pos, *worker;
9072cf429b5SChristoph Hellwig
9082cf429b5SChristoph Hellwig spin_lock_irq(&lo->lo_work_lock);
9092cf429b5SChristoph Hellwig list_for_each_entry_safe(worker, pos, &lo->idle_worker_list,
9102cf429b5SChristoph Hellwig idle_list) {
9112cf429b5SChristoph Hellwig if (!delete_all &&
9122cf429b5SChristoph Hellwig time_is_after_jiffies(worker->last_ran_at +
9132cf429b5SChristoph Hellwig LOOP_IDLE_WORKER_TIMEOUT))
9142cf429b5SChristoph Hellwig break;
9152cf429b5SChristoph Hellwig list_del(&worker->idle_list);
9162cf429b5SChristoph Hellwig rb_erase(&worker->rb_node, &lo->worker_tree);
9172cf429b5SChristoph Hellwig css_put(worker->blkcg_css);
9182cf429b5SChristoph Hellwig kfree(worker);
9192cf429b5SChristoph Hellwig }
9202cf429b5SChristoph Hellwig if (!list_empty(&lo->idle_worker_list))
9212cf429b5SChristoph Hellwig loop_set_timer(lo);
9222cf429b5SChristoph Hellwig spin_unlock_irq(&lo->lo_work_lock);
9232cf429b5SChristoph Hellwig }
9242cf429b5SChristoph Hellwig
loop_free_idle_workers_timer(struct timer_list * timer)9252cf429b5SChristoph Hellwig static void loop_free_idle_workers_timer(struct timer_list *timer)
9262cf429b5SChristoph Hellwig {
9272cf429b5SChristoph Hellwig struct loop_device *lo = container_of(timer, struct loop_device, timer);
9282cf429b5SChristoph Hellwig
9292cf429b5SChristoph Hellwig return loop_free_idle_workers(lo, false);
9302cf429b5SChristoph Hellwig }
9312cf429b5SChristoph Hellwig
loop_update_rotational(struct loop_device * lo)93256a85fd8SHolger Hoffstätte static void loop_update_rotational(struct loop_device *lo)
93356a85fd8SHolger Hoffstätte {
93456a85fd8SHolger Hoffstätte struct file *file = lo->lo_backing_file;
93556a85fd8SHolger Hoffstätte struct inode *file_inode = file->f_mapping->host;
93656a85fd8SHolger Hoffstätte struct block_device *file_bdev = file_inode->i_sb->s_bdev;
93756a85fd8SHolger Hoffstätte struct request_queue *q = lo->lo_queue;
93856a85fd8SHolger Hoffstätte bool nonrot = true;
93956a85fd8SHolger Hoffstätte
94056a85fd8SHolger Hoffstätte /* not all filesystems (e.g. tmpfs) have a sb->s_bdev */
94156a85fd8SHolger Hoffstätte if (file_bdev)
94210f0d2a5SChristoph Hellwig nonrot = bdev_nonrot(file_bdev);
94356a85fd8SHolger Hoffstätte
94456a85fd8SHolger Hoffstätte if (nonrot)
94556a85fd8SHolger Hoffstätte blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
94656a85fd8SHolger Hoffstätte else
94756a85fd8SHolger Hoffstätte blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
94856a85fd8SHolger Hoffstätte }
94956a85fd8SHolger Hoffstätte
95062ab466cSMartijn Coenen /**
95162ab466cSMartijn Coenen * loop_set_status_from_info - configure device from loop_info
95262ab466cSMartijn Coenen * @lo: struct loop_device to configure
95362ab466cSMartijn Coenen * @info: struct loop_info64 to configure the device with
95462ab466cSMartijn Coenen *
95562ab466cSMartijn Coenen * Configures the loop device parameters according to the passed
95662ab466cSMartijn Coenen * in loop_info64 configuration.
95762ab466cSMartijn Coenen */
95862ab466cSMartijn Coenen static int
loop_set_status_from_info(struct loop_device * lo,const struct loop_info64 * info)95962ab466cSMartijn Coenen loop_set_status_from_info(struct loop_device *lo,
96062ab466cSMartijn Coenen const struct loop_info64 *info)
96162ab466cSMartijn Coenen {
96262ab466cSMartijn Coenen if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE)
96362ab466cSMartijn Coenen return -EINVAL;
96462ab466cSMartijn Coenen
96547e96246SChristoph Hellwig switch (info->lo_encrypt_type) {
96647e96246SChristoph Hellwig case LO_CRYPT_NONE:
96747e96246SChristoph Hellwig break;
96847e96246SChristoph Hellwig case LO_CRYPT_XOR:
96947e96246SChristoph Hellwig pr_warn("support for the xor transformation has been removed.\n");
97062ab466cSMartijn Coenen return -EINVAL;
97147e96246SChristoph Hellwig case LO_CRYPT_CRYPTOAPI:
97247e96246SChristoph Hellwig pr_warn("support for cryptoloop has been removed. Use dm-crypt instead.\n");
97362ab466cSMartijn Coenen return -EINVAL;
97447e96246SChristoph Hellwig default:
97547e96246SChristoph Hellwig return -EINVAL;
97647e96246SChristoph Hellwig }
97762ab466cSMartijn Coenen
9789f6ad5d5SZhong Jinghua /* Avoid assigning overflow values */
9799f6ad5d5SZhong Jinghua if (info->lo_offset > LLONG_MAX || info->lo_sizelimit > LLONG_MAX)
9809f6ad5d5SZhong Jinghua return -EOVERFLOW;
9819f6ad5d5SZhong Jinghua
98262ab466cSMartijn Coenen lo->lo_offset = info->lo_offset;
98362ab466cSMartijn Coenen lo->lo_sizelimit = info->lo_sizelimit;
984c490a0b5SSiddh Raman Pant
98562ab466cSMartijn Coenen memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
98662ab466cSMartijn Coenen lo->lo_file_name[LO_NAME_SIZE-1] = 0;
987faf1d254SMartijn Coenen lo->lo_flags = info->lo_flags;
98862ab466cSMartijn Coenen return 0;
98962ab466cSMartijn Coenen }
99062ab466cSMartijn Coenen
loop_configure(struct loop_device * lo,blk_mode_t mode,struct block_device * bdev,const struct loop_config * config)99105bdb996SChristoph Hellwig static int loop_configure(struct loop_device *lo, blk_mode_t mode,
9923448914eSMartijn Coenen struct block_device *bdev,
9933448914eSMartijn Coenen const struct loop_config *config)
9941da177e4SLinus Torvalds {
9953ce6e1f6STetsuo Handa struct file *file = fget(config->fd);
9961da177e4SLinus Torvalds struct inode *inode;
9971da177e4SLinus Torvalds struct address_space *mapping;
9981da177e4SLinus Torvalds int error;
9991da177e4SLinus Torvalds loff_t size;
100085b0a54aSJan Kara bool partscan;
10013448914eSMartijn Coenen unsigned short bsize;
10023ce6e1f6STetsuo Handa bool is_loop;
10033ce6e1f6STetsuo Handa
10043ce6e1f6STetsuo Handa if (!file)
10053ce6e1f6STetsuo Handa return -EBADF;
10063ce6e1f6STetsuo Handa is_loop = is_loop_device(file);
10071da177e4SLinus Torvalds
10081da177e4SLinus Torvalds /* This is safe, since we have a reference from open(). */
10091da177e4SLinus Torvalds __module_get(THIS_MODULE);
10101da177e4SLinus Torvalds
101133ec3e53SJan Kara /*
101233ec3e53SJan Kara * If we don't hold exclusive handle for the device, upgrade to it
101333ec3e53SJan Kara * here to avoid changing device under exclusive owner.
101433ec3e53SJan Kara */
101505bdb996SChristoph Hellwig if (!(mode & BLK_OPEN_EXCL)) {
10160718afd4SChristoph Hellwig error = bd_prepare_to_claim(bdev, loop_configure, NULL);
1017ecbe6bc0SChristoph Hellwig if (error)
1018757ecf40SJan Kara goto out_putf;
101933ec3e53SJan Kara }
102033ec3e53SJan Kara
10213ce6e1f6STetsuo Handa error = loop_global_lock_killable(lo, is_loop);
102233ec3e53SJan Kara if (error)
102333ec3e53SJan Kara goto out_bdev;
1024757ecf40SJan Kara
10251da177e4SLinus Torvalds error = -EBUSY;
10261da177e4SLinus Torvalds if (lo->lo_state != Lo_unbound)
1027757ecf40SJan Kara goto out_unlock;
10281da177e4SLinus Torvalds
1029d2ac838eSTheodore Ts'o error = loop_validate_file(file, bdev);
1030d2ac838eSTheodore Ts'o if (error)
1031757ecf40SJan Kara goto out_unlock;
10321da177e4SLinus Torvalds
10331da177e4SLinus Torvalds mapping = file->f_mapping;
10341da177e4SLinus Torvalds inode = mapping->host;
10351da177e4SLinus Torvalds
10363448914eSMartijn Coenen if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) {
10373448914eSMartijn Coenen error = -EINVAL;
10383448914eSMartijn Coenen goto out_unlock;
10393448914eSMartijn Coenen }
10403448914eSMartijn Coenen
10413448914eSMartijn Coenen if (config->block_size) {
1042af3c570fSXie Yongji error = blk_validate_block_size(config->block_size);
10433448914eSMartijn Coenen if (error)
10443448914eSMartijn Coenen goto out_unlock;
10453448914eSMartijn Coenen }
10463448914eSMartijn Coenen
10473448914eSMartijn Coenen error = loop_set_status_from_info(lo, &config->info);
10483448914eSMartijn Coenen if (error)
10493448914eSMartijn Coenen goto out_unlock;
10503448914eSMartijn Coenen
105105bdb996SChristoph Hellwig if (!(file->f_mode & FMODE_WRITE) || !(mode & BLK_OPEN_WRITE) ||
1052283e7e5dSAl Viro !file->f_op->write_iter)
10533448914eSMartijn Coenen lo->lo_flags |= LO_FLAGS_READ_ONLY;
1054083a6a50SMartijn Coenen
1055d292dc80SChristoph Hellwig if (!lo->workqueue) {
105687579e9bSDan Schatzberg lo->workqueue = alloc_workqueue("loop%d",
105787579e9bSDan Schatzberg WQ_UNBOUND | WQ_FREEZABLE,
1058d292dc80SChristoph Hellwig 0, lo->lo_number);
105987579e9bSDan Schatzberg if (!lo->workqueue) {
106087579e9bSDan Schatzberg error = -ENOMEM;
1061757ecf40SJan Kara goto out_unlock;
106287579e9bSDan Schatzberg }
1063d292dc80SChristoph Hellwig }
10641da177e4SLinus Torvalds
1065bb430b69SAlyssa Ross /* suppress uevents while reconfiguring the device */
1066bb430b69SAlyssa Ross dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 1);
1067bb430b69SAlyssa Ross
1068ab6860f6SChristoph Hellwig disk_force_media_change(lo->lo_disk);
10697a2f0ce1SChristoph Hellwig set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0);
10701da177e4SLinus Torvalds
10713448914eSMartijn Coenen lo->use_dio = lo->lo_flags & LO_FLAGS_DIRECT_IO;
10721da177e4SLinus Torvalds lo->lo_device = bdev;
10731da177e4SLinus Torvalds lo->lo_backing_file = file;
10741da177e4SLinus Torvalds lo->old_gfp_mask = mapping_gfp_mask(mapping);
10751da177e4SLinus Torvalds mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
10761da177e4SLinus Torvalds
10773448914eSMartijn Coenen if (!(lo->lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
107821d0727fSJens Axboe blk_queue_write_cache(lo->lo_queue, true, false);
107968db1961SNikanth Karthikesan
10803448914eSMartijn Coenen if (config->block_size)
10813448914eSMartijn Coenen bsize = config->block_size;
108296ed320dSLinus Torvalds else if ((lo->lo_backing_file->f_flags & O_DIRECT) && inode->i_sb->s_bdev)
108385560117SMartijn Coenen /* In case of direct I/O, match underlying block size */
10843448914eSMartijn Coenen bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
10853448914eSMartijn Coenen else
10863448914eSMartijn Coenen bsize = 512;
108785560117SMartijn Coenen
108885560117SMartijn Coenen blk_queue_logical_block_size(lo->lo_queue, bsize);
108985560117SMartijn Coenen blk_queue_physical_block_size(lo->lo_queue, bsize);
109085560117SMartijn Coenen blk_queue_io_min(lo->lo_queue, bsize);
109185560117SMartijn Coenen
10922b9ac22bSKristian Klausen loop_config_discard(lo);
109356a85fd8SHolger Hoffstätte loop_update_rotational(lo);
10942e5ab5f3SMing Lei loop_update_dio(lo);
1095ee862730SMilan Broz loop_sysfs_init(lo);
109679e5dc59SMartijn Coenen
109779e5dc59SMartijn Coenen size = get_loop_size(lo, file);
10985795b6f5SMartijn Coenen loop_set_size(lo, size);
10991da177e4SLinus Torvalds
11003ce6e1f6STetsuo Handa /* Order wrt reading lo_state in loop_validate_file(). */
11013ce6e1f6STetsuo Handa wmb();
11023ce6e1f6STetsuo Handa
11036c997918SSerge E. Hallyn lo->lo_state = Lo_bound;
1104e03c8dd1SKay Sievers if (part_shift)
1105e03c8dd1SKay Sievers lo->lo_flags |= LO_FLAGS_PARTSCAN;
110685b0a54aSJan Kara partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
1107fe6a8fc5SLennart Poettering if (partscan)
1108b9684a71SChristoph Hellwig clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
1109c1681bf8SAnatol Pomozov
1110bb430b69SAlyssa Ross /* enable and uncork uevent now that we are done */
1111bb430b69SAlyssa Ross dev_set_uevent_suppress(disk_to_dev(lo->lo_disk), 0);
1112bb430b69SAlyssa Ross
11133ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop);
111485b0a54aSJan Kara if (partscan)
11150384264eSChristoph Hellwig loop_reread_partitions(lo);
1116bb430b69SAlyssa Ross
111705bdb996SChristoph Hellwig if (!(mode & BLK_OPEN_EXCL))
111837c3fc9aSChristoph Hellwig bd_abort_claiming(bdev, loop_configure);
1119498ef5c7SChristoph Hellwig
1120bb430b69SAlyssa Ross return 0;
11211da177e4SLinus Torvalds
1122757ecf40SJan Kara out_unlock:
11233ce6e1f6STetsuo Handa loop_global_unlock(lo, is_loop);
112433ec3e53SJan Kara out_bdev:
112505bdb996SChristoph Hellwig if (!(mode & BLK_OPEN_EXCL))
112637c3fc9aSChristoph Hellwig bd_abort_claiming(bdev, loop_configure);
11271da177e4SLinus Torvalds out_putf:
11281da177e4SLinus Torvalds fput(file);
11291da177e4SLinus Torvalds /* This is safe: open() is still holding a reference. */
11301da177e4SLinus Torvalds module_put(THIS_MODULE);
1131bb430b69SAlyssa Ross return error;
11321da177e4SLinus Torvalds }
11331da177e4SLinus Torvalds
__loop_clr_fd(struct loop_device * lo,bool release)1134bf23747eSTetsuo Handa static void __loop_clr_fd(struct loop_device *lo, bool release)
11351da177e4SLinus Torvalds {
11366050fa4cSTetsuo Handa struct file *filp;
1137b4e3ca1aSAl Viro gfp_t gfp = lo->old_gfp_mask;
11381da177e4SLinus Torvalds
11394ceddce5SMauricio Faria de Oliveira if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags))
11404ceddce5SMauricio Faria de Oliveira blk_queue_write_cache(lo->lo_queue, false, false);
11414ceddce5SMauricio Faria de Oliveira
11421fe0b1acSChristoph Hellwig /*
11431fe0b1acSChristoph Hellwig * Freeze the request queue when unbinding on a live file descriptor and
11441fe0b1acSChristoph Hellwig * thus an open device. When called from ->release we are guaranteed
11451fe0b1acSChristoph Hellwig * that there is no I/O in progress already.
11461fe0b1acSChristoph Hellwig */
11471fe0b1acSChristoph Hellwig if (!release)
1148f8933667SMing Lei blk_mq_freeze_queue(lo->lo_queue);
1149f8933667SMing Lei
11501da177e4SLinus Torvalds spin_lock_irq(&lo->lo_lock);
11516050fa4cSTetsuo Handa filp = lo->lo_backing_file;
11521da177e4SLinus Torvalds lo->lo_backing_file = NULL;
115305eb0f25SKay Sievers spin_unlock_irq(&lo->lo_lock);
11541da177e4SLinus Torvalds
11551da177e4SLinus Torvalds lo->lo_device = NULL;
11561da177e4SLinus Torvalds lo->lo_offset = 0;
11571da177e4SLinus Torvalds lo->lo_sizelimit = 0;
11581da177e4SLinus Torvalds memset(lo->lo_file_name, 0, LO_NAME_SIZE);
115989e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, 512);
1160bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, 512);
1161bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, 512);
1162e515be8fSXie Yongji invalidate_disk(lo->lo_disk);
116351a0bb0cSMilan Broz loop_sysfs_exit(lo);
1164c3473c63SDavid Zeuthen /* let user-space know about this change */
116519f553dbSXie Yongji kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE);
11661da177e4SLinus Torvalds mapping_set_gfp_mask(filp->f_mapping, gfp);
1167bf23747eSTetsuo Handa /* This is safe: open() is still holding a reference. */
1168bf23747eSTetsuo Handa module_put(THIS_MODULE);
11691fe0b1acSChristoph Hellwig if (!release)
1170f8933667SMing Lei blk_mq_unfreeze_queue(lo->lo_queue);
1171f8933667SMing Lei
1172ab6860f6SChristoph Hellwig disk_force_media_change(lo->lo_disk);
11736050fa4cSTetsuo Handa
11746050fa4cSTetsuo Handa if (lo->lo_flags & LO_FLAGS_PARTSCAN) {
11756050fa4cSTetsuo Handa int err;
11766050fa4cSTetsuo Handa
1177bf23747eSTetsuo Handa /*
1178bf23747eSTetsuo Handa * open_mutex has been held already in release path, so don't
1179bf23747eSTetsuo Handa * acquire it if this function is called in such case.
1180bf23747eSTetsuo Handa *
1181bf23747eSTetsuo Handa * If the reread partition isn't from release path, lo_refcnt
1182bf23747eSTetsuo Handa * must be at least one and it can only become zero when the
1183bf23747eSTetsuo Handa * current holder is released.
1184bf23747eSTetsuo Handa */
1185bf23747eSTetsuo Handa if (!release)
11860384264eSChristoph Hellwig mutex_lock(&lo->lo_disk->open_mutex);
11870384264eSChristoph Hellwig err = bdev_disk_changed(lo->lo_disk, false);
1188bf23747eSTetsuo Handa if (!release)
11890384264eSChristoph Hellwig mutex_unlock(&lo->lo_disk->open_mutex);
119040853d6fSDongli Zhang if (err)
1191d57f3374SJan Kara pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
11926050fa4cSTetsuo Handa __func__, lo->lo_number, err);
1193d57f3374SJan Kara /* Device is gone, no point in returning error */
1194d57f3374SJan Kara }
1195758a58d0SDongli Zhang
1196bf23747eSTetsuo Handa /*
1197bf23747eSTetsuo Handa * lo->lo_state is set to Lo_unbound here after above partscan has
1198bf23747eSTetsuo Handa * finished. There cannot be anybody else entering __loop_clr_fd() as
1199bf23747eSTetsuo Handa * Lo_rundown state protects us from all the other places trying to
1200bf23747eSTetsuo Handa * change the 'lo' device.
1201bf23747eSTetsuo Handa */
1202758a58d0SDongli Zhang lo->lo_flags = 0;
1203758a58d0SDongli Zhang if (!part_shift)
1204b9684a71SChristoph Hellwig set_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
12056050fa4cSTetsuo Handa mutex_lock(&lo->lo_mutex);
1206758a58d0SDongli Zhang lo->lo_state = Lo_unbound;
12076cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
1208758a58d0SDongli Zhang
1209bf23747eSTetsuo Handa /*
1210bf23747eSTetsuo Handa * Need not hold lo_mutex to fput backing file. Calling fput holding
1211bf23747eSTetsuo Handa * lo_mutex triggers a circular lock dependency possibility warning as
1212bf23747eSTetsuo Handa * fput can take open_mutex which is usually taken before lo_mutex.
1213bf23747eSTetsuo Handa */
1214bf23747eSTetsuo Handa fput(filp);
12151da177e4SLinus Torvalds }
12161da177e4SLinus Torvalds
loop_clr_fd(struct loop_device * lo)1217a2505b79SJan Kara static int loop_clr_fd(struct loop_device *lo)
1218a2505b79SJan Kara {
12197ccd0791SJan Kara int err;
12207ccd0791SJan Kara
1221158eaebaSTetsuo Handa /*
1222158eaebaSTetsuo Handa * Since lo_ioctl() is called without locks held, it is possible that
1223158eaebaSTetsuo Handa * loop_configure()/loop_change_fd() and loop_clr_fd() run in parallel.
1224158eaebaSTetsuo Handa *
1225158eaebaSTetsuo Handa * Therefore, use global lock when setting Lo_rundown state in order to
1226158eaebaSTetsuo Handa * make sure that loop_validate_file() will fail if the "struct file"
1227158eaebaSTetsuo Handa * which loop_configure()/loop_change_fd() found via fget() was this
1228158eaebaSTetsuo Handa * loop device.
1229158eaebaSTetsuo Handa */
1230158eaebaSTetsuo Handa err = loop_global_lock_killable(lo, true);
12317ccd0791SJan Kara if (err)
12327ccd0791SJan Kara return err;
12337ccd0791SJan Kara if (lo->lo_state != Lo_bound) {
1234158eaebaSTetsuo Handa loop_global_unlock(lo, true);
1235a2505b79SJan Kara return -ENXIO;
12367ccd0791SJan Kara }
1237a2505b79SJan Kara /*
1238a2505b79SJan Kara * If we've explicitly asked to tear down the loop device,
1239a2505b79SJan Kara * and it has an elevated reference count, set it for auto-teardown when
1240a2505b79SJan Kara * the last reference goes away. This stops $!~#$@ udev from
1241a2505b79SJan Kara * preventing teardown because it decided that it needs to run blkid on
1242a2505b79SJan Kara * the loopback device whenever they appear. xfstests is notorious for
1243a2505b79SJan Kara * failing tests because blkid via udev races with a losetup
1244a2505b79SJan Kara * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
1245a2505b79SJan Kara * command to fail with EBUSY.
1246a2505b79SJan Kara */
1247a0e286b6SChristoph Hellwig if (disk_openers(lo->lo_disk) > 1) {
1248a2505b79SJan Kara lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
1249158eaebaSTetsuo Handa loop_global_unlock(lo, true);
1250a2505b79SJan Kara return 0;
1251a2505b79SJan Kara }
1252a2505b79SJan Kara lo->lo_state = Lo_rundown;
1253158eaebaSTetsuo Handa loop_global_unlock(lo, true);
1254a2505b79SJan Kara
1255bf23747eSTetsuo Handa __loop_clr_fd(lo, false);
12566050fa4cSTetsuo Handa return 0;
1257a2505b79SJan Kara }
1258a2505b79SJan Kara
12591da177e4SLinus Torvalds static int
loop_set_status(struct loop_device * lo,const struct loop_info64 * info)12601da177e4SLinus Torvalds loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
12611da177e4SLinus Torvalds {
12621da177e4SLinus Torvalds int err;
1263faf1d254SMartijn Coenen int prev_lo_flags;
126485b0a54aSJan Kara bool partscan = false;
12650c3796c2SMartijn Coenen bool size_changed = false;
12661da177e4SLinus Torvalds
12676cc8e743SPavel Tatashin err = mutex_lock_killable(&lo->lo_mutex);
1268550df5fdSJan Kara if (err)
1269550df5fdSJan Kara return err;
1270550df5fdSJan Kara if (lo->lo_state != Lo_bound) {
1271550df5fdSJan Kara err = -ENXIO;
1272550df5fdSJan Kara goto out_unlock;
1273550df5fdSJan Kara }
12741da177e4SLinus Torvalds
12755db470e2SJaegeuk Kim if (lo->lo_offset != info->lo_offset ||
12765db470e2SJaegeuk Kim lo->lo_sizelimit != info->lo_sizelimit) {
12770c3796c2SMartijn Coenen size_changed = true;
12785db470e2SJaegeuk Kim sync_blockdev(lo->lo_device);
1279f4bd34b1SZheng Bin invalidate_bdev(lo->lo_device);
12805db470e2SJaegeuk Kim }
12815db470e2SJaegeuk Kim
1282ecdd0959SMing Lei /* I/O need to be drained during transfer transition */
1283ecdd0959SMing Lei blk_mq_freeze_queue(lo->lo_queue);
1284ecdd0959SMing Lei
1285faf1d254SMartijn Coenen prev_lo_flags = lo->lo_flags;
1286faf1d254SMartijn Coenen
12870c3796c2SMartijn Coenen err = loop_set_status_from_info(lo, info);
12880c3796c2SMartijn Coenen if (err)
1289550df5fdSJan Kara goto out_unfreeze;
12900c3796c2SMartijn Coenen
1291faf1d254SMartijn Coenen /* Mask out flags that can't be set using LOOP_SET_STATUS. */
12926ac92fb5SMartijn Coenen lo->lo_flags &= LOOP_SET_STATUS_SETTABLE_FLAGS;
1293faf1d254SMartijn Coenen /* For those flags, use the previous values instead */
1294faf1d254SMartijn Coenen lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS;
1295faf1d254SMartijn Coenen /* For flags that can't be cleared, use previous values too */
1296faf1d254SMartijn Coenen lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_CLEARABLE_FLAGS;
1297faf1d254SMartijn Coenen
1298b0bd158dSMartijn Coenen if (size_changed) {
1299b0bd158dSMartijn Coenen loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit,
1300b0bd158dSMartijn Coenen lo->lo_backing_file);
1301b0bd158dSMartijn Coenen loop_set_size(lo, new_size);
1302b040ad9cSArnd Bergmann }
1303541c742aSGuo Chao
1304dfaa2ef6SLukas Czerner loop_config_discard(lo);
13051da177e4SLinus Torvalds
13062e5ab5f3SMing Lei /* update dio if lo_offset or transfer is changed */
13072e5ab5f3SMing Lei __loop_update_dio(lo, lo->use_dio);
13082e5ab5f3SMing Lei
1309550df5fdSJan Kara out_unfreeze:
1310ecdd0959SMing Lei blk_mq_unfreeze_queue(lo->lo_queue);
1311e02898b4SOmar Sandoval
1312faf1d254SMartijn Coenen if (!err && (lo->lo_flags & LO_FLAGS_PARTSCAN) &&
1313faf1d254SMartijn Coenen !(prev_lo_flags & LO_FLAGS_PARTSCAN)) {
1314b9684a71SChristoph Hellwig clear_bit(GD_SUPPRESS_PART_SCAN, &lo->lo_disk->state);
131585b0a54aSJan Kara partscan = true;
1316e02898b4SOmar Sandoval }
1317550df5fdSJan Kara out_unlock:
13186cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
131985b0a54aSJan Kara if (partscan)
13200384264eSChristoph Hellwig loop_reread_partitions(lo);
1321e02898b4SOmar Sandoval
1322ecdd0959SMing Lei return err;
13231da177e4SLinus Torvalds }
13241da177e4SLinus Torvalds
13251da177e4SLinus Torvalds static int
loop_get_status(struct loop_device * lo,struct loop_info64 * info)13261da177e4SLinus Torvalds loop_get_status(struct loop_device *lo, struct loop_info64 *info)
13271da177e4SLinus Torvalds {
1328b1ab5fa3STetsuo Handa struct path path;
13291da177e4SLinus Torvalds struct kstat stat;
13302d1d4c1eSOmar Sandoval int ret;
13311da177e4SLinus Torvalds
13326cc8e743SPavel Tatashin ret = mutex_lock_killable(&lo->lo_mutex);
13334a5ce9baSJan Kara if (ret)
13344a5ce9baSJan Kara return ret;
13352d1d4c1eSOmar Sandoval if (lo->lo_state != Lo_bound) {
13366cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
13371da177e4SLinus Torvalds return -ENXIO;
13382d1d4c1eSOmar Sandoval }
13392d1d4c1eSOmar Sandoval
13401da177e4SLinus Torvalds memset(info, 0, sizeof(*info));
13411da177e4SLinus Torvalds info->lo_number = lo->lo_number;
13421da177e4SLinus Torvalds info->lo_offset = lo->lo_offset;
13431da177e4SLinus Torvalds info->lo_sizelimit = lo->lo_sizelimit;
13441da177e4SLinus Torvalds info->lo_flags = lo->lo_flags;
13451da177e4SLinus Torvalds memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
13462d1d4c1eSOmar Sandoval
13476cc8e743SPavel Tatashin /* Drop lo_mutex while we call into the filesystem. */
1348b1ab5fa3STetsuo Handa path = lo->lo_backing_file->f_path;
1349b1ab5fa3STetsuo Handa path_get(&path);
13506cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
1351b1ab5fa3STetsuo Handa ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT);
13522d1d4c1eSOmar Sandoval if (!ret) {
13532d1d4c1eSOmar Sandoval info->lo_device = huge_encode_dev(stat.dev);
13542d1d4c1eSOmar Sandoval info->lo_inode = stat.ino;
13552d1d4c1eSOmar Sandoval info->lo_rdevice = huge_encode_dev(stat.rdev);
13562d1d4c1eSOmar Sandoval }
1357b1ab5fa3STetsuo Handa path_put(&path);
13582d1d4c1eSOmar Sandoval return ret;
13591da177e4SLinus Torvalds }
13601da177e4SLinus Torvalds
13611da177e4SLinus Torvalds static void
loop_info64_from_old(const struct loop_info * info,struct loop_info64 * info64)13621da177e4SLinus Torvalds loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
13631da177e4SLinus Torvalds {
13641da177e4SLinus Torvalds memset(info64, 0, sizeof(*info64));
13651da177e4SLinus Torvalds info64->lo_number = info->lo_number;
13661da177e4SLinus Torvalds info64->lo_device = info->lo_device;
13671da177e4SLinus Torvalds info64->lo_inode = info->lo_inode;
13681da177e4SLinus Torvalds info64->lo_rdevice = info->lo_rdevice;
13691da177e4SLinus Torvalds info64->lo_offset = info->lo_offset;
13701da177e4SLinus Torvalds info64->lo_sizelimit = 0;
13711da177e4SLinus Torvalds info64->lo_flags = info->lo_flags;
13721da177e4SLinus Torvalds memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
13731da177e4SLinus Torvalds }
13741da177e4SLinus Torvalds
13751da177e4SLinus Torvalds static int
loop_info64_to_old(const struct loop_info64 * info64,struct loop_info * info)13761da177e4SLinus Torvalds loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
13771da177e4SLinus Torvalds {
13781da177e4SLinus Torvalds memset(info, 0, sizeof(*info));
13791da177e4SLinus Torvalds info->lo_number = info64->lo_number;
13801da177e4SLinus Torvalds info->lo_device = info64->lo_device;
13811da177e4SLinus Torvalds info->lo_inode = info64->lo_inode;
13821da177e4SLinus Torvalds info->lo_rdevice = info64->lo_rdevice;
13831da177e4SLinus Torvalds info->lo_offset = info64->lo_offset;
13841da177e4SLinus Torvalds info->lo_flags = info64->lo_flags;
13851da177e4SLinus Torvalds memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
13861da177e4SLinus Torvalds
13871da177e4SLinus Torvalds /* error in case values were truncated */
13881da177e4SLinus Torvalds if (info->lo_device != info64->lo_device ||
13891da177e4SLinus Torvalds info->lo_rdevice != info64->lo_rdevice ||
13901da177e4SLinus Torvalds info->lo_inode != info64->lo_inode ||
13911da177e4SLinus Torvalds info->lo_offset != info64->lo_offset)
13921da177e4SLinus Torvalds return -EOVERFLOW;
13931da177e4SLinus Torvalds
13941da177e4SLinus Torvalds return 0;
13951da177e4SLinus Torvalds }
13961da177e4SLinus Torvalds
13971da177e4SLinus Torvalds static int
loop_set_status_old(struct loop_device * lo,const struct loop_info __user * arg)13981da177e4SLinus Torvalds loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
13991da177e4SLinus Torvalds {
14001da177e4SLinus Torvalds struct loop_info info;
14011da177e4SLinus Torvalds struct loop_info64 info64;
14021da177e4SLinus Torvalds
14031da177e4SLinus Torvalds if (copy_from_user(&info, arg, sizeof (struct loop_info)))
14041da177e4SLinus Torvalds return -EFAULT;
14051da177e4SLinus Torvalds loop_info64_from_old(&info, &info64);
14061da177e4SLinus Torvalds return loop_set_status(lo, &info64);
14071da177e4SLinus Torvalds }
14081da177e4SLinus Torvalds
14091da177e4SLinus Torvalds static int
loop_set_status64(struct loop_device * lo,const struct loop_info64 __user * arg)14101da177e4SLinus Torvalds loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
14111da177e4SLinus Torvalds {
14121da177e4SLinus Torvalds struct loop_info64 info64;
14131da177e4SLinus Torvalds
14141da177e4SLinus Torvalds if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
14151da177e4SLinus Torvalds return -EFAULT;
14161da177e4SLinus Torvalds return loop_set_status(lo, &info64);
14171da177e4SLinus Torvalds }
14181da177e4SLinus Torvalds
14191da177e4SLinus Torvalds static int
loop_get_status_old(struct loop_device * lo,struct loop_info __user * arg)14201da177e4SLinus Torvalds loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
14211da177e4SLinus Torvalds struct loop_info info;
14221da177e4SLinus Torvalds struct loop_info64 info64;
1423bdac616dSOmar Sandoval int err;
14241da177e4SLinus Torvalds
14254a5ce9baSJan Kara if (!arg)
1426bdac616dSOmar Sandoval return -EINVAL;
14271da177e4SLinus Torvalds err = loop_get_status(lo, &info64);
14281da177e4SLinus Torvalds if (!err)
14291da177e4SLinus Torvalds err = loop_info64_to_old(&info64, &info);
14301da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info, sizeof(info)))
14311da177e4SLinus Torvalds err = -EFAULT;
14321da177e4SLinus Torvalds
14331da177e4SLinus Torvalds return err;
14341da177e4SLinus Torvalds }
14351da177e4SLinus Torvalds
14361da177e4SLinus Torvalds static int
loop_get_status64(struct loop_device * lo,struct loop_info64 __user * arg)14371da177e4SLinus Torvalds loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
14381da177e4SLinus Torvalds struct loop_info64 info64;
1439bdac616dSOmar Sandoval int err;
14401da177e4SLinus Torvalds
14414a5ce9baSJan Kara if (!arg)
1442bdac616dSOmar Sandoval return -EINVAL;
14431da177e4SLinus Torvalds err = loop_get_status(lo, &info64);
14441da177e4SLinus Torvalds if (!err && copy_to_user(arg, &info64, sizeof(info64)))
14451da177e4SLinus Torvalds err = -EFAULT;
14461da177e4SLinus Torvalds
14471da177e4SLinus Torvalds return err;
14481da177e4SLinus Torvalds }
14491da177e4SLinus Torvalds
loop_set_capacity(struct loop_device * lo)145051001b7dSHannes Reinecke static int loop_set_capacity(struct loop_device *lo)
145153d66608SJ. R. Okajima {
14520a6ed1b5SMartijn Coenen loff_t size;
14530a6ed1b5SMartijn Coenen
145453d66608SJ. R. Okajima if (unlikely(lo->lo_state != Lo_bound))
14557b0576a3SGuo Chao return -ENXIO;
145653d66608SJ. R. Okajima
14570a6ed1b5SMartijn Coenen size = get_loop_size(lo, lo->lo_backing_file);
14580a6ed1b5SMartijn Coenen loop_set_size(lo, size);
1459083a6a50SMartijn Coenen
1460083a6a50SMartijn Coenen return 0;
146153d66608SJ. R. Okajima }
146253d66608SJ. R. Okajima
loop_set_dio(struct loop_device * lo,unsigned long arg)1463ab1cb278SMing Lei static int loop_set_dio(struct loop_device *lo, unsigned long arg)
1464ab1cb278SMing Lei {
1465ab1cb278SMing Lei int error = -ENXIO;
1466ab1cb278SMing Lei if (lo->lo_state != Lo_bound)
1467ab1cb278SMing Lei goto out;
1468ab1cb278SMing Lei
1469ab1cb278SMing Lei __loop_update_dio(lo, !!arg);
1470ab1cb278SMing Lei if (lo->use_dio == !!arg)
1471ab1cb278SMing Lei return 0;
1472ab1cb278SMing Lei error = -EINVAL;
1473ab1cb278SMing Lei out:
1474ab1cb278SMing Lei return error;
1475ab1cb278SMing Lei }
1476ab1cb278SMing Lei
loop_set_block_size(struct loop_device * lo,unsigned long arg)147789e4fdecSOmar Sandoval static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
147889e4fdecSOmar Sandoval {
14795db470e2SJaegeuk Kim int err = 0;
14805db470e2SJaegeuk Kim
148189e4fdecSOmar Sandoval if (lo->lo_state != Lo_bound)
148289e4fdecSOmar Sandoval return -ENXIO;
148389e4fdecSOmar Sandoval
1484af3c570fSXie Yongji err = blk_validate_block_size(arg);
14853448914eSMartijn Coenen if (err)
14863448914eSMartijn Coenen return err;
148789e4fdecSOmar Sandoval
14887e81f99aSMartijn Coenen if (lo->lo_queue->limits.logical_block_size == arg)
14897e81f99aSMartijn Coenen return 0;
14907e81f99aSMartijn Coenen
14915db470e2SJaegeuk Kim sync_blockdev(lo->lo_device);
1492f4bd34b1SZheng Bin invalidate_bdev(lo->lo_device);
14935db470e2SJaegeuk Kim
149489e4fdecSOmar Sandoval blk_mq_freeze_queue(lo->lo_queue);
149589e4fdecSOmar Sandoval blk_queue_logical_block_size(lo->lo_queue, arg);
1496bf093753SOmar Sandoval blk_queue_physical_block_size(lo->lo_queue, arg);
1497bf093753SOmar Sandoval blk_queue_io_min(lo->lo_queue, arg);
149889e4fdecSOmar Sandoval loop_update_dio(lo);
149989e4fdecSOmar Sandoval blk_mq_unfreeze_queue(lo->lo_queue);
150089e4fdecSOmar Sandoval
15015db470e2SJaegeuk Kim return err;
150289e4fdecSOmar Sandoval }
150389e4fdecSOmar Sandoval
lo_simple_ioctl(struct loop_device * lo,unsigned int cmd,unsigned long arg)1504a1316544SJan Kara static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
1505a1316544SJan Kara unsigned long arg)
15061da177e4SLinus Torvalds {
15071da177e4SLinus Torvalds int err;
15081da177e4SLinus Torvalds
15096cc8e743SPavel Tatashin err = mutex_lock_killable(&lo->lo_mutex);
15103148ffbdSOmar Sandoval if (err)
1511a1316544SJan Kara return err;
15121da177e4SLinus Torvalds switch (cmd) {
151353d66608SJ. R. Okajima case LOOP_SET_CAPACITY:
151451001b7dSHannes Reinecke err = loop_set_capacity(lo);
151553d66608SJ. R. Okajima break;
1516ab1cb278SMing Lei case LOOP_SET_DIRECT_IO:
1517ab1cb278SMing Lei err = loop_set_dio(lo, arg);
1518ab1cb278SMing Lei break;
151989e4fdecSOmar Sandoval case LOOP_SET_BLOCK_SIZE:
152089e4fdecSOmar Sandoval err = loop_set_block_size(lo, arg);
152189e4fdecSOmar Sandoval break;
15221da177e4SLinus Torvalds default:
152347e96246SChristoph Hellwig err = -EINVAL;
15241da177e4SLinus Torvalds }
15256cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
1526a1316544SJan Kara return err;
1527a1316544SJan Kara }
1528f028f3b2SNikanth Karthikesan
lo_ioctl(struct block_device * bdev,blk_mode_t mode,unsigned int cmd,unsigned long arg)152905bdb996SChristoph Hellwig static int lo_ioctl(struct block_device *bdev, blk_mode_t mode,
1530a1316544SJan Kara unsigned int cmd, unsigned long arg)
1531a1316544SJan Kara {
1532a1316544SJan Kara struct loop_device *lo = bdev->bd_disk->private_data;
1533571fae6eSMartijn Coenen void __user *argp = (void __user *) arg;
1534a1316544SJan Kara int err;
1535a1316544SJan Kara
1536a1316544SJan Kara switch (cmd) {
15373448914eSMartijn Coenen case LOOP_SET_FD: {
15383448914eSMartijn Coenen /*
15393448914eSMartijn Coenen * Legacy case - pass in a zeroed out struct loop_config with
15403448914eSMartijn Coenen * only the file descriptor set , which corresponds with the
15413448914eSMartijn Coenen * default parameters we'd have used otherwise.
15423448914eSMartijn Coenen */
15433448914eSMartijn Coenen struct loop_config config;
15443448914eSMartijn Coenen
15453448914eSMartijn Coenen memset(&config, 0, sizeof(config));
15463448914eSMartijn Coenen config.fd = arg;
15473448914eSMartijn Coenen
15483448914eSMartijn Coenen return loop_configure(lo, mode, bdev, &config);
15493448914eSMartijn Coenen }
15503448914eSMartijn Coenen case LOOP_CONFIGURE: {
15513448914eSMartijn Coenen struct loop_config config;
15523448914eSMartijn Coenen
15533448914eSMartijn Coenen if (copy_from_user(&config, argp, sizeof(config)))
15543448914eSMartijn Coenen return -EFAULT;
15553448914eSMartijn Coenen
15563448914eSMartijn Coenen return loop_configure(lo, mode, bdev, &config);
15573448914eSMartijn Coenen }
1558a1316544SJan Kara case LOOP_CHANGE_FD:
1559c3710770SJan Kara return loop_change_fd(lo, bdev, arg);
1560a1316544SJan Kara case LOOP_CLR_FD:
15617ccd0791SJan Kara return loop_clr_fd(lo);
1562a1316544SJan Kara case LOOP_SET_STATUS:
1563a1316544SJan Kara err = -EPERM;
156405bdb996SChristoph Hellwig if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
1565571fae6eSMartijn Coenen err = loop_set_status_old(lo, argp);
1566a1316544SJan Kara break;
1567a1316544SJan Kara case LOOP_GET_STATUS:
1568571fae6eSMartijn Coenen return loop_get_status_old(lo, argp);
1569a1316544SJan Kara case LOOP_SET_STATUS64:
1570a1316544SJan Kara err = -EPERM;
157105bdb996SChristoph Hellwig if ((mode & BLK_OPEN_WRITE) || capable(CAP_SYS_ADMIN))
1572571fae6eSMartijn Coenen err = loop_set_status64(lo, argp);
1573a1316544SJan Kara break;
1574a1316544SJan Kara case LOOP_GET_STATUS64:
1575571fae6eSMartijn Coenen return loop_get_status64(lo, argp);
1576a1316544SJan Kara case LOOP_SET_CAPACITY:
1577a1316544SJan Kara case LOOP_SET_DIRECT_IO:
1578a1316544SJan Kara case LOOP_SET_BLOCK_SIZE:
157905bdb996SChristoph Hellwig if (!(mode & BLK_OPEN_WRITE) && !capable(CAP_SYS_ADMIN))
1580a1316544SJan Kara return -EPERM;
1581df561f66SGustavo A. R. Silva fallthrough;
1582a1316544SJan Kara default:
1583a1316544SJan Kara err = lo_simple_ioctl(lo, cmd, arg);
1584a1316544SJan Kara break;
1585a1316544SJan Kara }
1586a1316544SJan Kara
15871da177e4SLinus Torvalds return err;
15881da177e4SLinus Torvalds }
15891da177e4SLinus Torvalds
1590863d5b82SDavid Howells #ifdef CONFIG_COMPAT
1591863d5b82SDavid Howells struct compat_loop_info {
1592863d5b82SDavid Howells compat_int_t lo_number; /* ioctl r/o */
1593863d5b82SDavid Howells compat_dev_t lo_device; /* ioctl r/o */
1594863d5b82SDavid Howells compat_ulong_t lo_inode; /* ioctl r/o */
1595863d5b82SDavid Howells compat_dev_t lo_rdevice; /* ioctl r/o */
1596863d5b82SDavid Howells compat_int_t lo_offset;
1597f941c51eSCarlos Llamas compat_int_t lo_encrypt_type; /* obsolete, ignored */
1598863d5b82SDavid Howells compat_int_t lo_encrypt_key_size; /* ioctl w/o */
1599863d5b82SDavid Howells compat_int_t lo_flags; /* ioctl r/o */
1600863d5b82SDavid Howells char lo_name[LO_NAME_SIZE];
1601863d5b82SDavid Howells unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
1602863d5b82SDavid Howells compat_ulong_t lo_init[2];
1603863d5b82SDavid Howells char reserved[4];
1604863d5b82SDavid Howells };
1605863d5b82SDavid Howells
1606863d5b82SDavid Howells /*
1607863d5b82SDavid Howells * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
1608863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver
1609863d5b82SDavid Howells */
1610863d5b82SDavid Howells static noinline int
loop_info64_from_compat(const struct compat_loop_info __user * arg,struct loop_info64 * info64)1611ba674cfcSAl Viro loop_info64_from_compat(const struct compat_loop_info __user *arg,
1612863d5b82SDavid Howells struct loop_info64 *info64)
1613863d5b82SDavid Howells {
1614863d5b82SDavid Howells struct compat_loop_info info;
1615863d5b82SDavid Howells
1616863d5b82SDavid Howells if (copy_from_user(&info, arg, sizeof(info)))
1617863d5b82SDavid Howells return -EFAULT;
1618863d5b82SDavid Howells
1619863d5b82SDavid Howells memset(info64, 0, sizeof(*info64));
1620863d5b82SDavid Howells info64->lo_number = info.lo_number;
1621863d5b82SDavid Howells info64->lo_device = info.lo_device;
1622863d5b82SDavid Howells info64->lo_inode = info.lo_inode;
1623863d5b82SDavid Howells info64->lo_rdevice = info.lo_rdevice;
1624863d5b82SDavid Howells info64->lo_offset = info.lo_offset;
1625863d5b82SDavid Howells info64->lo_sizelimit = 0;
1626863d5b82SDavid Howells info64->lo_flags = info.lo_flags;
1627863d5b82SDavid Howells memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
1628863d5b82SDavid Howells return 0;
1629863d5b82SDavid Howells }
1630863d5b82SDavid Howells
1631863d5b82SDavid Howells /*
1632863d5b82SDavid Howells * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
1633863d5b82SDavid Howells * - noinlined to reduce stack space usage in main part of driver
1634863d5b82SDavid Howells */
1635863d5b82SDavid Howells static noinline int
loop_info64_to_compat(const struct loop_info64 * info64,struct compat_loop_info __user * arg)1636863d5b82SDavid Howells loop_info64_to_compat(const struct loop_info64 *info64,
1637863d5b82SDavid Howells struct compat_loop_info __user *arg)
1638863d5b82SDavid Howells {
1639863d5b82SDavid Howells struct compat_loop_info info;
1640863d5b82SDavid Howells
1641863d5b82SDavid Howells memset(&info, 0, sizeof(info));
1642863d5b82SDavid Howells info.lo_number = info64->lo_number;
1643863d5b82SDavid Howells info.lo_device = info64->lo_device;
1644863d5b82SDavid Howells info.lo_inode = info64->lo_inode;
1645863d5b82SDavid Howells info.lo_rdevice = info64->lo_rdevice;
1646863d5b82SDavid Howells info.lo_offset = info64->lo_offset;
1647863d5b82SDavid Howells info.lo_flags = info64->lo_flags;
1648863d5b82SDavid Howells memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
1649863d5b82SDavid Howells
1650863d5b82SDavid Howells /* error in case values were truncated */
1651863d5b82SDavid Howells if (info.lo_device != info64->lo_device ||
1652863d5b82SDavid Howells info.lo_rdevice != info64->lo_rdevice ||
1653863d5b82SDavid Howells info.lo_inode != info64->lo_inode ||
165447e96246SChristoph Hellwig info.lo_offset != info64->lo_offset)
1655863d5b82SDavid Howells return -EOVERFLOW;
1656863d5b82SDavid Howells
1657863d5b82SDavid Howells if (copy_to_user(arg, &info, sizeof(info)))
1658863d5b82SDavid Howells return -EFAULT;
1659863d5b82SDavid Howells return 0;
1660863d5b82SDavid Howells }
1661863d5b82SDavid Howells
1662863d5b82SDavid Howells static int
loop_set_status_compat(struct loop_device * lo,const struct compat_loop_info __user * arg)1663863d5b82SDavid Howells loop_set_status_compat(struct loop_device *lo,
1664863d5b82SDavid Howells const struct compat_loop_info __user *arg)
1665863d5b82SDavid Howells {
1666863d5b82SDavid Howells struct loop_info64 info64;
1667863d5b82SDavid Howells int ret;
1668863d5b82SDavid Howells
1669863d5b82SDavid Howells ret = loop_info64_from_compat(arg, &info64);
1670863d5b82SDavid Howells if (ret < 0)
1671863d5b82SDavid Howells return ret;
1672863d5b82SDavid Howells return loop_set_status(lo, &info64);
1673863d5b82SDavid Howells }
1674863d5b82SDavid Howells
1675863d5b82SDavid Howells static int
loop_get_status_compat(struct loop_device * lo,struct compat_loop_info __user * arg)1676863d5b82SDavid Howells loop_get_status_compat(struct loop_device *lo,
1677863d5b82SDavid Howells struct compat_loop_info __user *arg)
1678863d5b82SDavid Howells {
1679863d5b82SDavid Howells struct loop_info64 info64;
1680bdac616dSOmar Sandoval int err;
1681863d5b82SDavid Howells
16824a5ce9baSJan Kara if (!arg)
1683bdac616dSOmar Sandoval return -EINVAL;
1684863d5b82SDavid Howells err = loop_get_status(lo, &info64);
1685863d5b82SDavid Howells if (!err)
1686863d5b82SDavid Howells err = loop_info64_to_compat(&info64, arg);
1687863d5b82SDavid Howells return err;
1688863d5b82SDavid Howells }
1689863d5b82SDavid Howells
lo_compat_ioctl(struct block_device * bdev,blk_mode_t mode,unsigned int cmd,unsigned long arg)169005bdb996SChristoph Hellwig static int lo_compat_ioctl(struct block_device *bdev, blk_mode_t mode,
1691bb214884SAl Viro unsigned int cmd, unsigned long arg)
1692863d5b82SDavid Howells {
1693bb214884SAl Viro struct loop_device *lo = bdev->bd_disk->private_data;
1694863d5b82SDavid Howells int err;
1695863d5b82SDavid Howells
1696863d5b82SDavid Howells switch(cmd) {
1697863d5b82SDavid Howells case LOOP_SET_STATUS:
16983148ffbdSOmar Sandoval err = loop_set_status_compat(lo,
16993148ffbdSOmar Sandoval (const struct compat_loop_info __user *)arg);
1700863d5b82SDavid Howells break;
1701863d5b82SDavid Howells case LOOP_GET_STATUS:
17023148ffbdSOmar Sandoval err = loop_get_status_compat(lo,
17033148ffbdSOmar Sandoval (struct compat_loop_info __user *)arg);
1704863d5b82SDavid Howells break;
170553d66608SJ. R. Okajima case LOOP_SET_CAPACITY:
1706863d5b82SDavid Howells case LOOP_CLR_FD:
1707863d5b82SDavid Howells case LOOP_GET_STATUS64:
1708863d5b82SDavid Howells case LOOP_SET_STATUS64:
17093448914eSMartijn Coenen case LOOP_CONFIGURE:
1710863d5b82SDavid Howells arg = (unsigned long) compat_ptr(arg);
1711df561f66SGustavo A. R. Silva fallthrough;
1712863d5b82SDavid Howells case LOOP_SET_FD:
1713863d5b82SDavid Howells case LOOP_CHANGE_FD:
17149fea4b39SEvan Green case LOOP_SET_BLOCK_SIZE:
1715fdbe4eeeSAlessio Balsini case LOOP_SET_DIRECT_IO:
1716bb214884SAl Viro err = lo_ioctl(bdev, mode, cmd, arg);
1717863d5b82SDavid Howells break;
1718863d5b82SDavid Howells default:
1719863d5b82SDavid Howells err = -ENOIOCTLCMD;
1720863d5b82SDavid Howells break;
1721863d5b82SDavid Howells }
1722863d5b82SDavid Howells return err;
1723863d5b82SDavid Howells }
1724863d5b82SDavid Howells #endif
1725863d5b82SDavid Howells
lo_release(struct gendisk * disk)1726ae220766SChristoph Hellwig static void lo_release(struct gendisk *disk)
17271da177e4SLinus Torvalds {
17286cc8e743SPavel Tatashin struct loop_device *lo = disk->private_data;
17291da177e4SLinus Torvalds
1730a0e286b6SChristoph Hellwig if (disk_openers(disk) > 0)
1731a0e286b6SChristoph Hellwig return;
1732f8933667SMing Lei
1733a0e286b6SChristoph Hellwig mutex_lock(&lo->lo_mutex);
1734a0e286b6SChristoph Hellwig if (lo->lo_state == Lo_bound && (lo->lo_flags & LO_FLAGS_AUTOCLEAR)) {
1735a2505b79SJan Kara lo->lo_state = Lo_rundown;
17366cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
173714f27939SMilan Broz /*
173814f27939SMilan Broz * In autoclear mode, stop the loop thread
173914f27939SMilan Broz * and remove configuration after last close.
174014f27939SMilan Broz */
1741bf23747eSTetsuo Handa __loop_clr_fd(lo, true);
17420a42e99bSJan Kara return;
174314f27939SMilan Broz }
17446cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
1745bb214884SAl Viro }
174696c58655SDavid Woodhouse
lo_free_disk(struct gendisk * disk)1747d2c7f56fSChristoph Hellwig static void lo_free_disk(struct gendisk *disk)
1748d2c7f56fSChristoph Hellwig {
1749d2c7f56fSChristoph Hellwig struct loop_device *lo = disk->private_data;
1750d2c7f56fSChristoph Hellwig
1751d292dc80SChristoph Hellwig if (lo->workqueue)
1752d292dc80SChristoph Hellwig destroy_workqueue(lo->workqueue);
1753d292dc80SChristoph Hellwig loop_free_idle_workers(lo, true);
1754292a089dSSteven Rostedt (Google) timer_shutdown_sync(&lo->timer);
1755d2c7f56fSChristoph Hellwig mutex_destroy(&lo->lo_mutex);
1756d2c7f56fSChristoph Hellwig kfree(lo);
1757ae665016SLinus Torvalds }
1758ae665016SLinus Torvalds
175983d5cde4SAlexey Dobriyan static const struct block_device_operations lo_fops = {
17601da177e4SLinus Torvalds .owner = THIS_MODULE,
1761bb214884SAl Viro .release = lo_release,
1762bb214884SAl Viro .ioctl = lo_ioctl,
1763863d5b82SDavid Howells #ifdef CONFIG_COMPAT
1764bb214884SAl Viro .compat_ioctl = lo_compat_ioctl,
1765863d5b82SDavid Howells #endif
1766d2c7f56fSChristoph Hellwig .free_disk = lo_free_disk,
17671da177e4SLinus Torvalds };
17681da177e4SLinus Torvalds
17691da177e4SLinus Torvalds /*
17701da177e4SLinus Torvalds * And now the modules code and kernel interface.
17711da177e4SLinus Torvalds */
177285c50197SIsaac J. Manjarres
177385c50197SIsaac J. Manjarres /*
177485c50197SIsaac J. Manjarres * If max_loop is specified, create that many devices upfront.
177585c50197SIsaac J. Manjarres * This also becomes a hard limit. If max_loop is not specified,
1776bb5faa99SMauricio Faria de Oliveira * the default isn't a hard limit (as before commit 85c50197716c
1777bb5faa99SMauricio Faria de Oliveira * changed the default value from 0 for max_loop=0 reasons), just
177885c50197SIsaac J. Manjarres * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
177985c50197SIsaac J. Manjarres * init time. Loop devices can be requested on-demand with the
178085c50197SIsaac J. Manjarres * /dev/loop-control interface, or be instantiated by accessing
178185c50197SIsaac J. Manjarres * a 'dead' device node.
178285c50197SIsaac J. Manjarres */
178385c50197SIsaac J. Manjarres static int max_loop = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
1784bb5faa99SMauricio Faria de Oliveira
1785bb5faa99SMauricio Faria de Oliveira #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
1786bb5faa99SMauricio Faria de Oliveira static bool max_loop_specified;
1787bb5faa99SMauricio Faria de Oliveira
max_loop_param_set_int(const char * val,const struct kernel_param * kp)1788bb5faa99SMauricio Faria de Oliveira static int max_loop_param_set_int(const char *val,
1789bb5faa99SMauricio Faria de Oliveira const struct kernel_param *kp)
1790bb5faa99SMauricio Faria de Oliveira {
1791bb5faa99SMauricio Faria de Oliveira int ret;
1792bb5faa99SMauricio Faria de Oliveira
1793bb5faa99SMauricio Faria de Oliveira ret = param_set_int(val, kp);
1794bb5faa99SMauricio Faria de Oliveira if (ret < 0)
1795bb5faa99SMauricio Faria de Oliveira return ret;
1796bb5faa99SMauricio Faria de Oliveira
1797bb5faa99SMauricio Faria de Oliveira max_loop_specified = true;
1798bb5faa99SMauricio Faria de Oliveira return 0;
1799bb5faa99SMauricio Faria de Oliveira }
1800bb5faa99SMauricio Faria de Oliveira
1801bb5faa99SMauricio Faria de Oliveira static const struct kernel_param_ops max_loop_param_ops = {
1802bb5faa99SMauricio Faria de Oliveira .set = max_loop_param_set_int,
1803bb5faa99SMauricio Faria de Oliveira .get = param_get_int,
1804bb5faa99SMauricio Faria de Oliveira };
1805bb5faa99SMauricio Faria de Oliveira
1806bb5faa99SMauricio Faria de Oliveira module_param_cb(max_loop, &max_loop_param_ops, &max_loop, 0444);
1807a47653fcSKen Chen MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
1808bb5faa99SMauricio Faria de Oliveira #else
1809bb5faa99SMauricio Faria de Oliveira module_param(max_loop, int, 0444);
1810bb5faa99SMauricio Faria de Oliveira MODULE_PARM_DESC(max_loop, "Initial number of loop devices");
1811bb5faa99SMauricio Faria de Oliveira #endif
1812bb5faa99SMauricio Faria de Oliveira
18135657a819SJoe Perches module_param(max_part, int, 0444);
1814476a4813SLaurent Vivier MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
1815ef44c508SChaitanya Kulkarni
1816ef44c508SChaitanya Kulkarni static int hw_queue_depth = LOOP_DEFAULT_HW_Q_DEPTH;
1817ef44c508SChaitanya Kulkarni
loop_set_hw_queue_depth(const char * s,const struct kernel_param * p)1818ef44c508SChaitanya Kulkarni static int loop_set_hw_queue_depth(const char *s, const struct kernel_param *p)
1819ef44c508SChaitanya Kulkarni {
1820e152a05fSBart Van Assche int qd, ret;
1821ef44c508SChaitanya Kulkarni
1822e152a05fSBart Van Assche ret = kstrtoint(s, 0, &qd);
1823e152a05fSBart Van Assche if (ret < 0)
1824e152a05fSBart Van Assche return ret;
1825e152a05fSBart Van Assche if (qd < 1)
1826e152a05fSBart Van Assche return -EINVAL;
1827e152a05fSBart Van Assche hw_queue_depth = qd;
1828e152a05fSBart Van Assche return 0;
1829ef44c508SChaitanya Kulkarni }
1830ef44c508SChaitanya Kulkarni
1831ef44c508SChaitanya Kulkarni static const struct kernel_param_ops loop_hw_qdepth_param_ops = {
1832ef44c508SChaitanya Kulkarni .set = loop_set_hw_queue_depth,
1833ef44c508SChaitanya Kulkarni .get = param_get_int,
1834ef44c508SChaitanya Kulkarni };
1835ef44c508SChaitanya Kulkarni
1836ef44c508SChaitanya Kulkarni device_param_cb(hw_queue_depth, &loop_hw_qdepth_param_ops, &hw_queue_depth, 0444);
1837e152a05fSBart Van Assche MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: " __stringify(LOOP_DEFAULT_HW_Q_DEPTH));
1838ef44c508SChaitanya Kulkarni
18391da177e4SLinus Torvalds MODULE_LICENSE("GPL");
18401da177e4SLinus Torvalds MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
18411da177e4SLinus Torvalds
loop_queue_rq(struct blk_mq_hw_ctx * hctx,const struct blk_mq_queue_data * bd)1842fc17b653SChristoph Hellwig static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
1843b5dd2f60SMing Lei const struct blk_mq_queue_data *bd)
1844b5dd2f60SMing Lei {
18451894e916SJens Axboe struct request *rq = bd->rq;
18461894e916SJens Axboe struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
18471894e916SJens Axboe struct loop_device *lo = rq->q->queuedata;
1848b5dd2f60SMing Lei
18491894e916SJens Axboe blk_mq_start_request(rq);
1850b5dd2f60SMing Lei
1851f4aa4c7bSMing Lei if (lo->lo_state != Lo_bound)
1852fc17b653SChristoph Hellwig return BLK_STS_IOERR;
1853f4aa4c7bSMing Lei
18541894e916SJens Axboe switch (req_op(rq)) {
1855f0225cacSChristoph Hellwig case REQ_OP_FLUSH:
1856f0225cacSChristoph Hellwig case REQ_OP_DISCARD:
185719372e27SChristoph Hellwig case REQ_OP_WRITE_ZEROES:
1858bc07c10aSMing Lei cmd->use_aio = false;
1859f0225cacSChristoph Hellwig break;
1860f0225cacSChristoph Hellwig default:
1861f0225cacSChristoph Hellwig cmd->use_aio = lo->use_dio;
1862f0225cacSChristoph Hellwig break;
1863f0225cacSChristoph Hellwig }
1864bc07c10aSMing Lei
1865d4478e92SShaohua Li /* always use the first bio's css */
1866c74d40e8SDan Schatzberg cmd->blkcg_css = NULL;
1867c74d40e8SDan Schatzberg cmd->memcg_css = NULL;
18680b508bc9SShaohua Li #ifdef CONFIG_BLK_CGROUP
1869bbb1ebe7SChristoph Hellwig if (rq->bio) {
1870bbb1ebe7SChristoph Hellwig cmd->blkcg_css = bio_blkcg_css(rq->bio);
1871c74d40e8SDan Schatzberg #ifdef CONFIG_MEMCG
1872bbb1ebe7SChristoph Hellwig if (cmd->blkcg_css) {
1873c74d40e8SDan Schatzberg cmd->memcg_css =
1874c74d40e8SDan Schatzberg cgroup_get_e_css(cmd->blkcg_css->cgroup,
1875c74d40e8SDan Schatzberg &memory_cgrp_subsys);
1876bbb1ebe7SChristoph Hellwig }
1877d4478e92SShaohua Li #endif
1878c74d40e8SDan Schatzberg }
1879c74d40e8SDan Schatzberg #endif
188087579e9bSDan Schatzberg loop_queue_work(lo, cmd);
1881b5dd2f60SMing Lei
1882fc17b653SChristoph Hellwig return BLK_STS_OK;
1883b5dd2f60SMing Lei }
1884b5dd2f60SMing Lei
loop_handle_cmd(struct loop_cmd * cmd)1885b5dd2f60SMing Lei static void loop_handle_cmd(struct loop_cmd *cmd)
1886b5dd2f60SMing Lei {
18879b0cb770SBart Van Assche struct cgroup_subsys_state *cmd_blkcg_css = cmd->blkcg_css;
18889b0cb770SBart Van Assche struct cgroup_subsys_state *cmd_memcg_css = cmd->memcg_css;
18891894e916SJens Axboe struct request *rq = blk_mq_rq_from_pdu(cmd);
18901894e916SJens Axboe const bool write = op_is_write(req_op(rq));
18911894e916SJens Axboe struct loop_device *lo = rq->q->queuedata;
1892f4829a9bSChristoph Hellwig int ret = 0;
1893c74d40e8SDan Schatzberg struct mem_cgroup *old_memcg = NULL;
18949b0cb770SBart Van Assche const bool use_aio = cmd->use_aio;
1895b5dd2f60SMing Lei
1896f4829a9bSChristoph Hellwig if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
1897f4829a9bSChristoph Hellwig ret = -EIO;
1898b5dd2f60SMing Lei goto failed;
1899f4829a9bSChristoph Hellwig }
1900b5dd2f60SMing Lei
19019b0cb770SBart Van Assche if (cmd_blkcg_css)
19029b0cb770SBart Van Assche kthread_associate_blkcg(cmd_blkcg_css);
19039b0cb770SBart Van Assche if (cmd_memcg_css)
1904c74d40e8SDan Schatzberg old_memcg = set_active_memcg(
19059b0cb770SBart Van Assche mem_cgroup_from_css(cmd_memcg_css));
1906c74d40e8SDan Schatzberg
19079b0cb770SBart Van Assche /*
19089b0cb770SBart Van Assche * do_req_filebacked() may call blk_mq_complete_request() synchronously
19099b0cb770SBart Van Assche * or asynchronously if using aio. Hence, do not touch 'cmd' after
19109b0cb770SBart Van Assche * do_req_filebacked() has returned unless we are sure that 'cmd' has
19119b0cb770SBart Van Assche * not yet been completed.
19129b0cb770SBart Van Assche */
19131894e916SJens Axboe ret = do_req_filebacked(lo, rq);
1914c74d40e8SDan Schatzberg
19159b0cb770SBart Van Assche if (cmd_blkcg_css)
1916c74d40e8SDan Schatzberg kthread_associate_blkcg(NULL);
1917c74d40e8SDan Schatzberg
19189b0cb770SBart Van Assche if (cmd_memcg_css) {
1919c74d40e8SDan Schatzberg set_active_memcg(old_memcg);
19209b0cb770SBart Van Assche css_put(cmd_memcg_css);
1921c74d40e8SDan Schatzberg }
1922b5dd2f60SMing Lei failed:
1923bc07c10aSMing Lei /* complete non-aio request */
19249b0cb770SBart Van Assche if (!use_aio || ret) {
19258cd55087SEvan Green if (ret == -EOPNOTSUPP)
19268cd55087SEvan Green cmd->ret = ret;
19278cd55087SEvan Green else
1928fe2cb290SChristoph Hellwig cmd->ret = ret ? -EIO : 0;
192915f73f5bSChristoph Hellwig if (likely(!blk_should_fake_timeout(rq->q)))
19301894e916SJens Axboe blk_mq_complete_request(rq);
1931fe2cb290SChristoph Hellwig }
1932b5dd2f60SMing Lei }
1933b5dd2f60SMing Lei
loop_process_work(struct loop_worker * worker,struct list_head * cmd_list,struct loop_device * lo)193487579e9bSDan Schatzberg static void loop_process_work(struct loop_worker *worker,
193587579e9bSDan Schatzberg struct list_head *cmd_list, struct loop_device *lo)
1936b5dd2f60SMing Lei {
193787579e9bSDan Schatzberg int orig_flags = current->flags;
193887579e9bSDan Schatzberg struct loop_cmd *cmd;
1939b5dd2f60SMing Lei
194087579e9bSDan Schatzberg current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO;
194187579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock);
194287579e9bSDan Schatzberg while (!list_empty(cmd_list)) {
194387579e9bSDan Schatzberg cmd = container_of(
194487579e9bSDan Schatzberg cmd_list->next, struct loop_cmd, list_entry);
194587579e9bSDan Schatzberg list_del(cmd_list->next);
194687579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock);
194787579e9bSDan Schatzberg
194887579e9bSDan Schatzberg loop_handle_cmd(cmd);
194987579e9bSDan Schatzberg cond_resched();
195087579e9bSDan Schatzberg
195187579e9bSDan Schatzberg spin_lock_irq(&lo->lo_work_lock);
195287579e9bSDan Schatzberg }
195387579e9bSDan Schatzberg
195487579e9bSDan Schatzberg /*
195587579e9bSDan Schatzberg * We only add to the idle list if there are no pending cmds
195687579e9bSDan Schatzberg * *and* the worker will not run again which ensures that it
195787579e9bSDan Schatzberg * is safe to free any worker on the idle list
195887579e9bSDan Schatzberg */
195987579e9bSDan Schatzberg if (worker && !work_pending(&worker->work)) {
196087579e9bSDan Schatzberg worker->last_ran_at = jiffies;
196187579e9bSDan Schatzberg list_add_tail(&worker->idle_list, &lo->idle_worker_list);
196287579e9bSDan Schatzberg loop_set_timer(lo);
196387579e9bSDan Schatzberg }
196487579e9bSDan Schatzberg spin_unlock_irq(&lo->lo_work_lock);
196587579e9bSDan Schatzberg current->flags = orig_flags;
196687579e9bSDan Schatzberg }
196787579e9bSDan Schatzberg
loop_workfn(struct work_struct * work)196887579e9bSDan Schatzberg static void loop_workfn(struct work_struct *work)
196987579e9bSDan Schatzberg {
197087579e9bSDan Schatzberg struct loop_worker *worker =
197187579e9bSDan Schatzberg container_of(work, struct loop_worker, work);
197287579e9bSDan Schatzberg loop_process_work(worker, &worker->cmd_list, worker->lo);
197387579e9bSDan Schatzberg }
197487579e9bSDan Schatzberg
loop_rootcg_workfn(struct work_struct * work)197587579e9bSDan Schatzberg static void loop_rootcg_workfn(struct work_struct *work)
197687579e9bSDan Schatzberg {
197787579e9bSDan Schatzberg struct loop_device *lo =
197887579e9bSDan Schatzberg container_of(work, struct loop_device, rootcg_work);
197987579e9bSDan Schatzberg loop_process_work(NULL, &lo->rootcg_cmd_list, lo);
198087579e9bSDan Schatzberg }
198187579e9bSDan Schatzberg
1982f363b089SEric Biggers static const struct blk_mq_ops loop_mq_ops = {
1983b5dd2f60SMing Lei .queue_rq = loop_queue_rq,
1984fe2cb290SChristoph Hellwig .complete = lo_complete_rq,
1985b5dd2f60SMing Lei };
1986b5dd2f60SMing Lei
loop_add(int i)1987d6da83d0SChristoph Hellwig static int loop_add(int i)
19881da177e4SLinus Torvalds {
198973285082SKen Chen struct loop_device *lo;
199073285082SKen Chen struct gendisk *disk;
199134dd82afSKay Sievers int err;
19921da177e4SLinus Torvalds
199334dd82afSKay Sievers err = -ENOMEM;
199468d740d7SSilva Paulo lo = kzalloc(sizeof(*lo), GFP_KERNEL);
199568d740d7SSilva Paulo if (!lo)
199673285082SKen Chen goto out;
1997b15ed546SChristoph Hellwig lo->worker_tree = RB_ROOT;
1998b15ed546SChristoph Hellwig INIT_LIST_HEAD(&lo->idle_worker_list);
1999b15ed546SChristoph Hellwig timer_setup(&lo->timer, loop_free_idle_workers_timer, TIMER_DEFERRABLE);
2000ef7e7c82SMikulas Patocka lo->lo_state = Lo_unbound;
2001ef7e7c82SMikulas Patocka
200218d1f200SChristoph Hellwig err = mutex_lock_killable(&loop_ctl_mutex);
200318d1f200SChristoph Hellwig if (err)
200418d1f200SChristoph Hellwig goto out_free_dev;
200518d1f200SChristoph Hellwig
2006c718aa65STejun Heo /* allocate id, if @id >= 0, we're requesting that specific id */
200734dd82afSKay Sievers if (i >= 0) {
2008c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
2009c718aa65STejun Heo if (err == -ENOSPC)
201034dd82afSKay Sievers err = -EEXIST;
201134dd82afSKay Sievers } else {
2012c718aa65STejun Heo err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
201334dd82afSKay Sievers }
20141c500ad7STetsuo Handa mutex_unlock(&loop_ctl_mutex);
201534dd82afSKay Sievers if (err < 0)
20161c500ad7STetsuo Handa goto out_free_dev;
2017c718aa65STejun Heo i = err;
20181da177e4SLinus Torvalds
2019b5dd2f60SMing Lei lo->tag_set.ops = &loop_mq_ops;
2020b5dd2f60SMing Lei lo->tag_set.nr_hw_queues = 1;
2021ef44c508SChaitanya Kulkarni lo->tag_set.queue_depth = hw_queue_depth;
2022b5dd2f60SMing Lei lo->tag_set.numa_node = NUMA_NO_NODE;
2023b5dd2f60SMing Lei lo->tag_set.cmd_size = sizeof(struct loop_cmd);
20242112f5c1SBart Van Assche lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING |
20252112f5c1SBart Van Assche BLK_MQ_F_NO_SCHED_BY_DEFAULT;
2026b5dd2f60SMing Lei lo->tag_set.driver_data = lo;
2027b5dd2f60SMing Lei
2028b5dd2f60SMing Lei err = blk_mq_alloc_tag_set(&lo->tag_set);
2029b5dd2f60SMing Lei if (err)
20303ec981e3SMikulas Patocka goto out_free_idr;
203173285082SKen Chen
20321c99502fSChristoph Hellwig disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, lo);
20331c99502fSChristoph Hellwig if (IS_ERR(disk)) {
20341c99502fSChristoph Hellwig err = PTR_ERR(disk);
2035b5dd2f60SMing Lei goto out_cleanup_tags;
2036b5dd2f60SMing Lei }
20371c99502fSChristoph Hellwig lo->lo_queue = lo->lo_disk->queue;
2038ef7e7c82SMikulas Patocka
203954bb0adeSShaohua Li blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS);
204040326d8aSShaohua Li
20415b5e20f4SMing Lei /*
204240326d8aSShaohua Li * By default, we do buffer IO, so it doesn't make sense to enable
204340326d8aSShaohua Li * merge because the I/O submitted to backing file is handled page by
204440326d8aSShaohua Li * page. For directio mode, merge does help to dispatch bigger request
204540326d8aSShaohua Li * to underlayer disk. We will enable merge once directio is enabled.
20465b5e20f4SMing Lei */
20478b904b5bSBart Van Assche blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
20485b5e20f4SMing Lei
2049e03c8dd1SKay Sievers /*
2050e03c8dd1SKay Sievers * Disable partition scanning by default. The in-kernel partition
2051e03c8dd1SKay Sievers * scanning can be requested individually per-device during its
2052e03c8dd1SKay Sievers * setup. Userspace can always add and remove partitions from all
2053e03c8dd1SKay Sievers * devices. The needed partition minors are allocated from the
2054e03c8dd1SKay Sievers * extended minor space, the main loop device numbers will continue
2055e03c8dd1SKay Sievers * to match the loop minors, regardless of the number of partitions
2056e03c8dd1SKay Sievers * used.
2057e03c8dd1SKay Sievers *
2058e03c8dd1SKay Sievers * If max_part is given, partition scanning is globally enabled for
2059e03c8dd1SKay Sievers * all loop devices. The minors for the main loop devices will be
2060e03c8dd1SKay Sievers * multiples of max_part.
2061e03c8dd1SKay Sievers *
2062e03c8dd1SKay Sievers * Note: Global-for-all-devices, set-only-at-init, read-only module
2063e03c8dd1SKay Sievers * parameteters like 'max_loop' and 'max_part' make things needlessly
2064e03c8dd1SKay Sievers * complicated, are too static, inflexible and may surprise
2065e03c8dd1SKay Sievers * userspace tools. Parameters like this in general should be avoided.
2066e03c8dd1SKay Sievers */
2067e03c8dd1SKay Sievers if (!part_shift)
2068b9684a71SChristoph Hellwig set_bit(GD_SUPPRESS_PART_SCAN, &disk->state);
20696cc8e743SPavel Tatashin mutex_init(&lo->lo_mutex);
20701da177e4SLinus Torvalds lo->lo_number = i;
20711da177e4SLinus Torvalds spin_lock_init(&lo->lo_lock);
207287579e9bSDan Schatzberg spin_lock_init(&lo->lo_work_lock);
2073d292dc80SChristoph Hellwig INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn);
2074d292dc80SChristoph Hellwig INIT_LIST_HEAD(&lo->rootcg_cmd_list);
20751da177e4SLinus Torvalds disk->major = LOOP_MAJOR;
2076476a4813SLaurent Vivier disk->first_minor = i << part_shift;
20771c99502fSChristoph Hellwig disk->minors = 1 << part_shift;
20781da177e4SLinus Torvalds disk->fops = &lo_fops;
20791da177e4SLinus Torvalds disk->private_data = lo;
20801da177e4SLinus Torvalds disk->queue = lo->lo_queue;
20819f65c489SMatteo Croce disk->events = DISK_EVENT_MEDIA_CHANGE;
20829f65c489SMatteo Croce disk->event_flags = DISK_EVENT_FLAG_UEVENT;
208373285082SKen Chen sprintf(disk->disk_name, "loop%d", i);
20841c500ad7STetsuo Handa /* Make this loop device reachable from pathname. */
2085905705f0SLuis Chamberlain err = add_disk(disk);
2086905705f0SLuis Chamberlain if (err)
2087905705f0SLuis Chamberlain goto out_cleanup_disk;
2088905705f0SLuis Chamberlain
20891c500ad7STetsuo Handa /* Show this loop device. */
20901c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex);
20911c500ad7STetsuo Handa lo->idr_visible = true;
209218d1f200SChristoph Hellwig mutex_unlock(&loop_ctl_mutex);
2093905705f0SLuis Chamberlain
209418d1f200SChristoph Hellwig return i;
209573285082SKen Chen
2096905705f0SLuis Chamberlain out_cleanup_disk:
20978b9ab626SChristoph Hellwig put_disk(disk);
2098b5dd2f60SMing Lei out_cleanup_tags:
2099b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set);
21003ec981e3SMikulas Patocka out_free_idr:
21011c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex);
21023ec981e3SMikulas Patocka idr_remove(&loop_index_idr, i);
210318d1f200SChristoph Hellwig mutex_unlock(&loop_ctl_mutex);
210473285082SKen Chen out_free_dev:
210573285082SKen Chen kfree(lo);
210673285082SKen Chen out:
210734dd82afSKay Sievers return err;
21081da177e4SLinus Torvalds }
21091da177e4SLinus Torvalds
loop_remove(struct loop_device * lo)211034dd82afSKay Sievers static void loop_remove(struct loop_device *lo)
211173285082SKen Chen {
21121c500ad7STetsuo Handa /* Make this loop device unreachable from pathname. */
21136cd18e71SNeilBrown del_gendisk(lo->lo_disk);
2114b5dd2f60SMing Lei blk_mq_free_tag_set(&lo->tag_set);
2115ce8d7861SChristoph Hellwig
21161c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex);
21171c500ad7STetsuo Handa idr_remove(&loop_index_idr, lo->lo_number);
21181c500ad7STetsuo Handa mutex_unlock(&loop_ctl_mutex);
2119d2c7f56fSChristoph Hellwig
2120d2c7f56fSChristoph Hellwig put_disk(lo->lo_disk);
212173285082SKen Chen }
212273285082SKen Chen
212323881aecSMauricio Faria de Oliveira #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
loop_probe(dev_t dev)21248410d38cSChristoph Hellwig static void loop_probe(dev_t dev)
212573285082SKen Chen {
21268410d38cSChristoph Hellwig int idx = MINOR(dev) >> part_shift;
21278410d38cSChristoph Hellwig
2128bb5faa99SMauricio Faria de Oliveira if (max_loop_specified && max_loop && idx >= max_loop)
21298410d38cSChristoph Hellwig return;
2130d6da83d0SChristoph Hellwig loop_add(idx);
2131f9d10764SChristoph Hellwig }
213223881aecSMauricio Faria de Oliveira #else
213323881aecSMauricio Faria de Oliveira #define loop_probe NULL
213423881aecSMauricio Faria de Oliveira #endif /* !CONFIG_BLOCK_LEGACY_AUTOLOAD */
2135f9d10764SChristoph Hellwig
loop_control_remove(int idx)2136f9d10764SChristoph Hellwig static int loop_control_remove(int idx)
2137770fe30aSKay Sievers {
2138770fe30aSKay Sievers struct loop_device *lo;
21390a42e99bSJan Kara int ret;
2140e5d66a10SChristoph Hellwig
2141e5d66a10SChristoph Hellwig if (idx < 0) {
2142e3f9387aSTetsuo Handa pr_warn_once("deleting an unspecified loop device is not supported.\n");
2143e5d66a10SChristoph Hellwig return -EINVAL;
2144e5d66a10SChristoph Hellwig }
2145770fe30aSKay Sievers
21461c500ad7STetsuo Handa /* Hide this loop device for serialization. */
21470a42e99bSJan Kara ret = mutex_lock_killable(&loop_ctl_mutex);
21480a42e99bSJan Kara if (ret)
21490a42e99bSJan Kara return ret;
2150b9848081SChristoph Hellwig lo = idr_find(&loop_index_idr, idx);
21511c500ad7STetsuo Handa if (!lo || !lo->idr_visible)
2152b9848081SChristoph Hellwig ret = -ENODEV;
21531c500ad7STetsuo Handa else
21541c500ad7STetsuo Handa lo->idr_visible = false;
21551c500ad7STetsuo Handa mutex_unlock(&loop_ctl_mutex);
21561c500ad7STetsuo Handa if (ret)
21571c500ad7STetsuo Handa return ret;
2158f9d10764SChristoph Hellwig
21591c500ad7STetsuo Handa /* Check whether this loop device can be removed. */
21606cc8e743SPavel Tatashin ret = mutex_lock_killable(&lo->lo_mutex);
21616cc8e743SPavel Tatashin if (ret)
21621c500ad7STetsuo Handa goto mark_visible;
2163a0e286b6SChristoph Hellwig if (lo->lo_state != Lo_unbound || disk_openers(lo->lo_disk) > 0) {
21646cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
2165770fe30aSKay Sievers ret = -EBUSY;
21661c500ad7STetsuo Handa goto mark_visible;
2167770fe30aSKay Sievers }
2168a0e286b6SChristoph Hellwig /* Mark this loop device as no more bound, but not quite unbound yet */
2169990e7811SChristoph Hellwig lo->lo_state = Lo_deleting;
21706cc8e743SPavel Tatashin mutex_unlock(&lo->lo_mutex);
2171f9d10764SChristoph Hellwig
2172770fe30aSKay Sievers loop_remove(lo);
21731c500ad7STetsuo Handa return 0;
21741c500ad7STetsuo Handa
21751c500ad7STetsuo Handa mark_visible:
21761c500ad7STetsuo Handa /* Show this loop device again. */
21771c500ad7STetsuo Handa mutex_lock(&loop_ctl_mutex);
21781c500ad7STetsuo Handa lo->idr_visible = true;
2179f9d10764SChristoph Hellwig mutex_unlock(&loop_ctl_mutex);
2180f9d10764SChristoph Hellwig return ret;
2181770fe30aSKay Sievers }
2182f9d10764SChristoph Hellwig
loop_control_get_free(int idx)2183f9d10764SChristoph Hellwig static int loop_control_get_free(int idx)
2184f9d10764SChristoph Hellwig {
2185f9d10764SChristoph Hellwig struct loop_device *lo;
2186b9848081SChristoph Hellwig int id, ret;
2187f9d10764SChristoph Hellwig
2188f9d10764SChristoph Hellwig ret = mutex_lock_killable(&loop_ctl_mutex);
2189f9d10764SChristoph Hellwig if (ret)
2190f9d10764SChristoph Hellwig return ret;
2191b9848081SChristoph Hellwig idr_for_each_entry(&loop_index_idr, lo, id) {
21921c500ad7STetsuo Handa /* Hitting a race results in creating a new loop device which is harmless. */
21931c500ad7STetsuo Handa if (lo->idr_visible && data_race(lo->lo_state) == Lo_unbound)
2194b9848081SChristoph Hellwig goto found;
2195b9848081SChristoph Hellwig }
21960a42e99bSJan Kara mutex_unlock(&loop_ctl_mutex);
219718d1f200SChristoph Hellwig return loop_add(-1);
2198b9848081SChristoph Hellwig found:
2199b9848081SChristoph Hellwig mutex_unlock(&loop_ctl_mutex);
2200b9848081SChristoph Hellwig return id;
2201770fe30aSKay Sievers }
2202770fe30aSKay Sievers
loop_control_ioctl(struct file * file,unsigned int cmd,unsigned long parm)2203f9d10764SChristoph Hellwig static long loop_control_ioctl(struct file *file, unsigned int cmd,
2204f9d10764SChristoph Hellwig unsigned long parm)
2205f9d10764SChristoph Hellwig {
2206f9d10764SChristoph Hellwig switch (cmd) {
2207f9d10764SChristoph Hellwig case LOOP_CTL_ADD:
220818d1f200SChristoph Hellwig return loop_add(parm);
2209f9d10764SChristoph Hellwig case LOOP_CTL_REMOVE:
2210f9d10764SChristoph Hellwig return loop_control_remove(parm);
2211f9d10764SChristoph Hellwig case LOOP_CTL_GET_FREE:
2212f9d10764SChristoph Hellwig return loop_control_get_free(parm);
2213f9d10764SChristoph Hellwig default:
2214f9d10764SChristoph Hellwig return -ENOSYS;
2215f9d10764SChristoph Hellwig }
2216f9d10764SChristoph Hellwig }
2217f9d10764SChristoph Hellwig
2218770fe30aSKay Sievers static const struct file_operations loop_ctl_fops = {
2219770fe30aSKay Sievers .open = nonseekable_open,
2220770fe30aSKay Sievers .unlocked_ioctl = loop_control_ioctl,
2221770fe30aSKay Sievers .compat_ioctl = loop_control_ioctl,
2222770fe30aSKay Sievers .owner = THIS_MODULE,
2223770fe30aSKay Sievers .llseek = noop_llseek,
2224770fe30aSKay Sievers };
2225770fe30aSKay Sievers
2226770fe30aSKay Sievers static struct miscdevice loop_misc = {
2227770fe30aSKay Sievers .minor = LOOP_CTRL_MINOR,
2228770fe30aSKay Sievers .name = "loop-control",
2229770fe30aSKay Sievers .fops = &loop_ctl_fops,
2230770fe30aSKay Sievers };
2231770fe30aSKay Sievers
2232770fe30aSKay Sievers MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
2233770fe30aSKay Sievers MODULE_ALIAS("devname:loop-control");
2234770fe30aSKay Sievers
loop_init(void)223573285082SKen Chen static int __init loop_init(void)
223673285082SKen Chen {
223785c50197SIsaac J. Manjarres int i;
2238770fe30aSKay Sievers int err;
2239a47653fcSKen Chen
2240476a4813SLaurent Vivier part_shift = 0;
2241ac04fee0SNamhyung Kim if (max_part > 0) {
2242476a4813SLaurent Vivier part_shift = fls(max_part);
2243476a4813SLaurent Vivier
2244ac04fee0SNamhyung Kim /*
2245ac04fee0SNamhyung Kim * Adjust max_part according to part_shift as it is exported
2246ac04fee0SNamhyung Kim * to user space so that user can decide correct minor number
2247ac04fee0SNamhyung Kim * if [s]he want to create more devices.
2248ac04fee0SNamhyung Kim *
2249ac04fee0SNamhyung Kim * Note that -1 is required because partition 0 is reserved
2250ac04fee0SNamhyung Kim * for the whole disk.
2251ac04fee0SNamhyung Kim */
2252ac04fee0SNamhyung Kim max_part = (1UL << part_shift) - 1;
2253ac04fee0SNamhyung Kim }
2254ac04fee0SNamhyung Kim
2255b1a66504SGuo Chao if ((1UL << part_shift) > DISK_MAX_PARTS) {
2256b1a66504SGuo Chao err = -EINVAL;
2257a8c1d064SAnton Volkov goto err_out;
2258b1a66504SGuo Chao }
225978f4bb36SNamhyung Kim
2260b1a66504SGuo Chao if (max_loop > 1UL << (MINORBITS - part_shift)) {
2261b1a66504SGuo Chao err = -EINVAL;
2262a8c1d064SAnton Volkov goto err_out;
2263b1a66504SGuo Chao }
226473285082SKen Chen
2265a8c1d064SAnton Volkov err = misc_register(&loop_misc);
2266a8c1d064SAnton Volkov if (err < 0)
2267a8c1d064SAnton Volkov goto err_out;
2268a8c1d064SAnton Volkov
2269a8c1d064SAnton Volkov
22708410d38cSChristoph Hellwig if (__register_blkdev(LOOP_MAJOR, "loop", loop_probe)) {
2271b1a66504SGuo Chao err = -EIO;
2272b1a66504SGuo Chao goto misc_out;
2273b1a66504SGuo Chao }
2274a47653fcSKen Chen
2275d134b00bSKay Sievers /* pre-create number of devices given by config or max_loop */
227685c50197SIsaac J. Manjarres for (i = 0; i < max_loop; i++)
2277d6da83d0SChristoph Hellwig loop_add(i);
227834dd82afSKay Sievers
227973285082SKen Chen printk(KERN_INFO "loop: module loaded\n");
22801da177e4SLinus Torvalds return 0;
2281b1a66504SGuo Chao
2282b1a66504SGuo Chao misc_out:
2283b1a66504SGuo Chao misc_deregister(&loop_misc);
2284a8c1d064SAnton Volkov err_out:
2285b1a66504SGuo Chao return err;
228634dd82afSKay Sievers }
2287a47653fcSKen Chen
loop_exit(void)228873285082SKen Chen static void __exit loop_exit(void)
22891da177e4SLinus Torvalds {
22908e60947dSChristoph Hellwig struct loop_device *lo;
22918e60947dSChristoph Hellwig int id;
22928e60947dSChristoph Hellwig
22938b52d8beSChristoph Hellwig unregister_blkdev(LOOP_MAJOR, "loop");
22948b52d8beSChristoph Hellwig misc_deregister(&loop_misc);
2295200f9337SLuis Chamberlain
22961c500ad7STetsuo Handa /*
22971c500ad7STetsuo Handa * There is no need to use loop_ctl_mutex here, for nobody else can
22981c500ad7STetsuo Handa * access loop_index_idr when this module is unloading (unless forced
22991c500ad7STetsuo Handa * module unloading is requested). If this is not a clean unloading,
23001c500ad7STetsuo Handa * we have no means to avoid kernel crash.
23011c500ad7STetsuo Handa */
23028e60947dSChristoph Hellwig idr_for_each_entry(&loop_index_idr, lo, id)
23038e60947dSChristoph Hellwig loop_remove(lo);
2304bd5c39edSChristoph Hellwig
2305bd5c39edSChristoph Hellwig idr_destroy(&loop_index_idr);
23061da177e4SLinus Torvalds }
23071da177e4SLinus Torvalds
23081da177e4SLinus Torvalds module_init(loop_init);
23091da177e4SLinus Torvalds module_exit(loop_exit);
23101da177e4SLinus Torvalds
23111da177e4SLinus Torvalds #ifndef MODULE
max_loop_setup(char * str)23121da177e4SLinus Torvalds static int __init max_loop_setup(char *str)
23131da177e4SLinus Torvalds {
23141da177e4SLinus Torvalds max_loop = simple_strtol(str, NULL, 0);
2315bb5faa99SMauricio Faria de Oliveira #ifdef CONFIG_BLOCK_LEGACY_AUTOLOAD
2316bb5faa99SMauricio Faria de Oliveira max_loop_specified = true;
2317bb5faa99SMauricio Faria de Oliveira #endif
23181da177e4SLinus Torvalds return 1;
23191da177e4SLinus Torvalds }
23201da177e4SLinus Torvalds
23211da177e4SLinus Torvalds __setup("max_loop=", max_loop_setup);
23221da177e4SLinus Torvalds #endif
2323