xref: /openbmc/linux/drivers/block/loop.c (revision 0c3df9ed)
1 /*
2  *  linux/drivers/block/loop.c
3  *
4  *  Written by Theodore Ts'o, 3/29/93
5  *
6  * Copyright 1993 by Theodore Ts'o.  Redistribution of this file is
7  * permitted under the GNU General Public License.
8  *
9  * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993
10  * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996
11  *
12  * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994
13  * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996
14  *
15  * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997
16  *
17  * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998
18  *
19  * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998
20  *
21  * Loadable modules and other fixes by AK, 1998
22  *
23  * Make real block number available to downstream transfer functions, enables
24  * CBC (and relatives) mode encryption requiring unique IVs per data block.
25  * Reed H. Petty, rhp@draper.net
26  *
27  * Maximum number of loop devices now dynamic via max_loop module parameter.
28  * Russell Kroll <rkroll@exploits.org> 19990701
29  *
30  * Maximum number of loop devices when compiled-in now selectable by passing
31  * max_loop=<1-255> to the kernel on boot.
32  * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
33  *
34  * Completely rewrite request handling to be make_request_fn style and
35  * non blocking, pushing work to a helper thread. Lots of fixes from
36  * Al Viro too.
37  * Jens Axboe <axboe@suse.de>, Nov 2000
38  *
39  * Support up to 256 loop devices
40  * Heinz Mauelshagen <mge@sistina.com>, Feb 2002
41  *
42  * Support for falling back on the write file operation when the address space
43  * operations write_begin is not available on the backing filesystem.
44  * Anton Altaparmakov, 16 Feb 2005
45  *
46  * Still To Fix:
47  * - Advisory locking is ignored here.
48  * - Should use an own CAP_* category instead of CAP_SYS_ADMIN
49  *
50  */
51 
52 #include <linux/module.h>
53 #include <linux/moduleparam.h>
54 #include <linux/sched.h>
55 #include <linux/fs.h>
56 #include <linux/file.h>
57 #include <linux/stat.h>
58 #include <linux/errno.h>
59 #include <linux/major.h>
60 #include <linux/wait.h>
61 #include <linux/blkdev.h>
62 #include <linux/blkpg.h>
63 #include <linux/init.h>
64 #include <linux/swap.h>
65 #include <linux/slab.h>
66 #include <linux/compat.h>
67 #include <linux/suspend.h>
68 #include <linux/freezer.h>
69 #include <linux/mutex.h>
70 #include <linux/writeback.h>
71 #include <linux/completion.h>
72 #include <linux/highmem.h>
73 #include <linux/kthread.h>
74 #include <linux/splice.h>
75 #include <linux/sysfs.h>
76 #include <linux/miscdevice.h>
77 #include <linux/falloc.h>
78 #include <linux/uio.h>
79 #include <linux/ioprio.h>
80 #include <linux/blk-cgroup.h>
81 
82 #include "loop.h"
83 
84 #include <linux/uaccess.h>
85 
86 static DEFINE_IDR(loop_index_idr);
87 static DEFINE_MUTEX(loop_ctl_mutex);
88 
89 static int max_part;
90 static int part_shift;
91 
92 static int transfer_xor(struct loop_device *lo, int cmd,
93 			struct page *raw_page, unsigned raw_off,
94 			struct page *loop_page, unsigned loop_off,
95 			int size, sector_t real_block)
96 {
97 	char *raw_buf = kmap_atomic(raw_page) + raw_off;
98 	char *loop_buf = kmap_atomic(loop_page) + loop_off;
99 	char *in, *out, *key;
100 	int i, keysize;
101 
102 	if (cmd == READ) {
103 		in = raw_buf;
104 		out = loop_buf;
105 	} else {
106 		in = loop_buf;
107 		out = raw_buf;
108 	}
109 
110 	key = lo->lo_encrypt_key;
111 	keysize = lo->lo_encrypt_key_size;
112 	for (i = 0; i < size; i++)
113 		*out++ = *in++ ^ key[(i & 511) % keysize];
114 
115 	kunmap_atomic(loop_buf);
116 	kunmap_atomic(raw_buf);
117 	cond_resched();
118 	return 0;
119 }
120 
121 static int xor_init(struct loop_device *lo, const struct loop_info64 *info)
122 {
123 	if (unlikely(info->lo_encrypt_key_size <= 0))
124 		return -EINVAL;
125 	return 0;
126 }
127 
128 static struct loop_func_table none_funcs = {
129 	.number = LO_CRYPT_NONE,
130 };
131 
132 static struct loop_func_table xor_funcs = {
133 	.number = LO_CRYPT_XOR,
134 	.transfer = transfer_xor,
135 	.init = xor_init
136 };
137 
138 /* xfer_funcs[0] is special - its release function is never called */
139 static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = {
140 	&none_funcs,
141 	&xor_funcs
142 };
143 
144 static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file)
145 {
146 	loff_t loopsize;
147 
148 	/* Compute loopsize in bytes */
149 	loopsize = i_size_read(file->f_mapping->host);
150 	if (offset > 0)
151 		loopsize -= offset;
152 	/* offset is beyond i_size, weird but possible */
153 	if (loopsize < 0)
154 		return 0;
155 
156 	if (sizelimit > 0 && sizelimit < loopsize)
157 		loopsize = sizelimit;
158 	/*
159 	 * Unfortunately, if we want to do I/O on the device,
160 	 * the number of 512-byte sectors has to fit into a sector_t.
161 	 */
162 	return loopsize >> 9;
163 }
164 
165 static loff_t get_loop_size(struct loop_device *lo, struct file *file)
166 {
167 	return get_size(lo->lo_offset, lo->lo_sizelimit, file);
168 }
169 
170 static void __loop_update_dio(struct loop_device *lo, bool dio)
171 {
172 	struct file *file = lo->lo_backing_file;
173 	struct address_space *mapping = file->f_mapping;
174 	struct inode *inode = mapping->host;
175 	unsigned short sb_bsize = 0;
176 	unsigned dio_align = 0;
177 	bool use_dio;
178 
179 	if (inode->i_sb->s_bdev) {
180 		sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev);
181 		dio_align = sb_bsize - 1;
182 	}
183 
184 	/*
185 	 * We support direct I/O only if lo_offset is aligned with the
186 	 * logical I/O size of backing device, and the logical block
187 	 * size of loop is bigger than the backing device's and the loop
188 	 * needn't transform transfer.
189 	 *
190 	 * TODO: the above condition may be loosed in the future, and
191 	 * direct I/O may be switched runtime at that time because most
192 	 * of requests in sane applications should be PAGE_SIZE aligned
193 	 */
194 	if (dio) {
195 		if (queue_logical_block_size(lo->lo_queue) >= sb_bsize &&
196 				!(lo->lo_offset & dio_align) &&
197 				mapping->a_ops->direct_IO &&
198 				!lo->transfer)
199 			use_dio = true;
200 		else
201 			use_dio = false;
202 	} else {
203 		use_dio = false;
204 	}
205 
206 	if (lo->use_dio == use_dio)
207 		return;
208 
209 	/* flush dirty pages before changing direct IO */
210 	vfs_fsync(file, 0);
211 
212 	/*
213 	 * The flag of LO_FLAGS_DIRECT_IO is handled similarly with
214 	 * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup
215 	 * will get updated by ioctl(LOOP_GET_STATUS)
216 	 */
217 	blk_mq_freeze_queue(lo->lo_queue);
218 	lo->use_dio = use_dio;
219 	if (use_dio) {
220 		blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue);
221 		lo->lo_flags |= LO_FLAGS_DIRECT_IO;
222 	} else {
223 		blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
224 		lo->lo_flags &= ~LO_FLAGS_DIRECT_IO;
225 	}
226 	blk_mq_unfreeze_queue(lo->lo_queue);
227 }
228 
229 static int
230 figure_loop_size(struct loop_device *lo, loff_t offset, loff_t sizelimit)
231 {
232 	loff_t size = get_size(offset, sizelimit, lo->lo_backing_file);
233 	sector_t x = (sector_t)size;
234 	struct block_device *bdev = lo->lo_device;
235 
236 	if (unlikely((loff_t)x != size))
237 		return -EFBIG;
238 	if (lo->lo_offset != offset)
239 		lo->lo_offset = offset;
240 	if (lo->lo_sizelimit != sizelimit)
241 		lo->lo_sizelimit = sizelimit;
242 	set_capacity(lo->lo_disk, x);
243 	bd_set_size(bdev, (loff_t)get_capacity(bdev->bd_disk) << 9);
244 	/* let user-space know about the new size */
245 	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
246 	return 0;
247 }
248 
249 static inline int
250 lo_do_transfer(struct loop_device *lo, int cmd,
251 	       struct page *rpage, unsigned roffs,
252 	       struct page *lpage, unsigned loffs,
253 	       int size, sector_t rblock)
254 {
255 	int ret;
256 
257 	ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock);
258 	if (likely(!ret))
259 		return 0;
260 
261 	printk_ratelimited(KERN_ERR
262 		"loop: Transfer error at byte offset %llu, length %i.\n",
263 		(unsigned long long)rblock << 9, size);
264 	return ret;
265 }
266 
267 static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos)
268 {
269 	struct iov_iter i;
270 	ssize_t bw;
271 
272 	iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len);
273 
274 	file_start_write(file);
275 	bw = vfs_iter_write(file, &i, ppos, 0);
276 	file_end_write(file);
277 
278 	if (likely(bw ==  bvec->bv_len))
279 		return 0;
280 
281 	printk_ratelimited(KERN_ERR
282 		"loop: Write error at byte offset %llu, length %i.\n",
283 		(unsigned long long)*ppos, bvec->bv_len);
284 	if (bw >= 0)
285 		bw = -EIO;
286 	return bw;
287 }
288 
289 static int lo_write_simple(struct loop_device *lo, struct request *rq,
290 		loff_t pos)
291 {
292 	struct bio_vec bvec;
293 	struct req_iterator iter;
294 	int ret = 0;
295 
296 	rq_for_each_segment(bvec, rq, iter) {
297 		ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos);
298 		if (ret < 0)
299 			break;
300 		cond_resched();
301 	}
302 
303 	return ret;
304 }
305 
306 /*
307  * This is the slow, transforming version that needs to double buffer the
308  * data as it cannot do the transformations in place without having direct
309  * access to the destination pages of the backing file.
310  */
311 static int lo_write_transfer(struct loop_device *lo, struct request *rq,
312 		loff_t pos)
313 {
314 	struct bio_vec bvec, b;
315 	struct req_iterator iter;
316 	struct page *page;
317 	int ret = 0;
318 
319 	page = alloc_page(GFP_NOIO);
320 	if (unlikely(!page))
321 		return -ENOMEM;
322 
323 	rq_for_each_segment(bvec, rq, iter) {
324 		ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page,
325 			bvec.bv_offset, bvec.bv_len, pos >> 9);
326 		if (unlikely(ret))
327 			break;
328 
329 		b.bv_page = page;
330 		b.bv_offset = 0;
331 		b.bv_len = bvec.bv_len;
332 		ret = lo_write_bvec(lo->lo_backing_file, &b, &pos);
333 		if (ret < 0)
334 			break;
335 	}
336 
337 	__free_page(page);
338 	return ret;
339 }
340 
341 static int lo_read_simple(struct loop_device *lo, struct request *rq,
342 		loff_t pos)
343 {
344 	struct bio_vec bvec;
345 	struct req_iterator iter;
346 	struct iov_iter i;
347 	ssize_t len;
348 
349 	rq_for_each_segment(bvec, rq, iter) {
350 		iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len);
351 		len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
352 		if (len < 0)
353 			return len;
354 
355 		flush_dcache_page(bvec.bv_page);
356 
357 		if (len != bvec.bv_len) {
358 			struct bio *bio;
359 
360 			__rq_for_each_bio(bio, rq)
361 				zero_fill_bio(bio);
362 			break;
363 		}
364 		cond_resched();
365 	}
366 
367 	return 0;
368 }
369 
370 static int lo_read_transfer(struct loop_device *lo, struct request *rq,
371 		loff_t pos)
372 {
373 	struct bio_vec bvec, b;
374 	struct req_iterator iter;
375 	struct iov_iter i;
376 	struct page *page;
377 	ssize_t len;
378 	int ret = 0;
379 
380 	page = alloc_page(GFP_NOIO);
381 	if (unlikely(!page))
382 		return -ENOMEM;
383 
384 	rq_for_each_segment(bvec, rq, iter) {
385 		loff_t offset = pos;
386 
387 		b.bv_page = page;
388 		b.bv_offset = 0;
389 		b.bv_len = bvec.bv_len;
390 
391 		iov_iter_bvec(&i, READ, &b, 1, b.bv_len);
392 		len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0);
393 		if (len < 0) {
394 			ret = len;
395 			goto out_free_page;
396 		}
397 
398 		ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page,
399 			bvec.bv_offset, len, offset >> 9);
400 		if (ret)
401 			goto out_free_page;
402 
403 		flush_dcache_page(bvec.bv_page);
404 
405 		if (len != bvec.bv_len) {
406 			struct bio *bio;
407 
408 			__rq_for_each_bio(bio, rq)
409 				zero_fill_bio(bio);
410 			break;
411 		}
412 	}
413 
414 	ret = 0;
415 out_free_page:
416 	__free_page(page);
417 	return ret;
418 }
419 
420 static int lo_discard(struct loop_device *lo, struct request *rq, loff_t pos)
421 {
422 	/*
423 	 * We use punch hole to reclaim the free space used by the
424 	 * image a.k.a. discard. However we do not support discard if
425 	 * encryption is enabled, because it may give an attacker
426 	 * useful information.
427 	 */
428 	struct file *file = lo->lo_backing_file;
429 	int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
430 	int ret;
431 
432 	if ((!file->f_op->fallocate) || lo->lo_encrypt_key_size) {
433 		ret = -EOPNOTSUPP;
434 		goto out;
435 	}
436 
437 	ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq));
438 	if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP))
439 		ret = -EIO;
440  out:
441 	return ret;
442 }
443 
444 static int lo_req_flush(struct loop_device *lo, struct request *rq)
445 {
446 	struct file *file = lo->lo_backing_file;
447 	int ret = vfs_fsync(file, 0);
448 	if (unlikely(ret && ret != -EINVAL))
449 		ret = -EIO;
450 
451 	return ret;
452 }
453 
454 static void lo_complete_rq(struct request *rq)
455 {
456 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
457 	blk_status_t ret = BLK_STS_OK;
458 
459 	if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) ||
460 	    req_op(rq) != REQ_OP_READ) {
461 		if (cmd->ret < 0)
462 			ret = BLK_STS_IOERR;
463 		goto end_io;
464 	}
465 
466 	/*
467 	 * Short READ - if we got some data, advance our request and
468 	 * retry it. If we got no data, end the rest with EIO.
469 	 */
470 	if (cmd->ret) {
471 		blk_update_request(rq, BLK_STS_OK, cmd->ret);
472 		cmd->ret = 0;
473 		blk_mq_requeue_request(rq, true);
474 	} else {
475 		if (cmd->use_aio) {
476 			struct bio *bio = rq->bio;
477 
478 			while (bio) {
479 				zero_fill_bio(bio);
480 				bio = bio->bi_next;
481 			}
482 		}
483 		ret = BLK_STS_IOERR;
484 end_io:
485 		blk_mq_end_request(rq, ret);
486 	}
487 }
488 
489 static void lo_rw_aio_do_completion(struct loop_cmd *cmd)
490 {
491 	struct request *rq = blk_mq_rq_from_pdu(cmd);
492 
493 	if (!atomic_dec_and_test(&cmd->ref))
494 		return;
495 	kfree(cmd->bvec);
496 	cmd->bvec = NULL;
497 	blk_mq_complete_request(rq);
498 }
499 
500 static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2)
501 {
502 	struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb);
503 
504 	if (cmd->css)
505 		css_put(cmd->css);
506 	cmd->ret = ret;
507 	lo_rw_aio_do_completion(cmd);
508 }
509 
510 static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd,
511 		     loff_t pos, bool rw)
512 {
513 	struct iov_iter iter;
514 	struct bio_vec *bvec;
515 	struct request *rq = blk_mq_rq_from_pdu(cmd);
516 	struct bio *bio = rq->bio;
517 	struct file *file = lo->lo_backing_file;
518 	unsigned int offset;
519 	int segments = 0;
520 	int ret;
521 
522 	if (rq->bio != rq->biotail) {
523 		struct req_iterator iter;
524 		struct bio_vec tmp;
525 
526 		__rq_for_each_bio(bio, rq)
527 			segments += bio_segments(bio);
528 		bvec = kmalloc_array(segments, sizeof(struct bio_vec),
529 				     GFP_NOIO);
530 		if (!bvec)
531 			return -EIO;
532 		cmd->bvec = bvec;
533 
534 		/*
535 		 * The bios of the request may be started from the middle of
536 		 * the 'bvec' because of bio splitting, so we can't directly
537 		 * copy bio->bi_iov_vec to new bvec. The rq_for_each_segment
538 		 * API will take care of all details for us.
539 		 */
540 		rq_for_each_segment(tmp, rq, iter) {
541 			*bvec = tmp;
542 			bvec++;
543 		}
544 		bvec = cmd->bvec;
545 		offset = 0;
546 	} else {
547 		/*
548 		 * Same here, this bio may be started from the middle of the
549 		 * 'bvec' because of bio splitting, so offset from the bvec
550 		 * must be passed to iov iterator
551 		 */
552 		offset = bio->bi_iter.bi_bvec_done;
553 		bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
554 		segments = bio_segments(bio);
555 	}
556 	atomic_set(&cmd->ref, 2);
557 
558 	iov_iter_bvec(&iter, rw, bvec, segments, blk_rq_bytes(rq));
559 	iter.iov_offset = offset;
560 
561 	cmd->iocb.ki_pos = pos;
562 	cmd->iocb.ki_filp = file;
563 	cmd->iocb.ki_complete = lo_rw_aio_complete;
564 	cmd->iocb.ki_flags = IOCB_DIRECT;
565 	cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0);
566 	if (cmd->css)
567 		kthread_associate_blkcg(cmd->css);
568 
569 	if (rw == WRITE)
570 		ret = call_write_iter(file, &cmd->iocb, &iter);
571 	else
572 		ret = call_read_iter(file, &cmd->iocb, &iter);
573 
574 	lo_rw_aio_do_completion(cmd);
575 	kthread_associate_blkcg(NULL);
576 
577 	if (ret != -EIOCBQUEUED)
578 		cmd->iocb.ki_complete(&cmd->iocb, ret, 0);
579 	return 0;
580 }
581 
582 static int do_req_filebacked(struct loop_device *lo, struct request *rq)
583 {
584 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
585 	loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset;
586 
587 	/*
588 	 * lo_write_simple and lo_read_simple should have been covered
589 	 * by io submit style function like lo_rw_aio(), one blocker
590 	 * is that lo_read_simple() need to call flush_dcache_page after
591 	 * the page is written from kernel, and it isn't easy to handle
592 	 * this in io submit style function which submits all segments
593 	 * of the req at one time. And direct read IO doesn't need to
594 	 * run flush_dcache_page().
595 	 */
596 	switch (req_op(rq)) {
597 	case REQ_OP_FLUSH:
598 		return lo_req_flush(lo, rq);
599 	case REQ_OP_DISCARD:
600 	case REQ_OP_WRITE_ZEROES:
601 		return lo_discard(lo, rq, pos);
602 	case REQ_OP_WRITE:
603 		if (lo->transfer)
604 			return lo_write_transfer(lo, rq, pos);
605 		else if (cmd->use_aio)
606 			return lo_rw_aio(lo, cmd, pos, WRITE);
607 		else
608 			return lo_write_simple(lo, rq, pos);
609 	case REQ_OP_READ:
610 		if (lo->transfer)
611 			return lo_read_transfer(lo, rq, pos);
612 		else if (cmd->use_aio)
613 			return lo_rw_aio(lo, cmd, pos, READ);
614 		else
615 			return lo_read_simple(lo, rq, pos);
616 	default:
617 		WARN_ON_ONCE(1);
618 		return -EIO;
619 	}
620 }
621 
622 static inline void loop_update_dio(struct loop_device *lo)
623 {
624 	__loop_update_dio(lo, io_is_direct(lo->lo_backing_file) |
625 			lo->use_dio);
626 }
627 
628 static void loop_reread_partitions(struct loop_device *lo,
629 				   struct block_device *bdev)
630 {
631 	int rc;
632 
633 	rc = blkdev_reread_part(bdev);
634 	if (rc)
635 		pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n",
636 			__func__, lo->lo_number, lo->lo_file_name, rc);
637 }
638 
639 static inline int is_loop_device(struct file *file)
640 {
641 	struct inode *i = file->f_mapping->host;
642 
643 	return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
644 }
645 
646 static int loop_validate_file(struct file *file, struct block_device *bdev)
647 {
648 	struct inode	*inode = file->f_mapping->host;
649 	struct file	*f = file;
650 
651 	/* Avoid recursion */
652 	while (is_loop_device(f)) {
653 		struct loop_device *l;
654 
655 		if (f->f_mapping->host->i_bdev == bdev)
656 			return -EBADF;
657 
658 		l = f->f_mapping->host->i_bdev->bd_disk->private_data;
659 		if (l->lo_state == Lo_unbound) {
660 			return -EINVAL;
661 		}
662 		f = l->lo_backing_file;
663 	}
664 	if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
665 		return -EINVAL;
666 	return 0;
667 }
668 
669 /*
670  * loop_change_fd switched the backing store of a loopback device to
671  * a new file. This is useful for operating system installers to free up
672  * the original file and in High Availability environments to switch to
673  * an alternative location for the content in case of server meltdown.
674  * This can only work if the loop device is used read-only, and if the
675  * new backing store is the same size and type as the old backing store.
676  */
677 static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
678 			  unsigned int arg)
679 {
680 	struct file	*file = NULL, *old_file;
681 	int		error;
682 	bool		partscan;
683 
684 	error = mutex_lock_killable(&loop_ctl_mutex);
685 	if (error)
686 		return error;
687 	error = -ENXIO;
688 	if (lo->lo_state != Lo_bound)
689 		goto out_err;
690 
691 	/* the loop device has to be read-only */
692 	error = -EINVAL;
693 	if (!(lo->lo_flags & LO_FLAGS_READ_ONLY))
694 		goto out_err;
695 
696 	error = -EBADF;
697 	file = fget(arg);
698 	if (!file)
699 		goto out_err;
700 
701 	error = loop_validate_file(file, bdev);
702 	if (error)
703 		goto out_err;
704 
705 	old_file = lo->lo_backing_file;
706 
707 	error = -EINVAL;
708 
709 	/* size of the new backing store needs to be the same */
710 	if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
711 		goto out_err;
712 
713 	/* and ... switch */
714 	blk_mq_freeze_queue(lo->lo_queue);
715 	mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
716 	lo->lo_backing_file = file;
717 	lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping);
718 	mapping_set_gfp_mask(file->f_mapping,
719 			     lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
720 	loop_update_dio(lo);
721 	blk_mq_unfreeze_queue(lo->lo_queue);
722 	partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
723 	mutex_unlock(&loop_ctl_mutex);
724 	/*
725 	 * We must drop file reference outside of loop_ctl_mutex as dropping
726 	 * the file ref can take bd_mutex which creates circular locking
727 	 * dependency.
728 	 */
729 	fput(old_file);
730 	if (partscan)
731 		loop_reread_partitions(lo, bdev);
732 	return 0;
733 
734 out_err:
735 	mutex_unlock(&loop_ctl_mutex);
736 	if (file)
737 		fput(file);
738 	return error;
739 }
740 
741 /* loop sysfs attributes */
742 
743 static ssize_t loop_attr_show(struct device *dev, char *page,
744 			      ssize_t (*callback)(struct loop_device *, char *))
745 {
746 	struct gendisk *disk = dev_to_disk(dev);
747 	struct loop_device *lo = disk->private_data;
748 
749 	return callback(lo, page);
750 }
751 
752 #define LOOP_ATTR_RO(_name)						\
753 static ssize_t loop_attr_##_name##_show(struct loop_device *, char *);	\
754 static ssize_t loop_attr_do_show_##_name(struct device *d,		\
755 				struct device_attribute *attr, char *b)	\
756 {									\
757 	return loop_attr_show(d, b, loop_attr_##_name##_show);		\
758 }									\
759 static struct device_attribute loop_attr_##_name =			\
760 	__ATTR(_name, 0444, loop_attr_do_show_##_name, NULL);
761 
762 static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf)
763 {
764 	ssize_t ret;
765 	char *p = NULL;
766 
767 	spin_lock_irq(&lo->lo_lock);
768 	if (lo->lo_backing_file)
769 		p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1);
770 	spin_unlock_irq(&lo->lo_lock);
771 
772 	if (IS_ERR_OR_NULL(p))
773 		ret = PTR_ERR(p);
774 	else {
775 		ret = strlen(p);
776 		memmove(buf, p, ret);
777 		buf[ret++] = '\n';
778 		buf[ret] = 0;
779 	}
780 
781 	return ret;
782 }
783 
784 static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf)
785 {
786 	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_offset);
787 }
788 
789 static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf)
790 {
791 	return sprintf(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit);
792 }
793 
794 static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf)
795 {
796 	int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR);
797 
798 	return sprintf(buf, "%s\n", autoclear ? "1" : "0");
799 }
800 
801 static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf)
802 {
803 	int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN);
804 
805 	return sprintf(buf, "%s\n", partscan ? "1" : "0");
806 }
807 
808 static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf)
809 {
810 	int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO);
811 
812 	return sprintf(buf, "%s\n", dio ? "1" : "0");
813 }
814 
815 LOOP_ATTR_RO(backing_file);
816 LOOP_ATTR_RO(offset);
817 LOOP_ATTR_RO(sizelimit);
818 LOOP_ATTR_RO(autoclear);
819 LOOP_ATTR_RO(partscan);
820 LOOP_ATTR_RO(dio);
821 
822 static struct attribute *loop_attrs[] = {
823 	&loop_attr_backing_file.attr,
824 	&loop_attr_offset.attr,
825 	&loop_attr_sizelimit.attr,
826 	&loop_attr_autoclear.attr,
827 	&loop_attr_partscan.attr,
828 	&loop_attr_dio.attr,
829 	NULL,
830 };
831 
832 static struct attribute_group loop_attribute_group = {
833 	.name = "loop",
834 	.attrs= loop_attrs,
835 };
836 
837 static void loop_sysfs_init(struct loop_device *lo)
838 {
839 	lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj,
840 						&loop_attribute_group);
841 }
842 
843 static void loop_sysfs_exit(struct loop_device *lo)
844 {
845 	if (lo->sysfs_inited)
846 		sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj,
847 				   &loop_attribute_group);
848 }
849 
850 static void loop_config_discard(struct loop_device *lo)
851 {
852 	struct file *file = lo->lo_backing_file;
853 	struct inode *inode = file->f_mapping->host;
854 	struct request_queue *q = lo->lo_queue;
855 
856 	/*
857 	 * We use punch hole to reclaim the free space used by the
858 	 * image a.k.a. discard. However we do not support discard if
859 	 * encryption is enabled, because it may give an attacker
860 	 * useful information.
861 	 */
862 	if ((!file->f_op->fallocate) ||
863 	    lo->lo_encrypt_key_size) {
864 		q->limits.discard_granularity = 0;
865 		q->limits.discard_alignment = 0;
866 		blk_queue_max_discard_sectors(q, 0);
867 		blk_queue_max_write_zeroes_sectors(q, 0);
868 		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
869 		return;
870 	}
871 
872 	q->limits.discard_granularity = inode->i_sb->s_blocksize;
873 	q->limits.discard_alignment = 0;
874 
875 	blk_queue_max_discard_sectors(q, UINT_MAX >> 9);
876 	blk_queue_max_write_zeroes_sectors(q, UINT_MAX >> 9);
877 	blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
878 }
879 
880 static void loop_unprepare_queue(struct loop_device *lo)
881 {
882 	kthread_flush_worker(&lo->worker);
883 	kthread_stop(lo->worker_task);
884 }
885 
886 static int loop_kthread_worker_fn(void *worker_ptr)
887 {
888 	current->flags |= PF_LESS_THROTTLE;
889 	return kthread_worker_fn(worker_ptr);
890 }
891 
892 static int loop_prepare_queue(struct loop_device *lo)
893 {
894 	kthread_init_worker(&lo->worker);
895 	lo->worker_task = kthread_run(loop_kthread_worker_fn,
896 			&lo->worker, "loop%d", lo->lo_number);
897 	if (IS_ERR(lo->worker_task))
898 		return -ENOMEM;
899 	set_user_nice(lo->worker_task, MIN_NICE);
900 	return 0;
901 }
902 
903 static int loop_set_fd(struct loop_device *lo, fmode_t mode,
904 		       struct block_device *bdev, unsigned int arg)
905 {
906 	struct file	*file;
907 	struct inode	*inode;
908 	struct address_space *mapping;
909 	int		lo_flags = 0;
910 	int		error;
911 	loff_t		size;
912 	bool		partscan;
913 
914 	/* This is safe, since we have a reference from open(). */
915 	__module_get(THIS_MODULE);
916 
917 	error = -EBADF;
918 	file = fget(arg);
919 	if (!file)
920 		goto out;
921 
922 	error = mutex_lock_killable(&loop_ctl_mutex);
923 	if (error)
924 		goto out_putf;
925 
926 	error = -EBUSY;
927 	if (lo->lo_state != Lo_unbound)
928 		goto out_unlock;
929 
930 	error = loop_validate_file(file, bdev);
931 	if (error)
932 		goto out_unlock;
933 
934 	mapping = file->f_mapping;
935 	inode = mapping->host;
936 
937 	if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) ||
938 	    !file->f_op->write_iter)
939 		lo_flags |= LO_FLAGS_READ_ONLY;
940 
941 	error = -EFBIG;
942 	size = get_loop_size(lo, file);
943 	if ((loff_t)(sector_t)size != size)
944 		goto out_unlock;
945 	error = loop_prepare_queue(lo);
946 	if (error)
947 		goto out_unlock;
948 
949 	error = 0;
950 
951 	set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
952 
953 	lo->use_dio = false;
954 	lo->lo_device = bdev;
955 	lo->lo_flags = lo_flags;
956 	lo->lo_backing_file = file;
957 	lo->transfer = NULL;
958 	lo->ioctl = NULL;
959 	lo->lo_sizelimit = 0;
960 	lo->old_gfp_mask = mapping_gfp_mask(mapping);
961 	mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
962 
963 	if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
964 		blk_queue_write_cache(lo->lo_queue, true, false);
965 
966 	loop_update_dio(lo);
967 	set_capacity(lo->lo_disk, size);
968 	bd_set_size(bdev, size << 9);
969 	loop_sysfs_init(lo);
970 	/* let user-space know about the new size */
971 	kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
972 
973 	set_blocksize(bdev, S_ISBLK(inode->i_mode) ?
974 		      block_size(inode->i_bdev) : PAGE_SIZE);
975 
976 	lo->lo_state = Lo_bound;
977 	if (part_shift)
978 		lo->lo_flags |= LO_FLAGS_PARTSCAN;
979 	partscan = lo->lo_flags & LO_FLAGS_PARTSCAN;
980 
981 	/* Grab the block_device to prevent its destruction after we
982 	 * put /dev/loopXX inode. Later in __loop_clr_fd() we bdput(bdev).
983 	 */
984 	bdgrab(bdev);
985 	mutex_unlock(&loop_ctl_mutex);
986 	if (partscan)
987 		loop_reread_partitions(lo, bdev);
988 	return 0;
989 
990 out_unlock:
991 	mutex_unlock(&loop_ctl_mutex);
992 out_putf:
993 	fput(file);
994 out:
995 	/* This is safe: open() is still holding a reference. */
996 	module_put(THIS_MODULE);
997 	return error;
998 }
999 
1000 static int
1001 loop_release_xfer(struct loop_device *lo)
1002 {
1003 	int err = 0;
1004 	struct loop_func_table *xfer = lo->lo_encryption;
1005 
1006 	if (xfer) {
1007 		if (xfer->release)
1008 			err = xfer->release(lo);
1009 		lo->transfer = NULL;
1010 		lo->lo_encryption = NULL;
1011 		module_put(xfer->owner);
1012 	}
1013 	return err;
1014 }
1015 
1016 static int
1017 loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer,
1018 	       const struct loop_info64 *i)
1019 {
1020 	int err = 0;
1021 
1022 	if (xfer) {
1023 		struct module *owner = xfer->owner;
1024 
1025 		if (!try_module_get(owner))
1026 			return -EINVAL;
1027 		if (xfer->init)
1028 			err = xfer->init(lo, i);
1029 		if (err)
1030 			module_put(owner);
1031 		else
1032 			lo->lo_encryption = xfer;
1033 	}
1034 	return err;
1035 }
1036 
1037 static int __loop_clr_fd(struct loop_device *lo, bool release)
1038 {
1039 	struct file *filp = NULL;
1040 	gfp_t gfp = lo->old_gfp_mask;
1041 	struct block_device *bdev = lo->lo_device;
1042 	int err = 0;
1043 	bool partscan = false;
1044 	int lo_number;
1045 
1046 	mutex_lock(&loop_ctl_mutex);
1047 	if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) {
1048 		err = -ENXIO;
1049 		goto out_unlock;
1050 	}
1051 
1052 	filp = lo->lo_backing_file;
1053 	if (filp == NULL) {
1054 		err = -EINVAL;
1055 		goto out_unlock;
1056 	}
1057 
1058 	/* freeze request queue during the transition */
1059 	blk_mq_freeze_queue(lo->lo_queue);
1060 
1061 	spin_lock_irq(&lo->lo_lock);
1062 	lo->lo_backing_file = NULL;
1063 	spin_unlock_irq(&lo->lo_lock);
1064 
1065 	loop_release_xfer(lo);
1066 	lo->transfer = NULL;
1067 	lo->ioctl = NULL;
1068 	lo->lo_device = NULL;
1069 	lo->lo_encryption = NULL;
1070 	lo->lo_offset = 0;
1071 	lo->lo_sizelimit = 0;
1072 	lo->lo_encrypt_key_size = 0;
1073 	memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
1074 	memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
1075 	memset(lo->lo_file_name, 0, LO_NAME_SIZE);
1076 	blk_queue_logical_block_size(lo->lo_queue, 512);
1077 	blk_queue_physical_block_size(lo->lo_queue, 512);
1078 	blk_queue_io_min(lo->lo_queue, 512);
1079 	if (bdev) {
1080 		bdput(bdev);
1081 		invalidate_bdev(bdev);
1082 		bdev->bd_inode->i_mapping->wb_err = 0;
1083 	}
1084 	set_capacity(lo->lo_disk, 0);
1085 	loop_sysfs_exit(lo);
1086 	if (bdev) {
1087 		bd_set_size(bdev, 0);
1088 		/* let user-space know about this change */
1089 		kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE);
1090 	}
1091 	mapping_set_gfp_mask(filp->f_mapping, gfp);
1092 	lo->lo_state = Lo_unbound;
1093 	/* This is safe: open() is still holding a reference. */
1094 	module_put(THIS_MODULE);
1095 	blk_mq_unfreeze_queue(lo->lo_queue);
1096 
1097 	partscan = lo->lo_flags & LO_FLAGS_PARTSCAN && bdev;
1098 	lo_number = lo->lo_number;
1099 	lo->lo_flags = 0;
1100 	if (!part_shift)
1101 		lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN;
1102 	loop_unprepare_queue(lo);
1103 out_unlock:
1104 	mutex_unlock(&loop_ctl_mutex);
1105 	if (partscan) {
1106 		/*
1107 		 * bd_mutex has been held already in release path, so don't
1108 		 * acquire it if this function is called in such case.
1109 		 *
1110 		 * If the reread partition isn't from release path, lo_refcnt
1111 		 * must be at least one and it can only become zero when the
1112 		 * current holder is released.
1113 		 */
1114 		if (release)
1115 			err = __blkdev_reread_part(bdev);
1116 		else
1117 			err = blkdev_reread_part(bdev);
1118 		pr_warn("%s: partition scan of loop%d failed (rc=%d)\n",
1119 			__func__, lo_number, err);
1120 		/* Device is gone, no point in returning error */
1121 		err = 0;
1122 	}
1123 	/*
1124 	 * Need not hold loop_ctl_mutex to fput backing file.
1125 	 * Calling fput holding loop_ctl_mutex triggers a circular
1126 	 * lock dependency possibility warning as fput can take
1127 	 * bd_mutex which is usually taken before loop_ctl_mutex.
1128 	 */
1129 	if (filp)
1130 		fput(filp);
1131 	return err;
1132 }
1133 
1134 static int loop_clr_fd(struct loop_device *lo)
1135 {
1136 	int err;
1137 
1138 	err = mutex_lock_killable(&loop_ctl_mutex);
1139 	if (err)
1140 		return err;
1141 	if (lo->lo_state != Lo_bound) {
1142 		mutex_unlock(&loop_ctl_mutex);
1143 		return -ENXIO;
1144 	}
1145 	/*
1146 	 * If we've explicitly asked to tear down the loop device,
1147 	 * and it has an elevated reference count, set it for auto-teardown when
1148 	 * the last reference goes away. This stops $!~#$@ udev from
1149 	 * preventing teardown because it decided that it needs to run blkid on
1150 	 * the loopback device whenever they appear. xfstests is notorious for
1151 	 * failing tests because blkid via udev races with a losetup
1152 	 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d
1153 	 * command to fail with EBUSY.
1154 	 */
1155 	if (atomic_read(&lo->lo_refcnt) > 1) {
1156 		lo->lo_flags |= LO_FLAGS_AUTOCLEAR;
1157 		mutex_unlock(&loop_ctl_mutex);
1158 		return 0;
1159 	}
1160 	lo->lo_state = Lo_rundown;
1161 	mutex_unlock(&loop_ctl_mutex);
1162 
1163 	return __loop_clr_fd(lo, false);
1164 }
1165 
1166 static int
1167 loop_set_status(struct loop_device *lo, const struct loop_info64 *info)
1168 {
1169 	int err;
1170 	struct loop_func_table *xfer;
1171 	kuid_t uid = current_uid();
1172 	struct block_device *bdev;
1173 	bool partscan = false;
1174 
1175 	err = mutex_lock_killable(&loop_ctl_mutex);
1176 	if (err)
1177 		return err;
1178 	if (lo->lo_encrypt_key_size &&
1179 	    !uid_eq(lo->lo_key_owner, uid) &&
1180 	    !capable(CAP_SYS_ADMIN)) {
1181 		err = -EPERM;
1182 		goto out_unlock;
1183 	}
1184 	if (lo->lo_state != Lo_bound) {
1185 		err = -ENXIO;
1186 		goto out_unlock;
1187 	}
1188 	if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) {
1189 		err = -EINVAL;
1190 		goto out_unlock;
1191 	}
1192 
1193 	/* I/O need to be drained during transfer transition */
1194 	blk_mq_freeze_queue(lo->lo_queue);
1195 
1196 	err = loop_release_xfer(lo);
1197 	if (err)
1198 		goto out_unfreeze;
1199 
1200 	if (info->lo_encrypt_type) {
1201 		unsigned int type = info->lo_encrypt_type;
1202 
1203 		if (type >= MAX_LO_CRYPT) {
1204 			err = -EINVAL;
1205 			goto out_unfreeze;
1206 		}
1207 		xfer = xfer_funcs[type];
1208 		if (xfer == NULL) {
1209 			err = -EINVAL;
1210 			goto out_unfreeze;
1211 		}
1212 	} else
1213 		xfer = NULL;
1214 
1215 	err = loop_init_xfer(lo, xfer, info);
1216 	if (err)
1217 		goto out_unfreeze;
1218 
1219 	if (lo->lo_offset != info->lo_offset ||
1220 	    lo->lo_sizelimit != info->lo_sizelimit) {
1221 		if (figure_loop_size(lo, info->lo_offset, info->lo_sizelimit)) {
1222 			err = -EFBIG;
1223 			goto out_unfreeze;
1224 		}
1225 	}
1226 
1227 	loop_config_discard(lo);
1228 
1229 	memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE);
1230 	memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE);
1231 	lo->lo_file_name[LO_NAME_SIZE-1] = 0;
1232 	lo->lo_crypt_name[LO_NAME_SIZE-1] = 0;
1233 
1234 	if (!xfer)
1235 		xfer = &none_funcs;
1236 	lo->transfer = xfer->transfer;
1237 	lo->ioctl = xfer->ioctl;
1238 
1239 	if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
1240 	     (info->lo_flags & LO_FLAGS_AUTOCLEAR))
1241 		lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;
1242 
1243 	lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
1244 	lo->lo_init[0] = info->lo_init[0];
1245 	lo->lo_init[1] = info->lo_init[1];
1246 	if (info->lo_encrypt_key_size) {
1247 		memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
1248 		       info->lo_encrypt_key_size);
1249 		lo->lo_key_owner = uid;
1250 	}
1251 
1252 	/* update dio if lo_offset or transfer is changed */
1253 	__loop_update_dio(lo, lo->use_dio);
1254 
1255 out_unfreeze:
1256 	blk_mq_unfreeze_queue(lo->lo_queue);
1257 
1258 	if (!err && (info->lo_flags & LO_FLAGS_PARTSCAN) &&
1259 	     !(lo->lo_flags & LO_FLAGS_PARTSCAN)) {
1260 		lo->lo_flags |= LO_FLAGS_PARTSCAN;
1261 		lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN;
1262 		bdev = lo->lo_device;
1263 		partscan = true;
1264 	}
1265 out_unlock:
1266 	mutex_unlock(&loop_ctl_mutex);
1267 	if (partscan)
1268 		loop_reread_partitions(lo, bdev);
1269 
1270 	return err;
1271 }
1272 
1273 static int
1274 loop_get_status(struct loop_device *lo, struct loop_info64 *info)
1275 {
1276 	struct path path;
1277 	struct kstat stat;
1278 	int ret;
1279 
1280 	ret = mutex_lock_killable(&loop_ctl_mutex);
1281 	if (ret)
1282 		return ret;
1283 	if (lo->lo_state != Lo_bound) {
1284 		mutex_unlock(&loop_ctl_mutex);
1285 		return -ENXIO;
1286 	}
1287 
1288 	memset(info, 0, sizeof(*info));
1289 	info->lo_number = lo->lo_number;
1290 	info->lo_offset = lo->lo_offset;
1291 	info->lo_sizelimit = lo->lo_sizelimit;
1292 	info->lo_flags = lo->lo_flags;
1293 	memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE);
1294 	memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE);
1295 	info->lo_encrypt_type =
1296 		lo->lo_encryption ? lo->lo_encryption->number : 0;
1297 	if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) {
1298 		info->lo_encrypt_key_size = lo->lo_encrypt_key_size;
1299 		memcpy(info->lo_encrypt_key, lo->lo_encrypt_key,
1300 		       lo->lo_encrypt_key_size);
1301 	}
1302 
1303 	/* Drop loop_ctl_mutex while we call into the filesystem. */
1304 	path = lo->lo_backing_file->f_path;
1305 	path_get(&path);
1306 	mutex_unlock(&loop_ctl_mutex);
1307 	ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT);
1308 	if (!ret) {
1309 		info->lo_device = huge_encode_dev(stat.dev);
1310 		info->lo_inode = stat.ino;
1311 		info->lo_rdevice = huge_encode_dev(stat.rdev);
1312 	}
1313 	path_put(&path);
1314 	return ret;
1315 }
1316 
1317 static void
1318 loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64)
1319 {
1320 	memset(info64, 0, sizeof(*info64));
1321 	info64->lo_number = info->lo_number;
1322 	info64->lo_device = info->lo_device;
1323 	info64->lo_inode = info->lo_inode;
1324 	info64->lo_rdevice = info->lo_rdevice;
1325 	info64->lo_offset = info->lo_offset;
1326 	info64->lo_sizelimit = 0;
1327 	info64->lo_encrypt_type = info->lo_encrypt_type;
1328 	info64->lo_encrypt_key_size = info->lo_encrypt_key_size;
1329 	info64->lo_flags = info->lo_flags;
1330 	info64->lo_init[0] = info->lo_init[0];
1331 	info64->lo_init[1] = info->lo_init[1];
1332 	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1333 		memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE);
1334 	else
1335 		memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE);
1336 	memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE);
1337 }
1338 
1339 static int
1340 loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info)
1341 {
1342 	memset(info, 0, sizeof(*info));
1343 	info->lo_number = info64->lo_number;
1344 	info->lo_device = info64->lo_device;
1345 	info->lo_inode = info64->lo_inode;
1346 	info->lo_rdevice = info64->lo_rdevice;
1347 	info->lo_offset = info64->lo_offset;
1348 	info->lo_encrypt_type = info64->lo_encrypt_type;
1349 	info->lo_encrypt_key_size = info64->lo_encrypt_key_size;
1350 	info->lo_flags = info64->lo_flags;
1351 	info->lo_init[0] = info64->lo_init[0];
1352 	info->lo_init[1] = info64->lo_init[1];
1353 	if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1354 		memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
1355 	else
1356 		memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE);
1357 	memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
1358 
1359 	/* error in case values were truncated */
1360 	if (info->lo_device != info64->lo_device ||
1361 	    info->lo_rdevice != info64->lo_rdevice ||
1362 	    info->lo_inode != info64->lo_inode ||
1363 	    info->lo_offset != info64->lo_offset)
1364 		return -EOVERFLOW;
1365 
1366 	return 0;
1367 }
1368 
1369 static int
1370 loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg)
1371 {
1372 	struct loop_info info;
1373 	struct loop_info64 info64;
1374 
1375 	if (copy_from_user(&info, arg, sizeof (struct loop_info)))
1376 		return -EFAULT;
1377 	loop_info64_from_old(&info, &info64);
1378 	return loop_set_status(lo, &info64);
1379 }
1380 
1381 static int
1382 loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg)
1383 {
1384 	struct loop_info64 info64;
1385 
1386 	if (copy_from_user(&info64, arg, sizeof (struct loop_info64)))
1387 		return -EFAULT;
1388 	return loop_set_status(lo, &info64);
1389 }
1390 
1391 static int
1392 loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) {
1393 	struct loop_info info;
1394 	struct loop_info64 info64;
1395 	int err;
1396 
1397 	if (!arg)
1398 		return -EINVAL;
1399 	err = loop_get_status(lo, &info64);
1400 	if (!err)
1401 		err = loop_info64_to_old(&info64, &info);
1402 	if (!err && copy_to_user(arg, &info, sizeof(info)))
1403 		err = -EFAULT;
1404 
1405 	return err;
1406 }
1407 
1408 static int
1409 loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) {
1410 	struct loop_info64 info64;
1411 	int err;
1412 
1413 	if (!arg)
1414 		return -EINVAL;
1415 	err = loop_get_status(lo, &info64);
1416 	if (!err && copy_to_user(arg, &info64, sizeof(info64)))
1417 		err = -EFAULT;
1418 
1419 	return err;
1420 }
1421 
1422 static int loop_set_capacity(struct loop_device *lo)
1423 {
1424 	if (unlikely(lo->lo_state != Lo_bound))
1425 		return -ENXIO;
1426 
1427 	return figure_loop_size(lo, lo->lo_offset, lo->lo_sizelimit);
1428 }
1429 
1430 static int loop_set_dio(struct loop_device *lo, unsigned long arg)
1431 {
1432 	int error = -ENXIO;
1433 	if (lo->lo_state != Lo_bound)
1434 		goto out;
1435 
1436 	__loop_update_dio(lo, !!arg);
1437 	if (lo->use_dio == !!arg)
1438 		return 0;
1439 	error = -EINVAL;
1440  out:
1441 	return error;
1442 }
1443 
1444 static int loop_set_block_size(struct loop_device *lo, unsigned long arg)
1445 {
1446 	if (lo->lo_state != Lo_bound)
1447 		return -ENXIO;
1448 
1449 	if (arg < 512 || arg > PAGE_SIZE || !is_power_of_2(arg))
1450 		return -EINVAL;
1451 
1452 	blk_mq_freeze_queue(lo->lo_queue);
1453 
1454 	blk_queue_logical_block_size(lo->lo_queue, arg);
1455 	blk_queue_physical_block_size(lo->lo_queue, arg);
1456 	blk_queue_io_min(lo->lo_queue, arg);
1457 	loop_update_dio(lo);
1458 
1459 	blk_mq_unfreeze_queue(lo->lo_queue);
1460 
1461 	return 0;
1462 }
1463 
1464 static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd,
1465 			   unsigned long arg)
1466 {
1467 	int err;
1468 
1469 	err = mutex_lock_killable(&loop_ctl_mutex);
1470 	if (err)
1471 		return err;
1472 	switch (cmd) {
1473 	case LOOP_SET_CAPACITY:
1474 		err = loop_set_capacity(lo);
1475 		break;
1476 	case LOOP_SET_DIRECT_IO:
1477 		err = loop_set_dio(lo, arg);
1478 		break;
1479 	case LOOP_SET_BLOCK_SIZE:
1480 		err = loop_set_block_size(lo, arg);
1481 		break;
1482 	default:
1483 		err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
1484 	}
1485 	mutex_unlock(&loop_ctl_mutex);
1486 	return err;
1487 }
1488 
1489 static int lo_ioctl(struct block_device *bdev, fmode_t mode,
1490 	unsigned int cmd, unsigned long arg)
1491 {
1492 	struct loop_device *lo = bdev->bd_disk->private_data;
1493 	int err;
1494 
1495 	switch (cmd) {
1496 	case LOOP_SET_FD:
1497 		return loop_set_fd(lo, mode, bdev, arg);
1498 	case LOOP_CHANGE_FD:
1499 		return loop_change_fd(lo, bdev, arg);
1500 	case LOOP_CLR_FD:
1501 		return loop_clr_fd(lo);
1502 	case LOOP_SET_STATUS:
1503 		err = -EPERM;
1504 		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) {
1505 			err = loop_set_status_old(lo,
1506 					(struct loop_info __user *)arg);
1507 		}
1508 		break;
1509 	case LOOP_GET_STATUS:
1510 		return loop_get_status_old(lo, (struct loop_info __user *) arg);
1511 	case LOOP_SET_STATUS64:
1512 		err = -EPERM;
1513 		if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) {
1514 			err = loop_set_status64(lo,
1515 					(struct loop_info64 __user *) arg);
1516 		}
1517 		break;
1518 	case LOOP_GET_STATUS64:
1519 		return loop_get_status64(lo, (struct loop_info64 __user *) arg);
1520 	case LOOP_SET_CAPACITY:
1521 	case LOOP_SET_DIRECT_IO:
1522 	case LOOP_SET_BLOCK_SIZE:
1523 		if (!(mode & FMODE_WRITE) && !capable(CAP_SYS_ADMIN))
1524 			return -EPERM;
1525 		/* Fall through */
1526 	default:
1527 		err = lo_simple_ioctl(lo, cmd, arg);
1528 		break;
1529 	}
1530 
1531 	return err;
1532 }
1533 
1534 #ifdef CONFIG_COMPAT
1535 struct compat_loop_info {
1536 	compat_int_t	lo_number;      /* ioctl r/o */
1537 	compat_dev_t	lo_device;      /* ioctl r/o */
1538 	compat_ulong_t	lo_inode;       /* ioctl r/o */
1539 	compat_dev_t	lo_rdevice;     /* ioctl r/o */
1540 	compat_int_t	lo_offset;
1541 	compat_int_t	lo_encrypt_type;
1542 	compat_int_t	lo_encrypt_key_size;    /* ioctl w/o */
1543 	compat_int_t	lo_flags;       /* ioctl r/o */
1544 	char		lo_name[LO_NAME_SIZE];
1545 	unsigned char	lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
1546 	compat_ulong_t	lo_init[2];
1547 	char		reserved[4];
1548 };
1549 
1550 /*
1551  * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
1552  * - noinlined to reduce stack space usage in main part of driver
1553  */
1554 static noinline int
1555 loop_info64_from_compat(const struct compat_loop_info __user *arg,
1556 			struct loop_info64 *info64)
1557 {
1558 	struct compat_loop_info info;
1559 
1560 	if (copy_from_user(&info, arg, sizeof(info)))
1561 		return -EFAULT;
1562 
1563 	memset(info64, 0, sizeof(*info64));
1564 	info64->lo_number = info.lo_number;
1565 	info64->lo_device = info.lo_device;
1566 	info64->lo_inode = info.lo_inode;
1567 	info64->lo_rdevice = info.lo_rdevice;
1568 	info64->lo_offset = info.lo_offset;
1569 	info64->lo_sizelimit = 0;
1570 	info64->lo_encrypt_type = info.lo_encrypt_type;
1571 	info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
1572 	info64->lo_flags = info.lo_flags;
1573 	info64->lo_init[0] = info.lo_init[0];
1574 	info64->lo_init[1] = info.lo_init[1];
1575 	if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1576 		memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
1577 	else
1578 		memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
1579 	memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
1580 	return 0;
1581 }
1582 
1583 /*
1584  * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
1585  * - noinlined to reduce stack space usage in main part of driver
1586  */
1587 static noinline int
1588 loop_info64_to_compat(const struct loop_info64 *info64,
1589 		      struct compat_loop_info __user *arg)
1590 {
1591 	struct compat_loop_info info;
1592 
1593 	memset(&info, 0, sizeof(info));
1594 	info.lo_number = info64->lo_number;
1595 	info.lo_device = info64->lo_device;
1596 	info.lo_inode = info64->lo_inode;
1597 	info.lo_rdevice = info64->lo_rdevice;
1598 	info.lo_offset = info64->lo_offset;
1599 	info.lo_encrypt_type = info64->lo_encrypt_type;
1600 	info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
1601 	info.lo_flags = info64->lo_flags;
1602 	info.lo_init[0] = info64->lo_init[0];
1603 	info.lo_init[1] = info64->lo_init[1];
1604 	if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
1605 		memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
1606 	else
1607 		memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
1608 	memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
1609 
1610 	/* error in case values were truncated */
1611 	if (info.lo_device != info64->lo_device ||
1612 	    info.lo_rdevice != info64->lo_rdevice ||
1613 	    info.lo_inode != info64->lo_inode ||
1614 	    info.lo_offset != info64->lo_offset ||
1615 	    info.lo_init[0] != info64->lo_init[0] ||
1616 	    info.lo_init[1] != info64->lo_init[1])
1617 		return -EOVERFLOW;
1618 
1619 	if (copy_to_user(arg, &info, sizeof(info)))
1620 		return -EFAULT;
1621 	return 0;
1622 }
1623 
1624 static int
1625 loop_set_status_compat(struct loop_device *lo,
1626 		       const struct compat_loop_info __user *arg)
1627 {
1628 	struct loop_info64 info64;
1629 	int ret;
1630 
1631 	ret = loop_info64_from_compat(arg, &info64);
1632 	if (ret < 0)
1633 		return ret;
1634 	return loop_set_status(lo, &info64);
1635 }
1636 
1637 static int
1638 loop_get_status_compat(struct loop_device *lo,
1639 		       struct compat_loop_info __user *arg)
1640 {
1641 	struct loop_info64 info64;
1642 	int err;
1643 
1644 	if (!arg)
1645 		return -EINVAL;
1646 	err = loop_get_status(lo, &info64);
1647 	if (!err)
1648 		err = loop_info64_to_compat(&info64, arg);
1649 	return err;
1650 }
1651 
1652 static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
1653 			   unsigned int cmd, unsigned long arg)
1654 {
1655 	struct loop_device *lo = bdev->bd_disk->private_data;
1656 	int err;
1657 
1658 	switch(cmd) {
1659 	case LOOP_SET_STATUS:
1660 		err = loop_set_status_compat(lo,
1661 			     (const struct compat_loop_info __user *)arg);
1662 		break;
1663 	case LOOP_GET_STATUS:
1664 		err = loop_get_status_compat(lo,
1665 				     (struct compat_loop_info __user *)arg);
1666 		break;
1667 	case LOOP_SET_CAPACITY:
1668 	case LOOP_CLR_FD:
1669 	case LOOP_GET_STATUS64:
1670 	case LOOP_SET_STATUS64:
1671 		arg = (unsigned long) compat_ptr(arg);
1672 		/* fall through */
1673 	case LOOP_SET_FD:
1674 	case LOOP_CHANGE_FD:
1675 	case LOOP_SET_BLOCK_SIZE:
1676 		err = lo_ioctl(bdev, mode, cmd, arg);
1677 		break;
1678 	default:
1679 		err = -ENOIOCTLCMD;
1680 		break;
1681 	}
1682 	return err;
1683 }
1684 #endif
1685 
1686 static int lo_open(struct block_device *bdev, fmode_t mode)
1687 {
1688 	struct loop_device *lo;
1689 	int err;
1690 
1691 	err = mutex_lock_killable(&loop_ctl_mutex);
1692 	if (err)
1693 		return err;
1694 	lo = bdev->bd_disk->private_data;
1695 	if (!lo) {
1696 		err = -ENXIO;
1697 		goto out;
1698 	}
1699 
1700 	atomic_inc(&lo->lo_refcnt);
1701 out:
1702 	mutex_unlock(&loop_ctl_mutex);
1703 	return err;
1704 }
1705 
1706 static void lo_release(struct gendisk *disk, fmode_t mode)
1707 {
1708 	struct loop_device *lo;
1709 
1710 	mutex_lock(&loop_ctl_mutex);
1711 	lo = disk->private_data;
1712 	if (atomic_dec_return(&lo->lo_refcnt))
1713 		goto out_unlock;
1714 
1715 	if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
1716 		if (lo->lo_state != Lo_bound)
1717 			goto out_unlock;
1718 		lo->lo_state = Lo_rundown;
1719 		mutex_unlock(&loop_ctl_mutex);
1720 		/*
1721 		 * In autoclear mode, stop the loop thread
1722 		 * and remove configuration after last close.
1723 		 */
1724 		__loop_clr_fd(lo, true);
1725 		return;
1726 	} else if (lo->lo_state == Lo_bound) {
1727 		/*
1728 		 * Otherwise keep thread (if running) and config,
1729 		 * but flush possible ongoing bios in thread.
1730 		 */
1731 		blk_mq_freeze_queue(lo->lo_queue);
1732 		blk_mq_unfreeze_queue(lo->lo_queue);
1733 	}
1734 
1735 out_unlock:
1736 	mutex_unlock(&loop_ctl_mutex);
1737 }
1738 
1739 static const struct block_device_operations lo_fops = {
1740 	.owner =	THIS_MODULE,
1741 	.open =		lo_open,
1742 	.release =	lo_release,
1743 	.ioctl =	lo_ioctl,
1744 #ifdef CONFIG_COMPAT
1745 	.compat_ioctl =	lo_compat_ioctl,
1746 #endif
1747 };
1748 
1749 /*
1750  * And now the modules code and kernel interface.
1751  */
1752 static int max_loop;
1753 module_param(max_loop, int, 0444);
1754 MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
1755 module_param(max_part, int, 0444);
1756 MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
1757 MODULE_LICENSE("GPL");
1758 MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
1759 
1760 int loop_register_transfer(struct loop_func_table *funcs)
1761 {
1762 	unsigned int n = funcs->number;
1763 
1764 	if (n >= MAX_LO_CRYPT || xfer_funcs[n])
1765 		return -EINVAL;
1766 	xfer_funcs[n] = funcs;
1767 	return 0;
1768 }
1769 
1770 static int unregister_transfer_cb(int id, void *ptr, void *data)
1771 {
1772 	struct loop_device *lo = ptr;
1773 	struct loop_func_table *xfer = data;
1774 
1775 	mutex_lock(&loop_ctl_mutex);
1776 	if (lo->lo_encryption == xfer)
1777 		loop_release_xfer(lo);
1778 	mutex_unlock(&loop_ctl_mutex);
1779 	return 0;
1780 }
1781 
1782 int loop_unregister_transfer(int number)
1783 {
1784 	unsigned int n = number;
1785 	struct loop_func_table *xfer;
1786 
1787 	if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL)
1788 		return -EINVAL;
1789 
1790 	xfer_funcs[n] = NULL;
1791 	idr_for_each(&loop_index_idr, &unregister_transfer_cb, xfer);
1792 	return 0;
1793 }
1794 
1795 EXPORT_SYMBOL(loop_register_transfer);
1796 EXPORT_SYMBOL(loop_unregister_transfer);
1797 
1798 static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx,
1799 		const struct blk_mq_queue_data *bd)
1800 {
1801 	struct request *rq = bd->rq;
1802 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
1803 	struct loop_device *lo = rq->q->queuedata;
1804 
1805 	blk_mq_start_request(rq);
1806 
1807 	if (lo->lo_state != Lo_bound)
1808 		return BLK_STS_IOERR;
1809 
1810 	switch (req_op(rq)) {
1811 	case REQ_OP_FLUSH:
1812 	case REQ_OP_DISCARD:
1813 	case REQ_OP_WRITE_ZEROES:
1814 		cmd->use_aio = false;
1815 		break;
1816 	default:
1817 		cmd->use_aio = lo->use_dio;
1818 		break;
1819 	}
1820 
1821 	/* always use the first bio's css */
1822 #ifdef CONFIG_BLK_CGROUP
1823 	if (cmd->use_aio && rq->bio && rq->bio->bi_blkg) {
1824 		cmd->css = &bio_blkcg(rq->bio)->css;
1825 		css_get(cmd->css);
1826 	} else
1827 #endif
1828 		cmd->css = NULL;
1829 	kthread_queue_work(&lo->worker, &cmd->work);
1830 
1831 	return BLK_STS_OK;
1832 }
1833 
1834 static void loop_handle_cmd(struct loop_cmd *cmd)
1835 {
1836 	struct request *rq = blk_mq_rq_from_pdu(cmd);
1837 	const bool write = op_is_write(req_op(rq));
1838 	struct loop_device *lo = rq->q->queuedata;
1839 	int ret = 0;
1840 
1841 	if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) {
1842 		ret = -EIO;
1843 		goto failed;
1844 	}
1845 
1846 	ret = do_req_filebacked(lo, rq);
1847  failed:
1848 	/* complete non-aio request */
1849 	if (!cmd->use_aio || ret) {
1850 		cmd->ret = ret ? -EIO : 0;
1851 		blk_mq_complete_request(rq);
1852 	}
1853 }
1854 
1855 static void loop_queue_work(struct kthread_work *work)
1856 {
1857 	struct loop_cmd *cmd =
1858 		container_of(work, struct loop_cmd, work);
1859 
1860 	loop_handle_cmd(cmd);
1861 }
1862 
1863 static int loop_init_request(struct blk_mq_tag_set *set, struct request *rq,
1864 		unsigned int hctx_idx, unsigned int numa_node)
1865 {
1866 	struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq);
1867 
1868 	kthread_init_work(&cmd->work, loop_queue_work);
1869 	return 0;
1870 }
1871 
1872 static const struct blk_mq_ops loop_mq_ops = {
1873 	.queue_rq       = loop_queue_rq,
1874 	.init_request	= loop_init_request,
1875 	.complete	= lo_complete_rq,
1876 };
1877 
1878 static int loop_add(struct loop_device **l, int i)
1879 {
1880 	struct loop_device *lo;
1881 	struct gendisk *disk;
1882 	int err;
1883 
1884 	err = -ENOMEM;
1885 	lo = kzalloc(sizeof(*lo), GFP_KERNEL);
1886 	if (!lo)
1887 		goto out;
1888 
1889 	lo->lo_state = Lo_unbound;
1890 
1891 	/* allocate id, if @id >= 0, we're requesting that specific id */
1892 	if (i >= 0) {
1893 		err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL);
1894 		if (err == -ENOSPC)
1895 			err = -EEXIST;
1896 	} else {
1897 		err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL);
1898 	}
1899 	if (err < 0)
1900 		goto out_free_dev;
1901 	i = err;
1902 
1903 	err = -ENOMEM;
1904 	lo->tag_set.ops = &loop_mq_ops;
1905 	lo->tag_set.nr_hw_queues = 1;
1906 	lo->tag_set.queue_depth = 128;
1907 	lo->tag_set.numa_node = NUMA_NO_NODE;
1908 	lo->tag_set.cmd_size = sizeof(struct loop_cmd);
1909 	lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_SG_MERGE;
1910 	lo->tag_set.driver_data = lo;
1911 
1912 	err = blk_mq_alloc_tag_set(&lo->tag_set);
1913 	if (err)
1914 		goto out_free_idr;
1915 
1916 	lo->lo_queue = blk_mq_init_queue(&lo->tag_set);
1917 	if (IS_ERR(lo->lo_queue)) {
1918 		err = PTR_ERR(lo->lo_queue);
1919 		goto out_cleanup_tags;
1920 	}
1921 	lo->lo_queue->queuedata = lo;
1922 
1923 	blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS);
1924 
1925 	/*
1926 	 * By default, we do buffer IO, so it doesn't make sense to enable
1927 	 * merge because the I/O submitted to backing file is handled page by
1928 	 * page. For directio mode, merge does help to dispatch bigger request
1929 	 * to underlayer disk. We will enable merge once directio is enabled.
1930 	 */
1931 	blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue);
1932 
1933 	err = -ENOMEM;
1934 	disk = lo->lo_disk = alloc_disk(1 << part_shift);
1935 	if (!disk)
1936 		goto out_free_queue;
1937 
1938 	/*
1939 	 * Disable partition scanning by default. The in-kernel partition
1940 	 * scanning can be requested individually per-device during its
1941 	 * setup. Userspace can always add and remove partitions from all
1942 	 * devices. The needed partition minors are allocated from the
1943 	 * extended minor space, the main loop device numbers will continue
1944 	 * to match the loop minors, regardless of the number of partitions
1945 	 * used.
1946 	 *
1947 	 * If max_part is given, partition scanning is globally enabled for
1948 	 * all loop devices. The minors for the main loop devices will be
1949 	 * multiples of max_part.
1950 	 *
1951 	 * Note: Global-for-all-devices, set-only-at-init, read-only module
1952 	 * parameteters like 'max_loop' and 'max_part' make things needlessly
1953 	 * complicated, are too static, inflexible and may surprise
1954 	 * userspace tools. Parameters like this in general should be avoided.
1955 	 */
1956 	if (!part_shift)
1957 		disk->flags |= GENHD_FL_NO_PART_SCAN;
1958 	disk->flags |= GENHD_FL_EXT_DEVT;
1959 	atomic_set(&lo->lo_refcnt, 0);
1960 	lo->lo_number		= i;
1961 	spin_lock_init(&lo->lo_lock);
1962 	disk->major		= LOOP_MAJOR;
1963 	disk->first_minor	= i << part_shift;
1964 	disk->fops		= &lo_fops;
1965 	disk->private_data	= lo;
1966 	disk->queue		= lo->lo_queue;
1967 	sprintf(disk->disk_name, "loop%d", i);
1968 	add_disk(disk);
1969 	*l = lo;
1970 	return lo->lo_number;
1971 
1972 out_free_queue:
1973 	blk_cleanup_queue(lo->lo_queue);
1974 out_cleanup_tags:
1975 	blk_mq_free_tag_set(&lo->tag_set);
1976 out_free_idr:
1977 	idr_remove(&loop_index_idr, i);
1978 out_free_dev:
1979 	kfree(lo);
1980 out:
1981 	return err;
1982 }
1983 
1984 static void loop_remove(struct loop_device *lo)
1985 {
1986 	del_gendisk(lo->lo_disk);
1987 	blk_cleanup_queue(lo->lo_queue);
1988 	blk_mq_free_tag_set(&lo->tag_set);
1989 	put_disk(lo->lo_disk);
1990 	kfree(lo);
1991 }
1992 
1993 static int find_free_cb(int id, void *ptr, void *data)
1994 {
1995 	struct loop_device *lo = ptr;
1996 	struct loop_device **l = data;
1997 
1998 	if (lo->lo_state == Lo_unbound) {
1999 		*l = lo;
2000 		return 1;
2001 	}
2002 	return 0;
2003 }
2004 
2005 static int loop_lookup(struct loop_device **l, int i)
2006 {
2007 	struct loop_device *lo;
2008 	int ret = -ENODEV;
2009 
2010 	if (i < 0) {
2011 		int err;
2012 
2013 		err = idr_for_each(&loop_index_idr, &find_free_cb, &lo);
2014 		if (err == 1) {
2015 			*l = lo;
2016 			ret = lo->lo_number;
2017 		}
2018 		goto out;
2019 	}
2020 
2021 	/* lookup and return a specific i */
2022 	lo = idr_find(&loop_index_idr, i);
2023 	if (lo) {
2024 		*l = lo;
2025 		ret = lo->lo_number;
2026 	}
2027 out:
2028 	return ret;
2029 }
2030 
2031 static struct kobject *loop_probe(dev_t dev, int *part, void *data)
2032 {
2033 	struct loop_device *lo;
2034 	struct kobject *kobj;
2035 	int err;
2036 
2037 	mutex_lock(&loop_ctl_mutex);
2038 	err = loop_lookup(&lo, MINOR(dev) >> part_shift);
2039 	if (err < 0)
2040 		err = loop_add(&lo, MINOR(dev) >> part_shift);
2041 	if (err < 0)
2042 		kobj = NULL;
2043 	else
2044 		kobj = get_disk_and_module(lo->lo_disk);
2045 	mutex_unlock(&loop_ctl_mutex);
2046 
2047 	*part = 0;
2048 	return kobj;
2049 }
2050 
2051 static long loop_control_ioctl(struct file *file, unsigned int cmd,
2052 			       unsigned long parm)
2053 {
2054 	struct loop_device *lo;
2055 	int ret;
2056 
2057 	ret = mutex_lock_killable(&loop_ctl_mutex);
2058 	if (ret)
2059 		return ret;
2060 
2061 	ret = -ENOSYS;
2062 	switch (cmd) {
2063 	case LOOP_CTL_ADD:
2064 		ret = loop_lookup(&lo, parm);
2065 		if (ret >= 0) {
2066 			ret = -EEXIST;
2067 			break;
2068 		}
2069 		ret = loop_add(&lo, parm);
2070 		break;
2071 	case LOOP_CTL_REMOVE:
2072 		ret = loop_lookup(&lo, parm);
2073 		if (ret < 0)
2074 			break;
2075 		if (lo->lo_state != Lo_unbound) {
2076 			ret = -EBUSY;
2077 			break;
2078 		}
2079 		if (atomic_read(&lo->lo_refcnt) > 0) {
2080 			ret = -EBUSY;
2081 			break;
2082 		}
2083 		lo->lo_disk->private_data = NULL;
2084 		idr_remove(&loop_index_idr, lo->lo_number);
2085 		loop_remove(lo);
2086 		break;
2087 	case LOOP_CTL_GET_FREE:
2088 		ret = loop_lookup(&lo, -1);
2089 		if (ret >= 0)
2090 			break;
2091 		ret = loop_add(&lo, -1);
2092 	}
2093 	mutex_unlock(&loop_ctl_mutex);
2094 
2095 	return ret;
2096 }
2097 
2098 static const struct file_operations loop_ctl_fops = {
2099 	.open		= nonseekable_open,
2100 	.unlocked_ioctl	= loop_control_ioctl,
2101 	.compat_ioctl	= loop_control_ioctl,
2102 	.owner		= THIS_MODULE,
2103 	.llseek		= noop_llseek,
2104 };
2105 
2106 static struct miscdevice loop_misc = {
2107 	.minor		= LOOP_CTRL_MINOR,
2108 	.name		= "loop-control",
2109 	.fops		= &loop_ctl_fops,
2110 };
2111 
2112 MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR);
2113 MODULE_ALIAS("devname:loop-control");
2114 
2115 static int __init loop_init(void)
2116 {
2117 	int i, nr;
2118 	unsigned long range;
2119 	struct loop_device *lo;
2120 	int err;
2121 
2122 	part_shift = 0;
2123 	if (max_part > 0) {
2124 		part_shift = fls(max_part);
2125 
2126 		/*
2127 		 * Adjust max_part according to part_shift as it is exported
2128 		 * to user space so that user can decide correct minor number
2129 		 * if [s]he want to create more devices.
2130 		 *
2131 		 * Note that -1 is required because partition 0 is reserved
2132 		 * for the whole disk.
2133 		 */
2134 		max_part = (1UL << part_shift) - 1;
2135 	}
2136 
2137 	if ((1UL << part_shift) > DISK_MAX_PARTS) {
2138 		err = -EINVAL;
2139 		goto err_out;
2140 	}
2141 
2142 	if (max_loop > 1UL << (MINORBITS - part_shift)) {
2143 		err = -EINVAL;
2144 		goto err_out;
2145 	}
2146 
2147 	/*
2148 	 * If max_loop is specified, create that many devices upfront.
2149 	 * This also becomes a hard limit. If max_loop is not specified,
2150 	 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module
2151 	 * init time. Loop devices can be requested on-demand with the
2152 	 * /dev/loop-control interface, or be instantiated by accessing
2153 	 * a 'dead' device node.
2154 	 */
2155 	if (max_loop) {
2156 		nr = max_loop;
2157 		range = max_loop << part_shift;
2158 	} else {
2159 		nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT;
2160 		range = 1UL << MINORBITS;
2161 	}
2162 
2163 	err = misc_register(&loop_misc);
2164 	if (err < 0)
2165 		goto err_out;
2166 
2167 
2168 	if (register_blkdev(LOOP_MAJOR, "loop")) {
2169 		err = -EIO;
2170 		goto misc_out;
2171 	}
2172 
2173 	blk_register_region(MKDEV(LOOP_MAJOR, 0), range,
2174 				  THIS_MODULE, loop_probe, NULL, NULL);
2175 
2176 	/* pre-create number of devices given by config or max_loop */
2177 	mutex_lock(&loop_ctl_mutex);
2178 	for (i = 0; i < nr; i++)
2179 		loop_add(&lo, i);
2180 	mutex_unlock(&loop_ctl_mutex);
2181 
2182 	printk(KERN_INFO "loop: module loaded\n");
2183 	return 0;
2184 
2185 misc_out:
2186 	misc_deregister(&loop_misc);
2187 err_out:
2188 	return err;
2189 }
2190 
2191 static int loop_exit_cb(int id, void *ptr, void *data)
2192 {
2193 	struct loop_device *lo = ptr;
2194 
2195 	loop_remove(lo);
2196 	return 0;
2197 }
2198 
2199 static void __exit loop_exit(void)
2200 {
2201 	unsigned long range;
2202 
2203 	range = max_loop ? max_loop << part_shift : 1UL << MINORBITS;
2204 
2205 	idr_for_each(&loop_index_idr, &loop_exit_cb, NULL);
2206 	idr_destroy(&loop_index_idr);
2207 
2208 	blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range);
2209 	unregister_blkdev(LOOP_MAJOR, "loop");
2210 
2211 	misc_deregister(&loop_misc);
2212 }
2213 
2214 module_init(loop_init);
2215 module_exit(loop_exit);
2216 
2217 #ifndef MODULE
2218 static int __init max_loop_setup(char *str)
2219 {
2220 	max_loop = simple_strtol(str, NULL, 0);
2221 	return 1;
2222 }
2223 
2224 __setup("max_loop=", max_loop_setup);
2225 #endif
2226