1 /* 2 * linux/drivers/block/loop.c 3 * 4 * Written by Theodore Ts'o, 3/29/93 5 * 6 * Copyright 1993 by Theodore Ts'o. Redistribution of this file is 7 * permitted under the GNU General Public License. 8 * 9 * DES encryption plus some minor changes by Werner Almesberger, 30-MAY-1993 10 * more DES encryption plus IDEA encryption by Nicholas J. Leon, June 20, 1996 11 * 12 * Modularized and updated for 1.1.16 kernel - Mitch Dsouza 28th May 1994 13 * Adapted for 1.3.59 kernel - Andries Brouwer, 1 Feb 1996 14 * 15 * Fixed do_loop_request() re-entrancy - Vincent.Renardias@waw.com Mar 20, 1997 16 * 17 * Added devfs support - Richard Gooch <rgooch@atnf.csiro.au> 16-Jan-1998 18 * 19 * Handle sparse backing files correctly - Kenn Humborg, Jun 28, 1998 20 * 21 * Loadable modules and other fixes by AK, 1998 22 * 23 * Make real block number available to downstream transfer functions, enables 24 * CBC (and relatives) mode encryption requiring unique IVs per data block. 25 * Reed H. Petty, rhp@draper.net 26 * 27 * Maximum number of loop devices now dynamic via max_loop module parameter. 28 * Russell Kroll <rkroll@exploits.org> 19990701 29 * 30 * Maximum number of loop devices when compiled-in now selectable by passing 31 * max_loop=<1-255> to the kernel on boot. 32 * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999 33 * 34 * Completely rewrite request handling to be make_request_fn style and 35 * non blocking, pushing work to a helper thread. Lots of fixes from 36 * Al Viro too. 37 * Jens Axboe <axboe@suse.de>, Nov 2000 38 * 39 * Support up to 256 loop devices 40 * Heinz Mauelshagen <mge@sistina.com>, Feb 2002 41 * 42 * Support for falling back on the write file operation when the address space 43 * operations write_begin is not available on the backing filesystem. 44 * Anton Altaparmakov, 16 Feb 2005 45 * 46 * Still To Fix: 47 * - Advisory locking is ignored here. 48 * - Should use an own CAP_* category instead of CAP_SYS_ADMIN 49 * 50 */ 51 52 #include <linux/module.h> 53 #include <linux/moduleparam.h> 54 #include <linux/sched.h> 55 #include <linux/fs.h> 56 #include <linux/pagemap.h> 57 #include <linux/file.h> 58 #include <linux/stat.h> 59 #include <linux/errno.h> 60 #include <linux/major.h> 61 #include <linux/wait.h> 62 #include <linux/blkdev.h> 63 #include <linux/blkpg.h> 64 #include <linux/init.h> 65 #include <linux/swap.h> 66 #include <linux/slab.h> 67 #include <linux/compat.h> 68 #include <linux/suspend.h> 69 #include <linux/freezer.h> 70 #include <linux/mutex.h> 71 #include <linux/writeback.h> 72 #include <linux/completion.h> 73 #include <linux/highmem.h> 74 #include <linux/splice.h> 75 #include <linux/sysfs.h> 76 #include <linux/miscdevice.h> 77 #include <linux/falloc.h> 78 #include <linux/uio.h> 79 #include <linux/ioprio.h> 80 #include <linux/blk-cgroup.h> 81 #include <linux/sched/mm.h> 82 #include <linux/statfs.h> 83 84 #include "loop.h" 85 86 #include <linux/uaccess.h> 87 88 #define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ) 89 90 static DEFINE_IDR(loop_index_idr); 91 static DEFINE_MUTEX(loop_ctl_mutex); 92 static DEFINE_MUTEX(loop_validate_mutex); 93 94 /** 95 * loop_global_lock_killable() - take locks for safe loop_validate_file() test 96 * 97 * @lo: struct loop_device 98 * @global: true if @lo is about to bind another "struct loop_device", false otherwise 99 * 100 * Returns 0 on success, -EINTR otherwise. 101 * 102 * Since loop_validate_file() traverses on other "struct loop_device" if 103 * is_loop_device() is true, we need a global lock for serializing concurrent 104 * loop_configure()/loop_change_fd()/__loop_clr_fd() calls. 105 */ 106 static int loop_global_lock_killable(struct loop_device *lo, bool global) 107 { 108 int err; 109 110 if (global) { 111 err = mutex_lock_killable(&loop_validate_mutex); 112 if (err) 113 return err; 114 } 115 err = mutex_lock_killable(&lo->lo_mutex); 116 if (err && global) 117 mutex_unlock(&loop_validate_mutex); 118 return err; 119 } 120 121 /** 122 * loop_global_unlock() - release locks taken by loop_global_lock_killable() 123 * 124 * @lo: struct loop_device 125 * @global: true if @lo was about to bind another "struct loop_device", false otherwise 126 */ 127 static void loop_global_unlock(struct loop_device *lo, bool global) 128 { 129 mutex_unlock(&lo->lo_mutex); 130 if (global) 131 mutex_unlock(&loop_validate_mutex); 132 } 133 134 static int max_part; 135 static int part_shift; 136 137 static int transfer_xor(struct loop_device *lo, int cmd, 138 struct page *raw_page, unsigned raw_off, 139 struct page *loop_page, unsigned loop_off, 140 int size, sector_t real_block) 141 { 142 char *raw_buf = kmap_atomic(raw_page) + raw_off; 143 char *loop_buf = kmap_atomic(loop_page) + loop_off; 144 char *in, *out, *key; 145 int i, keysize; 146 147 if (cmd == READ) { 148 in = raw_buf; 149 out = loop_buf; 150 } else { 151 in = loop_buf; 152 out = raw_buf; 153 } 154 155 key = lo->lo_encrypt_key; 156 keysize = lo->lo_encrypt_key_size; 157 for (i = 0; i < size; i++) 158 *out++ = *in++ ^ key[(i & 511) % keysize]; 159 160 kunmap_atomic(loop_buf); 161 kunmap_atomic(raw_buf); 162 cond_resched(); 163 return 0; 164 } 165 166 static int xor_init(struct loop_device *lo, const struct loop_info64 *info) 167 { 168 if (unlikely(info->lo_encrypt_key_size <= 0)) 169 return -EINVAL; 170 return 0; 171 } 172 173 static struct loop_func_table none_funcs = { 174 .number = LO_CRYPT_NONE, 175 }; 176 177 static struct loop_func_table xor_funcs = { 178 .number = LO_CRYPT_XOR, 179 .transfer = transfer_xor, 180 .init = xor_init 181 }; 182 183 /* xfer_funcs[0] is special - its release function is never called */ 184 static struct loop_func_table *xfer_funcs[MAX_LO_CRYPT] = { 185 &none_funcs, 186 &xor_funcs 187 }; 188 189 static loff_t get_size(loff_t offset, loff_t sizelimit, struct file *file) 190 { 191 loff_t loopsize; 192 193 /* Compute loopsize in bytes */ 194 loopsize = i_size_read(file->f_mapping->host); 195 if (offset > 0) 196 loopsize -= offset; 197 /* offset is beyond i_size, weird but possible */ 198 if (loopsize < 0) 199 return 0; 200 201 if (sizelimit > 0 && sizelimit < loopsize) 202 loopsize = sizelimit; 203 /* 204 * Unfortunately, if we want to do I/O on the device, 205 * the number of 512-byte sectors has to fit into a sector_t. 206 */ 207 return loopsize >> 9; 208 } 209 210 static loff_t get_loop_size(struct loop_device *lo, struct file *file) 211 { 212 return get_size(lo->lo_offset, lo->lo_sizelimit, file); 213 } 214 215 static void __loop_update_dio(struct loop_device *lo, bool dio) 216 { 217 struct file *file = lo->lo_backing_file; 218 struct address_space *mapping = file->f_mapping; 219 struct inode *inode = mapping->host; 220 unsigned short sb_bsize = 0; 221 unsigned dio_align = 0; 222 bool use_dio; 223 224 if (inode->i_sb->s_bdev) { 225 sb_bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 226 dio_align = sb_bsize - 1; 227 } 228 229 /* 230 * We support direct I/O only if lo_offset is aligned with the 231 * logical I/O size of backing device, and the logical block 232 * size of loop is bigger than the backing device's and the loop 233 * needn't transform transfer. 234 * 235 * TODO: the above condition may be loosed in the future, and 236 * direct I/O may be switched runtime at that time because most 237 * of requests in sane applications should be PAGE_SIZE aligned 238 */ 239 if (dio) { 240 if (queue_logical_block_size(lo->lo_queue) >= sb_bsize && 241 !(lo->lo_offset & dio_align) && 242 mapping->a_ops->direct_IO && 243 !lo->transfer) 244 use_dio = true; 245 else 246 use_dio = false; 247 } else { 248 use_dio = false; 249 } 250 251 if (lo->use_dio == use_dio) 252 return; 253 254 /* flush dirty pages before changing direct IO */ 255 vfs_fsync(file, 0); 256 257 /* 258 * The flag of LO_FLAGS_DIRECT_IO is handled similarly with 259 * LO_FLAGS_READ_ONLY, both are set from kernel, and losetup 260 * will get updated by ioctl(LOOP_GET_STATUS) 261 */ 262 if (lo->lo_state == Lo_bound) 263 blk_mq_freeze_queue(lo->lo_queue); 264 lo->use_dio = use_dio; 265 if (use_dio) { 266 blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, lo->lo_queue); 267 lo->lo_flags |= LO_FLAGS_DIRECT_IO; 268 } else { 269 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 270 lo->lo_flags &= ~LO_FLAGS_DIRECT_IO; 271 } 272 if (lo->lo_state == Lo_bound) 273 blk_mq_unfreeze_queue(lo->lo_queue); 274 } 275 276 /** 277 * loop_set_size() - sets device size and notifies userspace 278 * @lo: struct loop_device to set the size for 279 * @size: new size of the loop device 280 * 281 * Callers must validate that the size passed into this function fits into 282 * a sector_t, eg using loop_validate_size() 283 */ 284 static void loop_set_size(struct loop_device *lo, loff_t size) 285 { 286 if (!set_capacity_and_notify(lo->lo_disk, size)) 287 kobject_uevent(&disk_to_dev(lo->lo_disk)->kobj, KOBJ_CHANGE); 288 } 289 290 static inline int 291 lo_do_transfer(struct loop_device *lo, int cmd, 292 struct page *rpage, unsigned roffs, 293 struct page *lpage, unsigned loffs, 294 int size, sector_t rblock) 295 { 296 int ret; 297 298 ret = lo->transfer(lo, cmd, rpage, roffs, lpage, loffs, size, rblock); 299 if (likely(!ret)) 300 return 0; 301 302 printk_ratelimited(KERN_ERR 303 "loop: Transfer error at byte offset %llu, length %i.\n", 304 (unsigned long long)rblock << 9, size); 305 return ret; 306 } 307 308 static int lo_write_bvec(struct file *file, struct bio_vec *bvec, loff_t *ppos) 309 { 310 struct iov_iter i; 311 ssize_t bw; 312 313 iov_iter_bvec(&i, WRITE, bvec, 1, bvec->bv_len); 314 315 file_start_write(file); 316 bw = vfs_iter_write(file, &i, ppos, 0); 317 file_end_write(file); 318 319 if (likely(bw == bvec->bv_len)) 320 return 0; 321 322 printk_ratelimited(KERN_ERR 323 "loop: Write error at byte offset %llu, length %i.\n", 324 (unsigned long long)*ppos, bvec->bv_len); 325 if (bw >= 0) 326 bw = -EIO; 327 return bw; 328 } 329 330 static int lo_write_simple(struct loop_device *lo, struct request *rq, 331 loff_t pos) 332 { 333 struct bio_vec bvec; 334 struct req_iterator iter; 335 int ret = 0; 336 337 rq_for_each_segment(bvec, rq, iter) { 338 ret = lo_write_bvec(lo->lo_backing_file, &bvec, &pos); 339 if (ret < 0) 340 break; 341 cond_resched(); 342 } 343 344 return ret; 345 } 346 347 /* 348 * This is the slow, transforming version that needs to double buffer the 349 * data as it cannot do the transformations in place without having direct 350 * access to the destination pages of the backing file. 351 */ 352 static int lo_write_transfer(struct loop_device *lo, struct request *rq, 353 loff_t pos) 354 { 355 struct bio_vec bvec, b; 356 struct req_iterator iter; 357 struct page *page; 358 int ret = 0; 359 360 page = alloc_page(GFP_NOIO); 361 if (unlikely(!page)) 362 return -ENOMEM; 363 364 rq_for_each_segment(bvec, rq, iter) { 365 ret = lo_do_transfer(lo, WRITE, page, 0, bvec.bv_page, 366 bvec.bv_offset, bvec.bv_len, pos >> 9); 367 if (unlikely(ret)) 368 break; 369 370 b.bv_page = page; 371 b.bv_offset = 0; 372 b.bv_len = bvec.bv_len; 373 ret = lo_write_bvec(lo->lo_backing_file, &b, &pos); 374 if (ret < 0) 375 break; 376 } 377 378 __free_page(page); 379 return ret; 380 } 381 382 static int lo_read_simple(struct loop_device *lo, struct request *rq, 383 loff_t pos) 384 { 385 struct bio_vec bvec; 386 struct req_iterator iter; 387 struct iov_iter i; 388 ssize_t len; 389 390 rq_for_each_segment(bvec, rq, iter) { 391 iov_iter_bvec(&i, READ, &bvec, 1, bvec.bv_len); 392 len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 393 if (len < 0) 394 return len; 395 396 flush_dcache_page(bvec.bv_page); 397 398 if (len != bvec.bv_len) { 399 struct bio *bio; 400 401 __rq_for_each_bio(bio, rq) 402 zero_fill_bio(bio); 403 break; 404 } 405 cond_resched(); 406 } 407 408 return 0; 409 } 410 411 static int lo_read_transfer(struct loop_device *lo, struct request *rq, 412 loff_t pos) 413 { 414 struct bio_vec bvec, b; 415 struct req_iterator iter; 416 struct iov_iter i; 417 struct page *page; 418 ssize_t len; 419 int ret = 0; 420 421 page = alloc_page(GFP_NOIO); 422 if (unlikely(!page)) 423 return -ENOMEM; 424 425 rq_for_each_segment(bvec, rq, iter) { 426 loff_t offset = pos; 427 428 b.bv_page = page; 429 b.bv_offset = 0; 430 b.bv_len = bvec.bv_len; 431 432 iov_iter_bvec(&i, READ, &b, 1, b.bv_len); 433 len = vfs_iter_read(lo->lo_backing_file, &i, &pos, 0); 434 if (len < 0) { 435 ret = len; 436 goto out_free_page; 437 } 438 439 ret = lo_do_transfer(lo, READ, page, 0, bvec.bv_page, 440 bvec.bv_offset, len, offset >> 9); 441 if (ret) 442 goto out_free_page; 443 444 flush_dcache_page(bvec.bv_page); 445 446 if (len != bvec.bv_len) { 447 struct bio *bio; 448 449 __rq_for_each_bio(bio, rq) 450 zero_fill_bio(bio); 451 break; 452 } 453 } 454 455 ret = 0; 456 out_free_page: 457 __free_page(page); 458 return ret; 459 } 460 461 static int lo_fallocate(struct loop_device *lo, struct request *rq, loff_t pos, 462 int mode) 463 { 464 /* 465 * We use fallocate to manipulate the space mappings used by the image 466 * a.k.a. discard/zerorange. However we do not support this if 467 * encryption is enabled, because it may give an attacker useful 468 * information. 469 */ 470 struct file *file = lo->lo_backing_file; 471 struct request_queue *q = lo->lo_queue; 472 int ret; 473 474 mode |= FALLOC_FL_KEEP_SIZE; 475 476 if (!blk_queue_discard(q)) { 477 ret = -EOPNOTSUPP; 478 goto out; 479 } 480 481 ret = file->f_op->fallocate(file, mode, pos, blk_rq_bytes(rq)); 482 if (unlikely(ret && ret != -EINVAL && ret != -EOPNOTSUPP)) 483 ret = -EIO; 484 out: 485 return ret; 486 } 487 488 static int lo_req_flush(struct loop_device *lo, struct request *rq) 489 { 490 struct file *file = lo->lo_backing_file; 491 int ret = vfs_fsync(file, 0); 492 if (unlikely(ret && ret != -EINVAL)) 493 ret = -EIO; 494 495 return ret; 496 } 497 498 static void lo_complete_rq(struct request *rq) 499 { 500 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 501 blk_status_t ret = BLK_STS_OK; 502 503 if (!cmd->use_aio || cmd->ret < 0 || cmd->ret == blk_rq_bytes(rq) || 504 req_op(rq) != REQ_OP_READ) { 505 if (cmd->ret < 0) 506 ret = errno_to_blk_status(cmd->ret); 507 goto end_io; 508 } 509 510 /* 511 * Short READ - if we got some data, advance our request and 512 * retry it. If we got no data, end the rest with EIO. 513 */ 514 if (cmd->ret) { 515 blk_update_request(rq, BLK_STS_OK, cmd->ret); 516 cmd->ret = 0; 517 blk_mq_requeue_request(rq, true); 518 } else { 519 if (cmd->use_aio) { 520 struct bio *bio = rq->bio; 521 522 while (bio) { 523 zero_fill_bio(bio); 524 bio = bio->bi_next; 525 } 526 } 527 ret = BLK_STS_IOERR; 528 end_io: 529 blk_mq_end_request(rq, ret); 530 } 531 } 532 533 static void lo_rw_aio_do_completion(struct loop_cmd *cmd) 534 { 535 struct request *rq = blk_mq_rq_from_pdu(cmd); 536 537 if (!atomic_dec_and_test(&cmd->ref)) 538 return; 539 kfree(cmd->bvec); 540 cmd->bvec = NULL; 541 if (likely(!blk_should_fake_timeout(rq->q))) 542 blk_mq_complete_request(rq); 543 } 544 545 static void lo_rw_aio_complete(struct kiocb *iocb, long ret, long ret2) 546 { 547 struct loop_cmd *cmd = container_of(iocb, struct loop_cmd, iocb); 548 549 cmd->ret = ret; 550 lo_rw_aio_do_completion(cmd); 551 } 552 553 static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, 554 loff_t pos, bool rw) 555 { 556 struct iov_iter iter; 557 struct req_iterator rq_iter; 558 struct bio_vec *bvec; 559 struct request *rq = blk_mq_rq_from_pdu(cmd); 560 struct bio *bio = rq->bio; 561 struct file *file = lo->lo_backing_file; 562 struct bio_vec tmp; 563 unsigned int offset; 564 int nr_bvec = 0; 565 int ret; 566 567 rq_for_each_bvec(tmp, rq, rq_iter) 568 nr_bvec++; 569 570 if (rq->bio != rq->biotail) { 571 572 bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec), 573 GFP_NOIO); 574 if (!bvec) 575 return -EIO; 576 cmd->bvec = bvec; 577 578 /* 579 * The bios of the request may be started from the middle of 580 * the 'bvec' because of bio splitting, so we can't directly 581 * copy bio->bi_iov_vec to new bvec. The rq_for_each_bvec 582 * API will take care of all details for us. 583 */ 584 rq_for_each_bvec(tmp, rq, rq_iter) { 585 *bvec = tmp; 586 bvec++; 587 } 588 bvec = cmd->bvec; 589 offset = 0; 590 } else { 591 /* 592 * Same here, this bio may be started from the middle of the 593 * 'bvec' because of bio splitting, so offset from the bvec 594 * must be passed to iov iterator 595 */ 596 offset = bio->bi_iter.bi_bvec_done; 597 bvec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 598 } 599 atomic_set(&cmd->ref, 2); 600 601 iov_iter_bvec(&iter, rw, bvec, nr_bvec, blk_rq_bytes(rq)); 602 iter.iov_offset = offset; 603 604 cmd->iocb.ki_pos = pos; 605 cmd->iocb.ki_filp = file; 606 cmd->iocb.ki_complete = lo_rw_aio_complete; 607 cmd->iocb.ki_flags = IOCB_DIRECT; 608 cmd->iocb.ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); 609 610 if (rw == WRITE) 611 ret = call_write_iter(file, &cmd->iocb, &iter); 612 else 613 ret = call_read_iter(file, &cmd->iocb, &iter); 614 615 lo_rw_aio_do_completion(cmd); 616 617 if (ret != -EIOCBQUEUED) 618 cmd->iocb.ki_complete(&cmd->iocb, ret, 0); 619 return 0; 620 } 621 622 static int do_req_filebacked(struct loop_device *lo, struct request *rq) 623 { 624 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 625 loff_t pos = ((loff_t) blk_rq_pos(rq) << 9) + lo->lo_offset; 626 627 /* 628 * lo_write_simple and lo_read_simple should have been covered 629 * by io submit style function like lo_rw_aio(), one blocker 630 * is that lo_read_simple() need to call flush_dcache_page after 631 * the page is written from kernel, and it isn't easy to handle 632 * this in io submit style function which submits all segments 633 * of the req at one time. And direct read IO doesn't need to 634 * run flush_dcache_page(). 635 */ 636 switch (req_op(rq)) { 637 case REQ_OP_FLUSH: 638 return lo_req_flush(lo, rq); 639 case REQ_OP_WRITE_ZEROES: 640 /* 641 * If the caller doesn't want deallocation, call zeroout to 642 * write zeroes the range. Otherwise, punch them out. 643 */ 644 return lo_fallocate(lo, rq, pos, 645 (rq->cmd_flags & REQ_NOUNMAP) ? 646 FALLOC_FL_ZERO_RANGE : 647 FALLOC_FL_PUNCH_HOLE); 648 case REQ_OP_DISCARD: 649 return lo_fallocate(lo, rq, pos, FALLOC_FL_PUNCH_HOLE); 650 case REQ_OP_WRITE: 651 if (lo->transfer) 652 return lo_write_transfer(lo, rq, pos); 653 else if (cmd->use_aio) 654 return lo_rw_aio(lo, cmd, pos, WRITE); 655 else 656 return lo_write_simple(lo, rq, pos); 657 case REQ_OP_READ: 658 if (lo->transfer) 659 return lo_read_transfer(lo, rq, pos); 660 else if (cmd->use_aio) 661 return lo_rw_aio(lo, cmd, pos, READ); 662 else 663 return lo_read_simple(lo, rq, pos); 664 default: 665 WARN_ON_ONCE(1); 666 return -EIO; 667 } 668 } 669 670 static inline void loop_update_dio(struct loop_device *lo) 671 { 672 __loop_update_dio(lo, (lo->lo_backing_file->f_flags & O_DIRECT) | 673 lo->use_dio); 674 } 675 676 static void loop_reread_partitions(struct loop_device *lo) 677 { 678 int rc; 679 680 mutex_lock(&lo->lo_disk->open_mutex); 681 rc = bdev_disk_changed(lo->lo_disk, false); 682 mutex_unlock(&lo->lo_disk->open_mutex); 683 if (rc) 684 pr_warn("%s: partition scan of loop%d (%s) failed (rc=%d)\n", 685 __func__, lo->lo_number, lo->lo_file_name, rc); 686 } 687 688 static inline int is_loop_device(struct file *file) 689 { 690 struct inode *i = file->f_mapping->host; 691 692 return i && S_ISBLK(i->i_mode) && imajor(i) == LOOP_MAJOR; 693 } 694 695 static int loop_validate_file(struct file *file, struct block_device *bdev) 696 { 697 struct inode *inode = file->f_mapping->host; 698 struct file *f = file; 699 700 /* Avoid recursion */ 701 while (is_loop_device(f)) { 702 struct loop_device *l; 703 704 lockdep_assert_held(&loop_validate_mutex); 705 if (f->f_mapping->host->i_rdev == bdev->bd_dev) 706 return -EBADF; 707 708 l = I_BDEV(f->f_mapping->host)->bd_disk->private_data; 709 if (l->lo_state != Lo_bound) 710 return -EINVAL; 711 /* Order wrt setting lo->lo_backing_file in loop_configure(). */ 712 rmb(); 713 f = l->lo_backing_file; 714 } 715 if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode)) 716 return -EINVAL; 717 return 0; 718 } 719 720 /* 721 * loop_change_fd switched the backing store of a loopback device to 722 * a new file. This is useful for operating system installers to free up 723 * the original file and in High Availability environments to switch to 724 * an alternative location for the content in case of server meltdown. 725 * This can only work if the loop device is used read-only, and if the 726 * new backing store is the same size and type as the old backing store. 727 */ 728 static int loop_change_fd(struct loop_device *lo, struct block_device *bdev, 729 unsigned int arg) 730 { 731 struct file *file = fget(arg); 732 struct file *old_file; 733 int error; 734 bool partscan; 735 bool is_loop; 736 737 if (!file) 738 return -EBADF; 739 is_loop = is_loop_device(file); 740 error = loop_global_lock_killable(lo, is_loop); 741 if (error) 742 goto out_putf; 743 error = -ENXIO; 744 if (lo->lo_state != Lo_bound) 745 goto out_err; 746 747 /* the loop device has to be read-only */ 748 error = -EINVAL; 749 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY)) 750 goto out_err; 751 752 error = loop_validate_file(file, bdev); 753 if (error) 754 goto out_err; 755 756 old_file = lo->lo_backing_file; 757 758 error = -EINVAL; 759 760 /* size of the new backing store needs to be the same */ 761 if (get_loop_size(lo, file) != get_loop_size(lo, old_file)) 762 goto out_err; 763 764 /* and ... switch */ 765 disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE); 766 blk_mq_freeze_queue(lo->lo_queue); 767 mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask); 768 lo->lo_backing_file = file; 769 lo->old_gfp_mask = mapping_gfp_mask(file->f_mapping); 770 mapping_set_gfp_mask(file->f_mapping, 771 lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 772 loop_update_dio(lo); 773 blk_mq_unfreeze_queue(lo->lo_queue); 774 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 775 loop_global_unlock(lo, is_loop); 776 777 /* 778 * Flush loop_validate_file() before fput(), for l->lo_backing_file 779 * might be pointing at old_file which might be the last reference. 780 */ 781 if (!is_loop) { 782 mutex_lock(&loop_validate_mutex); 783 mutex_unlock(&loop_validate_mutex); 784 } 785 /* 786 * We must drop file reference outside of lo_mutex as dropping 787 * the file ref can take open_mutex which creates circular locking 788 * dependency. 789 */ 790 fput(old_file); 791 if (partscan) 792 loop_reread_partitions(lo); 793 return 0; 794 795 out_err: 796 loop_global_unlock(lo, is_loop); 797 out_putf: 798 fput(file); 799 return error; 800 } 801 802 /* loop sysfs attributes */ 803 804 static ssize_t loop_attr_show(struct device *dev, char *page, 805 ssize_t (*callback)(struct loop_device *, char *)) 806 { 807 struct gendisk *disk = dev_to_disk(dev); 808 struct loop_device *lo = disk->private_data; 809 810 return callback(lo, page); 811 } 812 813 #define LOOP_ATTR_RO(_name) \ 814 static ssize_t loop_attr_##_name##_show(struct loop_device *, char *); \ 815 static ssize_t loop_attr_do_show_##_name(struct device *d, \ 816 struct device_attribute *attr, char *b) \ 817 { \ 818 return loop_attr_show(d, b, loop_attr_##_name##_show); \ 819 } \ 820 static struct device_attribute loop_attr_##_name = \ 821 __ATTR(_name, 0444, loop_attr_do_show_##_name, NULL); 822 823 static ssize_t loop_attr_backing_file_show(struct loop_device *lo, char *buf) 824 { 825 ssize_t ret; 826 char *p = NULL; 827 828 spin_lock_irq(&lo->lo_lock); 829 if (lo->lo_backing_file) 830 p = file_path(lo->lo_backing_file, buf, PAGE_SIZE - 1); 831 spin_unlock_irq(&lo->lo_lock); 832 833 if (IS_ERR_OR_NULL(p)) 834 ret = PTR_ERR(p); 835 else { 836 ret = strlen(p); 837 memmove(buf, p, ret); 838 buf[ret++] = '\n'; 839 buf[ret] = 0; 840 } 841 842 return ret; 843 } 844 845 static ssize_t loop_attr_offset_show(struct loop_device *lo, char *buf) 846 { 847 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_offset); 848 } 849 850 static ssize_t loop_attr_sizelimit_show(struct loop_device *lo, char *buf) 851 { 852 return sysfs_emit(buf, "%llu\n", (unsigned long long)lo->lo_sizelimit); 853 } 854 855 static ssize_t loop_attr_autoclear_show(struct loop_device *lo, char *buf) 856 { 857 int autoclear = (lo->lo_flags & LO_FLAGS_AUTOCLEAR); 858 859 return sysfs_emit(buf, "%s\n", autoclear ? "1" : "0"); 860 } 861 862 static ssize_t loop_attr_partscan_show(struct loop_device *lo, char *buf) 863 { 864 int partscan = (lo->lo_flags & LO_FLAGS_PARTSCAN); 865 866 return sysfs_emit(buf, "%s\n", partscan ? "1" : "0"); 867 } 868 869 static ssize_t loop_attr_dio_show(struct loop_device *lo, char *buf) 870 { 871 int dio = (lo->lo_flags & LO_FLAGS_DIRECT_IO); 872 873 return sysfs_emit(buf, "%s\n", dio ? "1" : "0"); 874 } 875 876 LOOP_ATTR_RO(backing_file); 877 LOOP_ATTR_RO(offset); 878 LOOP_ATTR_RO(sizelimit); 879 LOOP_ATTR_RO(autoclear); 880 LOOP_ATTR_RO(partscan); 881 LOOP_ATTR_RO(dio); 882 883 static struct attribute *loop_attrs[] = { 884 &loop_attr_backing_file.attr, 885 &loop_attr_offset.attr, 886 &loop_attr_sizelimit.attr, 887 &loop_attr_autoclear.attr, 888 &loop_attr_partscan.attr, 889 &loop_attr_dio.attr, 890 NULL, 891 }; 892 893 static struct attribute_group loop_attribute_group = { 894 .name = "loop", 895 .attrs= loop_attrs, 896 }; 897 898 static void loop_sysfs_init(struct loop_device *lo) 899 { 900 lo->sysfs_inited = !sysfs_create_group(&disk_to_dev(lo->lo_disk)->kobj, 901 &loop_attribute_group); 902 } 903 904 static void loop_sysfs_exit(struct loop_device *lo) 905 { 906 if (lo->sysfs_inited) 907 sysfs_remove_group(&disk_to_dev(lo->lo_disk)->kobj, 908 &loop_attribute_group); 909 } 910 911 static void loop_config_discard(struct loop_device *lo) 912 { 913 struct file *file = lo->lo_backing_file; 914 struct inode *inode = file->f_mapping->host; 915 struct request_queue *q = lo->lo_queue; 916 u32 granularity, max_discard_sectors; 917 918 /* 919 * If the backing device is a block device, mirror its zeroing 920 * capability. Set the discard sectors to the block device's zeroing 921 * capabilities because loop discards result in blkdev_issue_zeroout(), 922 * not blkdev_issue_discard(). This maintains consistent behavior with 923 * file-backed loop devices: discarded regions read back as zero. 924 */ 925 if (S_ISBLK(inode->i_mode) && !lo->lo_encrypt_key_size) { 926 struct request_queue *backingq = bdev_get_queue(I_BDEV(inode)); 927 928 max_discard_sectors = backingq->limits.max_write_zeroes_sectors; 929 granularity = backingq->limits.discard_granularity ?: 930 queue_physical_block_size(backingq); 931 932 /* 933 * We use punch hole to reclaim the free space used by the 934 * image a.k.a. discard. However we do not support discard if 935 * encryption is enabled, because it may give an attacker 936 * useful information. 937 */ 938 } else if (!file->f_op->fallocate || lo->lo_encrypt_key_size) { 939 max_discard_sectors = 0; 940 granularity = 0; 941 942 } else { 943 struct kstatfs sbuf; 944 945 max_discard_sectors = UINT_MAX >> 9; 946 if (!vfs_statfs(&file->f_path, &sbuf)) 947 granularity = sbuf.f_bsize; 948 else 949 max_discard_sectors = 0; 950 } 951 952 if (max_discard_sectors) { 953 q->limits.discard_granularity = granularity; 954 blk_queue_max_discard_sectors(q, max_discard_sectors); 955 blk_queue_max_write_zeroes_sectors(q, max_discard_sectors); 956 blk_queue_flag_set(QUEUE_FLAG_DISCARD, q); 957 } else { 958 q->limits.discard_granularity = 0; 959 blk_queue_max_discard_sectors(q, 0); 960 blk_queue_max_write_zeroes_sectors(q, 0); 961 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q); 962 } 963 q->limits.discard_alignment = 0; 964 } 965 966 struct loop_worker { 967 struct rb_node rb_node; 968 struct work_struct work; 969 struct list_head cmd_list; 970 struct list_head idle_list; 971 struct loop_device *lo; 972 struct cgroup_subsys_state *blkcg_css; 973 unsigned long last_ran_at; 974 }; 975 976 static void loop_workfn(struct work_struct *work); 977 static void loop_rootcg_workfn(struct work_struct *work); 978 static void loop_free_idle_workers(struct timer_list *timer); 979 980 #ifdef CONFIG_BLK_CGROUP 981 static inline int queue_on_root_worker(struct cgroup_subsys_state *css) 982 { 983 return !css || css == blkcg_root_css; 984 } 985 #else 986 static inline int queue_on_root_worker(struct cgroup_subsys_state *css) 987 { 988 return !css; 989 } 990 #endif 991 992 static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd) 993 { 994 struct rb_node **node = &(lo->worker_tree.rb_node), *parent = NULL; 995 struct loop_worker *cur_worker, *worker = NULL; 996 struct work_struct *work; 997 struct list_head *cmd_list; 998 999 spin_lock_irq(&lo->lo_work_lock); 1000 1001 if (queue_on_root_worker(cmd->blkcg_css)) 1002 goto queue_work; 1003 1004 node = &lo->worker_tree.rb_node; 1005 1006 while (*node) { 1007 parent = *node; 1008 cur_worker = container_of(*node, struct loop_worker, rb_node); 1009 if (cur_worker->blkcg_css == cmd->blkcg_css) { 1010 worker = cur_worker; 1011 break; 1012 } else if ((long)cur_worker->blkcg_css < (long)cmd->blkcg_css) { 1013 node = &(*node)->rb_left; 1014 } else { 1015 node = &(*node)->rb_right; 1016 } 1017 } 1018 if (worker) 1019 goto queue_work; 1020 1021 worker = kzalloc(sizeof(struct loop_worker), GFP_NOWAIT | __GFP_NOWARN); 1022 /* 1023 * In the event we cannot allocate a worker, just queue on the 1024 * rootcg worker and issue the I/O as the rootcg 1025 */ 1026 if (!worker) { 1027 cmd->blkcg_css = NULL; 1028 if (cmd->memcg_css) 1029 css_put(cmd->memcg_css); 1030 cmd->memcg_css = NULL; 1031 goto queue_work; 1032 } 1033 1034 worker->blkcg_css = cmd->blkcg_css; 1035 css_get(worker->blkcg_css); 1036 INIT_WORK(&worker->work, loop_workfn); 1037 INIT_LIST_HEAD(&worker->cmd_list); 1038 INIT_LIST_HEAD(&worker->idle_list); 1039 worker->lo = lo; 1040 rb_link_node(&worker->rb_node, parent, node); 1041 rb_insert_color(&worker->rb_node, &lo->worker_tree); 1042 queue_work: 1043 if (worker) { 1044 /* 1045 * We need to remove from the idle list here while 1046 * holding the lock so that the idle timer doesn't 1047 * free the worker 1048 */ 1049 if (!list_empty(&worker->idle_list)) 1050 list_del_init(&worker->idle_list); 1051 work = &worker->work; 1052 cmd_list = &worker->cmd_list; 1053 } else { 1054 work = &lo->rootcg_work; 1055 cmd_list = &lo->rootcg_cmd_list; 1056 } 1057 list_add_tail(&cmd->list_entry, cmd_list); 1058 queue_work(lo->workqueue, work); 1059 spin_unlock_irq(&lo->lo_work_lock); 1060 } 1061 1062 static void loop_update_rotational(struct loop_device *lo) 1063 { 1064 struct file *file = lo->lo_backing_file; 1065 struct inode *file_inode = file->f_mapping->host; 1066 struct block_device *file_bdev = file_inode->i_sb->s_bdev; 1067 struct request_queue *q = lo->lo_queue; 1068 bool nonrot = true; 1069 1070 /* not all filesystems (e.g. tmpfs) have a sb->s_bdev */ 1071 if (file_bdev) 1072 nonrot = blk_queue_nonrot(bdev_get_queue(file_bdev)); 1073 1074 if (nonrot) 1075 blk_queue_flag_set(QUEUE_FLAG_NONROT, q); 1076 else 1077 blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); 1078 } 1079 1080 static int 1081 loop_release_xfer(struct loop_device *lo) 1082 { 1083 int err = 0; 1084 struct loop_func_table *xfer = lo->lo_encryption; 1085 1086 if (xfer) { 1087 if (xfer->release) 1088 err = xfer->release(lo); 1089 lo->transfer = NULL; 1090 lo->lo_encryption = NULL; 1091 module_put(xfer->owner); 1092 } 1093 return err; 1094 } 1095 1096 static int 1097 loop_init_xfer(struct loop_device *lo, struct loop_func_table *xfer, 1098 const struct loop_info64 *i) 1099 { 1100 int err = 0; 1101 1102 if (xfer) { 1103 struct module *owner = xfer->owner; 1104 1105 if (!try_module_get(owner)) 1106 return -EINVAL; 1107 if (xfer->init) 1108 err = xfer->init(lo, i); 1109 if (err) 1110 module_put(owner); 1111 else 1112 lo->lo_encryption = xfer; 1113 } 1114 return err; 1115 } 1116 1117 /** 1118 * loop_set_status_from_info - configure device from loop_info 1119 * @lo: struct loop_device to configure 1120 * @info: struct loop_info64 to configure the device with 1121 * 1122 * Configures the loop device parameters according to the passed 1123 * in loop_info64 configuration. 1124 */ 1125 static int 1126 loop_set_status_from_info(struct loop_device *lo, 1127 const struct loop_info64 *info) 1128 { 1129 int err; 1130 struct loop_func_table *xfer; 1131 kuid_t uid = current_uid(); 1132 1133 if ((unsigned int) info->lo_encrypt_key_size > LO_KEY_SIZE) 1134 return -EINVAL; 1135 1136 err = loop_release_xfer(lo); 1137 if (err) 1138 return err; 1139 1140 if (info->lo_encrypt_type) { 1141 unsigned int type = info->lo_encrypt_type; 1142 1143 if (type >= MAX_LO_CRYPT) 1144 return -EINVAL; 1145 xfer = xfer_funcs[type]; 1146 if (xfer == NULL) 1147 return -EINVAL; 1148 } else 1149 xfer = NULL; 1150 1151 err = loop_init_xfer(lo, xfer, info); 1152 if (err) 1153 return err; 1154 1155 lo->lo_offset = info->lo_offset; 1156 lo->lo_sizelimit = info->lo_sizelimit; 1157 1158 /* loff_t vars have been assigned __u64 */ 1159 if (lo->lo_offset < 0 || lo->lo_sizelimit < 0) 1160 return -EOVERFLOW; 1161 1162 memcpy(lo->lo_file_name, info->lo_file_name, LO_NAME_SIZE); 1163 memcpy(lo->lo_crypt_name, info->lo_crypt_name, LO_NAME_SIZE); 1164 lo->lo_file_name[LO_NAME_SIZE-1] = 0; 1165 lo->lo_crypt_name[LO_NAME_SIZE-1] = 0; 1166 1167 if (!xfer) 1168 xfer = &none_funcs; 1169 lo->transfer = xfer->transfer; 1170 lo->ioctl = xfer->ioctl; 1171 1172 lo->lo_flags = info->lo_flags; 1173 1174 lo->lo_encrypt_key_size = info->lo_encrypt_key_size; 1175 lo->lo_init[0] = info->lo_init[0]; 1176 lo->lo_init[1] = info->lo_init[1]; 1177 if (info->lo_encrypt_key_size) { 1178 memcpy(lo->lo_encrypt_key, info->lo_encrypt_key, 1179 info->lo_encrypt_key_size); 1180 lo->lo_key_owner = uid; 1181 } 1182 1183 return 0; 1184 } 1185 1186 static int loop_configure(struct loop_device *lo, fmode_t mode, 1187 struct block_device *bdev, 1188 const struct loop_config *config) 1189 { 1190 struct file *file = fget(config->fd); 1191 struct inode *inode; 1192 struct address_space *mapping; 1193 int error; 1194 loff_t size; 1195 bool partscan; 1196 unsigned short bsize; 1197 bool is_loop; 1198 1199 if (!file) 1200 return -EBADF; 1201 is_loop = is_loop_device(file); 1202 1203 /* This is safe, since we have a reference from open(). */ 1204 __module_get(THIS_MODULE); 1205 1206 /* 1207 * If we don't hold exclusive handle for the device, upgrade to it 1208 * here to avoid changing device under exclusive owner. 1209 */ 1210 if (!(mode & FMODE_EXCL)) { 1211 error = bd_prepare_to_claim(bdev, loop_configure); 1212 if (error) 1213 goto out_putf; 1214 } 1215 1216 error = loop_global_lock_killable(lo, is_loop); 1217 if (error) 1218 goto out_bdev; 1219 1220 error = -EBUSY; 1221 if (lo->lo_state != Lo_unbound) 1222 goto out_unlock; 1223 1224 error = loop_validate_file(file, bdev); 1225 if (error) 1226 goto out_unlock; 1227 1228 mapping = file->f_mapping; 1229 inode = mapping->host; 1230 1231 if ((config->info.lo_flags & ~LOOP_CONFIGURE_SETTABLE_FLAGS) != 0) { 1232 error = -EINVAL; 1233 goto out_unlock; 1234 } 1235 1236 if (config->block_size) { 1237 error = blk_validate_block_size(config->block_size); 1238 if (error) 1239 goto out_unlock; 1240 } 1241 1242 error = loop_set_status_from_info(lo, &config->info); 1243 if (error) 1244 goto out_unlock; 1245 1246 if (!(file->f_mode & FMODE_WRITE) || !(mode & FMODE_WRITE) || 1247 !file->f_op->write_iter) 1248 lo->lo_flags |= LO_FLAGS_READ_ONLY; 1249 1250 lo->workqueue = alloc_workqueue("loop%d", 1251 WQ_UNBOUND | WQ_FREEZABLE, 1252 0, 1253 lo->lo_number); 1254 if (!lo->workqueue) { 1255 error = -ENOMEM; 1256 goto out_unlock; 1257 } 1258 1259 disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE); 1260 set_disk_ro(lo->lo_disk, (lo->lo_flags & LO_FLAGS_READ_ONLY) != 0); 1261 1262 INIT_WORK(&lo->rootcg_work, loop_rootcg_workfn); 1263 INIT_LIST_HEAD(&lo->rootcg_cmd_list); 1264 INIT_LIST_HEAD(&lo->idle_worker_list); 1265 lo->worker_tree = RB_ROOT; 1266 timer_setup(&lo->timer, loop_free_idle_workers, 1267 TIMER_DEFERRABLE); 1268 lo->use_dio = lo->lo_flags & LO_FLAGS_DIRECT_IO; 1269 lo->lo_device = bdev; 1270 lo->lo_backing_file = file; 1271 lo->old_gfp_mask = mapping_gfp_mask(mapping); 1272 mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS)); 1273 1274 if (!(lo->lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync) 1275 blk_queue_write_cache(lo->lo_queue, true, false); 1276 1277 if (config->block_size) 1278 bsize = config->block_size; 1279 else if ((lo->lo_backing_file->f_flags & O_DIRECT) && inode->i_sb->s_bdev) 1280 /* In case of direct I/O, match underlying block size */ 1281 bsize = bdev_logical_block_size(inode->i_sb->s_bdev); 1282 else 1283 bsize = 512; 1284 1285 blk_queue_logical_block_size(lo->lo_queue, bsize); 1286 blk_queue_physical_block_size(lo->lo_queue, bsize); 1287 blk_queue_io_min(lo->lo_queue, bsize); 1288 1289 loop_config_discard(lo); 1290 loop_update_rotational(lo); 1291 loop_update_dio(lo); 1292 loop_sysfs_init(lo); 1293 1294 size = get_loop_size(lo, file); 1295 loop_set_size(lo, size); 1296 1297 /* Order wrt reading lo_state in loop_validate_file(). */ 1298 wmb(); 1299 1300 lo->lo_state = Lo_bound; 1301 if (part_shift) 1302 lo->lo_flags |= LO_FLAGS_PARTSCAN; 1303 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN; 1304 if (partscan) 1305 lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; 1306 1307 loop_global_unlock(lo, is_loop); 1308 if (partscan) 1309 loop_reread_partitions(lo); 1310 if (!(mode & FMODE_EXCL)) 1311 bd_abort_claiming(bdev, loop_configure); 1312 return 0; 1313 1314 out_unlock: 1315 loop_global_unlock(lo, is_loop); 1316 out_bdev: 1317 if (!(mode & FMODE_EXCL)) 1318 bd_abort_claiming(bdev, loop_configure); 1319 out_putf: 1320 fput(file); 1321 /* This is safe: open() is still holding a reference. */ 1322 module_put(THIS_MODULE); 1323 return error; 1324 } 1325 1326 static int __loop_clr_fd(struct loop_device *lo, bool release) 1327 { 1328 struct file *filp = NULL; 1329 gfp_t gfp = lo->old_gfp_mask; 1330 struct block_device *bdev = lo->lo_device; 1331 int err = 0; 1332 bool partscan = false; 1333 int lo_number; 1334 struct loop_worker *pos, *worker; 1335 1336 /* 1337 * Flush loop_configure() and loop_change_fd(). It is acceptable for 1338 * loop_validate_file() to succeed, for actual clear operation has not 1339 * started yet. 1340 */ 1341 mutex_lock(&loop_validate_mutex); 1342 mutex_unlock(&loop_validate_mutex); 1343 /* 1344 * loop_validate_file() now fails because l->lo_state != Lo_bound 1345 * became visible. 1346 */ 1347 1348 mutex_lock(&lo->lo_mutex); 1349 if (WARN_ON_ONCE(lo->lo_state != Lo_rundown)) { 1350 err = -ENXIO; 1351 goto out_unlock; 1352 } 1353 1354 filp = lo->lo_backing_file; 1355 if (filp == NULL) { 1356 err = -EINVAL; 1357 goto out_unlock; 1358 } 1359 1360 if (test_bit(QUEUE_FLAG_WC, &lo->lo_queue->queue_flags)) 1361 blk_queue_write_cache(lo->lo_queue, false, false); 1362 1363 /* freeze request queue during the transition */ 1364 blk_mq_freeze_queue(lo->lo_queue); 1365 1366 destroy_workqueue(lo->workqueue); 1367 spin_lock_irq(&lo->lo_work_lock); 1368 list_for_each_entry_safe(worker, pos, &lo->idle_worker_list, 1369 idle_list) { 1370 list_del(&worker->idle_list); 1371 rb_erase(&worker->rb_node, &lo->worker_tree); 1372 css_put(worker->blkcg_css); 1373 kfree(worker); 1374 } 1375 spin_unlock_irq(&lo->lo_work_lock); 1376 del_timer_sync(&lo->timer); 1377 1378 spin_lock_irq(&lo->lo_lock); 1379 lo->lo_backing_file = NULL; 1380 spin_unlock_irq(&lo->lo_lock); 1381 1382 loop_release_xfer(lo); 1383 lo->transfer = NULL; 1384 lo->ioctl = NULL; 1385 lo->lo_device = NULL; 1386 lo->lo_encryption = NULL; 1387 lo->lo_offset = 0; 1388 lo->lo_sizelimit = 0; 1389 lo->lo_encrypt_key_size = 0; 1390 memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE); 1391 memset(lo->lo_crypt_name, 0, LO_NAME_SIZE); 1392 memset(lo->lo_file_name, 0, LO_NAME_SIZE); 1393 blk_queue_logical_block_size(lo->lo_queue, 512); 1394 blk_queue_physical_block_size(lo->lo_queue, 512); 1395 blk_queue_io_min(lo->lo_queue, 512); 1396 if (bdev) { 1397 invalidate_bdev(bdev); 1398 bdev->bd_inode->i_mapping->wb_err = 0; 1399 } 1400 set_capacity(lo->lo_disk, 0); 1401 loop_sysfs_exit(lo); 1402 if (bdev) { 1403 /* let user-space know about this change */ 1404 kobject_uevent(&disk_to_dev(bdev->bd_disk)->kobj, KOBJ_CHANGE); 1405 } 1406 mapping_set_gfp_mask(filp->f_mapping, gfp); 1407 /* This is safe: open() is still holding a reference. */ 1408 module_put(THIS_MODULE); 1409 blk_mq_unfreeze_queue(lo->lo_queue); 1410 1411 partscan = lo->lo_flags & LO_FLAGS_PARTSCAN && bdev; 1412 lo_number = lo->lo_number; 1413 disk_force_media_change(lo->lo_disk, DISK_EVENT_MEDIA_CHANGE); 1414 out_unlock: 1415 mutex_unlock(&lo->lo_mutex); 1416 if (partscan) { 1417 /* 1418 * open_mutex has been held already in release path, so don't 1419 * acquire it if this function is called in such case. 1420 * 1421 * If the reread partition isn't from release path, lo_refcnt 1422 * must be at least one and it can only become zero when the 1423 * current holder is released. 1424 */ 1425 if (!release) 1426 mutex_lock(&lo->lo_disk->open_mutex); 1427 err = bdev_disk_changed(lo->lo_disk, false); 1428 if (!release) 1429 mutex_unlock(&lo->lo_disk->open_mutex); 1430 if (err) 1431 pr_warn("%s: partition scan of loop%d failed (rc=%d)\n", 1432 __func__, lo_number, err); 1433 /* Device is gone, no point in returning error */ 1434 err = 0; 1435 } 1436 1437 /* 1438 * lo->lo_state is set to Lo_unbound here after above partscan has 1439 * finished. 1440 * 1441 * There cannot be anybody else entering __loop_clr_fd() as 1442 * lo->lo_backing_file is already cleared and Lo_rundown state 1443 * protects us from all the other places trying to change the 'lo' 1444 * device. 1445 */ 1446 mutex_lock(&lo->lo_mutex); 1447 lo->lo_flags = 0; 1448 if (!part_shift) 1449 lo->lo_disk->flags |= GENHD_FL_NO_PART_SCAN; 1450 lo->lo_state = Lo_unbound; 1451 mutex_unlock(&lo->lo_mutex); 1452 1453 /* 1454 * Need not hold lo_mutex to fput backing file. Calling fput holding 1455 * lo_mutex triggers a circular lock dependency possibility warning as 1456 * fput can take open_mutex which is usually taken before lo_mutex. 1457 */ 1458 if (filp) 1459 fput(filp); 1460 return err; 1461 } 1462 1463 static int loop_clr_fd(struct loop_device *lo) 1464 { 1465 int err; 1466 1467 err = mutex_lock_killable(&lo->lo_mutex); 1468 if (err) 1469 return err; 1470 if (lo->lo_state != Lo_bound) { 1471 mutex_unlock(&lo->lo_mutex); 1472 return -ENXIO; 1473 } 1474 /* 1475 * If we've explicitly asked to tear down the loop device, 1476 * and it has an elevated reference count, set it for auto-teardown when 1477 * the last reference goes away. This stops $!~#$@ udev from 1478 * preventing teardown because it decided that it needs to run blkid on 1479 * the loopback device whenever they appear. xfstests is notorious for 1480 * failing tests because blkid via udev races with a losetup 1481 * <dev>/do something like mkfs/losetup -d <dev> causing the losetup -d 1482 * command to fail with EBUSY. 1483 */ 1484 if (atomic_read(&lo->lo_refcnt) > 1) { 1485 lo->lo_flags |= LO_FLAGS_AUTOCLEAR; 1486 mutex_unlock(&lo->lo_mutex); 1487 return 0; 1488 } 1489 lo->lo_state = Lo_rundown; 1490 mutex_unlock(&lo->lo_mutex); 1491 1492 return __loop_clr_fd(lo, false); 1493 } 1494 1495 static int 1496 loop_set_status(struct loop_device *lo, const struct loop_info64 *info) 1497 { 1498 int err; 1499 kuid_t uid = current_uid(); 1500 int prev_lo_flags; 1501 bool partscan = false; 1502 bool size_changed = false; 1503 1504 err = mutex_lock_killable(&lo->lo_mutex); 1505 if (err) 1506 return err; 1507 if (lo->lo_encrypt_key_size && 1508 !uid_eq(lo->lo_key_owner, uid) && 1509 !capable(CAP_SYS_ADMIN)) { 1510 err = -EPERM; 1511 goto out_unlock; 1512 } 1513 if (lo->lo_state != Lo_bound) { 1514 err = -ENXIO; 1515 goto out_unlock; 1516 } 1517 1518 if (lo->lo_offset != info->lo_offset || 1519 lo->lo_sizelimit != info->lo_sizelimit) { 1520 size_changed = true; 1521 sync_blockdev(lo->lo_device); 1522 invalidate_bdev(lo->lo_device); 1523 } 1524 1525 /* I/O need to be drained during transfer transition */ 1526 blk_mq_freeze_queue(lo->lo_queue); 1527 1528 if (size_changed && lo->lo_device->bd_inode->i_mapping->nrpages) { 1529 /* If any pages were dirtied after invalidate_bdev(), try again */ 1530 err = -EAGAIN; 1531 pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", 1532 __func__, lo->lo_number, lo->lo_file_name, 1533 lo->lo_device->bd_inode->i_mapping->nrpages); 1534 goto out_unfreeze; 1535 } 1536 1537 prev_lo_flags = lo->lo_flags; 1538 1539 err = loop_set_status_from_info(lo, info); 1540 if (err) 1541 goto out_unfreeze; 1542 1543 /* Mask out flags that can't be set using LOOP_SET_STATUS. */ 1544 lo->lo_flags &= LOOP_SET_STATUS_SETTABLE_FLAGS; 1545 /* For those flags, use the previous values instead */ 1546 lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS; 1547 /* For flags that can't be cleared, use previous values too */ 1548 lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_CLEARABLE_FLAGS; 1549 1550 if (size_changed) { 1551 loff_t new_size = get_size(lo->lo_offset, lo->lo_sizelimit, 1552 lo->lo_backing_file); 1553 loop_set_size(lo, new_size); 1554 } 1555 1556 loop_config_discard(lo); 1557 1558 /* update dio if lo_offset or transfer is changed */ 1559 __loop_update_dio(lo, lo->use_dio); 1560 1561 out_unfreeze: 1562 blk_mq_unfreeze_queue(lo->lo_queue); 1563 1564 if (!err && (lo->lo_flags & LO_FLAGS_PARTSCAN) && 1565 !(prev_lo_flags & LO_FLAGS_PARTSCAN)) { 1566 lo->lo_disk->flags &= ~GENHD_FL_NO_PART_SCAN; 1567 partscan = true; 1568 } 1569 out_unlock: 1570 mutex_unlock(&lo->lo_mutex); 1571 if (partscan) 1572 loop_reread_partitions(lo); 1573 1574 return err; 1575 } 1576 1577 static int 1578 loop_get_status(struct loop_device *lo, struct loop_info64 *info) 1579 { 1580 struct path path; 1581 struct kstat stat; 1582 int ret; 1583 1584 ret = mutex_lock_killable(&lo->lo_mutex); 1585 if (ret) 1586 return ret; 1587 if (lo->lo_state != Lo_bound) { 1588 mutex_unlock(&lo->lo_mutex); 1589 return -ENXIO; 1590 } 1591 1592 memset(info, 0, sizeof(*info)); 1593 info->lo_number = lo->lo_number; 1594 info->lo_offset = lo->lo_offset; 1595 info->lo_sizelimit = lo->lo_sizelimit; 1596 info->lo_flags = lo->lo_flags; 1597 memcpy(info->lo_file_name, lo->lo_file_name, LO_NAME_SIZE); 1598 memcpy(info->lo_crypt_name, lo->lo_crypt_name, LO_NAME_SIZE); 1599 info->lo_encrypt_type = 1600 lo->lo_encryption ? lo->lo_encryption->number : 0; 1601 if (lo->lo_encrypt_key_size && capable(CAP_SYS_ADMIN)) { 1602 info->lo_encrypt_key_size = lo->lo_encrypt_key_size; 1603 memcpy(info->lo_encrypt_key, lo->lo_encrypt_key, 1604 lo->lo_encrypt_key_size); 1605 } 1606 1607 /* Drop lo_mutex while we call into the filesystem. */ 1608 path = lo->lo_backing_file->f_path; 1609 path_get(&path); 1610 mutex_unlock(&lo->lo_mutex); 1611 ret = vfs_getattr(&path, &stat, STATX_INO, AT_STATX_SYNC_AS_STAT); 1612 if (!ret) { 1613 info->lo_device = huge_encode_dev(stat.dev); 1614 info->lo_inode = stat.ino; 1615 info->lo_rdevice = huge_encode_dev(stat.rdev); 1616 } 1617 path_put(&path); 1618 return ret; 1619 } 1620 1621 static void 1622 loop_info64_from_old(const struct loop_info *info, struct loop_info64 *info64) 1623 { 1624 memset(info64, 0, sizeof(*info64)); 1625 info64->lo_number = info->lo_number; 1626 info64->lo_device = info->lo_device; 1627 info64->lo_inode = info->lo_inode; 1628 info64->lo_rdevice = info->lo_rdevice; 1629 info64->lo_offset = info->lo_offset; 1630 info64->lo_sizelimit = 0; 1631 info64->lo_encrypt_type = info->lo_encrypt_type; 1632 info64->lo_encrypt_key_size = info->lo_encrypt_key_size; 1633 info64->lo_flags = info->lo_flags; 1634 info64->lo_init[0] = info->lo_init[0]; 1635 info64->lo_init[1] = info->lo_init[1]; 1636 if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1637 memcpy(info64->lo_crypt_name, info->lo_name, LO_NAME_SIZE); 1638 else 1639 memcpy(info64->lo_file_name, info->lo_name, LO_NAME_SIZE); 1640 memcpy(info64->lo_encrypt_key, info->lo_encrypt_key, LO_KEY_SIZE); 1641 } 1642 1643 static int 1644 loop_info64_to_old(const struct loop_info64 *info64, struct loop_info *info) 1645 { 1646 memset(info, 0, sizeof(*info)); 1647 info->lo_number = info64->lo_number; 1648 info->lo_device = info64->lo_device; 1649 info->lo_inode = info64->lo_inode; 1650 info->lo_rdevice = info64->lo_rdevice; 1651 info->lo_offset = info64->lo_offset; 1652 info->lo_encrypt_type = info64->lo_encrypt_type; 1653 info->lo_encrypt_key_size = info64->lo_encrypt_key_size; 1654 info->lo_flags = info64->lo_flags; 1655 info->lo_init[0] = info64->lo_init[0]; 1656 info->lo_init[1] = info64->lo_init[1]; 1657 if (info->lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1658 memcpy(info->lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 1659 else 1660 memcpy(info->lo_name, info64->lo_file_name, LO_NAME_SIZE); 1661 memcpy(info->lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 1662 1663 /* error in case values were truncated */ 1664 if (info->lo_device != info64->lo_device || 1665 info->lo_rdevice != info64->lo_rdevice || 1666 info->lo_inode != info64->lo_inode || 1667 info->lo_offset != info64->lo_offset) 1668 return -EOVERFLOW; 1669 1670 return 0; 1671 } 1672 1673 static int 1674 loop_set_status_old(struct loop_device *lo, const struct loop_info __user *arg) 1675 { 1676 struct loop_info info; 1677 struct loop_info64 info64; 1678 1679 if (copy_from_user(&info, arg, sizeof (struct loop_info))) 1680 return -EFAULT; 1681 loop_info64_from_old(&info, &info64); 1682 return loop_set_status(lo, &info64); 1683 } 1684 1685 static int 1686 loop_set_status64(struct loop_device *lo, const struct loop_info64 __user *arg) 1687 { 1688 struct loop_info64 info64; 1689 1690 if (copy_from_user(&info64, arg, sizeof (struct loop_info64))) 1691 return -EFAULT; 1692 return loop_set_status(lo, &info64); 1693 } 1694 1695 static int 1696 loop_get_status_old(struct loop_device *lo, struct loop_info __user *arg) { 1697 struct loop_info info; 1698 struct loop_info64 info64; 1699 int err; 1700 1701 if (!arg) 1702 return -EINVAL; 1703 err = loop_get_status(lo, &info64); 1704 if (!err) 1705 err = loop_info64_to_old(&info64, &info); 1706 if (!err && copy_to_user(arg, &info, sizeof(info))) 1707 err = -EFAULT; 1708 1709 return err; 1710 } 1711 1712 static int 1713 loop_get_status64(struct loop_device *lo, struct loop_info64 __user *arg) { 1714 struct loop_info64 info64; 1715 int err; 1716 1717 if (!arg) 1718 return -EINVAL; 1719 err = loop_get_status(lo, &info64); 1720 if (!err && copy_to_user(arg, &info64, sizeof(info64))) 1721 err = -EFAULT; 1722 1723 return err; 1724 } 1725 1726 static int loop_set_capacity(struct loop_device *lo) 1727 { 1728 loff_t size; 1729 1730 if (unlikely(lo->lo_state != Lo_bound)) 1731 return -ENXIO; 1732 1733 size = get_loop_size(lo, lo->lo_backing_file); 1734 loop_set_size(lo, size); 1735 1736 return 0; 1737 } 1738 1739 static int loop_set_dio(struct loop_device *lo, unsigned long arg) 1740 { 1741 int error = -ENXIO; 1742 if (lo->lo_state != Lo_bound) 1743 goto out; 1744 1745 __loop_update_dio(lo, !!arg); 1746 if (lo->use_dio == !!arg) 1747 return 0; 1748 error = -EINVAL; 1749 out: 1750 return error; 1751 } 1752 1753 static int loop_set_block_size(struct loop_device *lo, unsigned long arg) 1754 { 1755 int err = 0; 1756 1757 if (lo->lo_state != Lo_bound) 1758 return -ENXIO; 1759 1760 err = blk_validate_block_size(arg); 1761 if (err) 1762 return err; 1763 1764 if (lo->lo_queue->limits.logical_block_size == arg) 1765 return 0; 1766 1767 sync_blockdev(lo->lo_device); 1768 invalidate_bdev(lo->lo_device); 1769 1770 blk_mq_freeze_queue(lo->lo_queue); 1771 1772 /* invalidate_bdev should have truncated all the pages */ 1773 if (lo->lo_device->bd_inode->i_mapping->nrpages) { 1774 err = -EAGAIN; 1775 pr_warn("%s: loop%d (%s) has still dirty pages (nrpages=%lu)\n", 1776 __func__, lo->lo_number, lo->lo_file_name, 1777 lo->lo_device->bd_inode->i_mapping->nrpages); 1778 goto out_unfreeze; 1779 } 1780 1781 blk_queue_logical_block_size(lo->lo_queue, arg); 1782 blk_queue_physical_block_size(lo->lo_queue, arg); 1783 blk_queue_io_min(lo->lo_queue, arg); 1784 loop_update_dio(lo); 1785 out_unfreeze: 1786 blk_mq_unfreeze_queue(lo->lo_queue); 1787 1788 return err; 1789 } 1790 1791 static int lo_simple_ioctl(struct loop_device *lo, unsigned int cmd, 1792 unsigned long arg) 1793 { 1794 int err; 1795 1796 err = mutex_lock_killable(&lo->lo_mutex); 1797 if (err) 1798 return err; 1799 switch (cmd) { 1800 case LOOP_SET_CAPACITY: 1801 err = loop_set_capacity(lo); 1802 break; 1803 case LOOP_SET_DIRECT_IO: 1804 err = loop_set_dio(lo, arg); 1805 break; 1806 case LOOP_SET_BLOCK_SIZE: 1807 err = loop_set_block_size(lo, arg); 1808 break; 1809 default: 1810 err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL; 1811 } 1812 mutex_unlock(&lo->lo_mutex); 1813 return err; 1814 } 1815 1816 static int lo_ioctl(struct block_device *bdev, fmode_t mode, 1817 unsigned int cmd, unsigned long arg) 1818 { 1819 struct loop_device *lo = bdev->bd_disk->private_data; 1820 void __user *argp = (void __user *) arg; 1821 int err; 1822 1823 switch (cmd) { 1824 case LOOP_SET_FD: { 1825 /* 1826 * Legacy case - pass in a zeroed out struct loop_config with 1827 * only the file descriptor set , which corresponds with the 1828 * default parameters we'd have used otherwise. 1829 */ 1830 struct loop_config config; 1831 1832 memset(&config, 0, sizeof(config)); 1833 config.fd = arg; 1834 1835 return loop_configure(lo, mode, bdev, &config); 1836 } 1837 case LOOP_CONFIGURE: { 1838 struct loop_config config; 1839 1840 if (copy_from_user(&config, argp, sizeof(config))) 1841 return -EFAULT; 1842 1843 return loop_configure(lo, mode, bdev, &config); 1844 } 1845 case LOOP_CHANGE_FD: 1846 return loop_change_fd(lo, bdev, arg); 1847 case LOOP_CLR_FD: 1848 return loop_clr_fd(lo); 1849 case LOOP_SET_STATUS: 1850 err = -EPERM; 1851 if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { 1852 err = loop_set_status_old(lo, argp); 1853 } 1854 break; 1855 case LOOP_GET_STATUS: 1856 return loop_get_status_old(lo, argp); 1857 case LOOP_SET_STATUS64: 1858 err = -EPERM; 1859 if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN)) { 1860 err = loop_set_status64(lo, argp); 1861 } 1862 break; 1863 case LOOP_GET_STATUS64: 1864 return loop_get_status64(lo, argp); 1865 case LOOP_SET_CAPACITY: 1866 case LOOP_SET_DIRECT_IO: 1867 case LOOP_SET_BLOCK_SIZE: 1868 if (!(mode & FMODE_WRITE) && !capable(CAP_SYS_ADMIN)) 1869 return -EPERM; 1870 fallthrough; 1871 default: 1872 err = lo_simple_ioctl(lo, cmd, arg); 1873 break; 1874 } 1875 1876 return err; 1877 } 1878 1879 #ifdef CONFIG_COMPAT 1880 struct compat_loop_info { 1881 compat_int_t lo_number; /* ioctl r/o */ 1882 compat_dev_t lo_device; /* ioctl r/o */ 1883 compat_ulong_t lo_inode; /* ioctl r/o */ 1884 compat_dev_t lo_rdevice; /* ioctl r/o */ 1885 compat_int_t lo_offset; 1886 compat_int_t lo_encrypt_type; 1887 compat_int_t lo_encrypt_key_size; /* ioctl w/o */ 1888 compat_int_t lo_flags; /* ioctl r/o */ 1889 char lo_name[LO_NAME_SIZE]; 1890 unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */ 1891 compat_ulong_t lo_init[2]; 1892 char reserved[4]; 1893 }; 1894 1895 /* 1896 * Transfer 32-bit compatibility structure in userspace to 64-bit loop info 1897 * - noinlined to reduce stack space usage in main part of driver 1898 */ 1899 static noinline int 1900 loop_info64_from_compat(const struct compat_loop_info __user *arg, 1901 struct loop_info64 *info64) 1902 { 1903 struct compat_loop_info info; 1904 1905 if (copy_from_user(&info, arg, sizeof(info))) 1906 return -EFAULT; 1907 1908 memset(info64, 0, sizeof(*info64)); 1909 info64->lo_number = info.lo_number; 1910 info64->lo_device = info.lo_device; 1911 info64->lo_inode = info.lo_inode; 1912 info64->lo_rdevice = info.lo_rdevice; 1913 info64->lo_offset = info.lo_offset; 1914 info64->lo_sizelimit = 0; 1915 info64->lo_encrypt_type = info.lo_encrypt_type; 1916 info64->lo_encrypt_key_size = info.lo_encrypt_key_size; 1917 info64->lo_flags = info.lo_flags; 1918 info64->lo_init[0] = info.lo_init[0]; 1919 info64->lo_init[1] = info.lo_init[1]; 1920 if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1921 memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE); 1922 else 1923 memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE); 1924 memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE); 1925 return 0; 1926 } 1927 1928 /* 1929 * Transfer 64-bit loop info to 32-bit compatibility structure in userspace 1930 * - noinlined to reduce stack space usage in main part of driver 1931 */ 1932 static noinline int 1933 loop_info64_to_compat(const struct loop_info64 *info64, 1934 struct compat_loop_info __user *arg) 1935 { 1936 struct compat_loop_info info; 1937 1938 memset(&info, 0, sizeof(info)); 1939 info.lo_number = info64->lo_number; 1940 info.lo_device = info64->lo_device; 1941 info.lo_inode = info64->lo_inode; 1942 info.lo_rdevice = info64->lo_rdevice; 1943 info.lo_offset = info64->lo_offset; 1944 info.lo_encrypt_type = info64->lo_encrypt_type; 1945 info.lo_encrypt_key_size = info64->lo_encrypt_key_size; 1946 info.lo_flags = info64->lo_flags; 1947 info.lo_init[0] = info64->lo_init[0]; 1948 info.lo_init[1] = info64->lo_init[1]; 1949 if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI) 1950 memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE); 1951 else 1952 memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE); 1953 memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE); 1954 1955 /* error in case values were truncated */ 1956 if (info.lo_device != info64->lo_device || 1957 info.lo_rdevice != info64->lo_rdevice || 1958 info.lo_inode != info64->lo_inode || 1959 info.lo_offset != info64->lo_offset || 1960 info.lo_init[0] != info64->lo_init[0] || 1961 info.lo_init[1] != info64->lo_init[1]) 1962 return -EOVERFLOW; 1963 1964 if (copy_to_user(arg, &info, sizeof(info))) 1965 return -EFAULT; 1966 return 0; 1967 } 1968 1969 static int 1970 loop_set_status_compat(struct loop_device *lo, 1971 const struct compat_loop_info __user *arg) 1972 { 1973 struct loop_info64 info64; 1974 int ret; 1975 1976 ret = loop_info64_from_compat(arg, &info64); 1977 if (ret < 0) 1978 return ret; 1979 return loop_set_status(lo, &info64); 1980 } 1981 1982 static int 1983 loop_get_status_compat(struct loop_device *lo, 1984 struct compat_loop_info __user *arg) 1985 { 1986 struct loop_info64 info64; 1987 int err; 1988 1989 if (!arg) 1990 return -EINVAL; 1991 err = loop_get_status(lo, &info64); 1992 if (!err) 1993 err = loop_info64_to_compat(&info64, arg); 1994 return err; 1995 } 1996 1997 static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode, 1998 unsigned int cmd, unsigned long arg) 1999 { 2000 struct loop_device *lo = bdev->bd_disk->private_data; 2001 int err; 2002 2003 switch(cmd) { 2004 case LOOP_SET_STATUS: 2005 err = loop_set_status_compat(lo, 2006 (const struct compat_loop_info __user *)arg); 2007 break; 2008 case LOOP_GET_STATUS: 2009 err = loop_get_status_compat(lo, 2010 (struct compat_loop_info __user *)arg); 2011 break; 2012 case LOOP_SET_CAPACITY: 2013 case LOOP_CLR_FD: 2014 case LOOP_GET_STATUS64: 2015 case LOOP_SET_STATUS64: 2016 case LOOP_CONFIGURE: 2017 arg = (unsigned long) compat_ptr(arg); 2018 fallthrough; 2019 case LOOP_SET_FD: 2020 case LOOP_CHANGE_FD: 2021 case LOOP_SET_BLOCK_SIZE: 2022 case LOOP_SET_DIRECT_IO: 2023 err = lo_ioctl(bdev, mode, cmd, arg); 2024 break; 2025 default: 2026 err = -ENOIOCTLCMD; 2027 break; 2028 } 2029 return err; 2030 } 2031 #endif 2032 2033 static int lo_open(struct block_device *bdev, fmode_t mode) 2034 { 2035 struct loop_device *lo = bdev->bd_disk->private_data; 2036 int err; 2037 2038 err = mutex_lock_killable(&lo->lo_mutex); 2039 if (err) 2040 return err; 2041 if (lo->lo_state == Lo_deleting) 2042 err = -ENXIO; 2043 else 2044 atomic_inc(&lo->lo_refcnt); 2045 mutex_unlock(&lo->lo_mutex); 2046 return err; 2047 } 2048 2049 static void lo_release(struct gendisk *disk, fmode_t mode) 2050 { 2051 struct loop_device *lo = disk->private_data; 2052 2053 mutex_lock(&lo->lo_mutex); 2054 if (atomic_dec_return(&lo->lo_refcnt)) 2055 goto out_unlock; 2056 2057 if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) { 2058 if (lo->lo_state != Lo_bound) 2059 goto out_unlock; 2060 lo->lo_state = Lo_rundown; 2061 mutex_unlock(&lo->lo_mutex); 2062 /* 2063 * In autoclear mode, stop the loop thread 2064 * and remove configuration after last close. 2065 */ 2066 __loop_clr_fd(lo, true); 2067 return; 2068 } else if (lo->lo_state == Lo_bound) { 2069 /* 2070 * Otherwise keep thread (if running) and config, 2071 * but flush possible ongoing bios in thread. 2072 */ 2073 blk_mq_freeze_queue(lo->lo_queue); 2074 blk_mq_unfreeze_queue(lo->lo_queue); 2075 } 2076 2077 out_unlock: 2078 mutex_unlock(&lo->lo_mutex); 2079 } 2080 2081 static const struct block_device_operations lo_fops = { 2082 .owner = THIS_MODULE, 2083 .open = lo_open, 2084 .release = lo_release, 2085 .ioctl = lo_ioctl, 2086 #ifdef CONFIG_COMPAT 2087 .compat_ioctl = lo_compat_ioctl, 2088 #endif 2089 }; 2090 2091 /* 2092 * And now the modules code and kernel interface. 2093 */ 2094 static int max_loop; 2095 module_param(max_loop, int, 0444); 2096 MODULE_PARM_DESC(max_loop, "Maximum number of loop devices"); 2097 module_param(max_part, int, 0444); 2098 MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device"); 2099 MODULE_LICENSE("GPL"); 2100 MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR); 2101 2102 int loop_register_transfer(struct loop_func_table *funcs) 2103 { 2104 unsigned int n = funcs->number; 2105 2106 if (n >= MAX_LO_CRYPT || xfer_funcs[n]) 2107 return -EINVAL; 2108 xfer_funcs[n] = funcs; 2109 return 0; 2110 } 2111 2112 int loop_unregister_transfer(int number) 2113 { 2114 unsigned int n = number; 2115 struct loop_func_table *xfer; 2116 2117 if (n == 0 || n >= MAX_LO_CRYPT || (xfer = xfer_funcs[n]) == NULL) 2118 return -EINVAL; 2119 /* 2120 * This function is called from only cleanup_cryptoloop(). 2121 * Given that each loop device that has a transfer enabled holds a 2122 * reference to the module implementing it we should never get here 2123 * with a transfer that is set (unless forced module unloading is 2124 * requested). Thus, check module's refcount and warn if this is 2125 * not a clean unloading. 2126 */ 2127 #ifdef CONFIG_MODULE_UNLOAD 2128 if (xfer->owner && module_refcount(xfer->owner) != -1) 2129 pr_err("Danger! Unregistering an in use transfer function.\n"); 2130 #endif 2131 2132 xfer_funcs[n] = NULL; 2133 return 0; 2134 } 2135 2136 EXPORT_SYMBOL(loop_register_transfer); 2137 EXPORT_SYMBOL(loop_unregister_transfer); 2138 2139 static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, 2140 const struct blk_mq_queue_data *bd) 2141 { 2142 struct request *rq = bd->rq; 2143 struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); 2144 struct loop_device *lo = rq->q->queuedata; 2145 2146 blk_mq_start_request(rq); 2147 2148 if (lo->lo_state != Lo_bound) 2149 return BLK_STS_IOERR; 2150 2151 switch (req_op(rq)) { 2152 case REQ_OP_FLUSH: 2153 case REQ_OP_DISCARD: 2154 case REQ_OP_WRITE_ZEROES: 2155 cmd->use_aio = false; 2156 break; 2157 default: 2158 cmd->use_aio = lo->use_dio; 2159 break; 2160 } 2161 2162 /* always use the first bio's css */ 2163 cmd->blkcg_css = NULL; 2164 cmd->memcg_css = NULL; 2165 #ifdef CONFIG_BLK_CGROUP 2166 if (rq->bio && rq->bio->bi_blkg) { 2167 cmd->blkcg_css = &bio_blkcg(rq->bio)->css; 2168 #ifdef CONFIG_MEMCG 2169 cmd->memcg_css = 2170 cgroup_get_e_css(cmd->blkcg_css->cgroup, 2171 &memory_cgrp_subsys); 2172 #endif 2173 } 2174 #endif 2175 loop_queue_work(lo, cmd); 2176 2177 return BLK_STS_OK; 2178 } 2179 2180 static void loop_handle_cmd(struct loop_cmd *cmd) 2181 { 2182 struct request *rq = blk_mq_rq_from_pdu(cmd); 2183 const bool write = op_is_write(req_op(rq)); 2184 struct loop_device *lo = rq->q->queuedata; 2185 int ret = 0; 2186 struct mem_cgroup *old_memcg = NULL; 2187 2188 if (write && (lo->lo_flags & LO_FLAGS_READ_ONLY)) { 2189 ret = -EIO; 2190 goto failed; 2191 } 2192 2193 if (cmd->blkcg_css) 2194 kthread_associate_blkcg(cmd->blkcg_css); 2195 if (cmd->memcg_css) 2196 old_memcg = set_active_memcg( 2197 mem_cgroup_from_css(cmd->memcg_css)); 2198 2199 ret = do_req_filebacked(lo, rq); 2200 2201 if (cmd->blkcg_css) 2202 kthread_associate_blkcg(NULL); 2203 2204 if (cmd->memcg_css) { 2205 set_active_memcg(old_memcg); 2206 css_put(cmd->memcg_css); 2207 } 2208 failed: 2209 /* complete non-aio request */ 2210 if (!cmd->use_aio || ret) { 2211 if (ret == -EOPNOTSUPP) 2212 cmd->ret = ret; 2213 else 2214 cmd->ret = ret ? -EIO : 0; 2215 if (likely(!blk_should_fake_timeout(rq->q))) 2216 blk_mq_complete_request(rq); 2217 } 2218 } 2219 2220 static void loop_set_timer(struct loop_device *lo) 2221 { 2222 timer_reduce(&lo->timer, jiffies + LOOP_IDLE_WORKER_TIMEOUT); 2223 } 2224 2225 static void loop_process_work(struct loop_worker *worker, 2226 struct list_head *cmd_list, struct loop_device *lo) 2227 { 2228 int orig_flags = current->flags; 2229 struct loop_cmd *cmd; 2230 2231 current->flags |= PF_LOCAL_THROTTLE | PF_MEMALLOC_NOIO; 2232 spin_lock_irq(&lo->lo_work_lock); 2233 while (!list_empty(cmd_list)) { 2234 cmd = container_of( 2235 cmd_list->next, struct loop_cmd, list_entry); 2236 list_del(cmd_list->next); 2237 spin_unlock_irq(&lo->lo_work_lock); 2238 2239 loop_handle_cmd(cmd); 2240 cond_resched(); 2241 2242 spin_lock_irq(&lo->lo_work_lock); 2243 } 2244 2245 /* 2246 * We only add to the idle list if there are no pending cmds 2247 * *and* the worker will not run again which ensures that it 2248 * is safe to free any worker on the idle list 2249 */ 2250 if (worker && !work_pending(&worker->work)) { 2251 worker->last_ran_at = jiffies; 2252 list_add_tail(&worker->idle_list, &lo->idle_worker_list); 2253 loop_set_timer(lo); 2254 } 2255 spin_unlock_irq(&lo->lo_work_lock); 2256 current->flags = orig_flags; 2257 } 2258 2259 static void loop_workfn(struct work_struct *work) 2260 { 2261 struct loop_worker *worker = 2262 container_of(work, struct loop_worker, work); 2263 loop_process_work(worker, &worker->cmd_list, worker->lo); 2264 } 2265 2266 static void loop_rootcg_workfn(struct work_struct *work) 2267 { 2268 struct loop_device *lo = 2269 container_of(work, struct loop_device, rootcg_work); 2270 loop_process_work(NULL, &lo->rootcg_cmd_list, lo); 2271 } 2272 2273 static void loop_free_idle_workers(struct timer_list *timer) 2274 { 2275 struct loop_device *lo = container_of(timer, struct loop_device, timer); 2276 struct loop_worker *pos, *worker; 2277 2278 spin_lock_irq(&lo->lo_work_lock); 2279 list_for_each_entry_safe(worker, pos, &lo->idle_worker_list, 2280 idle_list) { 2281 if (time_is_after_jiffies(worker->last_ran_at + 2282 LOOP_IDLE_WORKER_TIMEOUT)) 2283 break; 2284 list_del(&worker->idle_list); 2285 rb_erase(&worker->rb_node, &lo->worker_tree); 2286 css_put(worker->blkcg_css); 2287 kfree(worker); 2288 } 2289 if (!list_empty(&lo->idle_worker_list)) 2290 loop_set_timer(lo); 2291 spin_unlock_irq(&lo->lo_work_lock); 2292 } 2293 2294 static const struct blk_mq_ops loop_mq_ops = { 2295 .queue_rq = loop_queue_rq, 2296 .complete = lo_complete_rq, 2297 }; 2298 2299 static int loop_add(int i) 2300 { 2301 struct loop_device *lo; 2302 struct gendisk *disk; 2303 int err; 2304 2305 err = -ENOMEM; 2306 lo = kzalloc(sizeof(*lo), GFP_KERNEL); 2307 if (!lo) 2308 goto out; 2309 lo->lo_state = Lo_unbound; 2310 2311 err = mutex_lock_killable(&loop_ctl_mutex); 2312 if (err) 2313 goto out_free_dev; 2314 2315 /* allocate id, if @id >= 0, we're requesting that specific id */ 2316 if (i >= 0) { 2317 err = idr_alloc(&loop_index_idr, lo, i, i + 1, GFP_KERNEL); 2318 if (err == -ENOSPC) 2319 err = -EEXIST; 2320 } else { 2321 err = idr_alloc(&loop_index_idr, lo, 0, 0, GFP_KERNEL); 2322 } 2323 mutex_unlock(&loop_ctl_mutex); 2324 if (err < 0) 2325 goto out_free_dev; 2326 i = err; 2327 2328 err = -ENOMEM; 2329 lo->tag_set.ops = &loop_mq_ops; 2330 lo->tag_set.nr_hw_queues = 1; 2331 lo->tag_set.queue_depth = 128; 2332 lo->tag_set.numa_node = NUMA_NO_NODE; 2333 lo->tag_set.cmd_size = sizeof(struct loop_cmd); 2334 lo->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_STACKING | 2335 BLK_MQ_F_NO_SCHED_BY_DEFAULT; 2336 lo->tag_set.driver_data = lo; 2337 2338 err = blk_mq_alloc_tag_set(&lo->tag_set); 2339 if (err) 2340 goto out_free_idr; 2341 2342 disk = lo->lo_disk = blk_mq_alloc_disk(&lo->tag_set, lo); 2343 if (IS_ERR(disk)) { 2344 err = PTR_ERR(disk); 2345 goto out_cleanup_tags; 2346 } 2347 lo->lo_queue = lo->lo_disk->queue; 2348 2349 blk_queue_max_hw_sectors(lo->lo_queue, BLK_DEF_MAX_SECTORS); 2350 2351 /* 2352 * By default, we do buffer IO, so it doesn't make sense to enable 2353 * merge because the I/O submitted to backing file is handled page by 2354 * page. For directio mode, merge does help to dispatch bigger request 2355 * to underlayer disk. We will enable merge once directio is enabled. 2356 */ 2357 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, lo->lo_queue); 2358 2359 /* 2360 * Disable partition scanning by default. The in-kernel partition 2361 * scanning can be requested individually per-device during its 2362 * setup. Userspace can always add and remove partitions from all 2363 * devices. The needed partition minors are allocated from the 2364 * extended minor space, the main loop device numbers will continue 2365 * to match the loop minors, regardless of the number of partitions 2366 * used. 2367 * 2368 * If max_part is given, partition scanning is globally enabled for 2369 * all loop devices. The minors for the main loop devices will be 2370 * multiples of max_part. 2371 * 2372 * Note: Global-for-all-devices, set-only-at-init, read-only module 2373 * parameteters like 'max_loop' and 'max_part' make things needlessly 2374 * complicated, are too static, inflexible and may surprise 2375 * userspace tools. Parameters like this in general should be avoided. 2376 */ 2377 if (!part_shift) 2378 disk->flags |= GENHD_FL_NO_PART_SCAN; 2379 disk->flags |= GENHD_FL_EXT_DEVT; 2380 atomic_set(&lo->lo_refcnt, 0); 2381 mutex_init(&lo->lo_mutex); 2382 lo->lo_number = i; 2383 spin_lock_init(&lo->lo_lock); 2384 spin_lock_init(&lo->lo_work_lock); 2385 disk->major = LOOP_MAJOR; 2386 disk->first_minor = i << part_shift; 2387 disk->minors = 1 << part_shift; 2388 disk->fops = &lo_fops; 2389 disk->private_data = lo; 2390 disk->queue = lo->lo_queue; 2391 disk->events = DISK_EVENT_MEDIA_CHANGE; 2392 disk->event_flags = DISK_EVENT_FLAG_UEVENT; 2393 sprintf(disk->disk_name, "loop%d", i); 2394 /* Make this loop device reachable from pathname. */ 2395 add_disk(disk); 2396 /* Show this loop device. */ 2397 mutex_lock(&loop_ctl_mutex); 2398 lo->idr_visible = true; 2399 mutex_unlock(&loop_ctl_mutex); 2400 return i; 2401 2402 out_cleanup_tags: 2403 blk_mq_free_tag_set(&lo->tag_set); 2404 out_free_idr: 2405 mutex_lock(&loop_ctl_mutex); 2406 idr_remove(&loop_index_idr, i); 2407 mutex_unlock(&loop_ctl_mutex); 2408 out_free_dev: 2409 kfree(lo); 2410 out: 2411 return err; 2412 } 2413 2414 static void loop_remove(struct loop_device *lo) 2415 { 2416 /* Make this loop device unreachable from pathname. */ 2417 del_gendisk(lo->lo_disk); 2418 blk_cleanup_disk(lo->lo_disk); 2419 blk_mq_free_tag_set(&lo->tag_set); 2420 mutex_lock(&loop_ctl_mutex); 2421 idr_remove(&loop_index_idr, lo->lo_number); 2422 mutex_unlock(&loop_ctl_mutex); 2423 /* There is no route which can find this loop device. */ 2424 mutex_destroy(&lo->lo_mutex); 2425 kfree(lo); 2426 } 2427 2428 static void loop_probe(dev_t dev) 2429 { 2430 int idx = MINOR(dev) >> part_shift; 2431 2432 if (max_loop && idx >= max_loop) 2433 return; 2434 loop_add(idx); 2435 } 2436 2437 static int loop_control_remove(int idx) 2438 { 2439 struct loop_device *lo; 2440 int ret; 2441 2442 if (idx < 0) { 2443 pr_warn_once("deleting an unspecified loop device is not supported.\n"); 2444 return -EINVAL; 2445 } 2446 2447 /* Hide this loop device for serialization. */ 2448 ret = mutex_lock_killable(&loop_ctl_mutex); 2449 if (ret) 2450 return ret; 2451 lo = idr_find(&loop_index_idr, idx); 2452 if (!lo || !lo->idr_visible) 2453 ret = -ENODEV; 2454 else 2455 lo->idr_visible = false; 2456 mutex_unlock(&loop_ctl_mutex); 2457 if (ret) 2458 return ret; 2459 2460 /* Check whether this loop device can be removed. */ 2461 ret = mutex_lock_killable(&lo->lo_mutex); 2462 if (ret) 2463 goto mark_visible; 2464 if (lo->lo_state != Lo_unbound || 2465 atomic_read(&lo->lo_refcnt) > 0) { 2466 mutex_unlock(&lo->lo_mutex); 2467 ret = -EBUSY; 2468 goto mark_visible; 2469 } 2470 /* Mark this loop device no longer open()-able. */ 2471 lo->lo_state = Lo_deleting; 2472 mutex_unlock(&lo->lo_mutex); 2473 2474 loop_remove(lo); 2475 return 0; 2476 2477 mark_visible: 2478 /* Show this loop device again. */ 2479 mutex_lock(&loop_ctl_mutex); 2480 lo->idr_visible = true; 2481 mutex_unlock(&loop_ctl_mutex); 2482 return ret; 2483 } 2484 2485 static int loop_control_get_free(int idx) 2486 { 2487 struct loop_device *lo; 2488 int id, ret; 2489 2490 ret = mutex_lock_killable(&loop_ctl_mutex); 2491 if (ret) 2492 return ret; 2493 idr_for_each_entry(&loop_index_idr, lo, id) { 2494 /* Hitting a race results in creating a new loop device which is harmless. */ 2495 if (lo->idr_visible && data_race(lo->lo_state) == Lo_unbound) 2496 goto found; 2497 } 2498 mutex_unlock(&loop_ctl_mutex); 2499 return loop_add(-1); 2500 found: 2501 mutex_unlock(&loop_ctl_mutex); 2502 return id; 2503 } 2504 2505 static long loop_control_ioctl(struct file *file, unsigned int cmd, 2506 unsigned long parm) 2507 { 2508 switch (cmd) { 2509 case LOOP_CTL_ADD: 2510 return loop_add(parm); 2511 case LOOP_CTL_REMOVE: 2512 return loop_control_remove(parm); 2513 case LOOP_CTL_GET_FREE: 2514 return loop_control_get_free(parm); 2515 default: 2516 return -ENOSYS; 2517 } 2518 } 2519 2520 static const struct file_operations loop_ctl_fops = { 2521 .open = nonseekable_open, 2522 .unlocked_ioctl = loop_control_ioctl, 2523 .compat_ioctl = loop_control_ioctl, 2524 .owner = THIS_MODULE, 2525 .llseek = noop_llseek, 2526 }; 2527 2528 static struct miscdevice loop_misc = { 2529 .minor = LOOP_CTRL_MINOR, 2530 .name = "loop-control", 2531 .fops = &loop_ctl_fops, 2532 }; 2533 2534 MODULE_ALIAS_MISCDEV(LOOP_CTRL_MINOR); 2535 MODULE_ALIAS("devname:loop-control"); 2536 2537 static int __init loop_init(void) 2538 { 2539 int i, nr; 2540 int err; 2541 2542 part_shift = 0; 2543 if (max_part > 0) { 2544 part_shift = fls(max_part); 2545 2546 /* 2547 * Adjust max_part according to part_shift as it is exported 2548 * to user space so that user can decide correct minor number 2549 * if [s]he want to create more devices. 2550 * 2551 * Note that -1 is required because partition 0 is reserved 2552 * for the whole disk. 2553 */ 2554 max_part = (1UL << part_shift) - 1; 2555 } 2556 2557 if ((1UL << part_shift) > DISK_MAX_PARTS) { 2558 err = -EINVAL; 2559 goto err_out; 2560 } 2561 2562 if (max_loop > 1UL << (MINORBITS - part_shift)) { 2563 err = -EINVAL; 2564 goto err_out; 2565 } 2566 2567 /* 2568 * If max_loop is specified, create that many devices upfront. 2569 * This also becomes a hard limit. If max_loop is not specified, 2570 * create CONFIG_BLK_DEV_LOOP_MIN_COUNT loop devices at module 2571 * init time. Loop devices can be requested on-demand with the 2572 * /dev/loop-control interface, or be instantiated by accessing 2573 * a 'dead' device node. 2574 */ 2575 if (max_loop) 2576 nr = max_loop; 2577 else 2578 nr = CONFIG_BLK_DEV_LOOP_MIN_COUNT; 2579 2580 err = misc_register(&loop_misc); 2581 if (err < 0) 2582 goto err_out; 2583 2584 2585 if (__register_blkdev(LOOP_MAJOR, "loop", loop_probe)) { 2586 err = -EIO; 2587 goto misc_out; 2588 } 2589 2590 /* pre-create number of devices given by config or max_loop */ 2591 for (i = 0; i < nr; i++) 2592 loop_add(i); 2593 2594 printk(KERN_INFO "loop: module loaded\n"); 2595 return 0; 2596 2597 misc_out: 2598 misc_deregister(&loop_misc); 2599 err_out: 2600 return err; 2601 } 2602 2603 static void __exit loop_exit(void) 2604 { 2605 struct loop_device *lo; 2606 int id; 2607 2608 unregister_blkdev(LOOP_MAJOR, "loop"); 2609 misc_deregister(&loop_misc); 2610 2611 /* 2612 * There is no need to use loop_ctl_mutex here, for nobody else can 2613 * access loop_index_idr when this module is unloading (unless forced 2614 * module unloading is requested). If this is not a clean unloading, 2615 * we have no means to avoid kernel crash. 2616 */ 2617 idr_for_each_entry(&loop_index_idr, lo, id) 2618 loop_remove(lo); 2619 2620 idr_destroy(&loop_index_idr); 2621 } 2622 2623 module_init(loop_init); 2624 module_exit(loop_exit); 2625 2626 #ifndef MODULE 2627 static int __init max_loop_setup(char *str) 2628 { 2629 max_loop = simple_strtol(str, NULL, 0); 2630 return 1; 2631 } 2632 2633 __setup("max_loop=", max_loop_setup); 2634 #endif 2635