1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Userspace block device - block device which IO is handled from userspace 4 * 5 * Take full use of io_uring passthrough command for communicating with 6 * ublk userspace daemon(ublksrvd) for handling basic IO request. 7 * 8 * Copyright 2022 Ming Lei <ming.lei@redhat.com> 9 * 10 * (part of code stolen from loop.c) 11 */ 12 #include <linux/module.h> 13 #include <linux/moduleparam.h> 14 #include <linux/sched.h> 15 #include <linux/fs.h> 16 #include <linux/pagemap.h> 17 #include <linux/file.h> 18 #include <linux/stat.h> 19 #include <linux/errno.h> 20 #include <linux/major.h> 21 #include <linux/wait.h> 22 #include <linux/blkdev.h> 23 #include <linux/init.h> 24 #include <linux/swap.h> 25 #include <linux/slab.h> 26 #include <linux/compat.h> 27 #include <linux/mutex.h> 28 #include <linux/writeback.h> 29 #include <linux/completion.h> 30 #include <linux/highmem.h> 31 #include <linux/sysfs.h> 32 #include <linux/miscdevice.h> 33 #include <linux/falloc.h> 34 #include <linux/uio.h> 35 #include <linux/ioprio.h> 36 #include <linux/sched/mm.h> 37 #include <linux/uaccess.h> 38 #include <linux/cdev.h> 39 #include <linux/io_uring.h> 40 #include <linux/blk-mq.h> 41 #include <linux/delay.h> 42 #include <linux/mm.h> 43 #include <asm/page.h> 44 #include <linux/task_work.h> 45 #include <uapi/linux/ublk_cmd.h> 46 47 #define UBLK_MINORS (1U << MINORBITS) 48 49 /* All UBLK_F_* have to be included into UBLK_F_ALL */ 50 #define UBLK_F_ALL (UBLK_F_SUPPORT_ZERO_COPY \ 51 | UBLK_F_URING_CMD_COMP_IN_TASK \ 52 | UBLK_F_NEED_GET_DATA) 53 54 /* All UBLK_PARAM_TYPE_* should be included here */ 55 #define UBLK_PARAM_TYPE_ALL (UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DISCARD) 56 57 struct ublk_rq_data { 58 struct callback_head work; 59 }; 60 61 struct ublk_uring_cmd_pdu { 62 struct request *req; 63 }; 64 65 /* 66 * io command is active: sqe cmd is received, and its cqe isn't done 67 * 68 * If the flag is set, the io command is owned by ublk driver, and waited 69 * for incoming blk-mq request from the ublk block device. 70 * 71 * If the flag is cleared, the io command will be completed, and owned by 72 * ublk server. 73 */ 74 #define UBLK_IO_FLAG_ACTIVE 0x01 75 76 /* 77 * IO command is completed via cqe, and it is being handled by ublksrv, and 78 * not committed yet 79 * 80 * Basically exclusively with UBLK_IO_FLAG_ACTIVE, so can be served for 81 * cross verification 82 */ 83 #define UBLK_IO_FLAG_OWNED_BY_SRV 0x02 84 85 /* 86 * IO command is aborted, so this flag is set in case of 87 * !UBLK_IO_FLAG_ACTIVE. 88 * 89 * After this flag is observed, any pending or new incoming request 90 * associated with this io command will be failed immediately 91 */ 92 #define UBLK_IO_FLAG_ABORTED 0x04 93 94 /* 95 * UBLK_IO_FLAG_NEED_GET_DATA is set because IO command requires 96 * get data buffer address from ublksrv. 97 * 98 * Then, bio data could be copied into this data buffer for a WRITE request 99 * after the IO command is issued again and UBLK_IO_FLAG_NEED_GET_DATA is unset. 100 */ 101 #define UBLK_IO_FLAG_NEED_GET_DATA 0x08 102 103 struct ublk_io { 104 /* userspace buffer address from io cmd */ 105 __u64 addr; 106 unsigned int flags; 107 int res; 108 109 struct io_uring_cmd *cmd; 110 }; 111 112 struct ublk_queue { 113 int q_id; 114 int q_depth; 115 116 unsigned long flags; 117 struct task_struct *ubq_daemon; 118 char *io_cmd_buf; 119 120 unsigned long io_addr; /* mapped vm address */ 121 unsigned int max_io_sz; 122 bool abort_work_pending; 123 unsigned short nr_io_ready; /* how many ios setup */ 124 struct ublk_device *dev; 125 struct ublk_io ios[0]; 126 }; 127 128 #define UBLK_DAEMON_MONITOR_PERIOD (5 * HZ) 129 130 struct ublk_device { 131 struct gendisk *ub_disk; 132 133 char *__queues; 134 135 unsigned short queue_size; 136 struct ublksrv_ctrl_dev_info dev_info; 137 138 struct blk_mq_tag_set tag_set; 139 140 struct cdev cdev; 141 struct device cdev_dev; 142 143 #define UB_STATE_OPEN 0 144 #define UB_STATE_USED 1 145 unsigned long state; 146 int ub_number; 147 148 struct mutex mutex; 149 150 spinlock_t mm_lock; 151 struct mm_struct *mm; 152 153 struct ublk_params params; 154 155 struct completion completion; 156 unsigned int nr_queues_ready; 157 atomic_t nr_aborted_queues; 158 159 /* 160 * Our ubq->daemon may be killed without any notification, so 161 * monitor each queue's daemon periodically 162 */ 163 struct delayed_work monitor_work; 164 struct work_struct stop_work; 165 }; 166 167 /* header of ublk_params */ 168 struct ublk_params_header { 169 __u32 len; 170 __u32 types; 171 }; 172 173 static dev_t ublk_chr_devt; 174 static struct class *ublk_chr_class; 175 176 static DEFINE_IDR(ublk_index_idr); 177 static DEFINE_SPINLOCK(ublk_idr_lock); 178 static wait_queue_head_t ublk_idr_wq; /* wait until one idr is freed */ 179 180 static DEFINE_MUTEX(ublk_ctl_mutex); 181 182 static struct miscdevice ublk_misc; 183 184 static void ublk_dev_param_basic_apply(struct ublk_device *ub) 185 { 186 struct request_queue *q = ub->ub_disk->queue; 187 const struct ublk_param_basic *p = &ub->params.basic; 188 189 blk_queue_logical_block_size(q, 1 << p->logical_bs_shift); 190 blk_queue_physical_block_size(q, 1 << p->physical_bs_shift); 191 blk_queue_io_min(q, 1 << p->io_min_shift); 192 blk_queue_io_opt(q, 1 << p->io_opt_shift); 193 194 blk_queue_write_cache(q, p->attrs & UBLK_ATTR_VOLATILE_CACHE, 195 p->attrs & UBLK_ATTR_FUA); 196 if (p->attrs & UBLK_ATTR_ROTATIONAL) 197 blk_queue_flag_clear(QUEUE_FLAG_NONROT, q); 198 else 199 blk_queue_flag_set(QUEUE_FLAG_NONROT, q); 200 201 blk_queue_max_hw_sectors(q, p->max_sectors); 202 blk_queue_chunk_sectors(q, p->chunk_sectors); 203 blk_queue_virt_boundary(q, p->virt_boundary_mask); 204 205 if (p->attrs & UBLK_ATTR_READ_ONLY) 206 set_disk_ro(ub->ub_disk, true); 207 208 set_capacity(ub->ub_disk, p->dev_sectors); 209 } 210 211 static void ublk_dev_param_discard_apply(struct ublk_device *ub) 212 { 213 struct request_queue *q = ub->ub_disk->queue; 214 const struct ublk_param_discard *p = &ub->params.discard; 215 216 q->limits.discard_alignment = p->discard_alignment; 217 q->limits.discard_granularity = p->discard_granularity; 218 blk_queue_max_discard_sectors(q, p->max_discard_sectors); 219 blk_queue_max_write_zeroes_sectors(q, 220 p->max_write_zeroes_sectors); 221 blk_queue_max_discard_segments(q, p->max_discard_segments); 222 } 223 224 static int ublk_validate_params(const struct ublk_device *ub) 225 { 226 /* basic param is the only one which must be set */ 227 if (ub->params.types & UBLK_PARAM_TYPE_BASIC) { 228 const struct ublk_param_basic *p = &ub->params.basic; 229 230 if (p->logical_bs_shift > PAGE_SHIFT) 231 return -EINVAL; 232 233 if (p->logical_bs_shift > p->physical_bs_shift) 234 return -EINVAL; 235 236 if (p->max_sectors > (ub->dev_info.max_io_buf_bytes >> 9)) 237 return -EINVAL; 238 } else 239 return -EINVAL; 240 241 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) { 242 const struct ublk_param_discard *p = &ub->params.discard; 243 244 /* So far, only support single segment discard */ 245 if (p->max_discard_sectors && p->max_discard_segments != 1) 246 return -EINVAL; 247 248 if (!p->discard_granularity) 249 return -EINVAL; 250 } 251 252 return 0; 253 } 254 255 static int ublk_apply_params(struct ublk_device *ub) 256 { 257 if (!(ub->params.types & UBLK_PARAM_TYPE_BASIC)) 258 return -EINVAL; 259 260 ublk_dev_param_basic_apply(ub); 261 262 if (ub->params.types & UBLK_PARAM_TYPE_DISCARD) 263 ublk_dev_param_discard_apply(ub); 264 265 return 0; 266 } 267 268 static inline bool ublk_can_use_task_work(const struct ublk_queue *ubq) 269 { 270 if (IS_BUILTIN(CONFIG_BLK_DEV_UBLK) && 271 !(ubq->flags & UBLK_F_URING_CMD_COMP_IN_TASK)) 272 return true; 273 return false; 274 } 275 276 static inline bool ublk_need_get_data(const struct ublk_queue *ubq) 277 { 278 if (ubq->flags & UBLK_F_NEED_GET_DATA) 279 return true; 280 return false; 281 } 282 283 static struct ublk_device *ublk_get_device(struct ublk_device *ub) 284 { 285 if (kobject_get_unless_zero(&ub->cdev_dev.kobj)) 286 return ub; 287 return NULL; 288 } 289 290 static void ublk_put_device(struct ublk_device *ub) 291 { 292 put_device(&ub->cdev_dev); 293 } 294 295 static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev, 296 int qid) 297 { 298 return (struct ublk_queue *)&(dev->__queues[qid * dev->queue_size]); 299 } 300 301 static inline bool ublk_rq_has_data(const struct request *rq) 302 { 303 return rq->bio && bio_has_data(rq->bio); 304 } 305 306 static inline struct ublksrv_io_desc *ublk_get_iod(struct ublk_queue *ubq, 307 int tag) 308 { 309 return (struct ublksrv_io_desc *) 310 &(ubq->io_cmd_buf[tag * sizeof(struct ublksrv_io_desc)]); 311 } 312 313 static inline char *ublk_queue_cmd_buf(struct ublk_device *ub, int q_id) 314 { 315 return ublk_get_queue(ub, q_id)->io_cmd_buf; 316 } 317 318 static inline int ublk_queue_cmd_buf_size(struct ublk_device *ub, int q_id) 319 { 320 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); 321 322 return round_up(ubq->q_depth * sizeof(struct ublksrv_io_desc), 323 PAGE_SIZE); 324 } 325 326 static void ublk_free_disk(struct gendisk *disk) 327 { 328 struct ublk_device *ub = disk->private_data; 329 330 clear_bit(UB_STATE_USED, &ub->state); 331 put_device(&ub->cdev_dev); 332 } 333 334 static const struct block_device_operations ub_fops = { 335 .owner = THIS_MODULE, 336 .free_disk = ublk_free_disk, 337 }; 338 339 #define UBLK_MAX_PIN_PAGES 32 340 341 struct ublk_map_data { 342 const struct ublk_queue *ubq; 343 const struct request *rq; 344 const struct ublk_io *io; 345 unsigned max_bytes; 346 }; 347 348 struct ublk_io_iter { 349 struct page *pages[UBLK_MAX_PIN_PAGES]; 350 unsigned pg_off; /* offset in the 1st page in pages */ 351 int nr_pages; /* how many page pointers in pages */ 352 struct bio *bio; 353 struct bvec_iter iter; 354 }; 355 356 static inline unsigned ublk_copy_io_pages(struct ublk_io_iter *data, 357 unsigned max_bytes, bool to_vm) 358 { 359 const unsigned total = min_t(unsigned, max_bytes, 360 PAGE_SIZE - data->pg_off + 361 ((data->nr_pages - 1) << PAGE_SHIFT)); 362 unsigned done = 0; 363 unsigned pg_idx = 0; 364 365 while (done < total) { 366 struct bio_vec bv = bio_iter_iovec(data->bio, data->iter); 367 const unsigned int bytes = min3(bv.bv_len, total - done, 368 (unsigned)(PAGE_SIZE - data->pg_off)); 369 void *bv_buf = bvec_kmap_local(&bv); 370 void *pg_buf = kmap_local_page(data->pages[pg_idx]); 371 372 if (to_vm) 373 memcpy(pg_buf + data->pg_off, bv_buf, bytes); 374 else 375 memcpy(bv_buf, pg_buf + data->pg_off, bytes); 376 377 kunmap_local(pg_buf); 378 kunmap_local(bv_buf); 379 380 /* advance page array */ 381 data->pg_off += bytes; 382 if (data->pg_off == PAGE_SIZE) { 383 pg_idx += 1; 384 data->pg_off = 0; 385 } 386 387 done += bytes; 388 389 /* advance bio */ 390 bio_advance_iter_single(data->bio, &data->iter, bytes); 391 if (!data->iter.bi_size) { 392 data->bio = data->bio->bi_next; 393 if (data->bio == NULL) 394 break; 395 data->iter = data->bio->bi_iter; 396 } 397 } 398 399 return done; 400 } 401 402 static inline int ublk_copy_user_pages(struct ublk_map_data *data, 403 bool to_vm) 404 { 405 const unsigned int gup_flags = to_vm ? FOLL_WRITE : 0; 406 const unsigned long start_vm = data->io->addr; 407 unsigned int done = 0; 408 struct ublk_io_iter iter = { 409 .pg_off = start_vm & (PAGE_SIZE - 1), 410 .bio = data->rq->bio, 411 .iter = data->rq->bio->bi_iter, 412 }; 413 const unsigned int nr_pages = round_up(data->max_bytes + 414 (start_vm & (PAGE_SIZE - 1)), PAGE_SIZE) >> PAGE_SHIFT; 415 416 while (done < nr_pages) { 417 const unsigned to_pin = min_t(unsigned, UBLK_MAX_PIN_PAGES, 418 nr_pages - done); 419 unsigned i, len; 420 421 iter.nr_pages = get_user_pages_fast(start_vm + 422 (done << PAGE_SHIFT), to_pin, gup_flags, 423 iter.pages); 424 if (iter.nr_pages <= 0) 425 return done == 0 ? iter.nr_pages : done; 426 len = ublk_copy_io_pages(&iter, data->max_bytes, to_vm); 427 for (i = 0; i < iter.nr_pages; i++) { 428 if (to_vm) 429 set_page_dirty(iter.pages[i]); 430 put_page(iter.pages[i]); 431 } 432 data->max_bytes -= len; 433 done += iter.nr_pages; 434 } 435 436 return done; 437 } 438 439 static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req, 440 struct ublk_io *io) 441 { 442 const unsigned int rq_bytes = blk_rq_bytes(req); 443 /* 444 * no zero copy, we delay copy WRITE request data into ublksrv 445 * context and the big benefit is that pinning pages in current 446 * context is pretty fast, see ublk_pin_user_pages 447 */ 448 if (req_op(req) != REQ_OP_WRITE && req_op(req) != REQ_OP_FLUSH) 449 return rq_bytes; 450 451 if (ublk_rq_has_data(req)) { 452 struct ublk_map_data data = { 453 .ubq = ubq, 454 .rq = req, 455 .io = io, 456 .max_bytes = rq_bytes, 457 }; 458 459 ublk_copy_user_pages(&data, true); 460 461 return rq_bytes - data.max_bytes; 462 } 463 return rq_bytes; 464 } 465 466 static int ublk_unmap_io(const struct ublk_queue *ubq, 467 const struct request *req, 468 struct ublk_io *io) 469 { 470 const unsigned int rq_bytes = blk_rq_bytes(req); 471 472 if (req_op(req) == REQ_OP_READ && ublk_rq_has_data(req)) { 473 struct ublk_map_data data = { 474 .ubq = ubq, 475 .rq = req, 476 .io = io, 477 .max_bytes = io->res, 478 }; 479 480 WARN_ON_ONCE(io->res > rq_bytes); 481 482 ublk_copy_user_pages(&data, false); 483 484 return io->res - data.max_bytes; 485 } 486 return rq_bytes; 487 } 488 489 static inline unsigned int ublk_req_build_flags(struct request *req) 490 { 491 unsigned flags = 0; 492 493 if (req->cmd_flags & REQ_FAILFAST_DEV) 494 flags |= UBLK_IO_F_FAILFAST_DEV; 495 496 if (req->cmd_flags & REQ_FAILFAST_TRANSPORT) 497 flags |= UBLK_IO_F_FAILFAST_TRANSPORT; 498 499 if (req->cmd_flags & REQ_FAILFAST_DRIVER) 500 flags |= UBLK_IO_F_FAILFAST_DRIVER; 501 502 if (req->cmd_flags & REQ_META) 503 flags |= UBLK_IO_F_META; 504 505 if (req->cmd_flags & REQ_FUA) 506 flags |= UBLK_IO_F_FUA; 507 508 if (req->cmd_flags & REQ_NOUNMAP) 509 flags |= UBLK_IO_F_NOUNMAP; 510 511 if (req->cmd_flags & REQ_SWAP) 512 flags |= UBLK_IO_F_SWAP; 513 514 return flags; 515 } 516 517 static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) 518 { 519 struct ublksrv_io_desc *iod = ublk_get_iod(ubq, req->tag); 520 struct ublk_io *io = &ubq->ios[req->tag]; 521 u32 ublk_op; 522 523 switch (req_op(req)) { 524 case REQ_OP_READ: 525 ublk_op = UBLK_IO_OP_READ; 526 break; 527 case REQ_OP_WRITE: 528 ublk_op = UBLK_IO_OP_WRITE; 529 break; 530 case REQ_OP_FLUSH: 531 ublk_op = UBLK_IO_OP_FLUSH; 532 break; 533 case REQ_OP_DISCARD: 534 ublk_op = UBLK_IO_OP_DISCARD; 535 break; 536 case REQ_OP_WRITE_ZEROES: 537 ublk_op = UBLK_IO_OP_WRITE_ZEROES; 538 break; 539 default: 540 return BLK_STS_IOERR; 541 } 542 543 /* need to translate since kernel may change */ 544 iod->op_flags = ublk_op | ublk_req_build_flags(req); 545 iod->nr_sectors = blk_rq_sectors(req); 546 iod->start_sector = blk_rq_pos(req); 547 iod->addr = io->addr; 548 549 return BLK_STS_OK; 550 } 551 552 static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu( 553 struct io_uring_cmd *ioucmd) 554 { 555 return (struct ublk_uring_cmd_pdu *)&ioucmd->pdu; 556 } 557 558 static bool ubq_daemon_is_dying(struct ublk_queue *ubq) 559 { 560 return ubq->ubq_daemon->flags & PF_EXITING; 561 } 562 563 /* todo: handle partial completion */ 564 static void ublk_complete_rq(struct request *req) 565 { 566 struct ublk_queue *ubq = req->mq_hctx->driver_data; 567 struct ublk_io *io = &ubq->ios[req->tag]; 568 unsigned int unmapped_bytes; 569 570 /* failed read IO if nothing is read */ 571 if (!io->res && req_op(req) == REQ_OP_READ) 572 io->res = -EIO; 573 574 if (io->res < 0) { 575 blk_mq_end_request(req, errno_to_blk_status(io->res)); 576 return; 577 } 578 579 /* 580 * FLUSH or DISCARD usually won't return bytes returned, so end them 581 * directly. 582 * 583 * Both the two needn't unmap. 584 */ 585 if (req_op(req) != REQ_OP_READ && req_op(req) != REQ_OP_WRITE) { 586 blk_mq_end_request(req, BLK_STS_OK); 587 return; 588 } 589 590 /* for READ request, writing data in iod->addr to rq buffers */ 591 unmapped_bytes = ublk_unmap_io(ubq, req, io); 592 593 /* 594 * Extremely impossible since we got data filled in just before 595 * 596 * Re-read simply for this unlikely case. 597 */ 598 if (unlikely(unmapped_bytes < io->res)) 599 io->res = unmapped_bytes; 600 601 if (blk_update_request(req, BLK_STS_OK, io->res)) 602 blk_mq_requeue_request(req, true); 603 else 604 __blk_mq_end_request(req, BLK_STS_OK); 605 } 606 607 /* 608 * __ublk_fail_req() may be called from abort context or ->ubq_daemon 609 * context during exiting, so lock is required. 610 * 611 * Also aborting may not be started yet, keep in mind that one failed 612 * request may be issued by block layer again. 613 */ 614 static void __ublk_fail_req(struct ublk_io *io, struct request *req) 615 { 616 WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_ACTIVE); 617 618 if (!(io->flags & UBLK_IO_FLAG_ABORTED)) { 619 io->flags |= UBLK_IO_FLAG_ABORTED; 620 blk_mq_end_request(req, BLK_STS_IOERR); 621 } 622 } 623 624 static void ubq_complete_io_cmd(struct ublk_io *io, int res) 625 { 626 /* mark this cmd owned by ublksrv */ 627 io->flags |= UBLK_IO_FLAG_OWNED_BY_SRV; 628 629 /* 630 * clear ACTIVE since we are done with this sqe/cmd slot 631 * We can only accept io cmd in case of being not active. 632 */ 633 io->flags &= ~UBLK_IO_FLAG_ACTIVE; 634 635 /* tell ublksrv one io request is coming */ 636 io_uring_cmd_done(io->cmd, res, 0); 637 } 638 639 #define UBLK_REQUEUE_DELAY_MS 3 640 641 static inline void __ublk_rq_task_work(struct request *req) 642 { 643 struct ublk_queue *ubq = req->mq_hctx->driver_data; 644 struct ublk_device *ub = ubq->dev; 645 int tag = req->tag; 646 struct ublk_io *io = &ubq->ios[tag]; 647 bool task_exiting = current != ubq->ubq_daemon || 648 (current->flags & PF_EXITING); 649 unsigned int mapped_bytes; 650 651 pr_devel("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n", 652 __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags, 653 ublk_get_iod(ubq, req->tag)->addr); 654 655 if (unlikely(task_exiting)) { 656 blk_mq_end_request(req, BLK_STS_IOERR); 657 mod_delayed_work(system_wq, &ub->monitor_work, 0); 658 return; 659 } 660 661 if (ublk_need_get_data(ubq) && 662 (req_op(req) == REQ_OP_WRITE || 663 req_op(req) == REQ_OP_FLUSH)) { 664 /* 665 * We have not handled UBLK_IO_NEED_GET_DATA command yet, 666 * so immepdately pass UBLK_IO_RES_NEED_GET_DATA to ublksrv 667 * and notify it. 668 */ 669 if (!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA)) { 670 io->flags |= UBLK_IO_FLAG_NEED_GET_DATA; 671 pr_devel("%s: need get data. op %d, qid %d tag %d io_flags %x\n", 672 __func__, io->cmd->cmd_op, ubq->q_id, 673 req->tag, io->flags); 674 ubq_complete_io_cmd(io, UBLK_IO_RES_NEED_GET_DATA); 675 return; 676 } 677 /* 678 * We have handled UBLK_IO_NEED_GET_DATA command, 679 * so clear UBLK_IO_FLAG_NEED_GET_DATA now and just 680 * do the copy work. 681 */ 682 io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA; 683 } 684 685 mapped_bytes = ublk_map_io(ubq, req, io); 686 687 /* partially mapped, update io descriptor */ 688 if (unlikely(mapped_bytes != blk_rq_bytes(req))) { 689 /* 690 * Nothing mapped, retry until we succeed. 691 * 692 * We may never succeed in mapping any bytes here because 693 * of OOM. TODO: reserve one buffer with single page pinned 694 * for providing forward progress guarantee. 695 */ 696 if (unlikely(!mapped_bytes)) { 697 blk_mq_requeue_request(req, false); 698 blk_mq_delay_kick_requeue_list(req->q, 699 UBLK_REQUEUE_DELAY_MS); 700 return; 701 } 702 703 ublk_get_iod(ubq, req->tag)->nr_sectors = 704 mapped_bytes >> 9; 705 } 706 707 ubq_complete_io_cmd(io, UBLK_IO_RES_OK); 708 } 709 710 static void ublk_rq_task_work_cb(struct io_uring_cmd *cmd) 711 { 712 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); 713 714 __ublk_rq_task_work(pdu->req); 715 } 716 717 static void ublk_rq_task_work_fn(struct callback_head *work) 718 { 719 struct ublk_rq_data *data = container_of(work, 720 struct ublk_rq_data, work); 721 struct request *req = blk_mq_rq_from_pdu(data); 722 723 __ublk_rq_task_work(req); 724 } 725 726 static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, 727 const struct blk_mq_queue_data *bd) 728 { 729 struct ublk_queue *ubq = hctx->driver_data; 730 struct request *rq = bd->rq; 731 blk_status_t res; 732 733 /* fill iod to slot in io cmd buffer */ 734 res = ublk_setup_iod(ubq, rq); 735 if (unlikely(res != BLK_STS_OK)) 736 return BLK_STS_IOERR; 737 738 blk_mq_start_request(bd->rq); 739 740 if (unlikely(ubq_daemon_is_dying(ubq))) { 741 fail: 742 mod_delayed_work(system_wq, &ubq->dev->monitor_work, 0); 743 return BLK_STS_IOERR; 744 } 745 746 if (ublk_can_use_task_work(ubq)) { 747 struct ublk_rq_data *data = blk_mq_rq_to_pdu(rq); 748 enum task_work_notify_mode notify_mode = bd->last ? 749 TWA_SIGNAL_NO_IPI : TWA_NONE; 750 751 if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode)) 752 goto fail; 753 } else { 754 struct io_uring_cmd *cmd = ubq->ios[rq->tag].cmd; 755 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); 756 757 pdu->req = rq; 758 io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb); 759 } 760 761 return BLK_STS_OK; 762 } 763 764 static void ublk_commit_rqs(struct blk_mq_hw_ctx *hctx) 765 { 766 struct ublk_queue *ubq = hctx->driver_data; 767 768 if (ublk_can_use_task_work(ubq)) 769 __set_notify_signal(ubq->ubq_daemon); 770 } 771 772 static int ublk_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data, 773 unsigned int hctx_idx) 774 { 775 struct ublk_device *ub = driver_data; 776 struct ublk_queue *ubq = ublk_get_queue(ub, hctx->queue_num); 777 778 hctx->driver_data = ubq; 779 return 0; 780 } 781 782 static int ublk_init_rq(struct blk_mq_tag_set *set, struct request *req, 783 unsigned int hctx_idx, unsigned int numa_node) 784 { 785 struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 786 787 init_task_work(&data->work, ublk_rq_task_work_fn); 788 return 0; 789 } 790 791 static const struct blk_mq_ops ublk_mq_ops = { 792 .queue_rq = ublk_queue_rq, 793 .commit_rqs = ublk_commit_rqs, 794 .init_hctx = ublk_init_hctx, 795 .init_request = ublk_init_rq, 796 }; 797 798 static int ublk_ch_open(struct inode *inode, struct file *filp) 799 { 800 struct ublk_device *ub = container_of(inode->i_cdev, 801 struct ublk_device, cdev); 802 803 if (test_and_set_bit(UB_STATE_OPEN, &ub->state)) 804 return -EBUSY; 805 filp->private_data = ub; 806 return 0; 807 } 808 809 static int ublk_ch_release(struct inode *inode, struct file *filp) 810 { 811 struct ublk_device *ub = filp->private_data; 812 813 clear_bit(UB_STATE_OPEN, &ub->state); 814 return 0; 815 } 816 817 /* map pre-allocated per-queue cmd buffer to ublksrv daemon */ 818 static int ublk_ch_mmap(struct file *filp, struct vm_area_struct *vma) 819 { 820 struct ublk_device *ub = filp->private_data; 821 size_t sz = vma->vm_end - vma->vm_start; 822 unsigned max_sz = UBLK_MAX_QUEUE_DEPTH * sizeof(struct ublksrv_io_desc); 823 unsigned long pfn, end, phys_off = vma->vm_pgoff << PAGE_SHIFT; 824 int q_id, ret = 0; 825 826 spin_lock(&ub->mm_lock); 827 if (!ub->mm) 828 ub->mm = current->mm; 829 if (current->mm != ub->mm) 830 ret = -EINVAL; 831 spin_unlock(&ub->mm_lock); 832 833 if (ret) 834 return ret; 835 836 if (vma->vm_flags & VM_WRITE) 837 return -EPERM; 838 839 end = UBLKSRV_CMD_BUF_OFFSET + ub->dev_info.nr_hw_queues * max_sz; 840 if (phys_off < UBLKSRV_CMD_BUF_OFFSET || phys_off >= end) 841 return -EINVAL; 842 843 q_id = (phys_off - UBLKSRV_CMD_BUF_OFFSET) / max_sz; 844 pr_devel("%s: qid %d, pid %d, addr %lx pg_off %lx sz %lu\n", 845 __func__, q_id, current->pid, vma->vm_start, 846 phys_off, (unsigned long)sz); 847 848 if (sz != ublk_queue_cmd_buf_size(ub, q_id)) 849 return -EINVAL; 850 851 pfn = virt_to_phys(ublk_queue_cmd_buf(ub, q_id)) >> PAGE_SHIFT; 852 return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot); 853 } 854 855 static void ublk_commit_completion(struct ublk_device *ub, 856 struct ublksrv_io_cmd *ub_cmd) 857 { 858 u32 qid = ub_cmd->q_id, tag = ub_cmd->tag; 859 struct ublk_queue *ubq = ublk_get_queue(ub, qid); 860 struct ublk_io *io = &ubq->ios[tag]; 861 struct request *req; 862 863 /* now this cmd slot is owned by nbd driver */ 864 io->flags &= ~UBLK_IO_FLAG_OWNED_BY_SRV; 865 io->res = ub_cmd->result; 866 867 /* find the io request and complete */ 868 req = blk_mq_tag_to_rq(ub->tag_set.tags[qid], tag); 869 870 if (req && likely(!blk_should_fake_timeout(req->q))) 871 ublk_complete_rq(req); 872 } 873 874 /* 875 * When ->ubq_daemon is exiting, either new request is ended immediately, 876 * or any queued io command is drained, so it is safe to abort queue 877 * lockless 878 */ 879 static void ublk_abort_queue(struct ublk_device *ub, struct ublk_queue *ubq) 880 { 881 int i; 882 883 if (!ublk_get_device(ub)) 884 return; 885 886 for (i = 0; i < ubq->q_depth; i++) { 887 struct ublk_io *io = &ubq->ios[i]; 888 889 if (!(io->flags & UBLK_IO_FLAG_ACTIVE)) { 890 struct request *rq; 891 892 /* 893 * Either we fail the request or ublk_rq_task_work_fn 894 * will do it 895 */ 896 rq = blk_mq_tag_to_rq(ub->tag_set.tags[ubq->q_id], i); 897 if (rq) 898 __ublk_fail_req(io, rq); 899 } 900 } 901 ublk_put_device(ub); 902 } 903 904 static void ublk_daemon_monitor_work(struct work_struct *work) 905 { 906 struct ublk_device *ub = 907 container_of(work, struct ublk_device, monitor_work.work); 908 int i; 909 910 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { 911 struct ublk_queue *ubq = ublk_get_queue(ub, i); 912 913 if (ubq_daemon_is_dying(ubq)) { 914 schedule_work(&ub->stop_work); 915 916 /* abort queue is for making forward progress */ 917 ublk_abort_queue(ub, ubq); 918 } 919 } 920 921 /* 922 * We can't schedule monitor work after ublk_remove() is started. 923 * 924 * No need ub->mutex, monitor work are canceled after state is marked 925 * as DEAD, so DEAD state is observed reliably. 926 */ 927 if (ub->dev_info.state != UBLK_S_DEV_DEAD) 928 schedule_delayed_work(&ub->monitor_work, 929 UBLK_DAEMON_MONITOR_PERIOD); 930 } 931 932 static inline bool ublk_queue_ready(struct ublk_queue *ubq) 933 { 934 return ubq->nr_io_ready == ubq->q_depth; 935 } 936 937 static void ublk_cancel_queue(struct ublk_queue *ubq) 938 { 939 int i; 940 941 if (!ublk_queue_ready(ubq)) 942 return; 943 944 for (i = 0; i < ubq->q_depth; i++) { 945 struct ublk_io *io = &ubq->ios[i]; 946 947 if (io->flags & UBLK_IO_FLAG_ACTIVE) 948 io_uring_cmd_done(io->cmd, UBLK_IO_RES_ABORT, 0); 949 } 950 951 /* all io commands are canceled */ 952 ubq->nr_io_ready = 0; 953 } 954 955 /* Cancel all pending commands, must be called after del_gendisk() returns */ 956 static void ublk_cancel_dev(struct ublk_device *ub) 957 { 958 int i; 959 960 for (i = 0; i < ub->dev_info.nr_hw_queues; i++) 961 ublk_cancel_queue(ublk_get_queue(ub, i)); 962 } 963 964 static void ublk_stop_dev(struct ublk_device *ub) 965 { 966 mutex_lock(&ub->mutex); 967 if (ub->dev_info.state != UBLK_S_DEV_LIVE) 968 goto unlock; 969 970 del_gendisk(ub->ub_disk); 971 ub->dev_info.state = UBLK_S_DEV_DEAD; 972 ub->dev_info.ublksrv_pid = -1; 973 put_disk(ub->ub_disk); 974 ub->ub_disk = NULL; 975 unlock: 976 ublk_cancel_dev(ub); 977 mutex_unlock(&ub->mutex); 978 cancel_delayed_work_sync(&ub->monitor_work); 979 } 980 981 /* device can only be started after all IOs are ready */ 982 static void ublk_mark_io_ready(struct ublk_device *ub, struct ublk_queue *ubq) 983 { 984 mutex_lock(&ub->mutex); 985 ubq->nr_io_ready++; 986 if (ublk_queue_ready(ubq)) { 987 ubq->ubq_daemon = current; 988 get_task_struct(ubq->ubq_daemon); 989 ub->nr_queues_ready++; 990 } 991 if (ub->nr_queues_ready == ub->dev_info.nr_hw_queues) 992 complete_all(&ub->completion); 993 mutex_unlock(&ub->mutex); 994 } 995 996 static void ublk_handle_need_get_data(struct ublk_device *ub, int q_id, 997 int tag, struct io_uring_cmd *cmd) 998 { 999 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); 1000 struct request *req = blk_mq_tag_to_rq(ub->tag_set.tags[q_id], tag); 1001 1002 if (ublk_can_use_task_work(ubq)) { 1003 struct ublk_rq_data *data = blk_mq_rq_to_pdu(req); 1004 1005 /* should not fail since we call it just in ubq->ubq_daemon */ 1006 task_work_add(ubq->ubq_daemon, &data->work, TWA_SIGNAL_NO_IPI); 1007 } else { 1008 struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); 1009 1010 pdu->req = req; 1011 io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb); 1012 } 1013 } 1014 1015 static int ublk_ch_uring_cmd(struct io_uring_cmd *cmd, unsigned int issue_flags) 1016 { 1017 struct ublksrv_io_cmd *ub_cmd = (struct ublksrv_io_cmd *)cmd->cmd; 1018 struct ublk_device *ub = cmd->file->private_data; 1019 struct ublk_queue *ubq; 1020 struct ublk_io *io; 1021 u32 cmd_op = cmd->cmd_op; 1022 unsigned tag = ub_cmd->tag; 1023 int ret = -EINVAL; 1024 1025 pr_devel("%s: received: cmd op %d queue %d tag %d result %d\n", 1026 __func__, cmd->cmd_op, ub_cmd->q_id, tag, 1027 ub_cmd->result); 1028 1029 if (!(issue_flags & IO_URING_F_SQE128)) 1030 goto out; 1031 1032 if (ub_cmd->q_id >= ub->dev_info.nr_hw_queues) 1033 goto out; 1034 1035 ubq = ublk_get_queue(ub, ub_cmd->q_id); 1036 if (!ubq || ub_cmd->q_id != ubq->q_id) 1037 goto out; 1038 1039 if (ubq->ubq_daemon && ubq->ubq_daemon != current) 1040 goto out; 1041 1042 if (tag >= ubq->q_depth) 1043 goto out; 1044 1045 io = &ubq->ios[tag]; 1046 1047 /* there is pending io cmd, something must be wrong */ 1048 if (io->flags & UBLK_IO_FLAG_ACTIVE) { 1049 ret = -EBUSY; 1050 goto out; 1051 } 1052 1053 /* 1054 * ensure that the user issues UBLK_IO_NEED_GET_DATA 1055 * iff the driver have set the UBLK_IO_FLAG_NEED_GET_DATA. 1056 */ 1057 if ((!!(io->flags & UBLK_IO_FLAG_NEED_GET_DATA)) 1058 ^ (cmd_op == UBLK_IO_NEED_GET_DATA)) 1059 goto out; 1060 1061 switch (cmd_op) { 1062 case UBLK_IO_FETCH_REQ: 1063 /* UBLK_IO_FETCH_REQ is only allowed before queue is setup */ 1064 if (ublk_queue_ready(ubq)) { 1065 ret = -EBUSY; 1066 goto out; 1067 } 1068 /* 1069 * The io is being handled by server, so COMMIT_RQ is expected 1070 * instead of FETCH_REQ 1071 */ 1072 if (io->flags & UBLK_IO_FLAG_OWNED_BY_SRV) 1073 goto out; 1074 /* FETCH_RQ has to provide IO buffer */ 1075 if (!ub_cmd->addr) 1076 goto out; 1077 io->cmd = cmd; 1078 io->flags |= UBLK_IO_FLAG_ACTIVE; 1079 io->addr = ub_cmd->addr; 1080 1081 ublk_mark_io_ready(ub, ubq); 1082 break; 1083 case UBLK_IO_COMMIT_AND_FETCH_REQ: 1084 /* FETCH_RQ has to provide IO buffer */ 1085 if (!ub_cmd->addr) 1086 goto out; 1087 if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) 1088 goto out; 1089 io->addr = ub_cmd->addr; 1090 io->flags |= UBLK_IO_FLAG_ACTIVE; 1091 io->cmd = cmd; 1092 ublk_commit_completion(ub, ub_cmd); 1093 break; 1094 case UBLK_IO_NEED_GET_DATA: 1095 if (!(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV)) 1096 goto out; 1097 io->addr = ub_cmd->addr; 1098 io->cmd = cmd; 1099 io->flags |= UBLK_IO_FLAG_ACTIVE; 1100 ublk_handle_need_get_data(ub, ub_cmd->q_id, ub_cmd->tag, cmd); 1101 break; 1102 default: 1103 goto out; 1104 } 1105 return -EIOCBQUEUED; 1106 1107 out: 1108 io_uring_cmd_done(cmd, ret, 0); 1109 pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n", 1110 __func__, cmd_op, tag, ret, io->flags); 1111 return -EIOCBQUEUED; 1112 } 1113 1114 static const struct file_operations ublk_ch_fops = { 1115 .owner = THIS_MODULE, 1116 .open = ublk_ch_open, 1117 .release = ublk_ch_release, 1118 .llseek = no_llseek, 1119 .uring_cmd = ublk_ch_uring_cmd, 1120 .mmap = ublk_ch_mmap, 1121 }; 1122 1123 static void ublk_deinit_queue(struct ublk_device *ub, int q_id) 1124 { 1125 int size = ublk_queue_cmd_buf_size(ub, q_id); 1126 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); 1127 1128 if (ubq->ubq_daemon) 1129 put_task_struct(ubq->ubq_daemon); 1130 if (ubq->io_cmd_buf) 1131 free_pages((unsigned long)ubq->io_cmd_buf, get_order(size)); 1132 } 1133 1134 static int ublk_init_queue(struct ublk_device *ub, int q_id) 1135 { 1136 struct ublk_queue *ubq = ublk_get_queue(ub, q_id); 1137 gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO; 1138 void *ptr; 1139 int size; 1140 1141 ubq->flags = ub->dev_info.flags; 1142 ubq->q_id = q_id; 1143 ubq->q_depth = ub->dev_info.queue_depth; 1144 size = ublk_queue_cmd_buf_size(ub, q_id); 1145 1146 ptr = (void *) __get_free_pages(gfp_flags, get_order(size)); 1147 if (!ptr) 1148 return -ENOMEM; 1149 1150 ubq->io_cmd_buf = ptr; 1151 ubq->dev = ub; 1152 return 0; 1153 } 1154 1155 static void ublk_deinit_queues(struct ublk_device *ub) 1156 { 1157 int nr_queues = ub->dev_info.nr_hw_queues; 1158 int i; 1159 1160 if (!ub->__queues) 1161 return; 1162 1163 for (i = 0; i < nr_queues; i++) 1164 ublk_deinit_queue(ub, i); 1165 kfree(ub->__queues); 1166 } 1167 1168 static int ublk_init_queues(struct ublk_device *ub) 1169 { 1170 int nr_queues = ub->dev_info.nr_hw_queues; 1171 int depth = ub->dev_info.queue_depth; 1172 int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io); 1173 int i, ret = -ENOMEM; 1174 1175 ub->queue_size = ubq_size; 1176 ub->__queues = kcalloc(nr_queues, ubq_size, GFP_KERNEL); 1177 if (!ub->__queues) 1178 return ret; 1179 1180 for (i = 0; i < nr_queues; i++) { 1181 if (ublk_init_queue(ub, i)) 1182 goto fail; 1183 } 1184 1185 init_completion(&ub->completion); 1186 return 0; 1187 1188 fail: 1189 ublk_deinit_queues(ub); 1190 return ret; 1191 } 1192 1193 static int ublk_alloc_dev_number(struct ublk_device *ub, int idx) 1194 { 1195 int i = idx; 1196 int err; 1197 1198 spin_lock(&ublk_idr_lock); 1199 /* allocate id, if @id >= 0, we're requesting that specific id */ 1200 if (i >= 0) { 1201 err = idr_alloc(&ublk_index_idr, ub, i, i + 1, GFP_NOWAIT); 1202 if (err == -ENOSPC) 1203 err = -EEXIST; 1204 } else { 1205 err = idr_alloc(&ublk_index_idr, ub, 0, 0, GFP_NOWAIT); 1206 } 1207 spin_unlock(&ublk_idr_lock); 1208 1209 if (err >= 0) 1210 ub->ub_number = err; 1211 1212 return err; 1213 } 1214 1215 static void ublk_free_dev_number(struct ublk_device *ub) 1216 { 1217 spin_lock(&ublk_idr_lock); 1218 idr_remove(&ublk_index_idr, ub->ub_number); 1219 wake_up_all(&ublk_idr_wq); 1220 spin_unlock(&ublk_idr_lock); 1221 } 1222 1223 static void ublk_cdev_rel(struct device *dev) 1224 { 1225 struct ublk_device *ub = container_of(dev, struct ublk_device, cdev_dev); 1226 1227 blk_mq_free_tag_set(&ub->tag_set); 1228 ublk_deinit_queues(ub); 1229 ublk_free_dev_number(ub); 1230 mutex_destroy(&ub->mutex); 1231 kfree(ub); 1232 } 1233 1234 static int ublk_add_chdev(struct ublk_device *ub) 1235 { 1236 struct device *dev = &ub->cdev_dev; 1237 int minor = ub->ub_number; 1238 int ret; 1239 1240 dev->parent = ublk_misc.this_device; 1241 dev->devt = MKDEV(MAJOR(ublk_chr_devt), minor); 1242 dev->class = ublk_chr_class; 1243 dev->release = ublk_cdev_rel; 1244 device_initialize(dev); 1245 1246 ret = dev_set_name(dev, "ublkc%d", minor); 1247 if (ret) 1248 goto fail; 1249 1250 cdev_init(&ub->cdev, &ublk_ch_fops); 1251 ret = cdev_device_add(&ub->cdev, dev); 1252 if (ret) 1253 goto fail; 1254 return 0; 1255 fail: 1256 put_device(dev); 1257 return ret; 1258 } 1259 1260 static void ublk_stop_work_fn(struct work_struct *work) 1261 { 1262 struct ublk_device *ub = 1263 container_of(work, struct ublk_device, stop_work); 1264 1265 ublk_stop_dev(ub); 1266 } 1267 1268 /* align max io buffer size with PAGE_SIZE */ 1269 static void ublk_align_max_io_size(struct ublk_device *ub) 1270 { 1271 unsigned int max_io_bytes = ub->dev_info.max_io_buf_bytes; 1272 1273 ub->dev_info.max_io_buf_bytes = 1274 round_down(max_io_bytes, PAGE_SIZE); 1275 } 1276 1277 static int ublk_add_tag_set(struct ublk_device *ub) 1278 { 1279 ub->tag_set.ops = &ublk_mq_ops; 1280 ub->tag_set.nr_hw_queues = ub->dev_info.nr_hw_queues; 1281 ub->tag_set.queue_depth = ub->dev_info.queue_depth; 1282 ub->tag_set.numa_node = NUMA_NO_NODE; 1283 ub->tag_set.cmd_size = sizeof(struct ublk_rq_data); 1284 ub->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 1285 ub->tag_set.driver_data = ub; 1286 return blk_mq_alloc_tag_set(&ub->tag_set); 1287 } 1288 1289 static void ublk_remove(struct ublk_device *ub) 1290 { 1291 ublk_stop_dev(ub); 1292 cancel_work_sync(&ub->stop_work); 1293 cdev_device_del(&ub->cdev, &ub->cdev_dev); 1294 put_device(&ub->cdev_dev); 1295 } 1296 1297 static struct ublk_device *ublk_get_device_from_id(int idx) 1298 { 1299 struct ublk_device *ub = NULL; 1300 1301 if (idx < 0) 1302 return NULL; 1303 1304 spin_lock(&ublk_idr_lock); 1305 ub = idr_find(&ublk_index_idr, idx); 1306 if (ub) 1307 ub = ublk_get_device(ub); 1308 spin_unlock(&ublk_idr_lock); 1309 1310 return ub; 1311 } 1312 1313 static int ublk_ctrl_start_dev(struct io_uring_cmd *cmd) 1314 { 1315 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1316 int ublksrv_pid = (int)header->data[0]; 1317 struct ublk_device *ub; 1318 struct gendisk *disk; 1319 int ret = -EINVAL; 1320 1321 if (ublksrv_pid <= 0) 1322 return -EINVAL; 1323 1324 ub = ublk_get_device_from_id(header->dev_id); 1325 if (!ub) 1326 return -EINVAL; 1327 1328 wait_for_completion_interruptible(&ub->completion); 1329 1330 schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); 1331 1332 mutex_lock(&ub->mutex); 1333 if (ub->dev_info.state == UBLK_S_DEV_LIVE || 1334 test_bit(UB_STATE_USED, &ub->state)) { 1335 ret = -EEXIST; 1336 goto out_unlock; 1337 } 1338 1339 disk = blk_mq_alloc_disk(&ub->tag_set, ub); 1340 if (IS_ERR(disk)) { 1341 ret = PTR_ERR(disk); 1342 goto out_unlock; 1343 } 1344 sprintf(disk->disk_name, "ublkb%d", ub->ub_number); 1345 disk->fops = &ub_fops; 1346 disk->private_data = ub; 1347 1348 ub->dev_info.ublksrv_pid = ublksrv_pid; 1349 ub->ub_disk = disk; 1350 1351 ret = ublk_apply_params(ub); 1352 if (ret) 1353 goto out_put_disk; 1354 1355 get_device(&ub->cdev_dev); 1356 ret = add_disk(disk); 1357 if (ret) { 1358 /* 1359 * Has to drop the reference since ->free_disk won't be 1360 * called in case of add_disk failure. 1361 */ 1362 ublk_put_device(ub); 1363 goto out_put_disk; 1364 } 1365 set_bit(UB_STATE_USED, &ub->state); 1366 ub->dev_info.state = UBLK_S_DEV_LIVE; 1367 out_put_disk: 1368 if (ret) 1369 put_disk(disk); 1370 out_unlock: 1371 mutex_unlock(&ub->mutex); 1372 ublk_put_device(ub); 1373 return ret; 1374 } 1375 1376 static int ublk_ctrl_get_queue_affinity(struct io_uring_cmd *cmd) 1377 { 1378 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1379 void __user *argp = (void __user *)(unsigned long)header->addr; 1380 struct ublk_device *ub; 1381 cpumask_var_t cpumask; 1382 unsigned long queue; 1383 unsigned int retlen; 1384 unsigned int i; 1385 int ret = -EINVAL; 1386 1387 if (header->len * BITS_PER_BYTE < nr_cpu_ids) 1388 return -EINVAL; 1389 if (header->len & (sizeof(unsigned long)-1)) 1390 return -EINVAL; 1391 if (!header->addr) 1392 return -EINVAL; 1393 1394 ub = ublk_get_device_from_id(header->dev_id); 1395 if (!ub) 1396 return -EINVAL; 1397 1398 queue = header->data[0]; 1399 if (queue >= ub->dev_info.nr_hw_queues) 1400 goto out_put_device; 1401 1402 ret = -ENOMEM; 1403 if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL)) 1404 goto out_put_device; 1405 1406 for_each_possible_cpu(i) { 1407 if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[i] == queue) 1408 cpumask_set_cpu(i, cpumask); 1409 } 1410 1411 ret = -EFAULT; 1412 retlen = min_t(unsigned short, header->len, cpumask_size()); 1413 if (copy_to_user(argp, cpumask, retlen)) 1414 goto out_free_cpumask; 1415 if (retlen != header->len && 1416 clear_user(argp + retlen, header->len - retlen)) 1417 goto out_free_cpumask; 1418 1419 ret = 0; 1420 out_free_cpumask: 1421 free_cpumask_var(cpumask); 1422 out_put_device: 1423 ublk_put_device(ub); 1424 return ret; 1425 } 1426 1427 static inline void ublk_dump_dev_info(struct ublksrv_ctrl_dev_info *info) 1428 { 1429 pr_devel("%s: dev id %d flags %llx\n", __func__, 1430 info->dev_id, info->flags); 1431 pr_devel("\t nr_hw_queues %d queue_depth %d\n", 1432 info->nr_hw_queues, info->queue_depth); 1433 } 1434 1435 static int ublk_ctrl_add_dev(struct io_uring_cmd *cmd) 1436 { 1437 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1438 void __user *argp = (void __user *)(unsigned long)header->addr; 1439 struct ublksrv_ctrl_dev_info info; 1440 struct ublk_device *ub; 1441 int ret = -EINVAL; 1442 1443 if (header->len < sizeof(info) || !header->addr) 1444 return -EINVAL; 1445 if (header->queue_id != (u16)-1) { 1446 pr_warn("%s: queue_id is wrong %x\n", 1447 __func__, header->queue_id); 1448 return -EINVAL; 1449 } 1450 if (copy_from_user(&info, argp, sizeof(info))) 1451 return -EFAULT; 1452 ublk_dump_dev_info(&info); 1453 if (header->dev_id != info.dev_id) { 1454 pr_warn("%s: dev id not match %u %u\n", 1455 __func__, header->dev_id, info.dev_id); 1456 return -EINVAL; 1457 } 1458 1459 ret = mutex_lock_killable(&ublk_ctl_mutex); 1460 if (ret) 1461 return ret; 1462 1463 ret = -ENOMEM; 1464 ub = kzalloc(sizeof(*ub), GFP_KERNEL); 1465 if (!ub) 1466 goto out_unlock; 1467 mutex_init(&ub->mutex); 1468 spin_lock_init(&ub->mm_lock); 1469 INIT_WORK(&ub->stop_work, ublk_stop_work_fn); 1470 INIT_DELAYED_WORK(&ub->monitor_work, ublk_daemon_monitor_work); 1471 1472 ret = ublk_alloc_dev_number(ub, header->dev_id); 1473 if (ret < 0) 1474 goto out_free_ub; 1475 1476 memcpy(&ub->dev_info, &info, sizeof(info)); 1477 1478 /* update device id */ 1479 ub->dev_info.dev_id = ub->ub_number; 1480 1481 /* 1482 * 64bit flags will be copied back to userspace as feature 1483 * negotiation result, so have to clear flags which driver 1484 * doesn't support yet, then userspace can get correct flags 1485 * (features) to handle. 1486 */ 1487 ub->dev_info.flags &= UBLK_F_ALL; 1488 1489 /* We are not ready to support zero copy */ 1490 ub->dev_info.flags &= ~UBLK_F_SUPPORT_ZERO_COPY; 1491 1492 ub->dev_info.nr_hw_queues = min_t(unsigned int, 1493 ub->dev_info.nr_hw_queues, nr_cpu_ids); 1494 ublk_align_max_io_size(ub); 1495 1496 ret = ublk_init_queues(ub); 1497 if (ret) 1498 goto out_free_dev_number; 1499 1500 ret = ublk_add_tag_set(ub); 1501 if (ret) 1502 goto out_deinit_queues; 1503 1504 ret = -EFAULT; 1505 if (copy_to_user(argp, &ub->dev_info, sizeof(info))) 1506 goto out_free_tag_set; 1507 1508 /* 1509 * Add the char dev so that ublksrv daemon can be setup. 1510 * ublk_add_chdev() will cleanup everything if it fails. 1511 */ 1512 ret = ublk_add_chdev(ub); 1513 goto out_unlock; 1514 1515 out_free_tag_set: 1516 blk_mq_free_tag_set(&ub->tag_set); 1517 out_deinit_queues: 1518 ublk_deinit_queues(ub); 1519 out_free_dev_number: 1520 ublk_free_dev_number(ub); 1521 out_free_ub: 1522 mutex_destroy(&ub->mutex); 1523 kfree(ub); 1524 out_unlock: 1525 mutex_unlock(&ublk_ctl_mutex); 1526 return ret; 1527 } 1528 1529 static inline bool ublk_idr_freed(int id) 1530 { 1531 void *ptr; 1532 1533 spin_lock(&ublk_idr_lock); 1534 ptr = idr_find(&ublk_index_idr, id); 1535 spin_unlock(&ublk_idr_lock); 1536 1537 return ptr == NULL; 1538 } 1539 1540 static int ublk_ctrl_del_dev(int idx) 1541 { 1542 struct ublk_device *ub; 1543 int ret; 1544 1545 ret = mutex_lock_killable(&ublk_ctl_mutex); 1546 if (ret) 1547 return ret; 1548 1549 ub = ublk_get_device_from_id(idx); 1550 if (ub) { 1551 ublk_remove(ub); 1552 ublk_put_device(ub); 1553 ret = 0; 1554 } else { 1555 ret = -ENODEV; 1556 } 1557 1558 /* 1559 * Wait until the idr is removed, then it can be reused after 1560 * DEL_DEV command is returned. 1561 */ 1562 if (!ret) 1563 wait_event(ublk_idr_wq, ublk_idr_freed(idx)); 1564 mutex_unlock(&ublk_ctl_mutex); 1565 1566 return ret; 1567 } 1568 1569 static inline void ublk_ctrl_cmd_dump(struct io_uring_cmd *cmd) 1570 { 1571 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1572 1573 pr_devel("%s: cmd_op %x, dev id %d qid %d data %llx buf %llx len %u\n", 1574 __func__, cmd->cmd_op, header->dev_id, header->queue_id, 1575 header->data[0], header->addr, header->len); 1576 } 1577 1578 static int ublk_ctrl_stop_dev(struct io_uring_cmd *cmd) 1579 { 1580 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1581 struct ublk_device *ub; 1582 1583 ub = ublk_get_device_from_id(header->dev_id); 1584 if (!ub) 1585 return -EINVAL; 1586 1587 ublk_stop_dev(ub); 1588 cancel_work_sync(&ub->stop_work); 1589 1590 ublk_put_device(ub); 1591 return 0; 1592 } 1593 1594 static int ublk_ctrl_get_dev_info(struct io_uring_cmd *cmd) 1595 { 1596 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1597 void __user *argp = (void __user *)(unsigned long)header->addr; 1598 struct ublk_device *ub; 1599 int ret = 0; 1600 1601 if (header->len < sizeof(struct ublksrv_ctrl_dev_info) || !header->addr) 1602 return -EINVAL; 1603 1604 ub = ublk_get_device_from_id(header->dev_id); 1605 if (!ub) 1606 return -EINVAL; 1607 1608 if (copy_to_user(argp, &ub->dev_info, sizeof(ub->dev_info))) 1609 ret = -EFAULT; 1610 ublk_put_device(ub); 1611 1612 return ret; 1613 } 1614 1615 static int ublk_ctrl_get_params(struct io_uring_cmd *cmd) 1616 { 1617 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1618 void __user *argp = (void __user *)(unsigned long)header->addr; 1619 struct ublk_params_header ph; 1620 struct ublk_device *ub; 1621 int ret; 1622 1623 if (header->len <= sizeof(ph) || !header->addr) 1624 return -EINVAL; 1625 1626 if (copy_from_user(&ph, argp, sizeof(ph))) 1627 return -EFAULT; 1628 1629 if (ph.len > header->len || !ph.len) 1630 return -EINVAL; 1631 1632 if (ph.len > sizeof(struct ublk_params)) 1633 ph.len = sizeof(struct ublk_params); 1634 1635 ub = ublk_get_device_from_id(header->dev_id); 1636 if (!ub) 1637 return -EINVAL; 1638 1639 mutex_lock(&ub->mutex); 1640 if (copy_to_user(argp, &ub->params, ph.len)) 1641 ret = -EFAULT; 1642 else 1643 ret = 0; 1644 mutex_unlock(&ub->mutex); 1645 1646 ublk_put_device(ub); 1647 return ret; 1648 } 1649 1650 static int ublk_ctrl_set_params(struct io_uring_cmd *cmd) 1651 { 1652 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1653 void __user *argp = (void __user *)(unsigned long)header->addr; 1654 struct ublk_params_header ph; 1655 struct ublk_device *ub; 1656 int ret = -EFAULT; 1657 1658 if (header->len <= sizeof(ph) || !header->addr) 1659 return -EINVAL; 1660 1661 if (copy_from_user(&ph, argp, sizeof(ph))) 1662 return -EFAULT; 1663 1664 if (ph.len > header->len || !ph.len || !ph.types) 1665 return -EINVAL; 1666 1667 if (ph.len > sizeof(struct ublk_params)) 1668 ph.len = sizeof(struct ublk_params); 1669 1670 ub = ublk_get_device_from_id(header->dev_id); 1671 if (!ub) 1672 return -EINVAL; 1673 1674 /* parameters can only be changed when device isn't live */ 1675 mutex_lock(&ub->mutex); 1676 if (ub->dev_info.state == UBLK_S_DEV_LIVE) { 1677 ret = -EACCES; 1678 } else if (copy_from_user(&ub->params, argp, ph.len)) { 1679 ret = -EFAULT; 1680 } else { 1681 /* clear all we don't support yet */ 1682 ub->params.types &= UBLK_PARAM_TYPE_ALL; 1683 ret = ublk_validate_params(ub); 1684 } 1685 mutex_unlock(&ub->mutex); 1686 ublk_put_device(ub); 1687 1688 return ret; 1689 } 1690 1691 static int ublk_ctrl_uring_cmd(struct io_uring_cmd *cmd, 1692 unsigned int issue_flags) 1693 { 1694 struct ublksrv_ctrl_cmd *header = (struct ublksrv_ctrl_cmd *)cmd->cmd; 1695 int ret = -EINVAL; 1696 1697 ublk_ctrl_cmd_dump(cmd); 1698 1699 if (!(issue_flags & IO_URING_F_SQE128)) 1700 goto out; 1701 1702 ret = -EPERM; 1703 if (!capable(CAP_SYS_ADMIN)) 1704 goto out; 1705 1706 ret = -ENODEV; 1707 switch (cmd->cmd_op) { 1708 case UBLK_CMD_START_DEV: 1709 ret = ublk_ctrl_start_dev(cmd); 1710 break; 1711 case UBLK_CMD_STOP_DEV: 1712 ret = ublk_ctrl_stop_dev(cmd); 1713 break; 1714 case UBLK_CMD_GET_DEV_INFO: 1715 ret = ublk_ctrl_get_dev_info(cmd); 1716 break; 1717 case UBLK_CMD_ADD_DEV: 1718 ret = ublk_ctrl_add_dev(cmd); 1719 break; 1720 case UBLK_CMD_DEL_DEV: 1721 ret = ublk_ctrl_del_dev(header->dev_id); 1722 break; 1723 case UBLK_CMD_GET_QUEUE_AFFINITY: 1724 ret = ublk_ctrl_get_queue_affinity(cmd); 1725 break; 1726 case UBLK_CMD_GET_PARAMS: 1727 ret = ublk_ctrl_get_params(cmd); 1728 break; 1729 case UBLK_CMD_SET_PARAMS: 1730 ret = ublk_ctrl_set_params(cmd); 1731 break; 1732 default: 1733 break; 1734 } 1735 out: 1736 io_uring_cmd_done(cmd, ret, 0); 1737 pr_devel("%s: cmd done ret %d cmd_op %x, dev id %d qid %d\n", 1738 __func__, ret, cmd->cmd_op, header->dev_id, header->queue_id); 1739 return -EIOCBQUEUED; 1740 } 1741 1742 static const struct file_operations ublk_ctl_fops = { 1743 .open = nonseekable_open, 1744 .uring_cmd = ublk_ctrl_uring_cmd, 1745 .owner = THIS_MODULE, 1746 .llseek = noop_llseek, 1747 }; 1748 1749 static struct miscdevice ublk_misc = { 1750 .minor = MISC_DYNAMIC_MINOR, 1751 .name = "ublk-control", 1752 .fops = &ublk_ctl_fops, 1753 }; 1754 1755 static int __init ublk_init(void) 1756 { 1757 int ret; 1758 1759 init_waitqueue_head(&ublk_idr_wq); 1760 1761 ret = misc_register(&ublk_misc); 1762 if (ret) 1763 return ret; 1764 1765 ret = alloc_chrdev_region(&ublk_chr_devt, 0, UBLK_MINORS, "ublk-char"); 1766 if (ret) 1767 goto unregister_mis; 1768 1769 ublk_chr_class = class_create(THIS_MODULE, "ublk-char"); 1770 if (IS_ERR(ublk_chr_class)) { 1771 ret = PTR_ERR(ublk_chr_class); 1772 goto free_chrdev_region; 1773 } 1774 return 0; 1775 1776 free_chrdev_region: 1777 unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS); 1778 unregister_mis: 1779 misc_deregister(&ublk_misc); 1780 return ret; 1781 } 1782 1783 static void __exit ublk_exit(void) 1784 { 1785 struct ublk_device *ub; 1786 int id; 1787 1788 class_destroy(ublk_chr_class); 1789 1790 misc_deregister(&ublk_misc); 1791 1792 idr_for_each_entry(&ublk_index_idr, ub, id) 1793 ublk_remove(ub); 1794 1795 idr_destroy(&ublk_index_idr); 1796 unregister_chrdev_region(ublk_chr_devt, UBLK_MINORS); 1797 } 1798 1799 module_init(ublk_init); 1800 module_exit(ublk_exit); 1801 1802 MODULE_AUTHOR("Ming Lei <ming.lei@redhat.com>"); 1803 MODULE_LICENSE("GPL"); 1804