1 /* 2 * Network block device - make block devices work over TCP 3 * 4 * Note that you can not swap over this thing, yet. Seems to work but 5 * deadlocks sometimes - you can not swap over TCP in general. 6 * 7 * Copyright 1997-2000, 2008 Pavel Machek <pavel@ucw.cz> 8 * Parts copyright 2001 Steven Whitehouse <steve@chygwyn.com> 9 * 10 * This file is released under GPLv2 or later. 11 * 12 * (part of code stolen from loop.c) 13 */ 14 15 #include <linux/major.h> 16 17 #include <linux/blkdev.h> 18 #include <linux/module.h> 19 #include <linux/init.h> 20 #include <linux/sched.h> 21 #include <linux/fs.h> 22 #include <linux/bio.h> 23 #include <linux/stat.h> 24 #include <linux/errno.h> 25 #include <linux/file.h> 26 #include <linux/ioctl.h> 27 #include <linux/mutex.h> 28 #include <linux/compiler.h> 29 #include <linux/err.h> 30 #include <linux/kernel.h> 31 #include <linux/slab.h> 32 #include <net/sock.h> 33 #include <linux/net.h> 34 #include <linux/kthread.h> 35 #include <linux/types.h> 36 #include <linux/debugfs.h> 37 #include <linux/blk-mq.h> 38 39 #include <linux/uaccess.h> 40 #include <asm/types.h> 41 42 #include <linux/nbd.h> 43 44 static DEFINE_IDR(nbd_index_idr); 45 static DEFINE_MUTEX(nbd_index_mutex); 46 47 struct nbd_sock { 48 struct socket *sock; 49 struct mutex tx_lock; 50 struct request *pending; 51 int sent; 52 }; 53 54 #define NBD_TIMEDOUT 0 55 #define NBD_DISCONNECT_REQUESTED 1 56 #define NBD_DISCONNECTED 2 57 #define NBD_RUNNING 3 58 59 struct nbd_device { 60 u32 flags; 61 unsigned long runtime_flags; 62 struct nbd_sock **socks; 63 int magic; 64 65 struct blk_mq_tag_set tag_set; 66 67 struct mutex config_lock; 68 struct gendisk *disk; 69 int num_connections; 70 atomic_t recv_threads; 71 wait_queue_head_t recv_wq; 72 loff_t blksize; 73 loff_t bytesize; 74 75 struct task_struct *task_recv; 76 struct task_struct *task_setup; 77 78 #if IS_ENABLED(CONFIG_DEBUG_FS) 79 struct dentry *dbg_dir; 80 #endif 81 }; 82 83 struct nbd_cmd { 84 struct nbd_device *nbd; 85 struct completion send_complete; 86 }; 87 88 #if IS_ENABLED(CONFIG_DEBUG_FS) 89 static struct dentry *nbd_dbg_dir; 90 #endif 91 92 #define nbd_name(nbd) ((nbd)->disk->disk_name) 93 94 #define NBD_MAGIC 0x68797548 95 96 static unsigned int nbds_max = 16; 97 static int max_part; 98 static struct workqueue_struct *recv_workqueue; 99 static int part_shift; 100 101 static int nbd_dev_dbg_init(struct nbd_device *nbd); 102 static void nbd_dev_dbg_close(struct nbd_device *nbd); 103 104 105 static inline struct device *nbd_to_dev(struct nbd_device *nbd) 106 { 107 return disk_to_dev(nbd->disk); 108 } 109 110 static bool nbd_is_connected(struct nbd_device *nbd) 111 { 112 return !!nbd->task_recv; 113 } 114 115 static const char *nbdcmd_to_ascii(int cmd) 116 { 117 switch (cmd) { 118 case NBD_CMD_READ: return "read"; 119 case NBD_CMD_WRITE: return "write"; 120 case NBD_CMD_DISC: return "disconnect"; 121 case NBD_CMD_FLUSH: return "flush"; 122 case NBD_CMD_TRIM: return "trim/discard"; 123 } 124 return "invalid"; 125 } 126 127 static int nbd_size_clear(struct nbd_device *nbd, struct block_device *bdev) 128 { 129 if (bdev->bd_openers <= 1) 130 bd_set_size(bdev, 0); 131 set_capacity(nbd->disk, 0); 132 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); 133 134 return 0; 135 } 136 137 static void nbd_size_update(struct nbd_device *nbd, struct block_device *bdev) 138 { 139 blk_queue_logical_block_size(nbd->disk->queue, nbd->blksize); 140 blk_queue_physical_block_size(nbd->disk->queue, nbd->blksize); 141 bd_set_size(bdev, nbd->bytesize); 142 set_capacity(nbd->disk, nbd->bytesize >> 9); 143 kobject_uevent(&nbd_to_dev(nbd)->kobj, KOBJ_CHANGE); 144 } 145 146 static void nbd_size_set(struct nbd_device *nbd, struct block_device *bdev, 147 loff_t blocksize, loff_t nr_blocks) 148 { 149 nbd->blksize = blocksize; 150 nbd->bytesize = blocksize * nr_blocks; 151 if (nbd_is_connected(nbd)) 152 nbd_size_update(nbd, bdev); 153 } 154 155 static void nbd_end_request(struct nbd_cmd *cmd) 156 { 157 struct nbd_device *nbd = cmd->nbd; 158 struct request *req = blk_mq_rq_from_pdu(cmd); 159 int error = req->errors ? -EIO : 0; 160 161 dev_dbg(nbd_to_dev(nbd), "request %p: %s\n", cmd, 162 error ? "failed" : "done"); 163 164 blk_mq_complete_request(req, error); 165 } 166 167 /* 168 * Forcibly shutdown the socket causing all listeners to error 169 */ 170 static void sock_shutdown(struct nbd_device *nbd) 171 { 172 int i; 173 174 if (nbd->num_connections == 0) 175 return; 176 if (test_and_set_bit(NBD_DISCONNECTED, &nbd->runtime_flags)) 177 return; 178 179 for (i = 0; i < nbd->num_connections; i++) { 180 struct nbd_sock *nsock = nbd->socks[i]; 181 mutex_lock(&nsock->tx_lock); 182 kernel_sock_shutdown(nsock->sock, SHUT_RDWR); 183 mutex_unlock(&nsock->tx_lock); 184 } 185 dev_warn(disk_to_dev(nbd->disk), "shutting down sockets\n"); 186 } 187 188 static enum blk_eh_timer_return nbd_xmit_timeout(struct request *req, 189 bool reserved) 190 { 191 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(req); 192 struct nbd_device *nbd = cmd->nbd; 193 194 dev_err(nbd_to_dev(nbd), "Connection timed out, shutting down connection\n"); 195 set_bit(NBD_TIMEDOUT, &nbd->runtime_flags); 196 req->errors = -EIO; 197 198 mutex_lock(&nbd->config_lock); 199 sock_shutdown(nbd); 200 mutex_unlock(&nbd->config_lock); 201 return BLK_EH_HANDLED; 202 } 203 204 /* 205 * Send or receive packet. 206 */ 207 static int sock_xmit(struct nbd_device *nbd, int index, int send, 208 struct iov_iter *iter, int msg_flags, int *sent) 209 { 210 struct socket *sock = nbd->socks[index]->sock; 211 int result; 212 struct msghdr msg; 213 unsigned long pflags = current->flags; 214 215 if (unlikely(!sock)) { 216 dev_err_ratelimited(disk_to_dev(nbd->disk), 217 "Attempted %s on closed socket in sock_xmit\n", 218 (send ? "send" : "recv")); 219 return -EINVAL; 220 } 221 222 msg.msg_iter = *iter; 223 224 current->flags |= PF_MEMALLOC; 225 do { 226 sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; 227 msg.msg_name = NULL; 228 msg.msg_namelen = 0; 229 msg.msg_control = NULL; 230 msg.msg_controllen = 0; 231 msg.msg_flags = msg_flags | MSG_NOSIGNAL; 232 233 if (send) 234 result = sock_sendmsg(sock, &msg); 235 else 236 result = sock_recvmsg(sock, &msg, msg.msg_flags); 237 238 if (result <= 0) { 239 if (result == 0) 240 result = -EPIPE; /* short read */ 241 break; 242 } 243 if (sent) 244 *sent += result; 245 } while (msg_data_left(&msg)); 246 247 tsk_restore_flags(current, pflags, PF_MEMALLOC); 248 249 return result; 250 } 251 252 /* always call with the tx_lock held */ 253 static int nbd_send_cmd(struct nbd_device *nbd, struct nbd_cmd *cmd, int index) 254 { 255 struct request *req = blk_mq_rq_from_pdu(cmd); 256 struct nbd_sock *nsock = nbd->socks[index]; 257 int result; 258 struct nbd_request request = {.magic = htonl(NBD_REQUEST_MAGIC)}; 259 struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)}; 260 struct iov_iter from; 261 unsigned long size = blk_rq_bytes(req); 262 struct bio *bio; 263 u32 type; 264 u32 tag = blk_mq_unique_tag(req); 265 int sent = nsock->sent, skip = 0; 266 267 iov_iter_kvec(&from, WRITE | ITER_KVEC, &iov, 1, sizeof(request)); 268 269 switch (req_op(req)) { 270 case REQ_OP_DISCARD: 271 type = NBD_CMD_TRIM; 272 break; 273 case REQ_OP_FLUSH: 274 type = NBD_CMD_FLUSH; 275 break; 276 case REQ_OP_WRITE: 277 type = NBD_CMD_WRITE; 278 break; 279 case REQ_OP_READ: 280 type = NBD_CMD_READ; 281 break; 282 default: 283 return -EIO; 284 } 285 286 if (rq_data_dir(req) == WRITE && 287 (nbd->flags & NBD_FLAG_READ_ONLY)) { 288 dev_err_ratelimited(disk_to_dev(nbd->disk), 289 "Write on read-only\n"); 290 return -EIO; 291 } 292 293 /* We did a partial send previously, and we at least sent the whole 294 * request struct, so just go and send the rest of the pages in the 295 * request. 296 */ 297 if (sent) { 298 if (sent >= sizeof(request)) { 299 skip = sent - sizeof(request); 300 goto send_pages; 301 } 302 iov_iter_advance(&from, sent); 303 } 304 request.type = htonl(type); 305 if (type != NBD_CMD_FLUSH) { 306 request.from = cpu_to_be64((u64)blk_rq_pos(req) << 9); 307 request.len = htonl(size); 308 } 309 memcpy(request.handle, &tag, sizeof(tag)); 310 311 dev_dbg(nbd_to_dev(nbd), "request %p: sending control (%s@%llu,%uB)\n", 312 cmd, nbdcmd_to_ascii(type), 313 (unsigned long long)blk_rq_pos(req) << 9, blk_rq_bytes(req)); 314 result = sock_xmit(nbd, index, 1, &from, 315 (type == NBD_CMD_WRITE) ? MSG_MORE : 0, &sent); 316 if (result <= 0) { 317 if (result == -ERESTARTSYS) { 318 /* If we havne't sent anything we can just return BUSY, 319 * however if we have sent something we need to make 320 * sure we only allow this req to be sent until we are 321 * completely done. 322 */ 323 if (sent) { 324 nsock->pending = req; 325 nsock->sent = sent; 326 } 327 return BLK_MQ_RQ_QUEUE_BUSY; 328 } 329 dev_err_ratelimited(disk_to_dev(nbd->disk), 330 "Send control failed (result %d)\n", result); 331 return -EIO; 332 } 333 send_pages: 334 if (type != NBD_CMD_WRITE) 335 goto out; 336 337 bio = req->bio; 338 while (bio) { 339 struct bio *next = bio->bi_next; 340 struct bvec_iter iter; 341 struct bio_vec bvec; 342 343 bio_for_each_segment(bvec, bio, iter) { 344 bool is_last = !next && bio_iter_last(bvec, iter); 345 int flags = is_last ? 0 : MSG_MORE; 346 347 dev_dbg(nbd_to_dev(nbd), "request %p: sending %d bytes data\n", 348 cmd, bvec.bv_len); 349 iov_iter_bvec(&from, ITER_BVEC | WRITE, 350 &bvec, 1, bvec.bv_len); 351 if (skip) { 352 if (skip >= iov_iter_count(&from)) { 353 skip -= iov_iter_count(&from); 354 continue; 355 } 356 iov_iter_advance(&from, skip); 357 skip = 0; 358 } 359 result = sock_xmit(nbd, index, 1, &from, flags, &sent); 360 if (result <= 0) { 361 if (result == -ERESTARTSYS) { 362 /* We've already sent the header, we 363 * have no choice but to set pending and 364 * return BUSY. 365 */ 366 nsock->pending = req; 367 nsock->sent = sent; 368 return BLK_MQ_RQ_QUEUE_BUSY; 369 } 370 dev_err(disk_to_dev(nbd->disk), 371 "Send data failed (result %d)\n", 372 result); 373 return -EIO; 374 } 375 /* 376 * The completion might already have come in, 377 * so break for the last one instead of letting 378 * the iterator do it. This prevents use-after-free 379 * of the bio. 380 */ 381 if (is_last) 382 break; 383 } 384 bio = next; 385 } 386 out: 387 nsock->pending = NULL; 388 nsock->sent = 0; 389 return 0; 390 } 391 392 /* NULL returned = something went wrong, inform userspace */ 393 static struct nbd_cmd *nbd_read_stat(struct nbd_device *nbd, int index) 394 { 395 int result; 396 struct nbd_reply reply; 397 struct nbd_cmd *cmd; 398 struct request *req = NULL; 399 u16 hwq; 400 u32 tag; 401 struct kvec iov = {.iov_base = &reply, .iov_len = sizeof(reply)}; 402 struct iov_iter to; 403 404 reply.magic = 0; 405 iov_iter_kvec(&to, READ | ITER_KVEC, &iov, 1, sizeof(reply)); 406 result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); 407 if (result <= 0) { 408 if (!test_bit(NBD_DISCONNECTED, &nbd->runtime_flags) && 409 !test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags)) 410 dev_err(disk_to_dev(nbd->disk), 411 "Receive control failed (result %d)\n", result); 412 return ERR_PTR(result); 413 } 414 415 if (ntohl(reply.magic) != NBD_REPLY_MAGIC) { 416 dev_err(disk_to_dev(nbd->disk), "Wrong magic (0x%lx)\n", 417 (unsigned long)ntohl(reply.magic)); 418 return ERR_PTR(-EPROTO); 419 } 420 421 memcpy(&tag, reply.handle, sizeof(u32)); 422 423 hwq = blk_mq_unique_tag_to_hwq(tag); 424 if (hwq < nbd->tag_set.nr_hw_queues) 425 req = blk_mq_tag_to_rq(nbd->tag_set.tags[hwq], 426 blk_mq_unique_tag_to_tag(tag)); 427 if (!req || !blk_mq_request_started(req)) { 428 dev_err(disk_to_dev(nbd->disk), "Unexpected reply (%d) %p\n", 429 tag, req); 430 return ERR_PTR(-ENOENT); 431 } 432 cmd = blk_mq_rq_to_pdu(req); 433 if (ntohl(reply.error)) { 434 dev_err(disk_to_dev(nbd->disk), "Other side returned error (%d)\n", 435 ntohl(reply.error)); 436 req->errors = -EIO; 437 return cmd; 438 } 439 440 dev_dbg(nbd_to_dev(nbd), "request %p: got reply\n", cmd); 441 if (rq_data_dir(req) != WRITE) { 442 struct req_iterator iter; 443 struct bio_vec bvec; 444 445 rq_for_each_segment(bvec, req, iter) { 446 iov_iter_bvec(&to, ITER_BVEC | READ, 447 &bvec, 1, bvec.bv_len); 448 result = sock_xmit(nbd, index, 0, &to, MSG_WAITALL, NULL); 449 if (result <= 0) { 450 dev_err(disk_to_dev(nbd->disk), "Receive data failed (result %d)\n", 451 result); 452 req->errors = -EIO; 453 return cmd; 454 } 455 dev_dbg(nbd_to_dev(nbd), "request %p: got %d bytes data\n", 456 cmd, bvec.bv_len); 457 } 458 } else { 459 /* See the comment in nbd_queue_rq. */ 460 wait_for_completion(&cmd->send_complete); 461 } 462 return cmd; 463 } 464 465 static ssize_t pid_show(struct device *dev, 466 struct device_attribute *attr, char *buf) 467 { 468 struct gendisk *disk = dev_to_disk(dev); 469 struct nbd_device *nbd = (struct nbd_device *)disk->private_data; 470 471 return sprintf(buf, "%d\n", task_pid_nr(nbd->task_recv)); 472 } 473 474 static struct device_attribute pid_attr = { 475 .attr = { .name = "pid", .mode = S_IRUGO}, 476 .show = pid_show, 477 }; 478 479 struct recv_thread_args { 480 struct work_struct work; 481 struct nbd_device *nbd; 482 int index; 483 }; 484 485 static void recv_work(struct work_struct *work) 486 { 487 struct recv_thread_args *args = container_of(work, 488 struct recv_thread_args, 489 work); 490 struct nbd_device *nbd = args->nbd; 491 struct nbd_cmd *cmd; 492 int ret = 0; 493 494 BUG_ON(nbd->magic != NBD_MAGIC); 495 while (1) { 496 cmd = nbd_read_stat(nbd, args->index); 497 if (IS_ERR(cmd)) { 498 ret = PTR_ERR(cmd); 499 break; 500 } 501 502 nbd_end_request(cmd); 503 } 504 505 /* 506 * We got an error, shut everybody down if this wasn't the result of a 507 * disconnect request. 508 */ 509 if (ret && !test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags)) 510 sock_shutdown(nbd); 511 atomic_dec(&nbd->recv_threads); 512 wake_up(&nbd->recv_wq); 513 } 514 515 static void nbd_clear_req(struct request *req, void *data, bool reserved) 516 { 517 struct nbd_cmd *cmd; 518 519 if (!blk_mq_request_started(req)) 520 return; 521 cmd = blk_mq_rq_to_pdu(req); 522 req->errors = -EIO; 523 nbd_end_request(cmd); 524 } 525 526 static void nbd_clear_que(struct nbd_device *nbd) 527 { 528 BUG_ON(nbd->magic != NBD_MAGIC); 529 530 blk_mq_tagset_busy_iter(&nbd->tag_set, nbd_clear_req, NULL); 531 dev_dbg(disk_to_dev(nbd->disk), "queue cleared\n"); 532 } 533 534 535 static int nbd_handle_cmd(struct nbd_cmd *cmd, int index) 536 { 537 struct request *req = blk_mq_rq_from_pdu(cmd); 538 struct nbd_device *nbd = cmd->nbd; 539 struct nbd_sock *nsock; 540 int ret; 541 542 if (index >= nbd->num_connections) { 543 dev_err_ratelimited(disk_to_dev(nbd->disk), 544 "Attempted send on invalid socket\n"); 545 return -EINVAL; 546 } 547 548 if (test_bit(NBD_DISCONNECTED, &nbd->runtime_flags)) { 549 dev_err_ratelimited(disk_to_dev(nbd->disk), 550 "Attempted send on closed socket\n"); 551 return -EINVAL; 552 } 553 554 req->errors = 0; 555 556 nsock = nbd->socks[index]; 557 mutex_lock(&nsock->tx_lock); 558 if (unlikely(!nsock->sock)) { 559 mutex_unlock(&nsock->tx_lock); 560 dev_err_ratelimited(disk_to_dev(nbd->disk), 561 "Attempted send on closed socket\n"); 562 return -EINVAL; 563 } 564 565 /* Handle the case that we have a pending request that was partially 566 * transmitted that _has_ to be serviced first. We need to call requeue 567 * here so that it gets put _after_ the request that is already on the 568 * dispatch list. 569 */ 570 if (unlikely(nsock->pending && nsock->pending != req)) { 571 blk_mq_requeue_request(req, true); 572 ret = 0; 573 goto out; 574 } 575 ret = nbd_send_cmd(nbd, cmd, index); 576 out: 577 mutex_unlock(&nsock->tx_lock); 578 return ret; 579 } 580 581 static int nbd_queue_rq(struct blk_mq_hw_ctx *hctx, 582 const struct blk_mq_queue_data *bd) 583 { 584 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); 585 int ret; 586 587 /* 588 * Since we look at the bio's to send the request over the network we 589 * need to make sure the completion work doesn't mark this request done 590 * before we are done doing our send. This keeps us from dereferencing 591 * freed data if we have particularly fast completions (ie we get the 592 * completion before we exit sock_xmit on the last bvec) or in the case 593 * that the server is misbehaving (or there was an error) before we're 594 * done sending everything over the wire. 595 */ 596 init_completion(&cmd->send_complete); 597 blk_mq_start_request(bd->rq); 598 599 /* We can be called directly from the user space process, which means we 600 * could possibly have signals pending so our sendmsg will fail. In 601 * this case we need to return that we are busy, otherwise error out as 602 * appropriate. 603 */ 604 ret = nbd_handle_cmd(cmd, hctx->queue_num); 605 if (ret < 0) 606 ret = BLK_MQ_RQ_QUEUE_ERROR; 607 if (!ret) 608 ret = BLK_MQ_RQ_QUEUE_OK; 609 complete(&cmd->send_complete); 610 611 return ret; 612 } 613 614 static int nbd_add_socket(struct nbd_device *nbd, struct block_device *bdev, 615 unsigned long arg) 616 { 617 struct socket *sock; 618 struct nbd_sock **socks; 619 struct nbd_sock *nsock; 620 int err; 621 622 sock = sockfd_lookup(arg, &err); 623 if (!sock) 624 return err; 625 626 if (!nbd->task_setup) 627 nbd->task_setup = current; 628 if (nbd->task_setup != current) { 629 dev_err(disk_to_dev(nbd->disk), 630 "Device being setup by another task"); 631 return -EINVAL; 632 } 633 634 socks = krealloc(nbd->socks, (nbd->num_connections + 1) * 635 sizeof(struct nbd_sock *), GFP_KERNEL); 636 if (!socks) 637 return -ENOMEM; 638 nsock = kzalloc(sizeof(struct nbd_sock), GFP_KERNEL); 639 if (!nsock) 640 return -ENOMEM; 641 642 nbd->socks = socks; 643 644 mutex_init(&nsock->tx_lock); 645 nsock->sock = sock; 646 nsock->pending = NULL; 647 nsock->sent = 0; 648 socks[nbd->num_connections++] = nsock; 649 650 if (max_part) 651 bdev->bd_invalidated = 1; 652 return 0; 653 } 654 655 /* Reset all properties of an NBD device */ 656 static void nbd_reset(struct nbd_device *nbd) 657 { 658 nbd->runtime_flags = 0; 659 nbd->blksize = 1024; 660 nbd->bytesize = 0; 661 set_capacity(nbd->disk, 0); 662 nbd->flags = 0; 663 nbd->tag_set.timeout = 0; 664 queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue); 665 } 666 667 static void nbd_bdev_reset(struct block_device *bdev) 668 { 669 if (bdev->bd_openers > 1) 670 return; 671 set_device_ro(bdev, false); 672 bdev->bd_inode->i_size = 0; 673 if (max_part > 0) { 674 blkdev_reread_part(bdev); 675 bdev->bd_invalidated = 1; 676 } 677 } 678 679 static void nbd_parse_flags(struct nbd_device *nbd, struct block_device *bdev) 680 { 681 if (nbd->flags & NBD_FLAG_READ_ONLY) 682 set_device_ro(bdev, true); 683 if (nbd->flags & NBD_FLAG_SEND_TRIM) 684 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, nbd->disk->queue); 685 if (nbd->flags & NBD_FLAG_SEND_FLUSH) 686 blk_queue_write_cache(nbd->disk->queue, true, false); 687 else 688 blk_queue_write_cache(nbd->disk->queue, false, false); 689 } 690 691 static void send_disconnects(struct nbd_device *nbd) 692 { 693 struct nbd_request request = { 694 .magic = htonl(NBD_REQUEST_MAGIC), 695 .type = htonl(NBD_CMD_DISC), 696 }; 697 struct kvec iov = {.iov_base = &request, .iov_len = sizeof(request)}; 698 struct iov_iter from; 699 int i, ret; 700 701 for (i = 0; i < nbd->num_connections; i++) { 702 iov_iter_kvec(&from, WRITE | ITER_KVEC, &iov, 1, sizeof(request)); 703 ret = sock_xmit(nbd, i, 1, &from, 0, NULL); 704 if (ret <= 0) 705 dev_err(disk_to_dev(nbd->disk), 706 "Send disconnect failed %d\n", ret); 707 } 708 } 709 710 static int nbd_disconnect(struct nbd_device *nbd, struct block_device *bdev) 711 { 712 dev_info(disk_to_dev(nbd->disk), "NBD_DISCONNECT\n"); 713 if (!nbd->socks) 714 return -EINVAL; 715 716 mutex_unlock(&nbd->config_lock); 717 fsync_bdev(bdev); 718 mutex_lock(&nbd->config_lock); 719 720 /* Check again after getting mutex back. */ 721 if (!nbd->socks) 722 return -EINVAL; 723 724 if (!test_and_set_bit(NBD_DISCONNECT_REQUESTED, 725 &nbd->runtime_flags)) 726 send_disconnects(nbd); 727 return 0; 728 } 729 730 static int nbd_clear_sock(struct nbd_device *nbd, struct block_device *bdev) 731 { 732 sock_shutdown(nbd); 733 nbd_clear_que(nbd); 734 735 __invalidate_device(bdev, true); 736 nbd_bdev_reset(bdev); 737 /* 738 * We want to give the run thread a chance to wait for everybody 739 * to clean up and then do it's own cleanup. 740 */ 741 if (!test_bit(NBD_RUNNING, &nbd->runtime_flags) && 742 nbd->num_connections) { 743 int i; 744 745 for (i = 0; i < nbd->num_connections; i++) { 746 sockfd_put(nbd->socks[i]->sock); 747 kfree(nbd->socks[i]); 748 } 749 kfree(nbd->socks); 750 nbd->socks = NULL; 751 nbd->num_connections = 0; 752 } 753 nbd->task_setup = NULL; 754 755 return 0; 756 } 757 758 static int nbd_start_device(struct nbd_device *nbd, struct block_device *bdev) 759 { 760 struct recv_thread_args *args; 761 int num_connections = nbd->num_connections; 762 int error = 0, i; 763 764 if (nbd->task_recv) 765 return -EBUSY; 766 if (!nbd->socks) 767 return -EINVAL; 768 if (num_connections > 1 && 769 !(nbd->flags & NBD_FLAG_CAN_MULTI_CONN)) { 770 dev_err(disk_to_dev(nbd->disk), "server does not support multiple connections per device.\n"); 771 error = -EINVAL; 772 goto out_err; 773 } 774 775 set_bit(NBD_RUNNING, &nbd->runtime_flags); 776 blk_mq_update_nr_hw_queues(&nbd->tag_set, nbd->num_connections); 777 args = kcalloc(num_connections, sizeof(*args), GFP_KERNEL); 778 if (!args) { 779 error = -ENOMEM; 780 goto out_err; 781 } 782 nbd->task_recv = current; 783 mutex_unlock(&nbd->config_lock); 784 785 nbd_parse_flags(nbd, bdev); 786 787 error = device_create_file(disk_to_dev(nbd->disk), &pid_attr); 788 if (error) { 789 dev_err(disk_to_dev(nbd->disk), "device_create_file failed!\n"); 790 goto out_recv; 791 } 792 793 nbd_size_update(nbd, bdev); 794 795 nbd_dev_dbg_init(nbd); 796 for (i = 0; i < num_connections; i++) { 797 sk_set_memalloc(nbd->socks[i]->sock->sk); 798 atomic_inc(&nbd->recv_threads); 799 INIT_WORK(&args[i].work, recv_work); 800 args[i].nbd = nbd; 801 args[i].index = i; 802 queue_work(recv_workqueue, &args[i].work); 803 } 804 wait_event_interruptible(nbd->recv_wq, 805 atomic_read(&nbd->recv_threads) == 0); 806 for (i = 0; i < num_connections; i++) 807 flush_work(&args[i].work); 808 nbd_dev_dbg_close(nbd); 809 nbd_size_clear(nbd, bdev); 810 device_remove_file(disk_to_dev(nbd->disk), &pid_attr); 811 out_recv: 812 mutex_lock(&nbd->config_lock); 813 nbd->task_recv = NULL; 814 out_err: 815 clear_bit(NBD_RUNNING, &nbd->runtime_flags); 816 nbd_clear_sock(nbd, bdev); 817 818 /* user requested, ignore socket errors */ 819 if (test_bit(NBD_DISCONNECT_REQUESTED, &nbd->runtime_flags)) 820 error = 0; 821 if (test_bit(NBD_TIMEDOUT, &nbd->runtime_flags)) 822 error = -ETIMEDOUT; 823 824 nbd_reset(nbd); 825 return error; 826 } 827 828 /* Must be called with config_lock held */ 829 static int __nbd_ioctl(struct block_device *bdev, struct nbd_device *nbd, 830 unsigned int cmd, unsigned long arg) 831 { 832 switch (cmd) { 833 case NBD_DISCONNECT: 834 return nbd_disconnect(nbd, bdev); 835 case NBD_CLEAR_SOCK: 836 return nbd_clear_sock(nbd, bdev); 837 case NBD_SET_SOCK: 838 return nbd_add_socket(nbd, bdev, arg); 839 case NBD_SET_BLKSIZE: 840 nbd_size_set(nbd, bdev, arg, 841 div_s64(nbd->bytesize, arg)); 842 return 0; 843 case NBD_SET_SIZE: 844 nbd_size_set(nbd, bdev, nbd->blksize, 845 div_s64(arg, nbd->blksize)); 846 return 0; 847 case NBD_SET_SIZE_BLOCKS: 848 nbd_size_set(nbd, bdev, nbd->blksize, arg); 849 return 0; 850 case NBD_SET_TIMEOUT: 851 if (arg) { 852 nbd->tag_set.timeout = arg * HZ; 853 blk_queue_rq_timeout(nbd->disk->queue, arg * HZ); 854 } 855 return 0; 856 857 case NBD_SET_FLAGS: 858 nbd->flags = arg; 859 return 0; 860 case NBD_DO_IT: 861 return nbd_start_device(nbd, bdev); 862 case NBD_CLEAR_QUE: 863 /* 864 * This is for compatibility only. The queue is always cleared 865 * by NBD_DO_IT or NBD_CLEAR_SOCK. 866 */ 867 return 0; 868 case NBD_PRINT_DEBUG: 869 /* 870 * For compatibility only, we no longer keep a list of 871 * outstanding requests. 872 */ 873 return 0; 874 } 875 return -ENOTTY; 876 } 877 878 static int nbd_ioctl(struct block_device *bdev, fmode_t mode, 879 unsigned int cmd, unsigned long arg) 880 { 881 struct nbd_device *nbd = bdev->bd_disk->private_data; 882 int error; 883 884 if (!capable(CAP_SYS_ADMIN)) 885 return -EPERM; 886 887 BUG_ON(nbd->magic != NBD_MAGIC); 888 889 mutex_lock(&nbd->config_lock); 890 error = __nbd_ioctl(bdev, nbd, cmd, arg); 891 mutex_unlock(&nbd->config_lock); 892 893 return error; 894 } 895 896 static const struct block_device_operations nbd_fops = 897 { 898 .owner = THIS_MODULE, 899 .ioctl = nbd_ioctl, 900 .compat_ioctl = nbd_ioctl, 901 }; 902 903 #if IS_ENABLED(CONFIG_DEBUG_FS) 904 905 static int nbd_dbg_tasks_show(struct seq_file *s, void *unused) 906 { 907 struct nbd_device *nbd = s->private; 908 909 if (nbd->task_recv) 910 seq_printf(s, "recv: %d\n", task_pid_nr(nbd->task_recv)); 911 912 return 0; 913 } 914 915 static int nbd_dbg_tasks_open(struct inode *inode, struct file *file) 916 { 917 return single_open(file, nbd_dbg_tasks_show, inode->i_private); 918 } 919 920 static const struct file_operations nbd_dbg_tasks_ops = { 921 .open = nbd_dbg_tasks_open, 922 .read = seq_read, 923 .llseek = seq_lseek, 924 .release = single_release, 925 }; 926 927 static int nbd_dbg_flags_show(struct seq_file *s, void *unused) 928 { 929 struct nbd_device *nbd = s->private; 930 u32 flags = nbd->flags; 931 932 seq_printf(s, "Hex: 0x%08x\n\n", flags); 933 934 seq_puts(s, "Known flags:\n"); 935 936 if (flags & NBD_FLAG_HAS_FLAGS) 937 seq_puts(s, "NBD_FLAG_HAS_FLAGS\n"); 938 if (flags & NBD_FLAG_READ_ONLY) 939 seq_puts(s, "NBD_FLAG_READ_ONLY\n"); 940 if (flags & NBD_FLAG_SEND_FLUSH) 941 seq_puts(s, "NBD_FLAG_SEND_FLUSH\n"); 942 if (flags & NBD_FLAG_SEND_TRIM) 943 seq_puts(s, "NBD_FLAG_SEND_TRIM\n"); 944 945 return 0; 946 } 947 948 static int nbd_dbg_flags_open(struct inode *inode, struct file *file) 949 { 950 return single_open(file, nbd_dbg_flags_show, inode->i_private); 951 } 952 953 static const struct file_operations nbd_dbg_flags_ops = { 954 .open = nbd_dbg_flags_open, 955 .read = seq_read, 956 .llseek = seq_lseek, 957 .release = single_release, 958 }; 959 960 static int nbd_dev_dbg_init(struct nbd_device *nbd) 961 { 962 struct dentry *dir; 963 964 if (!nbd_dbg_dir) 965 return -EIO; 966 967 dir = debugfs_create_dir(nbd_name(nbd), nbd_dbg_dir); 968 if (!dir) { 969 dev_err(nbd_to_dev(nbd), "Failed to create debugfs dir for '%s'\n", 970 nbd_name(nbd)); 971 return -EIO; 972 } 973 nbd->dbg_dir = dir; 974 975 debugfs_create_file("tasks", 0444, dir, nbd, &nbd_dbg_tasks_ops); 976 debugfs_create_u64("size_bytes", 0444, dir, &nbd->bytesize); 977 debugfs_create_u32("timeout", 0444, dir, &nbd->tag_set.timeout); 978 debugfs_create_u64("blocksize", 0444, dir, &nbd->blksize); 979 debugfs_create_file("flags", 0444, dir, nbd, &nbd_dbg_flags_ops); 980 981 return 0; 982 } 983 984 static void nbd_dev_dbg_close(struct nbd_device *nbd) 985 { 986 debugfs_remove_recursive(nbd->dbg_dir); 987 } 988 989 static int nbd_dbg_init(void) 990 { 991 struct dentry *dbg_dir; 992 993 dbg_dir = debugfs_create_dir("nbd", NULL); 994 if (!dbg_dir) 995 return -EIO; 996 997 nbd_dbg_dir = dbg_dir; 998 999 return 0; 1000 } 1001 1002 static void nbd_dbg_close(void) 1003 { 1004 debugfs_remove_recursive(nbd_dbg_dir); 1005 } 1006 1007 #else /* IS_ENABLED(CONFIG_DEBUG_FS) */ 1008 1009 static int nbd_dev_dbg_init(struct nbd_device *nbd) 1010 { 1011 return 0; 1012 } 1013 1014 static void nbd_dev_dbg_close(struct nbd_device *nbd) 1015 { 1016 } 1017 1018 static int nbd_dbg_init(void) 1019 { 1020 return 0; 1021 } 1022 1023 static void nbd_dbg_close(void) 1024 { 1025 } 1026 1027 #endif 1028 1029 static int nbd_init_request(void *data, struct request *rq, 1030 unsigned int hctx_idx, unsigned int request_idx, 1031 unsigned int numa_node) 1032 { 1033 struct nbd_cmd *cmd = blk_mq_rq_to_pdu(rq); 1034 cmd->nbd = data; 1035 return 0; 1036 } 1037 1038 static struct blk_mq_ops nbd_mq_ops = { 1039 .queue_rq = nbd_queue_rq, 1040 .init_request = nbd_init_request, 1041 .timeout = nbd_xmit_timeout, 1042 }; 1043 1044 static void nbd_dev_remove(struct nbd_device *nbd) 1045 { 1046 struct gendisk *disk = nbd->disk; 1047 nbd->magic = 0; 1048 if (disk) { 1049 del_gendisk(disk); 1050 blk_cleanup_queue(disk->queue); 1051 blk_mq_free_tag_set(&nbd->tag_set); 1052 put_disk(disk); 1053 } 1054 kfree(nbd); 1055 } 1056 1057 static int nbd_dev_add(int index) 1058 { 1059 struct nbd_device *nbd; 1060 struct gendisk *disk; 1061 struct request_queue *q; 1062 int err = -ENOMEM; 1063 1064 nbd = kzalloc(sizeof(struct nbd_device), GFP_KERNEL); 1065 if (!nbd) 1066 goto out; 1067 1068 disk = alloc_disk(1 << part_shift); 1069 if (!disk) 1070 goto out_free_nbd; 1071 1072 if (index >= 0) { 1073 err = idr_alloc(&nbd_index_idr, nbd, index, index + 1, 1074 GFP_KERNEL); 1075 if (err == -ENOSPC) 1076 err = -EEXIST; 1077 } else { 1078 err = idr_alloc(&nbd_index_idr, nbd, 0, 0, GFP_KERNEL); 1079 if (err >= 0) 1080 index = err; 1081 } 1082 if (err < 0) 1083 goto out_free_disk; 1084 1085 nbd->disk = disk; 1086 nbd->tag_set.ops = &nbd_mq_ops; 1087 nbd->tag_set.nr_hw_queues = 1; 1088 nbd->tag_set.queue_depth = 128; 1089 nbd->tag_set.numa_node = NUMA_NO_NODE; 1090 nbd->tag_set.cmd_size = sizeof(struct nbd_cmd); 1091 nbd->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | 1092 BLK_MQ_F_SG_MERGE | BLK_MQ_F_BLOCKING; 1093 nbd->tag_set.driver_data = nbd; 1094 1095 err = blk_mq_alloc_tag_set(&nbd->tag_set); 1096 if (err) 1097 goto out_free_idr; 1098 1099 q = blk_mq_init_queue(&nbd->tag_set); 1100 if (IS_ERR(q)) { 1101 err = PTR_ERR(q); 1102 goto out_free_tags; 1103 } 1104 disk->queue = q; 1105 1106 /* 1107 * Tell the block layer that we are not a rotational device 1108 */ 1109 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); 1110 queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, disk->queue); 1111 disk->queue->limits.discard_granularity = 512; 1112 blk_queue_max_discard_sectors(disk->queue, UINT_MAX); 1113 disk->queue->limits.discard_zeroes_data = 0; 1114 blk_queue_max_hw_sectors(disk->queue, 65536); 1115 disk->queue->limits.max_sectors = 256; 1116 1117 nbd->magic = NBD_MAGIC; 1118 mutex_init(&nbd->config_lock); 1119 disk->major = NBD_MAJOR; 1120 disk->first_minor = index << part_shift; 1121 disk->fops = &nbd_fops; 1122 disk->private_data = nbd; 1123 sprintf(disk->disk_name, "nbd%d", index); 1124 init_waitqueue_head(&nbd->recv_wq); 1125 nbd_reset(nbd); 1126 add_disk(disk); 1127 return index; 1128 1129 out_free_tags: 1130 blk_mq_free_tag_set(&nbd->tag_set); 1131 out_free_idr: 1132 idr_remove(&nbd_index_idr, index); 1133 out_free_disk: 1134 put_disk(disk); 1135 out_free_nbd: 1136 kfree(nbd); 1137 out: 1138 return err; 1139 } 1140 1141 /* 1142 * And here should be modules and kernel interface 1143 * (Just smiley confuses emacs :-) 1144 */ 1145 1146 static int __init nbd_init(void) 1147 { 1148 int i; 1149 1150 BUILD_BUG_ON(sizeof(struct nbd_request) != 28); 1151 1152 if (max_part < 0) { 1153 printk(KERN_ERR "nbd: max_part must be >= 0\n"); 1154 return -EINVAL; 1155 } 1156 1157 part_shift = 0; 1158 if (max_part > 0) { 1159 part_shift = fls(max_part); 1160 1161 /* 1162 * Adjust max_part according to part_shift as it is exported 1163 * to user space so that user can know the max number of 1164 * partition kernel should be able to manage. 1165 * 1166 * Note that -1 is required because partition 0 is reserved 1167 * for the whole disk. 1168 */ 1169 max_part = (1UL << part_shift) - 1; 1170 } 1171 1172 if ((1UL << part_shift) > DISK_MAX_PARTS) 1173 return -EINVAL; 1174 1175 if (nbds_max > 1UL << (MINORBITS - part_shift)) 1176 return -EINVAL; 1177 recv_workqueue = alloc_workqueue("knbd-recv", 1178 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 1179 if (!recv_workqueue) 1180 return -ENOMEM; 1181 1182 if (register_blkdev(NBD_MAJOR, "nbd")) { 1183 destroy_workqueue(recv_workqueue); 1184 return -EIO; 1185 } 1186 1187 nbd_dbg_init(); 1188 1189 mutex_lock(&nbd_index_mutex); 1190 for (i = 0; i < nbds_max; i++) 1191 nbd_dev_add(i); 1192 mutex_unlock(&nbd_index_mutex); 1193 return 0; 1194 } 1195 1196 static int nbd_exit_cb(int id, void *ptr, void *data) 1197 { 1198 struct nbd_device *nbd = ptr; 1199 nbd_dev_remove(nbd); 1200 return 0; 1201 } 1202 1203 static void __exit nbd_cleanup(void) 1204 { 1205 nbd_dbg_close(); 1206 1207 idr_for_each(&nbd_index_idr, &nbd_exit_cb, NULL); 1208 idr_destroy(&nbd_index_idr); 1209 destroy_workqueue(recv_workqueue); 1210 unregister_blkdev(NBD_MAJOR, "nbd"); 1211 } 1212 1213 module_init(nbd_init); 1214 module_exit(nbd_cleanup); 1215 1216 MODULE_DESCRIPTION("Network Block Device"); 1217 MODULE_LICENSE("GPL"); 1218 1219 module_param(nbds_max, int, 0444); 1220 MODULE_PARM_DESC(nbds_max, "number of network block devices to initialize (default: 16)"); 1221 module_param(max_part, int, 0444); 1222 MODULE_PARM_DESC(max_part, "number of partitions per device (default: 0)"); 1223