1 /* 2 * Common code for the NVMe target. 3 * Copyright (c) 2015-2016 HGST, a Western Digital Company. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 15 #include <linux/module.h> 16 #include <linux/random.h> 17 #include <linux/rculist.h> 18 19 #include "nvmet.h" 20 21 static struct nvmet_fabrics_ops *nvmet_transports[NVMF_TRTYPE_MAX]; 22 static DEFINE_IDA(cntlid_ida); 23 24 /* 25 * This read/write semaphore is used to synchronize access to configuration 26 * information on a target system that will result in discovery log page 27 * information change for at least one host. 28 * The full list of resources to protected by this semaphore is: 29 * 30 * - subsystems list 31 * - per-subsystem allowed hosts list 32 * - allow_any_host subsystem attribute 33 * - nvmet_genctr 34 * - the nvmet_transports array 35 * 36 * When updating any of those lists/structures write lock should be obtained, 37 * while when reading (popolating discovery log page or checking host-subsystem 38 * link) read lock is obtained to allow concurrent reads. 39 */ 40 DECLARE_RWSEM(nvmet_config_sem); 41 42 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, 43 const char *subsysnqn); 44 45 u16 nvmet_copy_to_sgl(struct nvmet_req *req, off_t off, const void *buf, 46 size_t len) 47 { 48 if (sg_pcopy_from_buffer(req->sg, req->sg_cnt, buf, len, off) != len) 49 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR; 50 return 0; 51 } 52 53 u16 nvmet_copy_from_sgl(struct nvmet_req *req, off_t off, void *buf, size_t len) 54 { 55 if (sg_pcopy_to_buffer(req->sg, req->sg_cnt, buf, len, off) != len) 56 return NVME_SC_SGL_INVALID_DATA | NVME_SC_DNR; 57 return 0; 58 } 59 60 static u32 nvmet_async_event_result(struct nvmet_async_event *aen) 61 { 62 return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16); 63 } 64 65 static void nvmet_async_events_free(struct nvmet_ctrl *ctrl) 66 { 67 struct nvmet_req *req; 68 69 while (1) { 70 mutex_lock(&ctrl->lock); 71 if (!ctrl->nr_async_event_cmds) { 72 mutex_unlock(&ctrl->lock); 73 return; 74 } 75 76 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; 77 mutex_unlock(&ctrl->lock); 78 nvmet_req_complete(req, NVME_SC_INTERNAL | NVME_SC_DNR); 79 } 80 } 81 82 static void nvmet_async_event_work(struct work_struct *work) 83 { 84 struct nvmet_ctrl *ctrl = 85 container_of(work, struct nvmet_ctrl, async_event_work); 86 struct nvmet_async_event *aen; 87 struct nvmet_req *req; 88 89 while (1) { 90 mutex_lock(&ctrl->lock); 91 aen = list_first_entry_or_null(&ctrl->async_events, 92 struct nvmet_async_event, entry); 93 if (!aen || !ctrl->nr_async_event_cmds) { 94 mutex_unlock(&ctrl->lock); 95 return; 96 } 97 98 req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; 99 nvmet_set_result(req, nvmet_async_event_result(aen)); 100 101 list_del(&aen->entry); 102 kfree(aen); 103 104 mutex_unlock(&ctrl->lock); 105 nvmet_req_complete(req, 0); 106 } 107 } 108 109 static void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, 110 u8 event_info, u8 log_page) 111 { 112 struct nvmet_async_event *aen; 113 114 aen = kmalloc(sizeof(*aen), GFP_KERNEL); 115 if (!aen) 116 return; 117 118 aen->event_type = event_type; 119 aen->event_info = event_info; 120 aen->log_page = log_page; 121 122 mutex_lock(&ctrl->lock); 123 list_add_tail(&aen->entry, &ctrl->async_events); 124 mutex_unlock(&ctrl->lock); 125 126 schedule_work(&ctrl->async_event_work); 127 } 128 129 int nvmet_register_transport(struct nvmet_fabrics_ops *ops) 130 { 131 int ret = 0; 132 133 down_write(&nvmet_config_sem); 134 if (nvmet_transports[ops->type]) 135 ret = -EINVAL; 136 else 137 nvmet_transports[ops->type] = ops; 138 up_write(&nvmet_config_sem); 139 140 return ret; 141 } 142 EXPORT_SYMBOL_GPL(nvmet_register_transport); 143 144 void nvmet_unregister_transport(struct nvmet_fabrics_ops *ops) 145 { 146 down_write(&nvmet_config_sem); 147 nvmet_transports[ops->type] = NULL; 148 up_write(&nvmet_config_sem); 149 } 150 EXPORT_SYMBOL_GPL(nvmet_unregister_transport); 151 152 int nvmet_enable_port(struct nvmet_port *port) 153 { 154 struct nvmet_fabrics_ops *ops; 155 int ret; 156 157 lockdep_assert_held(&nvmet_config_sem); 158 159 ops = nvmet_transports[port->disc_addr.trtype]; 160 if (!ops) { 161 up_write(&nvmet_config_sem); 162 request_module("nvmet-transport-%d", port->disc_addr.trtype); 163 down_write(&nvmet_config_sem); 164 ops = nvmet_transports[port->disc_addr.trtype]; 165 if (!ops) { 166 pr_err("transport type %d not supported\n", 167 port->disc_addr.trtype); 168 return -EINVAL; 169 } 170 } 171 172 if (!try_module_get(ops->owner)) 173 return -EINVAL; 174 175 ret = ops->add_port(port); 176 if (ret) { 177 module_put(ops->owner); 178 return ret; 179 } 180 181 port->enabled = true; 182 return 0; 183 } 184 185 void nvmet_disable_port(struct nvmet_port *port) 186 { 187 struct nvmet_fabrics_ops *ops; 188 189 lockdep_assert_held(&nvmet_config_sem); 190 191 port->enabled = false; 192 193 ops = nvmet_transports[port->disc_addr.trtype]; 194 ops->remove_port(port); 195 module_put(ops->owner); 196 } 197 198 static void nvmet_keep_alive_timer(struct work_struct *work) 199 { 200 struct nvmet_ctrl *ctrl = container_of(to_delayed_work(work), 201 struct nvmet_ctrl, ka_work); 202 203 pr_err("ctrl %d keep-alive timer (%d seconds) expired!\n", 204 ctrl->cntlid, ctrl->kato); 205 206 nvmet_ctrl_fatal_error(ctrl); 207 } 208 209 static void nvmet_start_keep_alive_timer(struct nvmet_ctrl *ctrl) 210 { 211 pr_debug("ctrl %d start keep-alive timer for %d secs\n", 212 ctrl->cntlid, ctrl->kato); 213 214 INIT_DELAYED_WORK(&ctrl->ka_work, nvmet_keep_alive_timer); 215 schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ); 216 } 217 218 static void nvmet_stop_keep_alive_timer(struct nvmet_ctrl *ctrl) 219 { 220 pr_debug("ctrl %d stop keep-alive\n", ctrl->cntlid); 221 222 cancel_delayed_work_sync(&ctrl->ka_work); 223 } 224 225 static struct nvmet_ns *__nvmet_find_namespace(struct nvmet_ctrl *ctrl, 226 __le32 nsid) 227 { 228 struct nvmet_ns *ns; 229 230 list_for_each_entry_rcu(ns, &ctrl->subsys->namespaces, dev_link) { 231 if (ns->nsid == le32_to_cpu(nsid)) 232 return ns; 233 } 234 235 return NULL; 236 } 237 238 struct nvmet_ns *nvmet_find_namespace(struct nvmet_ctrl *ctrl, __le32 nsid) 239 { 240 struct nvmet_ns *ns; 241 242 rcu_read_lock(); 243 ns = __nvmet_find_namespace(ctrl, nsid); 244 if (ns) 245 percpu_ref_get(&ns->ref); 246 rcu_read_unlock(); 247 248 return ns; 249 } 250 251 static void nvmet_destroy_namespace(struct percpu_ref *ref) 252 { 253 struct nvmet_ns *ns = container_of(ref, struct nvmet_ns, ref); 254 255 complete(&ns->disable_done); 256 } 257 258 void nvmet_put_namespace(struct nvmet_ns *ns) 259 { 260 percpu_ref_put(&ns->ref); 261 } 262 263 int nvmet_ns_enable(struct nvmet_ns *ns) 264 { 265 struct nvmet_subsys *subsys = ns->subsys; 266 struct nvmet_ctrl *ctrl; 267 int ret = 0; 268 269 mutex_lock(&subsys->lock); 270 if (ns->enabled) 271 goto out_unlock; 272 273 ns->bdev = blkdev_get_by_path(ns->device_path, FMODE_READ | FMODE_WRITE, 274 NULL); 275 if (IS_ERR(ns->bdev)) { 276 pr_err("failed to open block device %s: (%ld)\n", 277 ns->device_path, PTR_ERR(ns->bdev)); 278 ret = PTR_ERR(ns->bdev); 279 ns->bdev = NULL; 280 goto out_unlock; 281 } 282 283 ns->size = i_size_read(ns->bdev->bd_inode); 284 ns->blksize_shift = blksize_bits(bdev_logical_block_size(ns->bdev)); 285 286 ret = percpu_ref_init(&ns->ref, nvmet_destroy_namespace, 287 0, GFP_KERNEL); 288 if (ret) 289 goto out_blkdev_put; 290 291 if (ns->nsid > subsys->max_nsid) 292 subsys->max_nsid = ns->nsid; 293 294 /* 295 * The namespaces list needs to be sorted to simplify the implementation 296 * of the Identify Namepace List subcommand. 297 */ 298 if (list_empty(&subsys->namespaces)) { 299 list_add_tail_rcu(&ns->dev_link, &subsys->namespaces); 300 } else { 301 struct nvmet_ns *old; 302 303 list_for_each_entry_rcu(old, &subsys->namespaces, dev_link) { 304 BUG_ON(ns->nsid == old->nsid); 305 if (ns->nsid < old->nsid) 306 break; 307 } 308 309 list_add_tail_rcu(&ns->dev_link, &old->dev_link); 310 } 311 312 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 313 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0); 314 315 ns->enabled = true; 316 ret = 0; 317 out_unlock: 318 mutex_unlock(&subsys->lock); 319 return ret; 320 out_blkdev_put: 321 blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ); 322 ns->bdev = NULL; 323 goto out_unlock; 324 } 325 326 void nvmet_ns_disable(struct nvmet_ns *ns) 327 { 328 struct nvmet_subsys *subsys = ns->subsys; 329 struct nvmet_ctrl *ctrl; 330 331 mutex_lock(&subsys->lock); 332 if (!ns->enabled) 333 goto out_unlock; 334 335 ns->enabled = false; 336 list_del_rcu(&ns->dev_link); 337 mutex_unlock(&subsys->lock); 338 339 /* 340 * Now that we removed the namespaces from the lookup list, we 341 * can kill the per_cpu ref and wait for any remaining references 342 * to be dropped, as well as a RCU grace period for anyone only 343 * using the namepace under rcu_read_lock(). Note that we can't 344 * use call_rcu here as we need to ensure the namespaces have 345 * been fully destroyed before unloading the module. 346 */ 347 percpu_ref_kill(&ns->ref); 348 synchronize_rcu(); 349 wait_for_completion(&ns->disable_done); 350 percpu_ref_exit(&ns->ref); 351 352 mutex_lock(&subsys->lock); 353 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 354 nvmet_add_async_event(ctrl, NVME_AER_TYPE_NOTICE, 0, 0); 355 356 if (ns->bdev) 357 blkdev_put(ns->bdev, FMODE_WRITE|FMODE_READ); 358 out_unlock: 359 mutex_unlock(&subsys->lock); 360 } 361 362 void nvmet_ns_free(struct nvmet_ns *ns) 363 { 364 nvmet_ns_disable(ns); 365 366 kfree(ns->device_path); 367 kfree(ns); 368 } 369 370 struct nvmet_ns *nvmet_ns_alloc(struct nvmet_subsys *subsys, u32 nsid) 371 { 372 struct nvmet_ns *ns; 373 374 ns = kzalloc(sizeof(*ns), GFP_KERNEL); 375 if (!ns) 376 return NULL; 377 378 INIT_LIST_HEAD(&ns->dev_link); 379 init_completion(&ns->disable_done); 380 381 ns->nsid = nsid; 382 ns->subsys = subsys; 383 384 return ns; 385 } 386 387 static void __nvmet_req_complete(struct nvmet_req *req, u16 status) 388 { 389 if (status) 390 nvmet_set_status(req, status); 391 392 /* XXX: need to fill in something useful for sq_head */ 393 req->rsp->sq_head = 0; 394 if (likely(req->sq)) /* may happen during early failure */ 395 req->rsp->sq_id = cpu_to_le16(req->sq->qid); 396 req->rsp->command_id = req->cmd->common.command_id; 397 398 if (req->ns) 399 nvmet_put_namespace(req->ns); 400 req->ops->queue_response(req); 401 } 402 403 void nvmet_req_complete(struct nvmet_req *req, u16 status) 404 { 405 __nvmet_req_complete(req, status); 406 percpu_ref_put(&req->sq->ref); 407 } 408 EXPORT_SYMBOL_GPL(nvmet_req_complete); 409 410 void nvmet_cq_setup(struct nvmet_ctrl *ctrl, struct nvmet_cq *cq, 411 u16 qid, u16 size) 412 { 413 cq->qid = qid; 414 cq->size = size; 415 416 ctrl->cqs[qid] = cq; 417 } 418 419 void nvmet_sq_setup(struct nvmet_ctrl *ctrl, struct nvmet_sq *sq, 420 u16 qid, u16 size) 421 { 422 sq->qid = qid; 423 sq->size = size; 424 425 ctrl->sqs[qid] = sq; 426 } 427 428 static void nvmet_confirm_sq(struct percpu_ref *ref) 429 { 430 struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref); 431 432 complete(&sq->confirm_done); 433 } 434 435 void nvmet_sq_destroy(struct nvmet_sq *sq) 436 { 437 /* 438 * If this is the admin queue, complete all AERs so that our 439 * queue doesn't have outstanding requests on it. 440 */ 441 if (sq->ctrl && sq->ctrl->sqs && sq->ctrl->sqs[0] == sq) 442 nvmet_async_events_free(sq->ctrl); 443 percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); 444 wait_for_completion(&sq->confirm_done); 445 wait_for_completion(&sq->free_done); 446 percpu_ref_exit(&sq->ref); 447 448 if (sq->ctrl) { 449 nvmet_ctrl_put(sq->ctrl); 450 sq->ctrl = NULL; /* allows reusing the queue later */ 451 } 452 } 453 EXPORT_SYMBOL_GPL(nvmet_sq_destroy); 454 455 static void nvmet_sq_free(struct percpu_ref *ref) 456 { 457 struct nvmet_sq *sq = container_of(ref, struct nvmet_sq, ref); 458 459 complete(&sq->free_done); 460 } 461 462 int nvmet_sq_init(struct nvmet_sq *sq) 463 { 464 int ret; 465 466 ret = percpu_ref_init(&sq->ref, nvmet_sq_free, 0, GFP_KERNEL); 467 if (ret) { 468 pr_err("percpu_ref init failed!\n"); 469 return ret; 470 } 471 init_completion(&sq->free_done); 472 init_completion(&sq->confirm_done); 473 474 return 0; 475 } 476 EXPORT_SYMBOL_GPL(nvmet_sq_init); 477 478 bool nvmet_req_init(struct nvmet_req *req, struct nvmet_cq *cq, 479 struct nvmet_sq *sq, struct nvmet_fabrics_ops *ops) 480 { 481 u8 flags = req->cmd->common.flags; 482 u16 status; 483 484 req->cq = cq; 485 req->sq = sq; 486 req->ops = ops; 487 req->sg = NULL; 488 req->sg_cnt = 0; 489 req->rsp->status = 0; 490 491 /* no support for fused commands yet */ 492 if (unlikely(flags & (NVME_CMD_FUSE_FIRST | NVME_CMD_FUSE_SECOND))) { 493 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; 494 goto fail; 495 } 496 497 /* either variant of SGLs is fine, as we don't support metadata */ 498 if (unlikely((flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METABUF && 499 (flags & NVME_CMD_SGL_ALL) != NVME_CMD_SGL_METASEG)) { 500 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; 501 goto fail; 502 } 503 504 if (unlikely(!req->sq->ctrl)) 505 /* will return an error for any Non-connect command: */ 506 status = nvmet_parse_connect_cmd(req); 507 else if (likely(req->sq->qid != 0)) 508 status = nvmet_parse_io_cmd(req); 509 else if (req->cmd->common.opcode == nvme_fabrics_command) 510 status = nvmet_parse_fabrics_cmd(req); 511 else if (req->sq->ctrl->subsys->type == NVME_NQN_DISC) 512 status = nvmet_parse_discovery_cmd(req); 513 else 514 status = nvmet_parse_admin_cmd(req); 515 516 if (status) 517 goto fail; 518 519 if (unlikely(!percpu_ref_tryget_live(&sq->ref))) { 520 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; 521 goto fail; 522 } 523 524 return true; 525 526 fail: 527 __nvmet_req_complete(req, status); 528 return false; 529 } 530 EXPORT_SYMBOL_GPL(nvmet_req_init); 531 532 void nvmet_req_uninit(struct nvmet_req *req) 533 { 534 percpu_ref_put(&req->sq->ref); 535 } 536 EXPORT_SYMBOL_GPL(nvmet_req_uninit); 537 538 static inline bool nvmet_cc_en(u32 cc) 539 { 540 return cc & 0x1; 541 } 542 543 static inline u8 nvmet_cc_css(u32 cc) 544 { 545 return (cc >> 4) & 0x7; 546 } 547 548 static inline u8 nvmet_cc_mps(u32 cc) 549 { 550 return (cc >> 7) & 0xf; 551 } 552 553 static inline u8 nvmet_cc_ams(u32 cc) 554 { 555 return (cc >> 11) & 0x7; 556 } 557 558 static inline u8 nvmet_cc_shn(u32 cc) 559 { 560 return (cc >> 14) & 0x3; 561 } 562 563 static inline u8 nvmet_cc_iosqes(u32 cc) 564 { 565 return (cc >> 16) & 0xf; 566 } 567 568 static inline u8 nvmet_cc_iocqes(u32 cc) 569 { 570 return (cc >> 20) & 0xf; 571 } 572 573 static void nvmet_start_ctrl(struct nvmet_ctrl *ctrl) 574 { 575 lockdep_assert_held(&ctrl->lock); 576 577 if (nvmet_cc_iosqes(ctrl->cc) != NVME_NVM_IOSQES || 578 nvmet_cc_iocqes(ctrl->cc) != NVME_NVM_IOCQES || 579 nvmet_cc_mps(ctrl->cc) != 0 || 580 nvmet_cc_ams(ctrl->cc) != 0 || 581 nvmet_cc_css(ctrl->cc) != 0) { 582 ctrl->csts = NVME_CSTS_CFS; 583 return; 584 } 585 586 ctrl->csts = NVME_CSTS_RDY; 587 } 588 589 static void nvmet_clear_ctrl(struct nvmet_ctrl *ctrl) 590 { 591 lockdep_assert_held(&ctrl->lock); 592 593 /* XXX: tear down queues? */ 594 ctrl->csts &= ~NVME_CSTS_RDY; 595 ctrl->cc = 0; 596 } 597 598 void nvmet_update_cc(struct nvmet_ctrl *ctrl, u32 new) 599 { 600 u32 old; 601 602 mutex_lock(&ctrl->lock); 603 old = ctrl->cc; 604 ctrl->cc = new; 605 606 if (nvmet_cc_en(new) && !nvmet_cc_en(old)) 607 nvmet_start_ctrl(ctrl); 608 if (!nvmet_cc_en(new) && nvmet_cc_en(old)) 609 nvmet_clear_ctrl(ctrl); 610 if (nvmet_cc_shn(new) && !nvmet_cc_shn(old)) { 611 nvmet_clear_ctrl(ctrl); 612 ctrl->csts |= NVME_CSTS_SHST_CMPLT; 613 } 614 if (!nvmet_cc_shn(new) && nvmet_cc_shn(old)) 615 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT; 616 mutex_unlock(&ctrl->lock); 617 } 618 619 static void nvmet_init_cap(struct nvmet_ctrl *ctrl) 620 { 621 /* command sets supported: NVMe command set: */ 622 ctrl->cap = (1ULL << 37); 623 /* CC.EN timeout in 500msec units: */ 624 ctrl->cap |= (15ULL << 24); 625 /* maximum queue entries supported: */ 626 ctrl->cap |= NVMET_QUEUE_SIZE - 1; 627 } 628 629 u16 nvmet_ctrl_find_get(const char *subsysnqn, const char *hostnqn, u16 cntlid, 630 struct nvmet_req *req, struct nvmet_ctrl **ret) 631 { 632 struct nvmet_subsys *subsys; 633 struct nvmet_ctrl *ctrl; 634 u16 status = 0; 635 636 subsys = nvmet_find_get_subsys(req->port, subsysnqn); 637 if (!subsys) { 638 pr_warn("connect request for invalid subsystem %s!\n", 639 subsysnqn); 640 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn); 641 return NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; 642 } 643 644 mutex_lock(&subsys->lock); 645 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) { 646 if (ctrl->cntlid == cntlid) { 647 if (strncmp(hostnqn, ctrl->hostnqn, NVMF_NQN_SIZE)) { 648 pr_warn("hostnqn mismatch.\n"); 649 continue; 650 } 651 if (!kref_get_unless_zero(&ctrl->ref)) 652 continue; 653 654 *ret = ctrl; 655 goto out; 656 } 657 } 658 659 pr_warn("could not find controller %d for subsys %s / host %s\n", 660 cntlid, subsysnqn, hostnqn); 661 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(cntlid); 662 status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; 663 664 out: 665 mutex_unlock(&subsys->lock); 666 nvmet_subsys_put(subsys); 667 return status; 668 } 669 670 u16 nvmet_check_ctrl_status(struct nvmet_req *req, struct nvme_command *cmd) 671 { 672 if (unlikely(!(req->sq->ctrl->cc & NVME_CC_ENABLE))) { 673 pr_err("got io cmd %d while CC.EN == 0 on qid = %d\n", 674 cmd->common.opcode, req->sq->qid); 675 return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR; 676 } 677 678 if (unlikely(!(req->sq->ctrl->csts & NVME_CSTS_RDY))) { 679 pr_err("got io cmd %d while CSTS.RDY == 0 on qid = %d\n", 680 cmd->common.opcode, req->sq->qid); 681 req->ns = NULL; 682 return NVME_SC_CMD_SEQ_ERROR | NVME_SC_DNR; 683 } 684 return 0; 685 } 686 687 static bool __nvmet_host_allowed(struct nvmet_subsys *subsys, 688 const char *hostnqn) 689 { 690 struct nvmet_host_link *p; 691 692 if (subsys->allow_any_host) 693 return true; 694 695 list_for_each_entry(p, &subsys->hosts, entry) { 696 if (!strcmp(nvmet_host_name(p->host), hostnqn)) 697 return true; 698 } 699 700 return false; 701 } 702 703 static bool nvmet_host_discovery_allowed(struct nvmet_req *req, 704 const char *hostnqn) 705 { 706 struct nvmet_subsys_link *s; 707 708 list_for_each_entry(s, &req->port->subsystems, entry) { 709 if (__nvmet_host_allowed(s->subsys, hostnqn)) 710 return true; 711 } 712 713 return false; 714 } 715 716 bool nvmet_host_allowed(struct nvmet_req *req, struct nvmet_subsys *subsys, 717 const char *hostnqn) 718 { 719 lockdep_assert_held(&nvmet_config_sem); 720 721 if (subsys->type == NVME_NQN_DISC) 722 return nvmet_host_discovery_allowed(req, hostnqn); 723 else 724 return __nvmet_host_allowed(subsys, hostnqn); 725 } 726 727 u16 nvmet_alloc_ctrl(const char *subsysnqn, const char *hostnqn, 728 struct nvmet_req *req, u32 kato, struct nvmet_ctrl **ctrlp) 729 { 730 struct nvmet_subsys *subsys; 731 struct nvmet_ctrl *ctrl; 732 int ret; 733 u16 status; 734 735 status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; 736 subsys = nvmet_find_get_subsys(req->port, subsysnqn); 737 if (!subsys) { 738 pr_warn("connect request for invalid subsystem %s!\n", 739 subsysnqn); 740 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(subsysnqn); 741 goto out; 742 } 743 744 status = NVME_SC_CONNECT_INVALID_PARAM | NVME_SC_DNR; 745 down_read(&nvmet_config_sem); 746 if (!nvmet_host_allowed(req, subsys, hostnqn)) { 747 pr_info("connect by host %s for subsystem %s not allowed\n", 748 hostnqn, subsysnqn); 749 req->rsp->result.u32 = IPO_IATTR_CONNECT_DATA(hostnqn); 750 up_read(&nvmet_config_sem); 751 goto out_put_subsystem; 752 } 753 up_read(&nvmet_config_sem); 754 755 status = NVME_SC_INTERNAL; 756 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 757 if (!ctrl) 758 goto out_put_subsystem; 759 mutex_init(&ctrl->lock); 760 761 nvmet_init_cap(ctrl); 762 763 INIT_WORK(&ctrl->async_event_work, nvmet_async_event_work); 764 INIT_LIST_HEAD(&ctrl->async_events); 765 766 memcpy(ctrl->subsysnqn, subsysnqn, NVMF_NQN_SIZE); 767 memcpy(ctrl->hostnqn, hostnqn, NVMF_NQN_SIZE); 768 769 /* generate a random serial number as our controllers are ephemeral: */ 770 get_random_bytes(&ctrl->serial, sizeof(ctrl->serial)); 771 772 kref_init(&ctrl->ref); 773 ctrl->subsys = subsys; 774 775 ctrl->cqs = kcalloc(subsys->max_qid + 1, 776 sizeof(struct nvmet_cq *), 777 GFP_KERNEL); 778 if (!ctrl->cqs) 779 goto out_free_ctrl; 780 781 ctrl->sqs = kcalloc(subsys->max_qid + 1, 782 sizeof(struct nvmet_sq *), 783 GFP_KERNEL); 784 if (!ctrl->sqs) 785 goto out_free_cqs; 786 787 ret = ida_simple_get(&cntlid_ida, 788 NVME_CNTLID_MIN, NVME_CNTLID_MAX, 789 GFP_KERNEL); 790 if (ret < 0) { 791 status = NVME_SC_CONNECT_CTRL_BUSY | NVME_SC_DNR; 792 goto out_free_sqs; 793 } 794 ctrl->cntlid = ret; 795 796 ctrl->ops = req->ops; 797 if (ctrl->subsys->type == NVME_NQN_DISC) { 798 /* Don't accept keep-alive timeout for discovery controllers */ 799 if (kato) { 800 status = NVME_SC_INVALID_FIELD | NVME_SC_DNR; 801 goto out_free_sqs; 802 } 803 804 /* 805 * Discovery controllers use some arbitrary high value in order 806 * to cleanup stale discovery sessions 807 * 808 * From the latest base diff RC: 809 * "The Keep Alive command is not supported by 810 * Discovery controllers. A transport may specify a 811 * fixed Discovery controller activity timeout value 812 * (e.g., 2 minutes). If no commands are received 813 * by a Discovery controller within that time 814 * period, the controller may perform the 815 * actions for Keep Alive Timer expiration". 816 */ 817 ctrl->kato = NVMET_DISC_KATO; 818 } else { 819 /* keep-alive timeout in seconds */ 820 ctrl->kato = DIV_ROUND_UP(kato, 1000); 821 } 822 nvmet_start_keep_alive_timer(ctrl); 823 824 mutex_lock(&subsys->lock); 825 list_add_tail(&ctrl->subsys_entry, &subsys->ctrls); 826 mutex_unlock(&subsys->lock); 827 828 *ctrlp = ctrl; 829 return 0; 830 831 out_free_sqs: 832 kfree(ctrl->sqs); 833 out_free_cqs: 834 kfree(ctrl->cqs); 835 out_free_ctrl: 836 kfree(ctrl); 837 out_put_subsystem: 838 nvmet_subsys_put(subsys); 839 out: 840 return status; 841 } 842 843 static void nvmet_ctrl_free(struct kref *ref) 844 { 845 struct nvmet_ctrl *ctrl = container_of(ref, struct nvmet_ctrl, ref); 846 struct nvmet_subsys *subsys = ctrl->subsys; 847 848 nvmet_stop_keep_alive_timer(ctrl); 849 850 mutex_lock(&subsys->lock); 851 list_del(&ctrl->subsys_entry); 852 mutex_unlock(&subsys->lock); 853 854 flush_work(&ctrl->async_event_work); 855 cancel_work_sync(&ctrl->fatal_err_work); 856 857 ida_simple_remove(&cntlid_ida, ctrl->cntlid); 858 nvmet_subsys_put(subsys); 859 860 kfree(ctrl->sqs); 861 kfree(ctrl->cqs); 862 kfree(ctrl); 863 } 864 865 void nvmet_ctrl_put(struct nvmet_ctrl *ctrl) 866 { 867 kref_put(&ctrl->ref, nvmet_ctrl_free); 868 } 869 870 static void nvmet_fatal_error_handler(struct work_struct *work) 871 { 872 struct nvmet_ctrl *ctrl = 873 container_of(work, struct nvmet_ctrl, fatal_err_work); 874 875 pr_err("ctrl %d fatal error occurred!\n", ctrl->cntlid); 876 ctrl->ops->delete_ctrl(ctrl); 877 } 878 879 void nvmet_ctrl_fatal_error(struct nvmet_ctrl *ctrl) 880 { 881 mutex_lock(&ctrl->lock); 882 if (!(ctrl->csts & NVME_CSTS_CFS)) { 883 ctrl->csts |= NVME_CSTS_CFS; 884 INIT_WORK(&ctrl->fatal_err_work, nvmet_fatal_error_handler); 885 schedule_work(&ctrl->fatal_err_work); 886 } 887 mutex_unlock(&ctrl->lock); 888 } 889 EXPORT_SYMBOL_GPL(nvmet_ctrl_fatal_error); 890 891 static struct nvmet_subsys *nvmet_find_get_subsys(struct nvmet_port *port, 892 const char *subsysnqn) 893 { 894 struct nvmet_subsys_link *p; 895 896 if (!port) 897 return NULL; 898 899 if (!strncmp(NVME_DISC_SUBSYS_NAME, subsysnqn, 900 NVMF_NQN_SIZE)) { 901 if (!kref_get_unless_zero(&nvmet_disc_subsys->ref)) 902 return NULL; 903 return nvmet_disc_subsys; 904 } 905 906 down_read(&nvmet_config_sem); 907 list_for_each_entry(p, &port->subsystems, entry) { 908 if (!strncmp(p->subsys->subsysnqn, subsysnqn, 909 NVMF_NQN_SIZE)) { 910 if (!kref_get_unless_zero(&p->subsys->ref)) 911 break; 912 up_read(&nvmet_config_sem); 913 return p->subsys; 914 } 915 } 916 up_read(&nvmet_config_sem); 917 return NULL; 918 } 919 920 struct nvmet_subsys *nvmet_subsys_alloc(const char *subsysnqn, 921 enum nvme_subsys_type type) 922 { 923 struct nvmet_subsys *subsys; 924 925 subsys = kzalloc(sizeof(*subsys), GFP_KERNEL); 926 if (!subsys) 927 return NULL; 928 929 subsys->ver = NVME_VS(1, 2, 1); /* NVMe 1.2.1 */ 930 931 switch (type) { 932 case NVME_NQN_NVME: 933 subsys->max_qid = NVMET_NR_QUEUES; 934 break; 935 case NVME_NQN_DISC: 936 subsys->max_qid = 0; 937 break; 938 default: 939 pr_err("%s: Unknown Subsystem type - %d\n", __func__, type); 940 kfree(subsys); 941 return NULL; 942 } 943 subsys->type = type; 944 subsys->subsysnqn = kstrndup(subsysnqn, NVMF_NQN_SIZE, 945 GFP_KERNEL); 946 if (!subsys->subsysnqn) { 947 kfree(subsys); 948 return NULL; 949 } 950 951 kref_init(&subsys->ref); 952 953 mutex_init(&subsys->lock); 954 INIT_LIST_HEAD(&subsys->namespaces); 955 INIT_LIST_HEAD(&subsys->ctrls); 956 INIT_LIST_HEAD(&subsys->hosts); 957 958 return subsys; 959 } 960 961 static void nvmet_subsys_free(struct kref *ref) 962 { 963 struct nvmet_subsys *subsys = 964 container_of(ref, struct nvmet_subsys, ref); 965 966 WARN_ON_ONCE(!list_empty(&subsys->namespaces)); 967 968 kfree(subsys->subsysnqn); 969 kfree(subsys); 970 } 971 972 void nvmet_subsys_del_ctrls(struct nvmet_subsys *subsys) 973 { 974 struct nvmet_ctrl *ctrl; 975 976 mutex_lock(&subsys->lock); 977 list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) 978 ctrl->ops->delete_ctrl(ctrl); 979 mutex_unlock(&subsys->lock); 980 } 981 982 void nvmet_subsys_put(struct nvmet_subsys *subsys) 983 { 984 kref_put(&subsys->ref, nvmet_subsys_free); 985 } 986 987 static int __init nvmet_init(void) 988 { 989 int error; 990 991 error = nvmet_init_discovery(); 992 if (error) 993 goto out; 994 995 error = nvmet_init_configfs(); 996 if (error) 997 goto out_exit_discovery; 998 return 0; 999 1000 out_exit_discovery: 1001 nvmet_exit_discovery(); 1002 out: 1003 return error; 1004 } 1005 1006 static void __exit nvmet_exit(void) 1007 { 1008 nvmet_exit_configfs(); 1009 nvmet_exit_discovery(); 1010 ida_destroy(&cntlid_ida); 1011 1012 BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_entry) != 1024); 1013 BUILD_BUG_ON(sizeof(struct nvmf_disc_rsp_page_hdr) != 1024); 1014 } 1015 1016 module_init(nvmet_init); 1017 module_exit(nvmet_exit); 1018 1019 MODULE_LICENSE("GPL v2"); 1020