13a85a5deSChristoph Hellwig /* 23a85a5deSChristoph Hellwig * NVMe over Fabrics loopback device. 33a85a5deSChristoph Hellwig * Copyright (c) 2015-2016 HGST, a Western Digital Company. 43a85a5deSChristoph Hellwig * 53a85a5deSChristoph Hellwig * This program is free software; you can redistribute it and/or modify it 63a85a5deSChristoph Hellwig * under the terms and conditions of the GNU General Public License, 73a85a5deSChristoph Hellwig * version 2, as published by the Free Software Foundation. 83a85a5deSChristoph Hellwig * 93a85a5deSChristoph Hellwig * This program is distributed in the hope it will be useful, but WITHOUT 103a85a5deSChristoph Hellwig * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 113a85a5deSChristoph Hellwig * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 123a85a5deSChristoph Hellwig * more details. 133a85a5deSChristoph Hellwig */ 143a85a5deSChristoph Hellwig #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 153a85a5deSChristoph Hellwig #include <linux/scatterlist.h> 163a85a5deSChristoph Hellwig #include <linux/blk-mq.h> 173a85a5deSChristoph Hellwig #include <linux/nvme.h> 183a85a5deSChristoph Hellwig #include <linux/module.h> 193a85a5deSChristoph Hellwig #include <linux/parser.h> 203a85a5deSChristoph Hellwig #include "nvmet.h" 213a85a5deSChristoph Hellwig #include "../host/nvme.h" 223a85a5deSChristoph Hellwig #include "../host/fabrics.h" 233a85a5deSChristoph Hellwig 243a85a5deSChristoph Hellwig #define NVME_LOOP_MAX_SEGMENTS 256 253a85a5deSChristoph Hellwig 263a85a5deSChristoph Hellwig /* 273a85a5deSChristoph Hellwig * We handle AEN commands ourselves and don't even let the 283a85a5deSChristoph Hellwig * block layer know about them. 293a85a5deSChristoph Hellwig */ 303a85a5deSChristoph Hellwig #define NVME_LOOP_NR_AEN_COMMANDS 1 313a85a5deSChristoph Hellwig #define NVME_LOOP_AQ_BLKMQ_DEPTH \ 327aa1f427SSagi Grimberg (NVME_AQ_DEPTH - NVME_LOOP_NR_AEN_COMMANDS) 333a85a5deSChristoph Hellwig 343a85a5deSChristoph Hellwig struct nvme_loop_iod { 35d49187e9SChristoph Hellwig struct nvme_request nvme_req; 363a85a5deSChristoph Hellwig struct nvme_command cmd; 373a85a5deSChristoph Hellwig struct nvme_completion rsp; 383a85a5deSChristoph Hellwig struct nvmet_req req; 393a85a5deSChristoph Hellwig struct nvme_loop_queue *queue; 403a85a5deSChristoph Hellwig struct work_struct work; 413a85a5deSChristoph Hellwig struct sg_table sg_table; 423a85a5deSChristoph Hellwig struct scatterlist first_sgl[]; 433a85a5deSChristoph Hellwig }; 443a85a5deSChristoph Hellwig 453a85a5deSChristoph Hellwig struct nvme_loop_ctrl { 463a85a5deSChristoph Hellwig struct nvme_loop_queue *queues; 473a85a5deSChristoph Hellwig 483a85a5deSChristoph Hellwig struct blk_mq_tag_set admin_tag_set; 493a85a5deSChristoph Hellwig 503a85a5deSChristoph Hellwig struct list_head list; 513a85a5deSChristoph Hellwig struct blk_mq_tag_set tag_set; 523a85a5deSChristoph Hellwig struct nvme_loop_iod async_event_iod; 533a85a5deSChristoph Hellwig struct nvme_ctrl ctrl; 543a85a5deSChristoph Hellwig 553a85a5deSChristoph Hellwig struct nvmet_ctrl *target_ctrl; 563a85a5deSChristoph Hellwig struct work_struct delete_work; 573a85a5deSChristoph Hellwig }; 583a85a5deSChristoph Hellwig 593a85a5deSChristoph Hellwig static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl) 603a85a5deSChristoph Hellwig { 613a85a5deSChristoph Hellwig return container_of(ctrl, struct nvme_loop_ctrl, ctrl); 623a85a5deSChristoph Hellwig } 633a85a5deSChristoph Hellwig 643a85a5deSChristoph Hellwig struct nvme_loop_queue { 653a85a5deSChristoph Hellwig struct nvmet_cq nvme_cq; 663a85a5deSChristoph Hellwig struct nvmet_sq nvme_sq; 673a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl; 683a85a5deSChristoph Hellwig }; 693a85a5deSChristoph Hellwig 703a85a5deSChristoph Hellwig static struct nvmet_port *nvmet_loop_port; 713a85a5deSChristoph Hellwig 723a85a5deSChristoph Hellwig static LIST_HEAD(nvme_loop_ctrl_list); 733a85a5deSChristoph Hellwig static DEFINE_MUTEX(nvme_loop_ctrl_mutex); 743a85a5deSChristoph Hellwig 753a85a5deSChristoph Hellwig static void nvme_loop_queue_response(struct nvmet_req *nvme_req); 763a85a5deSChristoph Hellwig static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl); 773a85a5deSChristoph Hellwig 783a85a5deSChristoph Hellwig static struct nvmet_fabrics_ops nvme_loop_ops; 793a85a5deSChristoph Hellwig 803a85a5deSChristoph Hellwig static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue) 813a85a5deSChristoph Hellwig { 823a85a5deSChristoph Hellwig return queue - queue->ctrl->queues; 833a85a5deSChristoph Hellwig } 843a85a5deSChristoph Hellwig 853a85a5deSChristoph Hellwig static void nvme_loop_complete_rq(struct request *req) 863a85a5deSChristoph Hellwig { 873a85a5deSChristoph Hellwig struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); 883a85a5deSChristoph Hellwig 893a85a5deSChristoph Hellwig nvme_cleanup_cmd(req); 903a85a5deSChristoph Hellwig sg_free_table_chained(&iod->sg_table, true); 9177f02a7aSChristoph Hellwig nvme_complete_rq(req); 923a85a5deSChristoph Hellwig } 933a85a5deSChristoph Hellwig 943b068376SSagi Grimberg static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue) 953b068376SSagi Grimberg { 963b068376SSagi Grimberg u32 queue_idx = nvme_loop_queue_idx(queue); 973a85a5deSChristoph Hellwig 983b068376SSagi Grimberg if (queue_idx == 0) 993b068376SSagi Grimberg return queue->ctrl->admin_tag_set.tags[queue_idx]; 1003b068376SSagi Grimberg return queue->ctrl->tag_set.tags[queue_idx - 1]; 1013a85a5deSChristoph Hellwig } 1023a85a5deSChristoph Hellwig 103d49187e9SChristoph Hellwig static void nvme_loop_queue_response(struct nvmet_req *req) 1043a85a5deSChristoph Hellwig { 1053b068376SSagi Grimberg struct nvme_loop_queue *queue = 1063b068376SSagi Grimberg container_of(req->sq, struct nvme_loop_queue, nvme_sq); 1073b068376SSagi Grimberg struct nvme_completion *cqe = req->rsp; 1083a85a5deSChristoph Hellwig 1093a85a5deSChristoph Hellwig /* 1103a85a5deSChristoph Hellwig * AEN requests are special as they don't time out and can 1113a85a5deSChristoph Hellwig * survive any kind of queue freeze and often don't respond to 1123a85a5deSChristoph Hellwig * aborts. We don't even bother to allocate a struct request 1133a85a5deSChristoph Hellwig * for them but rather special case them here. 1143a85a5deSChristoph Hellwig */ 1153b068376SSagi Grimberg if (unlikely(nvme_loop_queue_idx(queue) == 0 && 1163a85a5deSChristoph Hellwig cqe->command_id >= NVME_LOOP_AQ_BLKMQ_DEPTH)) { 1173b068376SSagi Grimberg nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 1187bf58533SChristoph Hellwig &cqe->result); 1193a85a5deSChristoph Hellwig } else { 1203b068376SSagi Grimberg struct request *rq; 1213a85a5deSChristoph Hellwig 1223b068376SSagi Grimberg rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id); 1233b068376SSagi Grimberg if (!rq) { 1243b068376SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 1253b068376SSagi Grimberg "tag 0x%x on queue %d not found\n", 1263b068376SSagi Grimberg cqe->command_id, nvme_loop_queue_idx(queue)); 1273b068376SSagi Grimberg return; 1283b068376SSagi Grimberg } 1293b068376SSagi Grimberg 13027fa9bc5SChristoph Hellwig nvme_end_request(rq, cqe->status, cqe->result); 1313a85a5deSChristoph Hellwig } 1323a85a5deSChristoph Hellwig } 1333a85a5deSChristoph Hellwig 1343a85a5deSChristoph Hellwig static void nvme_loop_execute_work(struct work_struct *work) 1353a85a5deSChristoph Hellwig { 1363a85a5deSChristoph Hellwig struct nvme_loop_iod *iod = 1373a85a5deSChristoph Hellwig container_of(work, struct nvme_loop_iod, work); 1383a85a5deSChristoph Hellwig 1393a85a5deSChristoph Hellwig iod->req.execute(&iod->req); 1403a85a5deSChristoph Hellwig } 1413a85a5deSChristoph Hellwig 1423a85a5deSChristoph Hellwig static enum blk_eh_timer_return 1433a85a5deSChristoph Hellwig nvme_loop_timeout(struct request *rq, bool reserved) 1443a85a5deSChristoph Hellwig { 1453a85a5deSChristoph Hellwig struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq); 1463a85a5deSChristoph Hellwig 1473a85a5deSChristoph Hellwig /* queue error recovery */ 148d86c4d8eSChristoph Hellwig nvme_reset_ctrl(&iod->queue->ctrl->ctrl); 1493a85a5deSChristoph Hellwig 1503a85a5deSChristoph Hellwig /* fail with DNR on admin cmd timeout */ 15127fa9bc5SChristoph Hellwig nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR; 1523a85a5deSChristoph Hellwig 1533a85a5deSChristoph Hellwig return BLK_EH_HANDLED; 1543a85a5deSChristoph Hellwig } 1553a85a5deSChristoph Hellwig 156fc17b653SChristoph Hellwig static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx, 1573a85a5deSChristoph Hellwig const struct blk_mq_queue_data *bd) 1583a85a5deSChristoph Hellwig { 1593a85a5deSChristoph Hellwig struct nvme_ns *ns = hctx->queue->queuedata; 1603a85a5deSChristoph Hellwig struct nvme_loop_queue *queue = hctx->driver_data; 1613a85a5deSChristoph Hellwig struct request *req = bd->rq; 1623a85a5deSChristoph Hellwig struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req); 163fc17b653SChristoph Hellwig blk_status_t ret; 1643a85a5deSChristoph Hellwig 1653a85a5deSChristoph Hellwig ret = nvme_setup_cmd(ns, req, &iod->cmd); 166fc17b653SChristoph Hellwig if (ret) 1673a85a5deSChristoph Hellwig return ret; 1683a85a5deSChristoph Hellwig 1693a85a5deSChristoph Hellwig iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; 1703a85a5deSChristoph Hellwig iod->req.port = nvmet_loop_port; 1713a85a5deSChristoph Hellwig if (!nvmet_req_init(&iod->req, &queue->nvme_cq, 1723a85a5deSChristoph Hellwig &queue->nvme_sq, &nvme_loop_ops)) { 1733a85a5deSChristoph Hellwig nvme_cleanup_cmd(req); 1743a85a5deSChristoph Hellwig blk_mq_start_request(req); 1753a85a5deSChristoph Hellwig nvme_loop_queue_response(&iod->req); 176fc17b653SChristoph Hellwig return BLK_STS_OK; 1773a85a5deSChristoph Hellwig } 1783a85a5deSChristoph Hellwig 1793a85a5deSChristoph Hellwig if (blk_rq_bytes(req)) { 1803a85a5deSChristoph Hellwig iod->sg_table.sgl = iod->first_sgl; 181fc17b653SChristoph Hellwig if (sg_alloc_table_chained(&iod->sg_table, 182f9d03f96SChristoph Hellwig blk_rq_nr_phys_segments(req), 183fc17b653SChristoph Hellwig iod->sg_table.sgl)) 184fc17b653SChristoph Hellwig return BLK_STS_RESOURCE; 1853a85a5deSChristoph Hellwig 1863a85a5deSChristoph Hellwig iod->req.sg = iod->sg_table.sgl; 1873a85a5deSChristoph Hellwig iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl); 1883a85a5deSChristoph Hellwig } 1893a85a5deSChristoph Hellwig 1903a85a5deSChristoph Hellwig blk_mq_start_request(req); 1913a85a5deSChristoph Hellwig 1923a85a5deSChristoph Hellwig schedule_work(&iod->work); 193fc17b653SChristoph Hellwig return BLK_STS_OK; 1943a85a5deSChristoph Hellwig } 1953a85a5deSChristoph Hellwig 1963a85a5deSChristoph Hellwig static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx) 1973a85a5deSChristoph Hellwig { 1983a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg); 1993a85a5deSChristoph Hellwig struct nvme_loop_queue *queue = &ctrl->queues[0]; 2003a85a5deSChristoph Hellwig struct nvme_loop_iod *iod = &ctrl->async_event_iod; 2013a85a5deSChristoph Hellwig 2023a85a5deSChristoph Hellwig memset(&iod->cmd, 0, sizeof(iod->cmd)); 2033a85a5deSChristoph Hellwig iod->cmd.common.opcode = nvme_admin_async_event; 2043a85a5deSChristoph Hellwig iod->cmd.common.command_id = NVME_LOOP_AQ_BLKMQ_DEPTH; 2053a85a5deSChristoph Hellwig iod->cmd.common.flags |= NVME_CMD_SGL_METABUF; 2063a85a5deSChristoph Hellwig 2073a85a5deSChristoph Hellwig if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq, 2083a85a5deSChristoph Hellwig &nvme_loop_ops)) { 2093a85a5deSChristoph Hellwig dev_err(ctrl->ctrl.device, "failed async event work\n"); 2103a85a5deSChristoph Hellwig return; 2113a85a5deSChristoph Hellwig } 2123a85a5deSChristoph Hellwig 2133a85a5deSChristoph Hellwig schedule_work(&iod->work); 2143a85a5deSChristoph Hellwig } 2153a85a5deSChristoph Hellwig 2163a85a5deSChristoph Hellwig static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl, 2173a85a5deSChristoph Hellwig struct nvme_loop_iod *iod, unsigned int queue_idx) 2183a85a5deSChristoph Hellwig { 2193a85a5deSChristoph Hellwig iod->req.cmd = &iod->cmd; 2203a85a5deSChristoph Hellwig iod->req.rsp = &iod->rsp; 2213a85a5deSChristoph Hellwig iod->queue = &ctrl->queues[queue_idx]; 2223a85a5deSChristoph Hellwig INIT_WORK(&iod->work, nvme_loop_execute_work); 2233a85a5deSChristoph Hellwig return 0; 2243a85a5deSChristoph Hellwig } 2253a85a5deSChristoph Hellwig 226d6296d39SChristoph Hellwig static int nvme_loop_init_request(struct blk_mq_tag_set *set, 227d6296d39SChristoph Hellwig struct request *req, unsigned int hctx_idx, 2283a85a5deSChristoph Hellwig unsigned int numa_node) 2293a85a5deSChristoph Hellwig { 23062b83b18SChristoph Hellwig struct nvme_loop_ctrl *ctrl = set->driver_data; 2313a85a5deSChristoph Hellwig 23262b83b18SChristoph Hellwig return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req), 23362b83b18SChristoph Hellwig (set == &ctrl->tag_set) ? hctx_idx + 1 : 0); 2343a85a5deSChristoph Hellwig } 2353a85a5deSChristoph Hellwig 2363a85a5deSChristoph Hellwig static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 2373a85a5deSChristoph Hellwig unsigned int hctx_idx) 2383a85a5deSChristoph Hellwig { 2393a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl = data; 2403a85a5deSChristoph Hellwig struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1]; 2413a85a5deSChristoph Hellwig 242d858e5f0SSagi Grimberg BUG_ON(hctx_idx >= ctrl->ctrl.queue_count); 2433a85a5deSChristoph Hellwig 2443a85a5deSChristoph Hellwig hctx->driver_data = queue; 2453a85a5deSChristoph Hellwig return 0; 2463a85a5deSChristoph Hellwig } 2473a85a5deSChristoph Hellwig 2483a85a5deSChristoph Hellwig static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 2493a85a5deSChristoph Hellwig unsigned int hctx_idx) 2503a85a5deSChristoph Hellwig { 2513a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl = data; 2523a85a5deSChristoph Hellwig struct nvme_loop_queue *queue = &ctrl->queues[0]; 2533a85a5deSChristoph Hellwig 2543a85a5deSChristoph Hellwig BUG_ON(hctx_idx != 0); 2553a85a5deSChristoph Hellwig 2563a85a5deSChristoph Hellwig hctx->driver_data = queue; 2573a85a5deSChristoph Hellwig return 0; 2583a85a5deSChristoph Hellwig } 2593a85a5deSChristoph Hellwig 260f363b089SEric Biggers static const struct blk_mq_ops nvme_loop_mq_ops = { 2613a85a5deSChristoph Hellwig .queue_rq = nvme_loop_queue_rq, 2623a85a5deSChristoph Hellwig .complete = nvme_loop_complete_rq, 2633a85a5deSChristoph Hellwig .init_request = nvme_loop_init_request, 2643a85a5deSChristoph Hellwig .init_hctx = nvme_loop_init_hctx, 2653a85a5deSChristoph Hellwig .timeout = nvme_loop_timeout, 2663a85a5deSChristoph Hellwig }; 2673a85a5deSChristoph Hellwig 268f363b089SEric Biggers static const struct blk_mq_ops nvme_loop_admin_mq_ops = { 2693a85a5deSChristoph Hellwig .queue_rq = nvme_loop_queue_rq, 2703a85a5deSChristoph Hellwig .complete = nvme_loop_complete_rq, 27162b83b18SChristoph Hellwig .init_request = nvme_loop_init_request, 2723a85a5deSChristoph Hellwig .init_hctx = nvme_loop_init_admin_hctx, 2733a85a5deSChristoph Hellwig .timeout = nvme_loop_timeout, 2743a85a5deSChristoph Hellwig }; 2753a85a5deSChristoph Hellwig 2763a85a5deSChristoph Hellwig static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl) 2773a85a5deSChristoph Hellwig { 278e4c5d376SSagi Grimberg nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); 2793a85a5deSChristoph Hellwig blk_cleanup_queue(ctrl->ctrl.admin_q); 2803a85a5deSChristoph Hellwig blk_mq_free_tag_set(&ctrl->admin_tag_set); 2813a85a5deSChristoph Hellwig } 2823a85a5deSChristoph Hellwig 2833a85a5deSChristoph Hellwig static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl) 2843a85a5deSChristoph Hellwig { 2853a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); 2863a85a5deSChristoph Hellwig 2873a85a5deSChristoph Hellwig if (list_empty(&ctrl->list)) 2883a85a5deSChristoph Hellwig goto free_ctrl; 2893a85a5deSChristoph Hellwig 2903a85a5deSChristoph Hellwig mutex_lock(&nvme_loop_ctrl_mutex); 2913a85a5deSChristoph Hellwig list_del(&ctrl->list); 2923a85a5deSChristoph Hellwig mutex_unlock(&nvme_loop_ctrl_mutex); 2933a85a5deSChristoph Hellwig 2943a85a5deSChristoph Hellwig if (nctrl->tagset) { 2953a85a5deSChristoph Hellwig blk_cleanup_queue(ctrl->ctrl.connect_q); 2963a85a5deSChristoph Hellwig blk_mq_free_tag_set(&ctrl->tag_set); 2973a85a5deSChristoph Hellwig } 2983a85a5deSChristoph Hellwig kfree(ctrl->queues); 2993a85a5deSChristoph Hellwig nvmf_free_options(nctrl->opts); 3003a85a5deSChristoph Hellwig free_ctrl: 3013a85a5deSChristoph Hellwig kfree(ctrl); 3023a85a5deSChristoph Hellwig } 3033a85a5deSChristoph Hellwig 304945dd5baSSagi Grimberg static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl) 305945dd5baSSagi Grimberg { 306945dd5baSSagi Grimberg int i; 307945dd5baSSagi Grimberg 308d858e5f0SSagi Grimberg for (i = 1; i < ctrl->ctrl.queue_count; i++) 309945dd5baSSagi Grimberg nvmet_sq_destroy(&ctrl->queues[i].nvme_sq); 310945dd5baSSagi Grimberg } 311945dd5baSSagi Grimberg 312945dd5baSSagi Grimberg static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl) 313945dd5baSSagi Grimberg { 314945dd5baSSagi Grimberg struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 315945dd5baSSagi Grimberg unsigned int nr_io_queues; 316945dd5baSSagi Grimberg int ret, i; 317945dd5baSSagi Grimberg 318945dd5baSSagi Grimberg nr_io_queues = min(opts->nr_io_queues, num_online_cpus()); 319945dd5baSSagi Grimberg ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues); 320945dd5baSSagi Grimberg if (ret || !nr_io_queues) 321945dd5baSSagi Grimberg return ret; 322945dd5baSSagi Grimberg 323945dd5baSSagi Grimberg dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues); 324945dd5baSSagi Grimberg 325945dd5baSSagi Grimberg for (i = 1; i <= nr_io_queues; i++) { 326945dd5baSSagi Grimberg ctrl->queues[i].ctrl = ctrl; 327945dd5baSSagi Grimberg ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq); 328945dd5baSSagi Grimberg if (ret) 329945dd5baSSagi Grimberg goto out_destroy_queues; 330945dd5baSSagi Grimberg 331d858e5f0SSagi Grimberg ctrl->ctrl.queue_count++; 332945dd5baSSagi Grimberg } 333945dd5baSSagi Grimberg 334945dd5baSSagi Grimberg return 0; 335945dd5baSSagi Grimberg 336945dd5baSSagi Grimberg out_destroy_queues: 337945dd5baSSagi Grimberg nvme_loop_destroy_io_queues(ctrl); 338945dd5baSSagi Grimberg return ret; 339945dd5baSSagi Grimberg } 340945dd5baSSagi Grimberg 341297186d6SSagi Grimberg static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl) 342297186d6SSagi Grimberg { 343297186d6SSagi Grimberg int i, ret; 344297186d6SSagi Grimberg 345d858e5f0SSagi Grimberg for (i = 1; i < ctrl->ctrl.queue_count; i++) { 346297186d6SSagi Grimberg ret = nvmf_connect_io_queue(&ctrl->ctrl, i); 347297186d6SSagi Grimberg if (ret) 348297186d6SSagi Grimberg return ret; 349297186d6SSagi Grimberg } 350297186d6SSagi Grimberg 351297186d6SSagi Grimberg return 0; 352297186d6SSagi Grimberg } 353297186d6SSagi Grimberg 3543a85a5deSChristoph Hellwig static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl) 3553a85a5deSChristoph Hellwig { 3563a85a5deSChristoph Hellwig int error; 3573a85a5deSChristoph Hellwig 3583a85a5deSChristoph Hellwig memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set)); 3593a85a5deSChristoph Hellwig ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops; 3603a85a5deSChristoph Hellwig ctrl->admin_tag_set.queue_depth = NVME_LOOP_AQ_BLKMQ_DEPTH; 3613a85a5deSChristoph Hellwig ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */ 3623a85a5deSChristoph Hellwig ctrl->admin_tag_set.numa_node = NUMA_NO_NODE; 3633a85a5deSChristoph Hellwig ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) + 3643a85a5deSChristoph Hellwig SG_CHUNK_SIZE * sizeof(struct scatterlist); 3653a85a5deSChristoph Hellwig ctrl->admin_tag_set.driver_data = ctrl; 3663a85a5deSChristoph Hellwig ctrl->admin_tag_set.nr_hw_queues = 1; 3673a85a5deSChristoph Hellwig ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT; 3683a85a5deSChristoph Hellwig 3693a85a5deSChristoph Hellwig ctrl->queues[0].ctrl = ctrl; 3703a85a5deSChristoph Hellwig error = nvmet_sq_init(&ctrl->queues[0].nvme_sq); 3713a85a5deSChristoph Hellwig if (error) 3723a85a5deSChristoph Hellwig return error; 373d858e5f0SSagi Grimberg ctrl->ctrl.queue_count = 1; 3743a85a5deSChristoph Hellwig 3753a85a5deSChristoph Hellwig error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set); 3763a85a5deSChristoph Hellwig if (error) 3773a85a5deSChristoph Hellwig goto out_free_sq; 3783a85a5deSChristoph Hellwig 3793a85a5deSChristoph Hellwig ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set); 3803a85a5deSChristoph Hellwig if (IS_ERR(ctrl->ctrl.admin_q)) { 3813a85a5deSChristoph Hellwig error = PTR_ERR(ctrl->ctrl.admin_q); 3823a85a5deSChristoph Hellwig goto out_free_tagset; 3833a85a5deSChristoph Hellwig } 3843a85a5deSChristoph Hellwig 3853a85a5deSChristoph Hellwig error = nvmf_connect_admin_queue(&ctrl->ctrl); 3863a85a5deSChristoph Hellwig if (error) 3873a85a5deSChristoph Hellwig goto out_cleanup_queue; 3883a85a5deSChristoph Hellwig 38920d0dfe6SSagi Grimberg error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap); 3903a85a5deSChristoph Hellwig if (error) { 3913a85a5deSChristoph Hellwig dev_err(ctrl->ctrl.device, 3923a85a5deSChristoph Hellwig "prop_get NVME_REG_CAP failed\n"); 3933a85a5deSChristoph Hellwig goto out_cleanup_queue; 3943a85a5deSChristoph Hellwig } 3953a85a5deSChristoph Hellwig 3963a85a5deSChristoph Hellwig ctrl->ctrl.sqsize = 39720d0dfe6SSagi Grimberg min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize); 3983a85a5deSChristoph Hellwig 39920d0dfe6SSagi Grimberg error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap); 4003a85a5deSChristoph Hellwig if (error) 4013a85a5deSChristoph Hellwig goto out_cleanup_queue; 4023a85a5deSChristoph Hellwig 4033a85a5deSChristoph Hellwig ctrl->ctrl.max_hw_sectors = 4043a85a5deSChristoph Hellwig (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9); 4053a85a5deSChristoph Hellwig 4063a85a5deSChristoph Hellwig error = nvme_init_identify(&ctrl->ctrl); 4073a85a5deSChristoph Hellwig if (error) 4083a85a5deSChristoph Hellwig goto out_cleanup_queue; 4093a85a5deSChristoph Hellwig 4103a85a5deSChristoph Hellwig return 0; 4113a85a5deSChristoph Hellwig 4123a85a5deSChristoph Hellwig out_cleanup_queue: 4133a85a5deSChristoph Hellwig blk_cleanup_queue(ctrl->ctrl.admin_q); 4143a85a5deSChristoph Hellwig out_free_tagset: 4153a85a5deSChristoph Hellwig blk_mq_free_tag_set(&ctrl->admin_tag_set); 4163a85a5deSChristoph Hellwig out_free_sq: 4173a85a5deSChristoph Hellwig nvmet_sq_destroy(&ctrl->queues[0].nvme_sq); 4183a85a5deSChristoph Hellwig return error; 4193a85a5deSChristoph Hellwig } 4203a85a5deSChristoph Hellwig 4213a85a5deSChristoph Hellwig static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl) 4223a85a5deSChristoph Hellwig { 423d858e5f0SSagi Grimberg if (ctrl->ctrl.queue_count > 1) { 4243a85a5deSChristoph Hellwig nvme_stop_queues(&ctrl->ctrl); 4253a85a5deSChristoph Hellwig blk_mq_tagset_busy_iter(&ctrl->tag_set, 4263a85a5deSChristoph Hellwig nvme_cancel_request, &ctrl->ctrl); 427945dd5baSSagi Grimberg nvme_loop_destroy_io_queues(ctrl); 4283a85a5deSChristoph Hellwig } 4293a85a5deSChristoph Hellwig 4303a85a5deSChristoph Hellwig if (ctrl->ctrl.state == NVME_CTRL_LIVE) 4313a85a5deSChristoph Hellwig nvme_shutdown_ctrl(&ctrl->ctrl); 4323a85a5deSChristoph Hellwig 433c1c0ffffSSagi Grimberg blk_mq_quiesce_queue(ctrl->ctrl.admin_q); 4343a85a5deSChristoph Hellwig blk_mq_tagset_busy_iter(&ctrl->admin_tag_set, 4353a85a5deSChristoph Hellwig nvme_cancel_request, &ctrl->ctrl); 436c1c0ffffSSagi Grimberg blk_mq_unquiesce_queue(ctrl->ctrl.admin_q); 4373a85a5deSChristoph Hellwig nvme_loop_destroy_admin_queue(ctrl); 4383a85a5deSChristoph Hellwig } 4393a85a5deSChristoph Hellwig 4403a85a5deSChristoph Hellwig static void nvme_loop_del_ctrl_work(struct work_struct *work) 4413a85a5deSChristoph Hellwig { 4423a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl = container_of(work, 4433a85a5deSChristoph Hellwig struct nvme_loop_ctrl, delete_work); 4443a85a5deSChristoph Hellwig 445*d09f2b45SSagi Grimberg nvme_stop_ctrl(&ctrl->ctrl); 446*d09f2b45SSagi Grimberg nvme_remove_namespaces(&ctrl->ctrl); 447a159c64dSSagi Grimberg nvme_loop_shutdown_ctrl(ctrl); 448*d09f2b45SSagi Grimberg nvme_uninit_ctrl(&ctrl->ctrl); 4493a85a5deSChristoph Hellwig nvme_put_ctrl(&ctrl->ctrl); 4503a85a5deSChristoph Hellwig } 4513a85a5deSChristoph Hellwig 4523a85a5deSChristoph Hellwig static int __nvme_loop_del_ctrl(struct nvme_loop_ctrl *ctrl) 4533a85a5deSChristoph Hellwig { 4543a85a5deSChristoph Hellwig if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING)) 4553a85a5deSChristoph Hellwig return -EBUSY; 4563a85a5deSChristoph Hellwig 4579a6327d2SSagi Grimberg if (!queue_work(nvme_wq, &ctrl->delete_work)) 4583a85a5deSChristoph Hellwig return -EBUSY; 4593a85a5deSChristoph Hellwig 4603a85a5deSChristoph Hellwig return 0; 4613a85a5deSChristoph Hellwig } 4623a85a5deSChristoph Hellwig 4633a85a5deSChristoph Hellwig static int nvme_loop_del_ctrl(struct nvme_ctrl *nctrl) 4643a85a5deSChristoph Hellwig { 4653a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl); 4663a85a5deSChristoph Hellwig int ret; 4673a85a5deSChristoph Hellwig 4683a85a5deSChristoph Hellwig ret = __nvme_loop_del_ctrl(ctrl); 4693a85a5deSChristoph Hellwig if (ret) 4703a85a5deSChristoph Hellwig return ret; 4713a85a5deSChristoph Hellwig 4723a85a5deSChristoph Hellwig flush_work(&ctrl->delete_work); 4733a85a5deSChristoph Hellwig 4743a85a5deSChristoph Hellwig return 0; 4753a85a5deSChristoph Hellwig } 4763a85a5deSChristoph Hellwig 4773a85a5deSChristoph Hellwig static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl) 4783a85a5deSChristoph Hellwig { 4793a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl; 4803a85a5deSChristoph Hellwig 4813a85a5deSChristoph Hellwig mutex_lock(&nvme_loop_ctrl_mutex); 4823a85a5deSChristoph Hellwig list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) { 4833a85a5deSChristoph Hellwig if (ctrl->ctrl.cntlid == nctrl->cntlid) 4843a85a5deSChristoph Hellwig __nvme_loop_del_ctrl(ctrl); 4853a85a5deSChristoph Hellwig } 4863a85a5deSChristoph Hellwig mutex_unlock(&nvme_loop_ctrl_mutex); 4873a85a5deSChristoph Hellwig } 4883a85a5deSChristoph Hellwig 4893a85a5deSChristoph Hellwig static void nvme_loop_reset_ctrl_work(struct work_struct *work) 4903a85a5deSChristoph Hellwig { 491d86c4d8eSChristoph Hellwig struct nvme_loop_ctrl *ctrl = 492d86c4d8eSChristoph Hellwig container_of(work, struct nvme_loop_ctrl, ctrl.reset_work); 4933a85a5deSChristoph Hellwig bool changed; 494297186d6SSagi Grimberg int ret; 4953a85a5deSChristoph Hellwig 496*d09f2b45SSagi Grimberg nvme_stop_ctrl(&ctrl->ctrl); 4973a85a5deSChristoph Hellwig nvme_loop_shutdown_ctrl(ctrl); 4983a85a5deSChristoph Hellwig 4993a85a5deSChristoph Hellwig ret = nvme_loop_configure_admin_queue(ctrl); 5003a85a5deSChristoph Hellwig if (ret) 5013a85a5deSChristoph Hellwig goto out_disable; 5023a85a5deSChristoph Hellwig 503945dd5baSSagi Grimberg ret = nvme_loop_init_io_queues(ctrl); 5043a85a5deSChristoph Hellwig if (ret) 505945dd5baSSagi Grimberg goto out_destroy_admin; 5063a85a5deSChristoph Hellwig 507297186d6SSagi Grimberg ret = nvme_loop_connect_io_queues(ctrl); 5083a85a5deSChristoph Hellwig if (ret) 509945dd5baSSagi Grimberg goto out_destroy_io; 5103a85a5deSChristoph Hellwig 5114368c39bSSagi Grimberg blk_mq_update_nr_hw_queues(&ctrl->tag_set, 5124368c39bSSagi Grimberg ctrl->ctrl.queue_count - 1); 5134368c39bSSagi Grimberg 5143a85a5deSChristoph Hellwig changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE); 5153a85a5deSChristoph Hellwig WARN_ON_ONCE(!changed); 5163a85a5deSChristoph Hellwig 517*d09f2b45SSagi Grimberg nvme_start_ctrl(&ctrl->ctrl); 5183a85a5deSChristoph Hellwig 5193a85a5deSChristoph Hellwig return; 5203a85a5deSChristoph Hellwig 521945dd5baSSagi Grimberg out_destroy_io: 522945dd5baSSagi Grimberg nvme_loop_destroy_io_queues(ctrl); 523945dd5baSSagi Grimberg out_destroy_admin: 5243a85a5deSChristoph Hellwig nvme_loop_destroy_admin_queue(ctrl); 5253a85a5deSChristoph Hellwig out_disable: 5263a85a5deSChristoph Hellwig dev_warn(ctrl->ctrl.device, "Removing after reset failure\n"); 5273a85a5deSChristoph Hellwig nvme_uninit_ctrl(&ctrl->ctrl); 5283a85a5deSChristoph Hellwig nvme_put_ctrl(&ctrl->ctrl); 5293a85a5deSChristoph Hellwig } 5303a85a5deSChristoph Hellwig 5313a85a5deSChristoph Hellwig static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = { 5323a85a5deSChristoph Hellwig .name = "loop", 5333a85a5deSChristoph Hellwig .module = THIS_MODULE, 534d3d5b87dSChristoph Hellwig .flags = NVME_F_FABRICS, 5353a85a5deSChristoph Hellwig .reg_read32 = nvmf_reg_read32, 5363a85a5deSChristoph Hellwig .reg_read64 = nvmf_reg_read64, 5373a85a5deSChristoph Hellwig .reg_write32 = nvmf_reg_write32, 5383a85a5deSChristoph Hellwig .free_ctrl = nvme_loop_free_ctrl, 5393a85a5deSChristoph Hellwig .submit_async_event = nvme_loop_submit_async_event, 5403a85a5deSChristoph Hellwig .delete_ctrl = nvme_loop_del_ctrl, 5413a85a5deSChristoph Hellwig }; 5423a85a5deSChristoph Hellwig 5433a85a5deSChristoph Hellwig static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) 5443a85a5deSChristoph Hellwig { 545297186d6SSagi Grimberg int ret; 5463a85a5deSChristoph Hellwig 547945dd5baSSagi Grimberg ret = nvme_loop_init_io_queues(ctrl); 5483a85a5deSChristoph Hellwig if (ret) 549945dd5baSSagi Grimberg return ret; 5503a85a5deSChristoph Hellwig 5513a85a5deSChristoph Hellwig memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set)); 5523a85a5deSChristoph Hellwig ctrl->tag_set.ops = &nvme_loop_mq_ops; 553eadb7cf4SJay Freyensee ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size; 5543a85a5deSChristoph Hellwig ctrl->tag_set.reserved_tags = 1; /* fabric connect */ 5553a85a5deSChristoph Hellwig ctrl->tag_set.numa_node = NUMA_NO_NODE; 5563a85a5deSChristoph Hellwig ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE; 5573a85a5deSChristoph Hellwig ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) + 5583a85a5deSChristoph Hellwig SG_CHUNK_SIZE * sizeof(struct scatterlist); 5593a85a5deSChristoph Hellwig ctrl->tag_set.driver_data = ctrl; 560d858e5f0SSagi Grimberg ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1; 5613a85a5deSChristoph Hellwig ctrl->tag_set.timeout = NVME_IO_TIMEOUT; 5623a85a5deSChristoph Hellwig ctrl->ctrl.tagset = &ctrl->tag_set; 5633a85a5deSChristoph Hellwig 5643a85a5deSChristoph Hellwig ret = blk_mq_alloc_tag_set(&ctrl->tag_set); 5653a85a5deSChristoph Hellwig if (ret) 5663a85a5deSChristoph Hellwig goto out_destroy_queues; 5673a85a5deSChristoph Hellwig 5683a85a5deSChristoph Hellwig ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set); 5693a85a5deSChristoph Hellwig if (IS_ERR(ctrl->ctrl.connect_q)) { 5703a85a5deSChristoph Hellwig ret = PTR_ERR(ctrl->ctrl.connect_q); 5713a85a5deSChristoph Hellwig goto out_free_tagset; 5723a85a5deSChristoph Hellwig } 5733a85a5deSChristoph Hellwig 574297186d6SSagi Grimberg ret = nvme_loop_connect_io_queues(ctrl); 5753a85a5deSChristoph Hellwig if (ret) 5763a85a5deSChristoph Hellwig goto out_cleanup_connect_q; 5773a85a5deSChristoph Hellwig 5783a85a5deSChristoph Hellwig return 0; 5793a85a5deSChristoph Hellwig 5803a85a5deSChristoph Hellwig out_cleanup_connect_q: 5813a85a5deSChristoph Hellwig blk_cleanup_queue(ctrl->ctrl.connect_q); 5823a85a5deSChristoph Hellwig out_free_tagset: 5833a85a5deSChristoph Hellwig blk_mq_free_tag_set(&ctrl->tag_set); 5843a85a5deSChristoph Hellwig out_destroy_queues: 585945dd5baSSagi Grimberg nvme_loop_destroy_io_queues(ctrl); 5863a85a5deSChristoph Hellwig return ret; 5873a85a5deSChristoph Hellwig } 5883a85a5deSChristoph Hellwig 5893a85a5deSChristoph Hellwig static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev, 5903a85a5deSChristoph Hellwig struct nvmf_ctrl_options *opts) 5913a85a5deSChristoph Hellwig { 5923a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl; 5933a85a5deSChristoph Hellwig bool changed; 5943a85a5deSChristoph Hellwig int ret; 5953a85a5deSChristoph Hellwig 5963a85a5deSChristoph Hellwig ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 5973a85a5deSChristoph Hellwig if (!ctrl) 5983a85a5deSChristoph Hellwig return ERR_PTR(-ENOMEM); 5993a85a5deSChristoph Hellwig ctrl->ctrl.opts = opts; 6003a85a5deSChristoph Hellwig INIT_LIST_HEAD(&ctrl->list); 6013a85a5deSChristoph Hellwig 6023a85a5deSChristoph Hellwig INIT_WORK(&ctrl->delete_work, nvme_loop_del_ctrl_work); 603d86c4d8eSChristoph Hellwig INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work); 6043a85a5deSChristoph Hellwig 6053a85a5deSChristoph Hellwig ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops, 6063a85a5deSChristoph Hellwig 0 /* no quirks, we're perfect! */); 6073a85a5deSChristoph Hellwig if (ret) 6083a85a5deSChristoph Hellwig goto out_put_ctrl; 6093a85a5deSChristoph Hellwig 6103a85a5deSChristoph Hellwig ret = -ENOMEM; 6113a85a5deSChristoph Hellwig 612eadb7cf4SJay Freyensee ctrl->ctrl.sqsize = opts->queue_size - 1; 6133a85a5deSChristoph Hellwig ctrl->ctrl.kato = opts->kato; 6143a85a5deSChristoph Hellwig 6153a85a5deSChristoph Hellwig ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues), 6163a85a5deSChristoph Hellwig GFP_KERNEL); 6173a85a5deSChristoph Hellwig if (!ctrl->queues) 6183a85a5deSChristoph Hellwig goto out_uninit_ctrl; 6193a85a5deSChristoph Hellwig 6203a85a5deSChristoph Hellwig ret = nvme_loop_configure_admin_queue(ctrl); 6213a85a5deSChristoph Hellwig if (ret) 6223a85a5deSChristoph Hellwig goto out_free_queues; 6233a85a5deSChristoph Hellwig 6243a85a5deSChristoph Hellwig if (opts->queue_size > ctrl->ctrl.maxcmd) { 6253a85a5deSChristoph Hellwig /* warn if maxcmd is lower than queue_size */ 6263a85a5deSChristoph Hellwig dev_warn(ctrl->ctrl.device, 6273a85a5deSChristoph Hellwig "queue_size %zu > ctrl maxcmd %u, clamping down\n", 6283a85a5deSChristoph Hellwig opts->queue_size, ctrl->ctrl.maxcmd); 6293a85a5deSChristoph Hellwig opts->queue_size = ctrl->ctrl.maxcmd; 6303a85a5deSChristoph Hellwig } 6313a85a5deSChristoph Hellwig 6323a85a5deSChristoph Hellwig if (opts->nr_io_queues) { 6333a85a5deSChristoph Hellwig ret = nvme_loop_create_io_queues(ctrl); 6343a85a5deSChristoph Hellwig if (ret) 6353a85a5deSChristoph Hellwig goto out_remove_admin_queue; 6363a85a5deSChristoph Hellwig } 6373a85a5deSChristoph Hellwig 6383a85a5deSChristoph Hellwig nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0); 6393a85a5deSChristoph Hellwig 6403a85a5deSChristoph Hellwig dev_info(ctrl->ctrl.device, 6413a85a5deSChristoph Hellwig "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn); 6423a85a5deSChristoph Hellwig 6433a85a5deSChristoph Hellwig kref_get(&ctrl->ctrl.kref); 6443a85a5deSChristoph Hellwig 6453a85a5deSChristoph Hellwig changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE); 6463a85a5deSChristoph Hellwig WARN_ON_ONCE(!changed); 6473a85a5deSChristoph Hellwig 6483a85a5deSChristoph Hellwig mutex_lock(&nvme_loop_ctrl_mutex); 6493a85a5deSChristoph Hellwig list_add_tail(&ctrl->list, &nvme_loop_ctrl_list); 6503a85a5deSChristoph Hellwig mutex_unlock(&nvme_loop_ctrl_mutex); 6513a85a5deSChristoph Hellwig 652*d09f2b45SSagi Grimberg nvme_start_ctrl(&ctrl->ctrl); 6533a85a5deSChristoph Hellwig 6543a85a5deSChristoph Hellwig return &ctrl->ctrl; 6553a85a5deSChristoph Hellwig 6563a85a5deSChristoph Hellwig out_remove_admin_queue: 6573a85a5deSChristoph Hellwig nvme_loop_destroy_admin_queue(ctrl); 6583a85a5deSChristoph Hellwig out_free_queues: 6593a85a5deSChristoph Hellwig kfree(ctrl->queues); 6603a85a5deSChristoph Hellwig out_uninit_ctrl: 6613a85a5deSChristoph Hellwig nvme_uninit_ctrl(&ctrl->ctrl); 6623a85a5deSChristoph Hellwig out_put_ctrl: 6633a85a5deSChristoph Hellwig nvme_put_ctrl(&ctrl->ctrl); 6643a85a5deSChristoph Hellwig if (ret > 0) 6653a85a5deSChristoph Hellwig ret = -EIO; 6663a85a5deSChristoph Hellwig return ERR_PTR(ret); 6673a85a5deSChristoph Hellwig } 6683a85a5deSChristoph Hellwig 6693a85a5deSChristoph Hellwig static int nvme_loop_add_port(struct nvmet_port *port) 6703a85a5deSChristoph Hellwig { 6713a85a5deSChristoph Hellwig /* 6723a85a5deSChristoph Hellwig * XXX: disalow adding more than one port so 6733a85a5deSChristoph Hellwig * there is no connection rejections when a 6743a85a5deSChristoph Hellwig * a subsystem is assigned to a port for which 6753a85a5deSChristoph Hellwig * loop doesn't have a pointer. 6763a85a5deSChristoph Hellwig * This scenario would be possible if we allowed 6773a85a5deSChristoph Hellwig * more than one port to be added and a subsystem 6783a85a5deSChristoph Hellwig * was assigned to a port other than nvmet_loop_port. 6793a85a5deSChristoph Hellwig */ 6803a85a5deSChristoph Hellwig 6813a85a5deSChristoph Hellwig if (nvmet_loop_port) 6823a85a5deSChristoph Hellwig return -EPERM; 6833a85a5deSChristoph Hellwig 6843a85a5deSChristoph Hellwig nvmet_loop_port = port; 6853a85a5deSChristoph Hellwig return 0; 6863a85a5deSChristoph Hellwig } 6873a85a5deSChristoph Hellwig 6883a85a5deSChristoph Hellwig static void nvme_loop_remove_port(struct nvmet_port *port) 6893a85a5deSChristoph Hellwig { 6903a85a5deSChristoph Hellwig if (port == nvmet_loop_port) 6913a85a5deSChristoph Hellwig nvmet_loop_port = NULL; 6923a85a5deSChristoph Hellwig } 6933a85a5deSChristoph Hellwig 6943a85a5deSChristoph Hellwig static struct nvmet_fabrics_ops nvme_loop_ops = { 6953a85a5deSChristoph Hellwig .owner = THIS_MODULE, 6963a85a5deSChristoph Hellwig .type = NVMF_TRTYPE_LOOP, 6973a85a5deSChristoph Hellwig .add_port = nvme_loop_add_port, 6983a85a5deSChristoph Hellwig .remove_port = nvme_loop_remove_port, 6993a85a5deSChristoph Hellwig .queue_response = nvme_loop_queue_response, 7003a85a5deSChristoph Hellwig .delete_ctrl = nvme_loop_delete_ctrl, 7013a85a5deSChristoph Hellwig }; 7023a85a5deSChristoph Hellwig 7033a85a5deSChristoph Hellwig static struct nvmf_transport_ops nvme_loop_transport = { 7043a85a5deSChristoph Hellwig .name = "loop", 7053a85a5deSChristoph Hellwig .create_ctrl = nvme_loop_create_ctrl, 7063a85a5deSChristoph Hellwig }; 7073a85a5deSChristoph Hellwig 7083a85a5deSChristoph Hellwig static int __init nvme_loop_init_module(void) 7093a85a5deSChristoph Hellwig { 7103a85a5deSChristoph Hellwig int ret; 7113a85a5deSChristoph Hellwig 7123a85a5deSChristoph Hellwig ret = nvmet_register_transport(&nvme_loop_ops); 7133a85a5deSChristoph Hellwig if (ret) 7143a85a5deSChristoph Hellwig return ret; 715d19eef02SSagi Grimberg 716d19eef02SSagi Grimberg ret = nvmf_register_transport(&nvme_loop_transport); 717d19eef02SSagi Grimberg if (ret) 718d19eef02SSagi Grimberg nvmet_unregister_transport(&nvme_loop_ops); 719d19eef02SSagi Grimberg 720d19eef02SSagi Grimberg return ret; 7213a85a5deSChristoph Hellwig } 7223a85a5deSChristoph Hellwig 7233a85a5deSChristoph Hellwig static void __exit nvme_loop_cleanup_module(void) 7243a85a5deSChristoph Hellwig { 7253a85a5deSChristoph Hellwig struct nvme_loop_ctrl *ctrl, *next; 7263a85a5deSChristoph Hellwig 7273a85a5deSChristoph Hellwig nvmf_unregister_transport(&nvme_loop_transport); 7283a85a5deSChristoph Hellwig nvmet_unregister_transport(&nvme_loop_ops); 7293a85a5deSChristoph Hellwig 7303a85a5deSChristoph Hellwig mutex_lock(&nvme_loop_ctrl_mutex); 7313a85a5deSChristoph Hellwig list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list) 7323a85a5deSChristoph Hellwig __nvme_loop_del_ctrl(ctrl); 7333a85a5deSChristoph Hellwig mutex_unlock(&nvme_loop_ctrl_mutex); 7343a85a5deSChristoph Hellwig 7359a6327d2SSagi Grimberg flush_workqueue(nvme_wq); 7363a85a5deSChristoph Hellwig } 7373a85a5deSChristoph Hellwig 7383a85a5deSChristoph Hellwig module_init(nvme_loop_init_module); 7393a85a5deSChristoph Hellwig module_exit(nvme_loop_cleanup_module); 7403a85a5deSChristoph Hellwig 7413a85a5deSChristoph Hellwig MODULE_LICENSE("GPL v2"); 7423a85a5deSChristoph Hellwig MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */ 743