xref: /openbmc/linux/drivers/nvme/target/loop.c (revision d09f2b45f346f0a9e5e1b5fcea531b1b393671dc)
13a85a5deSChristoph Hellwig /*
23a85a5deSChristoph Hellwig  * NVMe over Fabrics loopback device.
33a85a5deSChristoph Hellwig  * Copyright (c) 2015-2016 HGST, a Western Digital Company.
43a85a5deSChristoph Hellwig  *
53a85a5deSChristoph Hellwig  * This program is free software; you can redistribute it and/or modify it
63a85a5deSChristoph Hellwig  * under the terms and conditions of the GNU General Public License,
73a85a5deSChristoph Hellwig  * version 2, as published by the Free Software Foundation.
83a85a5deSChristoph Hellwig  *
93a85a5deSChristoph Hellwig  * This program is distributed in the hope it will be useful, but WITHOUT
103a85a5deSChristoph Hellwig  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
113a85a5deSChristoph Hellwig  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
123a85a5deSChristoph Hellwig  * more details.
133a85a5deSChristoph Hellwig  */
143a85a5deSChristoph Hellwig #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
153a85a5deSChristoph Hellwig #include <linux/scatterlist.h>
163a85a5deSChristoph Hellwig #include <linux/blk-mq.h>
173a85a5deSChristoph Hellwig #include <linux/nvme.h>
183a85a5deSChristoph Hellwig #include <linux/module.h>
193a85a5deSChristoph Hellwig #include <linux/parser.h>
203a85a5deSChristoph Hellwig #include "nvmet.h"
213a85a5deSChristoph Hellwig #include "../host/nvme.h"
223a85a5deSChristoph Hellwig #include "../host/fabrics.h"
233a85a5deSChristoph Hellwig 
243a85a5deSChristoph Hellwig #define NVME_LOOP_MAX_SEGMENTS		256
253a85a5deSChristoph Hellwig 
263a85a5deSChristoph Hellwig /*
273a85a5deSChristoph Hellwig  * We handle AEN commands ourselves and don't even let the
283a85a5deSChristoph Hellwig  * block layer know about them.
293a85a5deSChristoph Hellwig  */
303a85a5deSChristoph Hellwig #define NVME_LOOP_NR_AEN_COMMANDS	1
313a85a5deSChristoph Hellwig #define NVME_LOOP_AQ_BLKMQ_DEPTH	\
327aa1f427SSagi Grimberg 	(NVME_AQ_DEPTH - NVME_LOOP_NR_AEN_COMMANDS)
333a85a5deSChristoph Hellwig 
343a85a5deSChristoph Hellwig struct nvme_loop_iod {
35d49187e9SChristoph Hellwig 	struct nvme_request	nvme_req;
363a85a5deSChristoph Hellwig 	struct nvme_command	cmd;
373a85a5deSChristoph Hellwig 	struct nvme_completion	rsp;
383a85a5deSChristoph Hellwig 	struct nvmet_req	req;
393a85a5deSChristoph Hellwig 	struct nvme_loop_queue	*queue;
403a85a5deSChristoph Hellwig 	struct work_struct	work;
413a85a5deSChristoph Hellwig 	struct sg_table		sg_table;
423a85a5deSChristoph Hellwig 	struct scatterlist	first_sgl[];
433a85a5deSChristoph Hellwig };
443a85a5deSChristoph Hellwig 
453a85a5deSChristoph Hellwig struct nvme_loop_ctrl {
463a85a5deSChristoph Hellwig 	struct nvme_loop_queue	*queues;
473a85a5deSChristoph Hellwig 
483a85a5deSChristoph Hellwig 	struct blk_mq_tag_set	admin_tag_set;
493a85a5deSChristoph Hellwig 
503a85a5deSChristoph Hellwig 	struct list_head	list;
513a85a5deSChristoph Hellwig 	struct blk_mq_tag_set	tag_set;
523a85a5deSChristoph Hellwig 	struct nvme_loop_iod	async_event_iod;
533a85a5deSChristoph Hellwig 	struct nvme_ctrl	ctrl;
543a85a5deSChristoph Hellwig 
553a85a5deSChristoph Hellwig 	struct nvmet_ctrl	*target_ctrl;
563a85a5deSChristoph Hellwig 	struct work_struct	delete_work;
573a85a5deSChristoph Hellwig };
583a85a5deSChristoph Hellwig 
593a85a5deSChristoph Hellwig static inline struct nvme_loop_ctrl *to_loop_ctrl(struct nvme_ctrl *ctrl)
603a85a5deSChristoph Hellwig {
613a85a5deSChristoph Hellwig 	return container_of(ctrl, struct nvme_loop_ctrl, ctrl);
623a85a5deSChristoph Hellwig }
633a85a5deSChristoph Hellwig 
643a85a5deSChristoph Hellwig struct nvme_loop_queue {
653a85a5deSChristoph Hellwig 	struct nvmet_cq		nvme_cq;
663a85a5deSChristoph Hellwig 	struct nvmet_sq		nvme_sq;
673a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl	*ctrl;
683a85a5deSChristoph Hellwig };
693a85a5deSChristoph Hellwig 
703a85a5deSChristoph Hellwig static struct nvmet_port *nvmet_loop_port;
713a85a5deSChristoph Hellwig 
723a85a5deSChristoph Hellwig static LIST_HEAD(nvme_loop_ctrl_list);
733a85a5deSChristoph Hellwig static DEFINE_MUTEX(nvme_loop_ctrl_mutex);
743a85a5deSChristoph Hellwig 
753a85a5deSChristoph Hellwig static void nvme_loop_queue_response(struct nvmet_req *nvme_req);
763a85a5deSChristoph Hellwig static void nvme_loop_delete_ctrl(struct nvmet_ctrl *ctrl);
773a85a5deSChristoph Hellwig 
783a85a5deSChristoph Hellwig static struct nvmet_fabrics_ops nvme_loop_ops;
793a85a5deSChristoph Hellwig 
803a85a5deSChristoph Hellwig static inline int nvme_loop_queue_idx(struct nvme_loop_queue *queue)
813a85a5deSChristoph Hellwig {
823a85a5deSChristoph Hellwig 	return queue - queue->ctrl->queues;
833a85a5deSChristoph Hellwig }
843a85a5deSChristoph Hellwig 
853a85a5deSChristoph Hellwig static void nvme_loop_complete_rq(struct request *req)
863a85a5deSChristoph Hellwig {
873a85a5deSChristoph Hellwig 	struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
883a85a5deSChristoph Hellwig 
893a85a5deSChristoph Hellwig 	nvme_cleanup_cmd(req);
903a85a5deSChristoph Hellwig 	sg_free_table_chained(&iod->sg_table, true);
9177f02a7aSChristoph Hellwig 	nvme_complete_rq(req);
923a85a5deSChristoph Hellwig }
933a85a5deSChristoph Hellwig 
943b068376SSagi Grimberg static struct blk_mq_tags *nvme_loop_tagset(struct nvme_loop_queue *queue)
953b068376SSagi Grimberg {
963b068376SSagi Grimberg 	u32 queue_idx = nvme_loop_queue_idx(queue);
973a85a5deSChristoph Hellwig 
983b068376SSagi Grimberg 	if (queue_idx == 0)
993b068376SSagi Grimberg 		return queue->ctrl->admin_tag_set.tags[queue_idx];
1003b068376SSagi Grimberg 	return queue->ctrl->tag_set.tags[queue_idx - 1];
1013a85a5deSChristoph Hellwig }
1023a85a5deSChristoph Hellwig 
103d49187e9SChristoph Hellwig static void nvme_loop_queue_response(struct nvmet_req *req)
1043a85a5deSChristoph Hellwig {
1053b068376SSagi Grimberg 	struct nvme_loop_queue *queue =
1063b068376SSagi Grimberg 		container_of(req->sq, struct nvme_loop_queue, nvme_sq);
1073b068376SSagi Grimberg 	struct nvme_completion *cqe = req->rsp;
1083a85a5deSChristoph Hellwig 
1093a85a5deSChristoph Hellwig 	/*
1103a85a5deSChristoph Hellwig 	 * AEN requests are special as they don't time out and can
1113a85a5deSChristoph Hellwig 	 * survive any kind of queue freeze and often don't respond to
1123a85a5deSChristoph Hellwig 	 * aborts.  We don't even bother to allocate a struct request
1133a85a5deSChristoph Hellwig 	 * for them but rather special case them here.
1143a85a5deSChristoph Hellwig 	 */
1153b068376SSagi Grimberg 	if (unlikely(nvme_loop_queue_idx(queue) == 0 &&
1163a85a5deSChristoph Hellwig 			cqe->command_id >= NVME_LOOP_AQ_BLKMQ_DEPTH)) {
1173b068376SSagi Grimberg 		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
1187bf58533SChristoph Hellwig 				&cqe->result);
1193a85a5deSChristoph Hellwig 	} else {
1203b068376SSagi Grimberg 		struct request *rq;
1213a85a5deSChristoph Hellwig 
1223b068376SSagi Grimberg 		rq = blk_mq_tag_to_rq(nvme_loop_tagset(queue), cqe->command_id);
1233b068376SSagi Grimberg 		if (!rq) {
1243b068376SSagi Grimberg 			dev_err(queue->ctrl->ctrl.device,
1253b068376SSagi Grimberg 				"tag 0x%x on queue %d not found\n",
1263b068376SSagi Grimberg 				cqe->command_id, nvme_loop_queue_idx(queue));
1273b068376SSagi Grimberg 			return;
1283b068376SSagi Grimberg 		}
1293b068376SSagi Grimberg 
13027fa9bc5SChristoph Hellwig 		nvme_end_request(rq, cqe->status, cqe->result);
1313a85a5deSChristoph Hellwig 	}
1323a85a5deSChristoph Hellwig }
1333a85a5deSChristoph Hellwig 
1343a85a5deSChristoph Hellwig static void nvme_loop_execute_work(struct work_struct *work)
1353a85a5deSChristoph Hellwig {
1363a85a5deSChristoph Hellwig 	struct nvme_loop_iod *iod =
1373a85a5deSChristoph Hellwig 		container_of(work, struct nvme_loop_iod, work);
1383a85a5deSChristoph Hellwig 
1393a85a5deSChristoph Hellwig 	iod->req.execute(&iod->req);
1403a85a5deSChristoph Hellwig }
1413a85a5deSChristoph Hellwig 
1423a85a5deSChristoph Hellwig static enum blk_eh_timer_return
1433a85a5deSChristoph Hellwig nvme_loop_timeout(struct request *rq, bool reserved)
1443a85a5deSChristoph Hellwig {
1453a85a5deSChristoph Hellwig 	struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(rq);
1463a85a5deSChristoph Hellwig 
1473a85a5deSChristoph Hellwig 	/* queue error recovery */
148d86c4d8eSChristoph Hellwig 	nvme_reset_ctrl(&iod->queue->ctrl->ctrl);
1493a85a5deSChristoph Hellwig 
1503a85a5deSChristoph Hellwig 	/* fail with DNR on admin cmd timeout */
15127fa9bc5SChristoph Hellwig 	nvme_req(rq)->status = NVME_SC_ABORT_REQ | NVME_SC_DNR;
1523a85a5deSChristoph Hellwig 
1533a85a5deSChristoph Hellwig 	return BLK_EH_HANDLED;
1543a85a5deSChristoph Hellwig }
1553a85a5deSChristoph Hellwig 
156fc17b653SChristoph Hellwig static blk_status_t nvme_loop_queue_rq(struct blk_mq_hw_ctx *hctx,
1573a85a5deSChristoph Hellwig 		const struct blk_mq_queue_data *bd)
1583a85a5deSChristoph Hellwig {
1593a85a5deSChristoph Hellwig 	struct nvme_ns *ns = hctx->queue->queuedata;
1603a85a5deSChristoph Hellwig 	struct nvme_loop_queue *queue = hctx->driver_data;
1613a85a5deSChristoph Hellwig 	struct request *req = bd->rq;
1623a85a5deSChristoph Hellwig 	struct nvme_loop_iod *iod = blk_mq_rq_to_pdu(req);
163fc17b653SChristoph Hellwig 	blk_status_t ret;
1643a85a5deSChristoph Hellwig 
1653a85a5deSChristoph Hellwig 	ret = nvme_setup_cmd(ns, req, &iod->cmd);
166fc17b653SChristoph Hellwig 	if (ret)
1673a85a5deSChristoph Hellwig 		return ret;
1683a85a5deSChristoph Hellwig 
1693a85a5deSChristoph Hellwig 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
1703a85a5deSChristoph Hellwig 	iod->req.port = nvmet_loop_port;
1713a85a5deSChristoph Hellwig 	if (!nvmet_req_init(&iod->req, &queue->nvme_cq,
1723a85a5deSChristoph Hellwig 			&queue->nvme_sq, &nvme_loop_ops)) {
1733a85a5deSChristoph Hellwig 		nvme_cleanup_cmd(req);
1743a85a5deSChristoph Hellwig 		blk_mq_start_request(req);
1753a85a5deSChristoph Hellwig 		nvme_loop_queue_response(&iod->req);
176fc17b653SChristoph Hellwig 		return BLK_STS_OK;
1773a85a5deSChristoph Hellwig 	}
1783a85a5deSChristoph Hellwig 
1793a85a5deSChristoph Hellwig 	if (blk_rq_bytes(req)) {
1803a85a5deSChristoph Hellwig 		iod->sg_table.sgl = iod->first_sgl;
181fc17b653SChristoph Hellwig 		if (sg_alloc_table_chained(&iod->sg_table,
182f9d03f96SChristoph Hellwig 				blk_rq_nr_phys_segments(req),
183fc17b653SChristoph Hellwig 				iod->sg_table.sgl))
184fc17b653SChristoph Hellwig 			return BLK_STS_RESOURCE;
1853a85a5deSChristoph Hellwig 
1863a85a5deSChristoph Hellwig 		iod->req.sg = iod->sg_table.sgl;
1873a85a5deSChristoph Hellwig 		iod->req.sg_cnt = blk_rq_map_sg(req->q, req, iod->sg_table.sgl);
1883a85a5deSChristoph Hellwig 	}
1893a85a5deSChristoph Hellwig 
1903a85a5deSChristoph Hellwig 	blk_mq_start_request(req);
1913a85a5deSChristoph Hellwig 
1923a85a5deSChristoph Hellwig 	schedule_work(&iod->work);
193fc17b653SChristoph Hellwig 	return BLK_STS_OK;
1943a85a5deSChristoph Hellwig }
1953a85a5deSChristoph Hellwig 
1963a85a5deSChristoph Hellwig static void nvme_loop_submit_async_event(struct nvme_ctrl *arg, int aer_idx)
1973a85a5deSChristoph Hellwig {
1983a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(arg);
1993a85a5deSChristoph Hellwig 	struct nvme_loop_queue *queue = &ctrl->queues[0];
2003a85a5deSChristoph Hellwig 	struct nvme_loop_iod *iod = &ctrl->async_event_iod;
2013a85a5deSChristoph Hellwig 
2023a85a5deSChristoph Hellwig 	memset(&iod->cmd, 0, sizeof(iod->cmd));
2033a85a5deSChristoph Hellwig 	iod->cmd.common.opcode = nvme_admin_async_event;
2043a85a5deSChristoph Hellwig 	iod->cmd.common.command_id = NVME_LOOP_AQ_BLKMQ_DEPTH;
2053a85a5deSChristoph Hellwig 	iod->cmd.common.flags |= NVME_CMD_SGL_METABUF;
2063a85a5deSChristoph Hellwig 
2073a85a5deSChristoph Hellwig 	if (!nvmet_req_init(&iod->req, &queue->nvme_cq, &queue->nvme_sq,
2083a85a5deSChristoph Hellwig 			&nvme_loop_ops)) {
2093a85a5deSChristoph Hellwig 		dev_err(ctrl->ctrl.device, "failed async event work\n");
2103a85a5deSChristoph Hellwig 		return;
2113a85a5deSChristoph Hellwig 	}
2123a85a5deSChristoph Hellwig 
2133a85a5deSChristoph Hellwig 	schedule_work(&iod->work);
2143a85a5deSChristoph Hellwig }
2153a85a5deSChristoph Hellwig 
2163a85a5deSChristoph Hellwig static int nvme_loop_init_iod(struct nvme_loop_ctrl *ctrl,
2173a85a5deSChristoph Hellwig 		struct nvme_loop_iod *iod, unsigned int queue_idx)
2183a85a5deSChristoph Hellwig {
2193a85a5deSChristoph Hellwig 	iod->req.cmd = &iod->cmd;
2203a85a5deSChristoph Hellwig 	iod->req.rsp = &iod->rsp;
2213a85a5deSChristoph Hellwig 	iod->queue = &ctrl->queues[queue_idx];
2223a85a5deSChristoph Hellwig 	INIT_WORK(&iod->work, nvme_loop_execute_work);
2233a85a5deSChristoph Hellwig 	return 0;
2243a85a5deSChristoph Hellwig }
2253a85a5deSChristoph Hellwig 
226d6296d39SChristoph Hellwig static int nvme_loop_init_request(struct blk_mq_tag_set *set,
227d6296d39SChristoph Hellwig 		struct request *req, unsigned int hctx_idx,
2283a85a5deSChristoph Hellwig 		unsigned int numa_node)
2293a85a5deSChristoph Hellwig {
23062b83b18SChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = set->driver_data;
2313a85a5deSChristoph Hellwig 
23262b83b18SChristoph Hellwig 	return nvme_loop_init_iod(ctrl, blk_mq_rq_to_pdu(req),
23362b83b18SChristoph Hellwig 			(set == &ctrl->tag_set) ? hctx_idx + 1 : 0);
2343a85a5deSChristoph Hellwig }
2353a85a5deSChristoph Hellwig 
2363a85a5deSChristoph Hellwig static int nvme_loop_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2373a85a5deSChristoph Hellwig 		unsigned int hctx_idx)
2383a85a5deSChristoph Hellwig {
2393a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = data;
2403a85a5deSChristoph Hellwig 	struct nvme_loop_queue *queue = &ctrl->queues[hctx_idx + 1];
2413a85a5deSChristoph Hellwig 
242d858e5f0SSagi Grimberg 	BUG_ON(hctx_idx >= ctrl->ctrl.queue_count);
2433a85a5deSChristoph Hellwig 
2443a85a5deSChristoph Hellwig 	hctx->driver_data = queue;
2453a85a5deSChristoph Hellwig 	return 0;
2463a85a5deSChristoph Hellwig }
2473a85a5deSChristoph Hellwig 
2483a85a5deSChristoph Hellwig static int nvme_loop_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
2493a85a5deSChristoph Hellwig 		unsigned int hctx_idx)
2503a85a5deSChristoph Hellwig {
2513a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = data;
2523a85a5deSChristoph Hellwig 	struct nvme_loop_queue *queue = &ctrl->queues[0];
2533a85a5deSChristoph Hellwig 
2543a85a5deSChristoph Hellwig 	BUG_ON(hctx_idx != 0);
2553a85a5deSChristoph Hellwig 
2563a85a5deSChristoph Hellwig 	hctx->driver_data = queue;
2573a85a5deSChristoph Hellwig 	return 0;
2583a85a5deSChristoph Hellwig }
2593a85a5deSChristoph Hellwig 
260f363b089SEric Biggers static const struct blk_mq_ops nvme_loop_mq_ops = {
2613a85a5deSChristoph Hellwig 	.queue_rq	= nvme_loop_queue_rq,
2623a85a5deSChristoph Hellwig 	.complete	= nvme_loop_complete_rq,
2633a85a5deSChristoph Hellwig 	.init_request	= nvme_loop_init_request,
2643a85a5deSChristoph Hellwig 	.init_hctx	= nvme_loop_init_hctx,
2653a85a5deSChristoph Hellwig 	.timeout	= nvme_loop_timeout,
2663a85a5deSChristoph Hellwig };
2673a85a5deSChristoph Hellwig 
268f363b089SEric Biggers static const struct blk_mq_ops nvme_loop_admin_mq_ops = {
2693a85a5deSChristoph Hellwig 	.queue_rq	= nvme_loop_queue_rq,
2703a85a5deSChristoph Hellwig 	.complete	= nvme_loop_complete_rq,
27162b83b18SChristoph Hellwig 	.init_request	= nvme_loop_init_request,
2723a85a5deSChristoph Hellwig 	.init_hctx	= nvme_loop_init_admin_hctx,
2733a85a5deSChristoph Hellwig 	.timeout	= nvme_loop_timeout,
2743a85a5deSChristoph Hellwig };
2753a85a5deSChristoph Hellwig 
2763a85a5deSChristoph Hellwig static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
2773a85a5deSChristoph Hellwig {
278e4c5d376SSagi Grimberg 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
2793a85a5deSChristoph Hellwig 	blk_cleanup_queue(ctrl->ctrl.admin_q);
2803a85a5deSChristoph Hellwig 	blk_mq_free_tag_set(&ctrl->admin_tag_set);
2813a85a5deSChristoph Hellwig }
2823a85a5deSChristoph Hellwig 
2833a85a5deSChristoph Hellwig static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
2843a85a5deSChristoph Hellwig {
2853a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
2863a85a5deSChristoph Hellwig 
2873a85a5deSChristoph Hellwig 	if (list_empty(&ctrl->list))
2883a85a5deSChristoph Hellwig 		goto free_ctrl;
2893a85a5deSChristoph Hellwig 
2903a85a5deSChristoph Hellwig 	mutex_lock(&nvme_loop_ctrl_mutex);
2913a85a5deSChristoph Hellwig 	list_del(&ctrl->list);
2923a85a5deSChristoph Hellwig 	mutex_unlock(&nvme_loop_ctrl_mutex);
2933a85a5deSChristoph Hellwig 
2943a85a5deSChristoph Hellwig 	if (nctrl->tagset) {
2953a85a5deSChristoph Hellwig 		blk_cleanup_queue(ctrl->ctrl.connect_q);
2963a85a5deSChristoph Hellwig 		blk_mq_free_tag_set(&ctrl->tag_set);
2973a85a5deSChristoph Hellwig 	}
2983a85a5deSChristoph Hellwig 	kfree(ctrl->queues);
2993a85a5deSChristoph Hellwig 	nvmf_free_options(nctrl->opts);
3003a85a5deSChristoph Hellwig free_ctrl:
3013a85a5deSChristoph Hellwig 	kfree(ctrl);
3023a85a5deSChristoph Hellwig }
3033a85a5deSChristoph Hellwig 
304945dd5baSSagi Grimberg static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
305945dd5baSSagi Grimberg {
306945dd5baSSagi Grimberg 	int i;
307945dd5baSSagi Grimberg 
308d858e5f0SSagi Grimberg 	for (i = 1; i < ctrl->ctrl.queue_count; i++)
309945dd5baSSagi Grimberg 		nvmet_sq_destroy(&ctrl->queues[i].nvme_sq);
310945dd5baSSagi Grimberg }
311945dd5baSSagi Grimberg 
312945dd5baSSagi Grimberg static int nvme_loop_init_io_queues(struct nvme_loop_ctrl *ctrl)
313945dd5baSSagi Grimberg {
314945dd5baSSagi Grimberg 	struct nvmf_ctrl_options *opts = ctrl->ctrl.opts;
315945dd5baSSagi Grimberg 	unsigned int nr_io_queues;
316945dd5baSSagi Grimberg 	int ret, i;
317945dd5baSSagi Grimberg 
318945dd5baSSagi Grimberg 	nr_io_queues = min(opts->nr_io_queues, num_online_cpus());
319945dd5baSSagi Grimberg 	ret = nvme_set_queue_count(&ctrl->ctrl, &nr_io_queues);
320945dd5baSSagi Grimberg 	if (ret || !nr_io_queues)
321945dd5baSSagi Grimberg 		return ret;
322945dd5baSSagi Grimberg 
323945dd5baSSagi Grimberg 	dev_info(ctrl->ctrl.device, "creating %d I/O queues.\n", nr_io_queues);
324945dd5baSSagi Grimberg 
325945dd5baSSagi Grimberg 	for (i = 1; i <= nr_io_queues; i++) {
326945dd5baSSagi Grimberg 		ctrl->queues[i].ctrl = ctrl;
327945dd5baSSagi Grimberg 		ret = nvmet_sq_init(&ctrl->queues[i].nvme_sq);
328945dd5baSSagi Grimberg 		if (ret)
329945dd5baSSagi Grimberg 			goto out_destroy_queues;
330945dd5baSSagi Grimberg 
331d858e5f0SSagi Grimberg 		ctrl->ctrl.queue_count++;
332945dd5baSSagi Grimberg 	}
333945dd5baSSagi Grimberg 
334945dd5baSSagi Grimberg 	return 0;
335945dd5baSSagi Grimberg 
336945dd5baSSagi Grimberg out_destroy_queues:
337945dd5baSSagi Grimberg 	nvme_loop_destroy_io_queues(ctrl);
338945dd5baSSagi Grimberg 	return ret;
339945dd5baSSagi Grimberg }
340945dd5baSSagi Grimberg 
341297186d6SSagi Grimberg static int nvme_loop_connect_io_queues(struct nvme_loop_ctrl *ctrl)
342297186d6SSagi Grimberg {
343297186d6SSagi Grimberg 	int i, ret;
344297186d6SSagi Grimberg 
345d858e5f0SSagi Grimberg 	for (i = 1; i < ctrl->ctrl.queue_count; i++) {
346297186d6SSagi Grimberg 		ret = nvmf_connect_io_queue(&ctrl->ctrl, i);
347297186d6SSagi Grimberg 		if (ret)
348297186d6SSagi Grimberg 			return ret;
349297186d6SSagi Grimberg 	}
350297186d6SSagi Grimberg 
351297186d6SSagi Grimberg 	return 0;
352297186d6SSagi Grimberg }
353297186d6SSagi Grimberg 
3543a85a5deSChristoph Hellwig static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
3553a85a5deSChristoph Hellwig {
3563a85a5deSChristoph Hellwig 	int error;
3573a85a5deSChristoph Hellwig 
3583a85a5deSChristoph Hellwig 	memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
3593a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops;
3603a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.queue_depth = NVME_LOOP_AQ_BLKMQ_DEPTH;
3613a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.reserved_tags = 2; /* connect + keep-alive */
3623a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.numa_node = NUMA_NO_NODE;
3633a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
3643a85a5deSChristoph Hellwig 		SG_CHUNK_SIZE * sizeof(struct scatterlist);
3653a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.driver_data = ctrl;
3663a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.nr_hw_queues = 1;
3673a85a5deSChristoph Hellwig 	ctrl->admin_tag_set.timeout = ADMIN_TIMEOUT;
3683a85a5deSChristoph Hellwig 
3693a85a5deSChristoph Hellwig 	ctrl->queues[0].ctrl = ctrl;
3703a85a5deSChristoph Hellwig 	error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
3713a85a5deSChristoph Hellwig 	if (error)
3723a85a5deSChristoph Hellwig 		return error;
373d858e5f0SSagi Grimberg 	ctrl->ctrl.queue_count = 1;
3743a85a5deSChristoph Hellwig 
3753a85a5deSChristoph Hellwig 	error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
3763a85a5deSChristoph Hellwig 	if (error)
3773a85a5deSChristoph Hellwig 		goto out_free_sq;
3783a85a5deSChristoph Hellwig 
3793a85a5deSChristoph Hellwig 	ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
3803a85a5deSChristoph Hellwig 	if (IS_ERR(ctrl->ctrl.admin_q)) {
3813a85a5deSChristoph Hellwig 		error = PTR_ERR(ctrl->ctrl.admin_q);
3823a85a5deSChristoph Hellwig 		goto out_free_tagset;
3833a85a5deSChristoph Hellwig 	}
3843a85a5deSChristoph Hellwig 
3853a85a5deSChristoph Hellwig 	error = nvmf_connect_admin_queue(&ctrl->ctrl);
3863a85a5deSChristoph Hellwig 	if (error)
3873a85a5deSChristoph Hellwig 		goto out_cleanup_queue;
3883a85a5deSChristoph Hellwig 
38920d0dfe6SSagi Grimberg 	error = nvmf_reg_read64(&ctrl->ctrl, NVME_REG_CAP, &ctrl->ctrl.cap);
3903a85a5deSChristoph Hellwig 	if (error) {
3913a85a5deSChristoph Hellwig 		dev_err(ctrl->ctrl.device,
3923a85a5deSChristoph Hellwig 			"prop_get NVME_REG_CAP failed\n");
3933a85a5deSChristoph Hellwig 		goto out_cleanup_queue;
3943a85a5deSChristoph Hellwig 	}
3953a85a5deSChristoph Hellwig 
3963a85a5deSChristoph Hellwig 	ctrl->ctrl.sqsize =
39720d0dfe6SSagi Grimberg 		min_t(int, NVME_CAP_MQES(ctrl->ctrl.cap), ctrl->ctrl.sqsize);
3983a85a5deSChristoph Hellwig 
39920d0dfe6SSagi Grimberg 	error = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
4003a85a5deSChristoph Hellwig 	if (error)
4013a85a5deSChristoph Hellwig 		goto out_cleanup_queue;
4023a85a5deSChristoph Hellwig 
4033a85a5deSChristoph Hellwig 	ctrl->ctrl.max_hw_sectors =
4043a85a5deSChristoph Hellwig 		(NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9);
4053a85a5deSChristoph Hellwig 
4063a85a5deSChristoph Hellwig 	error = nvme_init_identify(&ctrl->ctrl);
4073a85a5deSChristoph Hellwig 	if (error)
4083a85a5deSChristoph Hellwig 		goto out_cleanup_queue;
4093a85a5deSChristoph Hellwig 
4103a85a5deSChristoph Hellwig 	return 0;
4113a85a5deSChristoph Hellwig 
4123a85a5deSChristoph Hellwig out_cleanup_queue:
4133a85a5deSChristoph Hellwig 	blk_cleanup_queue(ctrl->ctrl.admin_q);
4143a85a5deSChristoph Hellwig out_free_tagset:
4153a85a5deSChristoph Hellwig 	blk_mq_free_tag_set(&ctrl->admin_tag_set);
4163a85a5deSChristoph Hellwig out_free_sq:
4173a85a5deSChristoph Hellwig 	nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
4183a85a5deSChristoph Hellwig 	return error;
4193a85a5deSChristoph Hellwig }
4203a85a5deSChristoph Hellwig 
4213a85a5deSChristoph Hellwig static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
4223a85a5deSChristoph Hellwig {
423d858e5f0SSagi Grimberg 	if (ctrl->ctrl.queue_count > 1) {
4243a85a5deSChristoph Hellwig 		nvme_stop_queues(&ctrl->ctrl);
4253a85a5deSChristoph Hellwig 		blk_mq_tagset_busy_iter(&ctrl->tag_set,
4263a85a5deSChristoph Hellwig 					nvme_cancel_request, &ctrl->ctrl);
427945dd5baSSagi Grimberg 		nvme_loop_destroy_io_queues(ctrl);
4283a85a5deSChristoph Hellwig 	}
4293a85a5deSChristoph Hellwig 
4303a85a5deSChristoph Hellwig 	if (ctrl->ctrl.state == NVME_CTRL_LIVE)
4313a85a5deSChristoph Hellwig 		nvme_shutdown_ctrl(&ctrl->ctrl);
4323a85a5deSChristoph Hellwig 
433c1c0ffffSSagi Grimberg 	blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
4343a85a5deSChristoph Hellwig 	blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
4353a85a5deSChristoph Hellwig 				nvme_cancel_request, &ctrl->ctrl);
436c1c0ffffSSagi Grimberg 	blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
4373a85a5deSChristoph Hellwig 	nvme_loop_destroy_admin_queue(ctrl);
4383a85a5deSChristoph Hellwig }
4393a85a5deSChristoph Hellwig 
4403a85a5deSChristoph Hellwig static void nvme_loop_del_ctrl_work(struct work_struct *work)
4413a85a5deSChristoph Hellwig {
4423a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = container_of(work,
4433a85a5deSChristoph Hellwig 				struct nvme_loop_ctrl, delete_work);
4443a85a5deSChristoph Hellwig 
445*d09f2b45SSagi Grimberg 	nvme_stop_ctrl(&ctrl->ctrl);
446*d09f2b45SSagi Grimberg 	nvme_remove_namespaces(&ctrl->ctrl);
447a159c64dSSagi Grimberg 	nvme_loop_shutdown_ctrl(ctrl);
448*d09f2b45SSagi Grimberg 	nvme_uninit_ctrl(&ctrl->ctrl);
4493a85a5deSChristoph Hellwig 	nvme_put_ctrl(&ctrl->ctrl);
4503a85a5deSChristoph Hellwig }
4513a85a5deSChristoph Hellwig 
4523a85a5deSChristoph Hellwig static int __nvme_loop_del_ctrl(struct nvme_loop_ctrl *ctrl)
4533a85a5deSChristoph Hellwig {
4543a85a5deSChristoph Hellwig 	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_DELETING))
4553a85a5deSChristoph Hellwig 		return -EBUSY;
4563a85a5deSChristoph Hellwig 
4579a6327d2SSagi Grimberg 	if (!queue_work(nvme_wq, &ctrl->delete_work))
4583a85a5deSChristoph Hellwig 		return -EBUSY;
4593a85a5deSChristoph Hellwig 
4603a85a5deSChristoph Hellwig 	return 0;
4613a85a5deSChristoph Hellwig }
4623a85a5deSChristoph Hellwig 
4633a85a5deSChristoph Hellwig static int nvme_loop_del_ctrl(struct nvme_ctrl *nctrl)
4643a85a5deSChristoph Hellwig {
4653a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
4663a85a5deSChristoph Hellwig 	int ret;
4673a85a5deSChristoph Hellwig 
4683a85a5deSChristoph Hellwig 	ret = __nvme_loop_del_ctrl(ctrl);
4693a85a5deSChristoph Hellwig 	if (ret)
4703a85a5deSChristoph Hellwig 		return ret;
4713a85a5deSChristoph Hellwig 
4723a85a5deSChristoph Hellwig 	flush_work(&ctrl->delete_work);
4733a85a5deSChristoph Hellwig 
4743a85a5deSChristoph Hellwig 	return 0;
4753a85a5deSChristoph Hellwig }
4763a85a5deSChristoph Hellwig 
4773a85a5deSChristoph Hellwig static void nvme_loop_delete_ctrl(struct nvmet_ctrl *nctrl)
4783a85a5deSChristoph Hellwig {
4793a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl;
4803a85a5deSChristoph Hellwig 
4813a85a5deSChristoph Hellwig 	mutex_lock(&nvme_loop_ctrl_mutex);
4823a85a5deSChristoph Hellwig 	list_for_each_entry(ctrl, &nvme_loop_ctrl_list, list) {
4833a85a5deSChristoph Hellwig 		if (ctrl->ctrl.cntlid == nctrl->cntlid)
4843a85a5deSChristoph Hellwig 			__nvme_loop_del_ctrl(ctrl);
4853a85a5deSChristoph Hellwig 	}
4863a85a5deSChristoph Hellwig 	mutex_unlock(&nvme_loop_ctrl_mutex);
4873a85a5deSChristoph Hellwig }
4883a85a5deSChristoph Hellwig 
4893a85a5deSChristoph Hellwig static void nvme_loop_reset_ctrl_work(struct work_struct *work)
4903a85a5deSChristoph Hellwig {
491d86c4d8eSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl =
492d86c4d8eSChristoph Hellwig 		container_of(work, struct nvme_loop_ctrl, ctrl.reset_work);
4933a85a5deSChristoph Hellwig 	bool changed;
494297186d6SSagi Grimberg 	int ret;
4953a85a5deSChristoph Hellwig 
496*d09f2b45SSagi Grimberg 	nvme_stop_ctrl(&ctrl->ctrl);
4973a85a5deSChristoph Hellwig 	nvme_loop_shutdown_ctrl(ctrl);
4983a85a5deSChristoph Hellwig 
4993a85a5deSChristoph Hellwig 	ret = nvme_loop_configure_admin_queue(ctrl);
5003a85a5deSChristoph Hellwig 	if (ret)
5013a85a5deSChristoph Hellwig 		goto out_disable;
5023a85a5deSChristoph Hellwig 
503945dd5baSSagi Grimberg 	ret = nvme_loop_init_io_queues(ctrl);
5043a85a5deSChristoph Hellwig 	if (ret)
505945dd5baSSagi Grimberg 		goto out_destroy_admin;
5063a85a5deSChristoph Hellwig 
507297186d6SSagi Grimberg 	ret = nvme_loop_connect_io_queues(ctrl);
5083a85a5deSChristoph Hellwig 	if (ret)
509945dd5baSSagi Grimberg 		goto out_destroy_io;
5103a85a5deSChristoph Hellwig 
5114368c39bSSagi Grimberg 	blk_mq_update_nr_hw_queues(&ctrl->tag_set,
5124368c39bSSagi Grimberg 			ctrl->ctrl.queue_count - 1);
5134368c39bSSagi Grimberg 
5143a85a5deSChristoph Hellwig 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
5153a85a5deSChristoph Hellwig 	WARN_ON_ONCE(!changed);
5163a85a5deSChristoph Hellwig 
517*d09f2b45SSagi Grimberg 	nvme_start_ctrl(&ctrl->ctrl);
5183a85a5deSChristoph Hellwig 
5193a85a5deSChristoph Hellwig 	return;
5203a85a5deSChristoph Hellwig 
521945dd5baSSagi Grimberg out_destroy_io:
522945dd5baSSagi Grimberg 	nvme_loop_destroy_io_queues(ctrl);
523945dd5baSSagi Grimberg out_destroy_admin:
5243a85a5deSChristoph Hellwig 	nvme_loop_destroy_admin_queue(ctrl);
5253a85a5deSChristoph Hellwig out_disable:
5263a85a5deSChristoph Hellwig 	dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
5273a85a5deSChristoph Hellwig 	nvme_uninit_ctrl(&ctrl->ctrl);
5283a85a5deSChristoph Hellwig 	nvme_put_ctrl(&ctrl->ctrl);
5293a85a5deSChristoph Hellwig }
5303a85a5deSChristoph Hellwig 
5313a85a5deSChristoph Hellwig static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = {
5323a85a5deSChristoph Hellwig 	.name			= "loop",
5333a85a5deSChristoph Hellwig 	.module			= THIS_MODULE,
534d3d5b87dSChristoph Hellwig 	.flags			= NVME_F_FABRICS,
5353a85a5deSChristoph Hellwig 	.reg_read32		= nvmf_reg_read32,
5363a85a5deSChristoph Hellwig 	.reg_read64		= nvmf_reg_read64,
5373a85a5deSChristoph Hellwig 	.reg_write32		= nvmf_reg_write32,
5383a85a5deSChristoph Hellwig 	.free_ctrl		= nvme_loop_free_ctrl,
5393a85a5deSChristoph Hellwig 	.submit_async_event	= nvme_loop_submit_async_event,
5403a85a5deSChristoph Hellwig 	.delete_ctrl		= nvme_loop_del_ctrl,
5413a85a5deSChristoph Hellwig };
5423a85a5deSChristoph Hellwig 
5433a85a5deSChristoph Hellwig static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
5443a85a5deSChristoph Hellwig {
545297186d6SSagi Grimberg 	int ret;
5463a85a5deSChristoph Hellwig 
547945dd5baSSagi Grimberg 	ret = nvme_loop_init_io_queues(ctrl);
5483a85a5deSChristoph Hellwig 	if (ret)
549945dd5baSSagi Grimberg 		return ret;
5503a85a5deSChristoph Hellwig 
5513a85a5deSChristoph Hellwig 	memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
5523a85a5deSChristoph Hellwig 	ctrl->tag_set.ops = &nvme_loop_mq_ops;
553eadb7cf4SJay Freyensee 	ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
5543a85a5deSChristoph Hellwig 	ctrl->tag_set.reserved_tags = 1; /* fabric connect */
5553a85a5deSChristoph Hellwig 	ctrl->tag_set.numa_node = NUMA_NO_NODE;
5563a85a5deSChristoph Hellwig 	ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
5573a85a5deSChristoph Hellwig 	ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
5583a85a5deSChristoph Hellwig 		SG_CHUNK_SIZE * sizeof(struct scatterlist);
5593a85a5deSChristoph Hellwig 	ctrl->tag_set.driver_data = ctrl;
560d858e5f0SSagi Grimberg 	ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
5613a85a5deSChristoph Hellwig 	ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
5623a85a5deSChristoph Hellwig 	ctrl->ctrl.tagset = &ctrl->tag_set;
5633a85a5deSChristoph Hellwig 
5643a85a5deSChristoph Hellwig 	ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
5653a85a5deSChristoph Hellwig 	if (ret)
5663a85a5deSChristoph Hellwig 		goto out_destroy_queues;
5673a85a5deSChristoph Hellwig 
5683a85a5deSChristoph Hellwig 	ctrl->ctrl.connect_q = blk_mq_init_queue(&ctrl->tag_set);
5693a85a5deSChristoph Hellwig 	if (IS_ERR(ctrl->ctrl.connect_q)) {
5703a85a5deSChristoph Hellwig 		ret = PTR_ERR(ctrl->ctrl.connect_q);
5713a85a5deSChristoph Hellwig 		goto out_free_tagset;
5723a85a5deSChristoph Hellwig 	}
5733a85a5deSChristoph Hellwig 
574297186d6SSagi Grimberg 	ret = nvme_loop_connect_io_queues(ctrl);
5753a85a5deSChristoph Hellwig 	if (ret)
5763a85a5deSChristoph Hellwig 		goto out_cleanup_connect_q;
5773a85a5deSChristoph Hellwig 
5783a85a5deSChristoph Hellwig 	return 0;
5793a85a5deSChristoph Hellwig 
5803a85a5deSChristoph Hellwig out_cleanup_connect_q:
5813a85a5deSChristoph Hellwig 	blk_cleanup_queue(ctrl->ctrl.connect_q);
5823a85a5deSChristoph Hellwig out_free_tagset:
5833a85a5deSChristoph Hellwig 	blk_mq_free_tag_set(&ctrl->tag_set);
5843a85a5deSChristoph Hellwig out_destroy_queues:
585945dd5baSSagi Grimberg 	nvme_loop_destroy_io_queues(ctrl);
5863a85a5deSChristoph Hellwig 	return ret;
5873a85a5deSChristoph Hellwig }
5883a85a5deSChristoph Hellwig 
5893a85a5deSChristoph Hellwig static struct nvme_ctrl *nvme_loop_create_ctrl(struct device *dev,
5903a85a5deSChristoph Hellwig 		struct nvmf_ctrl_options *opts)
5913a85a5deSChristoph Hellwig {
5923a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl;
5933a85a5deSChristoph Hellwig 	bool changed;
5943a85a5deSChristoph Hellwig 	int ret;
5953a85a5deSChristoph Hellwig 
5963a85a5deSChristoph Hellwig 	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
5973a85a5deSChristoph Hellwig 	if (!ctrl)
5983a85a5deSChristoph Hellwig 		return ERR_PTR(-ENOMEM);
5993a85a5deSChristoph Hellwig 	ctrl->ctrl.opts = opts;
6003a85a5deSChristoph Hellwig 	INIT_LIST_HEAD(&ctrl->list);
6013a85a5deSChristoph Hellwig 
6023a85a5deSChristoph Hellwig 	INIT_WORK(&ctrl->delete_work, nvme_loop_del_ctrl_work);
603d86c4d8eSChristoph Hellwig 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_loop_reset_ctrl_work);
6043a85a5deSChristoph Hellwig 
6053a85a5deSChristoph Hellwig 	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_loop_ctrl_ops,
6063a85a5deSChristoph Hellwig 				0 /* no quirks, we're perfect! */);
6073a85a5deSChristoph Hellwig 	if (ret)
6083a85a5deSChristoph Hellwig 		goto out_put_ctrl;
6093a85a5deSChristoph Hellwig 
6103a85a5deSChristoph Hellwig 	ret = -ENOMEM;
6113a85a5deSChristoph Hellwig 
612eadb7cf4SJay Freyensee 	ctrl->ctrl.sqsize = opts->queue_size - 1;
6133a85a5deSChristoph Hellwig 	ctrl->ctrl.kato = opts->kato;
6143a85a5deSChristoph Hellwig 
6153a85a5deSChristoph Hellwig 	ctrl->queues = kcalloc(opts->nr_io_queues + 1, sizeof(*ctrl->queues),
6163a85a5deSChristoph Hellwig 			GFP_KERNEL);
6173a85a5deSChristoph Hellwig 	if (!ctrl->queues)
6183a85a5deSChristoph Hellwig 		goto out_uninit_ctrl;
6193a85a5deSChristoph Hellwig 
6203a85a5deSChristoph Hellwig 	ret = nvme_loop_configure_admin_queue(ctrl);
6213a85a5deSChristoph Hellwig 	if (ret)
6223a85a5deSChristoph Hellwig 		goto out_free_queues;
6233a85a5deSChristoph Hellwig 
6243a85a5deSChristoph Hellwig 	if (opts->queue_size > ctrl->ctrl.maxcmd) {
6253a85a5deSChristoph Hellwig 		/* warn if maxcmd is lower than queue_size */
6263a85a5deSChristoph Hellwig 		dev_warn(ctrl->ctrl.device,
6273a85a5deSChristoph Hellwig 			"queue_size %zu > ctrl maxcmd %u, clamping down\n",
6283a85a5deSChristoph Hellwig 			opts->queue_size, ctrl->ctrl.maxcmd);
6293a85a5deSChristoph Hellwig 		opts->queue_size = ctrl->ctrl.maxcmd;
6303a85a5deSChristoph Hellwig 	}
6313a85a5deSChristoph Hellwig 
6323a85a5deSChristoph Hellwig 	if (opts->nr_io_queues) {
6333a85a5deSChristoph Hellwig 		ret = nvme_loop_create_io_queues(ctrl);
6343a85a5deSChristoph Hellwig 		if (ret)
6353a85a5deSChristoph Hellwig 			goto out_remove_admin_queue;
6363a85a5deSChristoph Hellwig 	}
6373a85a5deSChristoph Hellwig 
6383a85a5deSChristoph Hellwig 	nvme_loop_init_iod(ctrl, &ctrl->async_event_iod, 0);
6393a85a5deSChristoph Hellwig 
6403a85a5deSChristoph Hellwig 	dev_info(ctrl->ctrl.device,
6413a85a5deSChristoph Hellwig 		 "new ctrl: \"%s\"\n", ctrl->ctrl.opts->subsysnqn);
6423a85a5deSChristoph Hellwig 
6433a85a5deSChristoph Hellwig 	kref_get(&ctrl->ctrl.kref);
6443a85a5deSChristoph Hellwig 
6453a85a5deSChristoph Hellwig 	changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
6463a85a5deSChristoph Hellwig 	WARN_ON_ONCE(!changed);
6473a85a5deSChristoph Hellwig 
6483a85a5deSChristoph Hellwig 	mutex_lock(&nvme_loop_ctrl_mutex);
6493a85a5deSChristoph Hellwig 	list_add_tail(&ctrl->list, &nvme_loop_ctrl_list);
6503a85a5deSChristoph Hellwig 	mutex_unlock(&nvme_loop_ctrl_mutex);
6513a85a5deSChristoph Hellwig 
652*d09f2b45SSagi Grimberg 	nvme_start_ctrl(&ctrl->ctrl);
6533a85a5deSChristoph Hellwig 
6543a85a5deSChristoph Hellwig 	return &ctrl->ctrl;
6553a85a5deSChristoph Hellwig 
6563a85a5deSChristoph Hellwig out_remove_admin_queue:
6573a85a5deSChristoph Hellwig 	nvme_loop_destroy_admin_queue(ctrl);
6583a85a5deSChristoph Hellwig out_free_queues:
6593a85a5deSChristoph Hellwig 	kfree(ctrl->queues);
6603a85a5deSChristoph Hellwig out_uninit_ctrl:
6613a85a5deSChristoph Hellwig 	nvme_uninit_ctrl(&ctrl->ctrl);
6623a85a5deSChristoph Hellwig out_put_ctrl:
6633a85a5deSChristoph Hellwig 	nvme_put_ctrl(&ctrl->ctrl);
6643a85a5deSChristoph Hellwig 	if (ret > 0)
6653a85a5deSChristoph Hellwig 		ret = -EIO;
6663a85a5deSChristoph Hellwig 	return ERR_PTR(ret);
6673a85a5deSChristoph Hellwig }
6683a85a5deSChristoph Hellwig 
6693a85a5deSChristoph Hellwig static int nvme_loop_add_port(struct nvmet_port *port)
6703a85a5deSChristoph Hellwig {
6713a85a5deSChristoph Hellwig 	/*
6723a85a5deSChristoph Hellwig 	 * XXX: disalow adding more than one port so
6733a85a5deSChristoph Hellwig 	 * there is no connection rejections when a
6743a85a5deSChristoph Hellwig 	 * a subsystem is assigned to a port for which
6753a85a5deSChristoph Hellwig 	 * loop doesn't have a pointer.
6763a85a5deSChristoph Hellwig 	 * This scenario would be possible if we allowed
6773a85a5deSChristoph Hellwig 	 * more than one port to be added and a subsystem
6783a85a5deSChristoph Hellwig 	 * was assigned to a port other than nvmet_loop_port.
6793a85a5deSChristoph Hellwig 	 */
6803a85a5deSChristoph Hellwig 
6813a85a5deSChristoph Hellwig 	if (nvmet_loop_port)
6823a85a5deSChristoph Hellwig 		return -EPERM;
6833a85a5deSChristoph Hellwig 
6843a85a5deSChristoph Hellwig 	nvmet_loop_port = port;
6853a85a5deSChristoph Hellwig 	return 0;
6863a85a5deSChristoph Hellwig }
6873a85a5deSChristoph Hellwig 
6883a85a5deSChristoph Hellwig static void nvme_loop_remove_port(struct nvmet_port *port)
6893a85a5deSChristoph Hellwig {
6903a85a5deSChristoph Hellwig 	if (port == nvmet_loop_port)
6913a85a5deSChristoph Hellwig 		nvmet_loop_port = NULL;
6923a85a5deSChristoph Hellwig }
6933a85a5deSChristoph Hellwig 
6943a85a5deSChristoph Hellwig static struct nvmet_fabrics_ops nvme_loop_ops = {
6953a85a5deSChristoph Hellwig 	.owner		= THIS_MODULE,
6963a85a5deSChristoph Hellwig 	.type		= NVMF_TRTYPE_LOOP,
6973a85a5deSChristoph Hellwig 	.add_port	= nvme_loop_add_port,
6983a85a5deSChristoph Hellwig 	.remove_port	= nvme_loop_remove_port,
6993a85a5deSChristoph Hellwig 	.queue_response = nvme_loop_queue_response,
7003a85a5deSChristoph Hellwig 	.delete_ctrl	= nvme_loop_delete_ctrl,
7013a85a5deSChristoph Hellwig };
7023a85a5deSChristoph Hellwig 
7033a85a5deSChristoph Hellwig static struct nvmf_transport_ops nvme_loop_transport = {
7043a85a5deSChristoph Hellwig 	.name		= "loop",
7053a85a5deSChristoph Hellwig 	.create_ctrl	= nvme_loop_create_ctrl,
7063a85a5deSChristoph Hellwig };
7073a85a5deSChristoph Hellwig 
7083a85a5deSChristoph Hellwig static int __init nvme_loop_init_module(void)
7093a85a5deSChristoph Hellwig {
7103a85a5deSChristoph Hellwig 	int ret;
7113a85a5deSChristoph Hellwig 
7123a85a5deSChristoph Hellwig 	ret = nvmet_register_transport(&nvme_loop_ops);
7133a85a5deSChristoph Hellwig 	if (ret)
7143a85a5deSChristoph Hellwig 		return ret;
715d19eef02SSagi Grimberg 
716d19eef02SSagi Grimberg 	ret = nvmf_register_transport(&nvme_loop_transport);
717d19eef02SSagi Grimberg 	if (ret)
718d19eef02SSagi Grimberg 		nvmet_unregister_transport(&nvme_loop_ops);
719d19eef02SSagi Grimberg 
720d19eef02SSagi Grimberg 	return ret;
7213a85a5deSChristoph Hellwig }
7223a85a5deSChristoph Hellwig 
7233a85a5deSChristoph Hellwig static void __exit nvme_loop_cleanup_module(void)
7243a85a5deSChristoph Hellwig {
7253a85a5deSChristoph Hellwig 	struct nvme_loop_ctrl *ctrl, *next;
7263a85a5deSChristoph Hellwig 
7273a85a5deSChristoph Hellwig 	nvmf_unregister_transport(&nvme_loop_transport);
7283a85a5deSChristoph Hellwig 	nvmet_unregister_transport(&nvme_loop_ops);
7293a85a5deSChristoph Hellwig 
7303a85a5deSChristoph Hellwig 	mutex_lock(&nvme_loop_ctrl_mutex);
7313a85a5deSChristoph Hellwig 	list_for_each_entry_safe(ctrl, next, &nvme_loop_ctrl_list, list)
7323a85a5deSChristoph Hellwig 		__nvme_loop_del_ctrl(ctrl);
7333a85a5deSChristoph Hellwig 	mutex_unlock(&nvme_loop_ctrl_mutex);
7343a85a5deSChristoph Hellwig 
7359a6327d2SSagi Grimberg 	flush_workqueue(nvme_wq);
7363a85a5deSChristoph Hellwig }
7373a85a5deSChristoph Hellwig 
7383a85a5deSChristoph Hellwig module_init(nvme_loop_init_module);
7393a85a5deSChristoph Hellwig module_exit(nvme_loop_cleanup_module);
7403a85a5deSChristoph Hellwig 
7413a85a5deSChristoph Hellwig MODULE_LICENSE("GPL v2");
7423a85a5deSChristoph Hellwig MODULE_ALIAS("nvmet-transport-254"); /* 254 == NVMF_TRTYPE_LOOP */
743