loop.c (2ade82213b7a35c761273fe0c982f4ab8707f30a) loop.c (ceee1953f9239b699d80c5ef366712c3fd865cc2)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics loopback device.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/scatterlist.h>
8#include <linux/blk-mq.h>

--- 252 unchanged lines hidden (view full) ---

261 .init_hctx = nvme_loop_init_admin_hctx,
262};
263
264static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
265{
266 if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
267 return;
268 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics loopback device.
4 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/scatterlist.h>
8#include <linux/blk-mq.h>

--- 252 unchanged lines hidden (view full) ---

261 .init_hctx = nvme_loop_init_admin_hctx,
262};
263
264static void nvme_loop_destroy_admin_queue(struct nvme_loop_ctrl *ctrl)
265{
266 if (!test_and_clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags))
267 return;
268 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
269 blk_mq_destroy_queue(ctrl->ctrl.admin_q);
270 blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
271 blk_mq_free_tag_set(&ctrl->admin_tag_set);
269 nvme_remove_admin_tag_set(&ctrl->ctrl);
272}
273
274static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
275{
276 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
277
278 if (list_empty(&ctrl->list))
279 goto free_ctrl;
280
281 mutex_lock(&nvme_loop_ctrl_mutex);
282 list_del(&ctrl->list);
283 mutex_unlock(&nvme_loop_ctrl_mutex);
284
270}
271
272static void nvme_loop_free_ctrl(struct nvme_ctrl *nctrl)
273{
274 struct nvme_loop_ctrl *ctrl = to_loop_ctrl(nctrl);
275
276 if (list_empty(&ctrl->list))
277 goto free_ctrl;
278
279 mutex_lock(&nvme_loop_ctrl_mutex);
280 list_del(&ctrl->list);
281 mutex_unlock(&nvme_loop_ctrl_mutex);
282
285 if (nctrl->tagset) {
286 blk_mq_destroy_queue(ctrl->ctrl.connect_q);
287 blk_mq_free_tag_set(&ctrl->tag_set);
288 }
283 if (nctrl->tagset)
284 nvme_remove_io_tag_set(nctrl);
289 kfree(ctrl->queues);
290 nvmf_free_options(nctrl->opts);
291free_ctrl:
292 kfree(ctrl);
293}
294
295static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
296{

--- 48 unchanged lines hidden (view full) ---

345
346 return 0;
347}
348
349static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
350{
351 int error;
352
285 kfree(ctrl->queues);
286 nvmf_free_options(nctrl->opts);
287free_ctrl:
288 kfree(ctrl);
289}
290
291static void nvme_loop_destroy_io_queues(struct nvme_loop_ctrl *ctrl)
292{

--- 48 unchanged lines hidden (view full) ---

341
342 return 0;
343}
344
345static int nvme_loop_configure_admin_queue(struct nvme_loop_ctrl *ctrl)
346{
347 int error;
348
353 memset(&ctrl->admin_tag_set, 0, sizeof(ctrl->admin_tag_set));
354 ctrl->admin_tag_set.ops = &nvme_loop_admin_mq_ops;
355 ctrl->admin_tag_set.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
356 ctrl->admin_tag_set.reserved_tags = NVMF_RESERVED_TAGS;
357 ctrl->admin_tag_set.numa_node = ctrl->ctrl.numa_node;
358 ctrl->admin_tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
359 NVME_INLINE_SG_CNT * sizeof(struct scatterlist);
360 ctrl->admin_tag_set.driver_data = &ctrl->ctrl;
361 ctrl->admin_tag_set.nr_hw_queues = 1;
362 ctrl->admin_tag_set.timeout = NVME_ADMIN_TIMEOUT;
363 ctrl->admin_tag_set.flags = BLK_MQ_F_NO_SCHED;
364
365 ctrl->queues[0].ctrl = ctrl;
366 error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
367 if (error)
368 return error;
369 ctrl->ctrl.queue_count = 1;
370
349 ctrl->queues[0].ctrl = ctrl;
350 error = nvmet_sq_init(&ctrl->queues[0].nvme_sq);
351 if (error)
352 return error;
353 ctrl->ctrl.queue_count = 1;
354
371 error = blk_mq_alloc_tag_set(&ctrl->admin_tag_set);
355 error = nvme_alloc_admin_tag_set(&ctrl->ctrl, &ctrl->admin_tag_set,
356 &nvme_loop_admin_mq_ops, BLK_MQ_F_NO_SCHED,
357 sizeof(struct nvme_loop_iod) +
358 NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
372 if (error)
373 goto out_free_sq;
359 if (error)
360 goto out_free_sq;
374 ctrl->ctrl.admin_tagset = &ctrl->admin_tag_set;
375
361
376 ctrl->ctrl.fabrics_q = blk_mq_init_queue(&ctrl->admin_tag_set);
377 if (IS_ERR(ctrl->ctrl.fabrics_q)) {
378 error = PTR_ERR(ctrl->ctrl.fabrics_q);
379 goto out_free_tagset;
380 }
381
382 ctrl->ctrl.admin_q = blk_mq_init_queue(&ctrl->admin_tag_set);
383 if (IS_ERR(ctrl->ctrl.admin_q)) {
384 error = PTR_ERR(ctrl->ctrl.admin_q);
385 goto out_cleanup_fabrics_q;
386 }
387 /* reset stopped state for the fresh admin queue */
388 clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);
389
390 error = nvmf_connect_admin_queue(&ctrl->ctrl);
391 if (error)
362 /* reset stopped state for the fresh admin queue */
363 clear_bit(NVME_CTRL_ADMIN_Q_STOPPED, &ctrl->ctrl.flags);
364
365 error = nvmf_connect_admin_queue(&ctrl->ctrl);
366 if (error)
392 goto out_cleanup_queue;
367 goto out_cleanup_tagset;
393
394 set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
395
396 error = nvme_enable_ctrl(&ctrl->ctrl);
397 if (error)
368
369 set_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
370
371 error = nvme_enable_ctrl(&ctrl->ctrl);
372 if (error)
398 goto out_cleanup_queue;
373 goto out_cleanup_tagset;
399
400 ctrl->ctrl.max_hw_sectors =
401 (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9);
402
403 nvme_start_admin_queue(&ctrl->ctrl);
404
405 error = nvme_init_ctrl_finish(&ctrl->ctrl);
406 if (error)
374
375 ctrl->ctrl.max_hw_sectors =
376 (NVME_LOOP_MAX_SEGMENTS - 1) << (PAGE_SHIFT - 9);
377
378 nvme_start_admin_queue(&ctrl->ctrl);
379
380 error = nvme_init_ctrl_finish(&ctrl->ctrl);
381 if (error)
407 goto out_cleanup_queue;
382 goto out_cleanup_tagset;
408
409 return 0;
410
383
384 return 0;
385
411out_cleanup_queue:
386out_cleanup_tagset:
412 clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
387 clear_bit(NVME_LOOP_Q_LIVE, &ctrl->queues[0].flags);
413 blk_mq_destroy_queue(ctrl->ctrl.admin_q);
414out_cleanup_fabrics_q:
415 blk_mq_destroy_queue(ctrl->ctrl.fabrics_q);
416out_free_tagset:
417 blk_mq_free_tag_set(&ctrl->admin_tag_set);
388 nvme_remove_admin_tag_set(&ctrl->ctrl);
418out_free_sq:
419 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
420 return error;
421}
422
423static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
424{
425 if (ctrl->ctrl.queue_count > 1) {

--- 91 unchanged lines hidden (view full) ---

517static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
518{
519 int ret;
520
521 ret = nvme_loop_init_io_queues(ctrl);
522 if (ret)
523 return ret;
524
389out_free_sq:
390 nvmet_sq_destroy(&ctrl->queues[0].nvme_sq);
391 return error;
392}
393
394static void nvme_loop_shutdown_ctrl(struct nvme_loop_ctrl *ctrl)
395{
396 if (ctrl->ctrl.queue_count > 1) {

--- 91 unchanged lines hidden (view full) ---

488static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl)
489{
490 int ret;
491
492 ret = nvme_loop_init_io_queues(ctrl);
493 if (ret)
494 return ret;
495
525 memset(&ctrl->tag_set, 0, sizeof(ctrl->tag_set));
526 ctrl->tag_set.ops = &nvme_loop_mq_ops;
527 ctrl->tag_set.queue_depth = ctrl->ctrl.opts->queue_size;
528 ctrl->tag_set.reserved_tags = NVMF_RESERVED_TAGS;
529 ctrl->tag_set.numa_node = ctrl->ctrl.numa_node;
530 ctrl->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
531 ctrl->tag_set.cmd_size = sizeof(struct nvme_loop_iod) +
532 NVME_INLINE_SG_CNT * sizeof(struct scatterlist);
533 ctrl->tag_set.driver_data = &ctrl->ctrl;
534 ctrl->tag_set.nr_hw_queues = ctrl->ctrl.queue_count - 1;
535 ctrl->tag_set.timeout = NVME_IO_TIMEOUT;
536 ctrl->ctrl.tagset = &ctrl->tag_set;
537
538 ret = blk_mq_alloc_tag_set(&ctrl->tag_set);
496 ret = nvme_alloc_io_tag_set(&ctrl->ctrl, &ctrl->tag_set,
497 &nvme_loop_mq_ops, BLK_MQ_F_SHOULD_MERGE,
498 sizeof(struct nvme_loop_iod) +
499 NVME_INLINE_SG_CNT * sizeof(struct scatterlist));
539 if (ret)
540 goto out_destroy_queues;
541
500 if (ret)
501 goto out_destroy_queues;
502
542 ret = nvme_ctrl_init_connect_q(&(ctrl->ctrl));
543 if (ret)
544 goto out_free_tagset;
545
546 ret = nvme_loop_connect_io_queues(ctrl);
547 if (ret)
503 ret = nvme_loop_connect_io_queues(ctrl);
504 if (ret)
548 goto out_cleanup_connect_q;
505 goto out_cleanup_tagset;
549
550 return 0;
551
506
507 return 0;
508
552out_cleanup_connect_q:
553 blk_mq_destroy_queue(ctrl->ctrl.connect_q);
554out_free_tagset:
555 blk_mq_free_tag_set(&ctrl->tag_set);
509out_cleanup_tagset:
510 nvme_remove_io_tag_set(&ctrl->ctrl);
556out_destroy_queues:
557 nvme_loop_destroy_io_queues(ctrl);
558 return ret;
559}
560
561static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
562{
563 struct nvmet_port *p, *found = NULL;

--- 170 unchanged lines hidden --- 		511out_destroy_queues:
512 nvme_loop_destroy_io_queues(ctrl);
513 return ret;
514}
515
516static struct nvmet_port *nvme_loop_find_port(struct nvme_ctrl *ctrl)
517{
518 struct nvmet_port *p, *found = NULL;

--- 170 unchanged lines hidden ---