rdma.c (fb051339727cd8134dd6ef0305a120e6e265dded) rdma.c (d09f2b45f346f0a9e5e1b5fcea531b1b393671dc)
1/*
2 * NVMe over Fabrics RDMA host code.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 718 unchanged lines hidden (view full) ---

727 goto requeue;
728
729 set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
730
731 ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
732 if (ret)
733 goto requeue;
734
1/*
2 * NVMe over Fabrics RDMA host code.
3 * Copyright (c) 2015-2016 HGST, a Western Digital Company.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *

--- 718 unchanged lines hidden (view full) ---

727 goto requeue;
728
729 set_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[0].flags);
730
731 ret = nvme_enable_ctrl(&ctrl->ctrl, ctrl->ctrl.cap);
732 if (ret)
733 goto requeue;
734
735 nvme_start_keep_alive(&ctrl->ctrl);
736
737 if (ctrl->ctrl.queue_count > 1) {
738 ret = nvme_rdma_init_io_queues(ctrl);
739 if (ret)
740 goto requeue;
741
742 ret = nvme_rdma_connect_io_queues(ctrl);
743 if (ret)
744 goto requeue;
745
746 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
747 ctrl->ctrl.queue_count - 1);
748 }
749
750 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
751 WARN_ON_ONCE(!changed);
752 ctrl->ctrl.nr_reconnects = 0;
753
735 if (ctrl->ctrl.queue_count > 1) {
736 ret = nvme_rdma_init_io_queues(ctrl);
737 if (ret)
738 goto requeue;
739
740 ret = nvme_rdma_connect_io_queues(ctrl);
741 if (ret)
742 goto requeue;
743
744 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
745 ctrl->ctrl.queue_count - 1);
746 }
747
748 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
749 WARN_ON_ONCE(!changed);
750 ctrl->ctrl.nr_reconnects = 0;
751
754 if (ctrl->ctrl.queue_count > 1) {
755 nvme_queue_scan(&ctrl->ctrl);
756 nvme_queue_async_events(&ctrl->ctrl);
757 }
752 nvme_start_ctrl(&ctrl->ctrl);
758
759 dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
760
761 return;
762
763requeue:
764 dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
765 ctrl->ctrl.nr_reconnects);
766 nvme_rdma_reconnect_or_remove(ctrl);
767}
768
769static void nvme_rdma_error_recovery_work(struct work_struct *work)
770{
771 struct nvme_rdma_ctrl *ctrl = container_of(work,
772 struct nvme_rdma_ctrl, err_work);
773 int i;
774
753
754 dev_info(ctrl->ctrl.device, "Successfully reconnected\n");
755
756 return;
757
758requeue:
759 dev_info(ctrl->ctrl.device, "Failed reconnect attempt %d\n",
760 ctrl->ctrl.nr_reconnects);
761 nvme_rdma_reconnect_or_remove(ctrl);
762}
763
764static void nvme_rdma_error_recovery_work(struct work_struct *work)
765{
766 struct nvme_rdma_ctrl *ctrl = container_of(work,
767 struct nvme_rdma_ctrl, err_work);
768 int i;
769
775 nvme_stop_keep_alive(&ctrl->ctrl);
770 nvme_stop_ctrl(&ctrl->ctrl);
776
777 for (i = 0; i < ctrl->ctrl.queue_count; i++)
778 clear_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
779
780 if (ctrl->ctrl.queue_count > 1)
781 nvme_stop_queues(&ctrl->ctrl);
782 blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
783

--- 814 unchanged lines hidden (view full) ---

1598 goto out_cleanup_queue;
1599
1600 error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
1601 &ctrl->async_event_sqe, sizeof(struct nvme_command),
1602 DMA_TO_DEVICE);
1603 if (error)
1604 goto out_cleanup_queue;
1605
771
772 for (i = 0; i < ctrl->ctrl.queue_count; i++)
773 clear_bit(NVME_RDMA_Q_LIVE, &ctrl->queues[i].flags);
774
775 if (ctrl->ctrl.queue_count > 1)
776 nvme_stop_queues(&ctrl->ctrl);
777 blk_mq_quiesce_queue(ctrl->ctrl.admin_q);
778

--- 814 unchanged lines hidden (view full) ---

1593 goto out_cleanup_queue;
1594
1595 error = nvme_rdma_alloc_qe(ctrl->queues[0].device->dev,
1596 &ctrl->async_event_sqe, sizeof(struct nvme_command),
1597 DMA_TO_DEVICE);
1598 if (error)
1599 goto out_cleanup_queue;
1600
1606 nvme_start_keep_alive(&ctrl->ctrl);
1607
1608 return 0;
1609
1610out_cleanup_queue:
1611 blk_cleanup_queue(ctrl->ctrl.admin_q);
1612out_free_tagset:
1613 /* disconnect and drain the queue before freeing the tagset */
1614 nvme_rdma_stop_queue(&ctrl->queues[0]);
1615 blk_mq_free_tag_set(&ctrl->admin_tag_set);
1616out_put_dev:
1617 nvme_rdma_dev_put(ctrl->device);
1618out_free_queue:
1619 nvme_rdma_free_queue(&ctrl->queues[0]);
1620 return error;
1621}
1622
1623static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
1624{
1601 return 0;
1602
1603out_cleanup_queue:
1604 blk_cleanup_queue(ctrl->ctrl.admin_q);
1605out_free_tagset:
1606 /* disconnect and drain the queue before freeing the tagset */
1607 nvme_rdma_stop_queue(&ctrl->queues[0]);
1608 blk_mq_free_tag_set(&ctrl->admin_tag_set);
1609out_put_dev:
1610 nvme_rdma_dev_put(ctrl->device);
1611out_free_queue:
1612 nvme_rdma_free_queue(&ctrl->queues[0]);
1613 return error;
1614}
1615
1616static void nvme_rdma_shutdown_ctrl(struct nvme_rdma_ctrl *ctrl)
1617{
1625 nvme_stop_keep_alive(&ctrl->ctrl);
1626 cancel_work_sync(&ctrl->err_work);
1627 cancel_delayed_work_sync(&ctrl->reconnect_work);
1628
1629 if (ctrl->ctrl.queue_count > 1) {
1630 nvme_stop_queues(&ctrl->ctrl);
1631 blk_mq_tagset_busy_iter(&ctrl->tag_set,
1632 nvme_cancel_request, &ctrl->ctrl);
1633 nvme_rdma_free_io_queues(ctrl);

--- 6 unchanged lines hidden (view full) ---

1640 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1641 nvme_cancel_request, &ctrl->ctrl);
1642 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
1643 nvme_rdma_destroy_admin_queue(ctrl);
1644}
1645
1646static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1647{
1618 cancel_work_sync(&ctrl->err_work);
1619 cancel_delayed_work_sync(&ctrl->reconnect_work);
1620
1621 if (ctrl->ctrl.queue_count > 1) {
1622 nvme_stop_queues(&ctrl->ctrl);
1623 blk_mq_tagset_busy_iter(&ctrl->tag_set,
1624 nvme_cancel_request, &ctrl->ctrl);
1625 nvme_rdma_free_io_queues(ctrl);

--- 6 unchanged lines hidden (view full) ---

1632 blk_mq_tagset_busy_iter(&ctrl->admin_tag_set,
1633 nvme_cancel_request, &ctrl->ctrl);
1634 blk_mq_unquiesce_queue(ctrl->ctrl.admin_q);
1635 nvme_rdma_destroy_admin_queue(ctrl);
1636}
1637
1638static void __nvme_rdma_remove_ctrl(struct nvme_rdma_ctrl *ctrl, bool shutdown)
1639{
1648 nvme_uninit_ctrl(&ctrl->ctrl);
1640 nvme_stop_ctrl(&ctrl->ctrl);
1641 nvme_remove_namespaces(&ctrl->ctrl);
1649 if (shutdown)
1650 nvme_rdma_shutdown_ctrl(ctrl);
1651
1642 if (shutdown)
1643 nvme_rdma_shutdown_ctrl(ctrl);
1644
1645 nvme_uninit_ctrl(&ctrl->ctrl);
1652 if (ctrl->ctrl.tagset) {
1653 blk_cleanup_queue(ctrl->ctrl.connect_q);
1654 blk_mq_free_tag_set(&ctrl->tag_set);
1655 nvme_rdma_dev_put(ctrl->device);
1656 }
1657
1658 nvme_put_ctrl(&ctrl->ctrl);
1659}

--- 45 unchanged lines hidden (view full) ---

1705
1706static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1707{
1708 struct nvme_rdma_ctrl *ctrl =
1709 container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
1710 int ret;
1711 bool changed;
1712
1646 if (ctrl->ctrl.tagset) {
1647 blk_cleanup_queue(ctrl->ctrl.connect_q);
1648 blk_mq_free_tag_set(&ctrl->tag_set);
1649 nvme_rdma_dev_put(ctrl->device);
1650 }
1651
1652 nvme_put_ctrl(&ctrl->ctrl);
1653}

--- 45 unchanged lines hidden (view full) ---

1699
1700static void nvme_rdma_reset_ctrl_work(struct work_struct *work)
1701{
1702 struct nvme_rdma_ctrl *ctrl =
1703 container_of(work, struct nvme_rdma_ctrl, ctrl.reset_work);
1704 int ret;
1705 bool changed;
1706
1707 nvme_stop_ctrl(&ctrl->ctrl);
1713 nvme_rdma_shutdown_ctrl(ctrl);
1714
1715 ret = nvme_rdma_configure_admin_queue(ctrl);
1716 if (ret) {
1717 /* ctrl is already shutdown, just remove the ctrl */
1718 INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
1719 goto del_dead_ctrl;
1720 }

--- 13 unchanged lines hidden (view full) ---

1734
1735 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
1736 ctrl->ctrl.queue_count - 1);
1737 }
1738
1739 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1740 WARN_ON_ONCE(!changed);
1741
1708 nvme_rdma_shutdown_ctrl(ctrl);
1709
1710 ret = nvme_rdma_configure_admin_queue(ctrl);
1711 if (ret) {
1712 /* ctrl is already shutdown, just remove the ctrl */
1713 INIT_WORK(&ctrl->delete_work, nvme_rdma_remove_ctrl_work);
1714 goto del_dead_ctrl;
1715 }

--- 13 unchanged lines hidden (view full) ---

1729
1730 blk_mq_update_nr_hw_queues(&ctrl->tag_set,
1731 ctrl->ctrl.queue_count - 1);
1732 }
1733
1734 changed = nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_LIVE);
1735 WARN_ON_ONCE(!changed);
1736
1742 if (ctrl->ctrl.queue_count > 1) {
1743 nvme_start_queues(&ctrl->ctrl);
1744 nvme_queue_scan(&ctrl->ctrl);
1745 nvme_queue_async_events(&ctrl->ctrl);
1746 }
1737 nvme_start_ctrl(&ctrl->ctrl);
1747
1748 return;
1749
1750del_dead_ctrl:
1751 /* Deleting this dead controller... */
1752 dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1753 WARN_ON(!queue_work(nvme_wq, &ctrl->delete_work));
1754}

--- 171 unchanged lines hidden (view full) ---

1926 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1927
1928 kref_get(&ctrl->ctrl.kref);
1929
1930 mutex_lock(&nvme_rdma_ctrl_mutex);
1931 list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
1932 mutex_unlock(&nvme_rdma_ctrl_mutex);
1933
1738
1739 return;
1740
1741del_dead_ctrl:
1742 /* Deleting this dead controller... */
1743 dev_warn(ctrl->ctrl.device, "Removing after reset failure\n");
1744 WARN_ON(!queue_work(nvme_wq, &ctrl->delete_work));
1745}

--- 171 unchanged lines hidden (view full) ---

1917 ctrl->ctrl.opts->subsysnqn, &ctrl->addr);
1918
1919 kref_get(&ctrl->ctrl.kref);
1920
1921 mutex_lock(&nvme_rdma_ctrl_mutex);
1922 list_add_tail(&ctrl->list, &nvme_rdma_ctrl_list);
1923 mutex_unlock(&nvme_rdma_ctrl_mutex);
1924
1934 if (ctrl->ctrl.queue_count > 1) {
1935 nvme_queue_scan(&ctrl->ctrl);
1936 nvme_queue_async_events(&ctrl->ctrl);
1937 }
1925 nvme_start_ctrl(&ctrl->ctrl);
1938
1939 return &ctrl->ctrl;
1940
1941out_remove_admin_queue:
1926
1927 return &ctrl->ctrl;
1928
1929out_remove_admin_queue:
1942 nvme_stop_keep_alive(&ctrl->ctrl);
1943 nvme_rdma_destroy_admin_queue(ctrl);
1944out_kfree_queues:
1945 kfree(ctrl->queues);
1946out_uninit_ctrl:
1947 nvme_uninit_ctrl(&ctrl->ctrl);
1948 nvme_put_ctrl(&ctrl->ctrl);
1949 if (ret > 0)
1950 ret = -EIO;

--- 72 unchanged lines hidden --- 		1930 nvme_rdma_destroy_admin_queue(ctrl);
1931out_kfree_queues:
1932 kfree(ctrl->queues);
1933out_uninit_ctrl:
1934 nvme_uninit_ctrl(&ctrl->ctrl);
1935 nvme_put_ctrl(&ctrl->ctrl);
1936 if (ret > 0)
1937 ret = -EIO;

--- 72 unchanged lines hidden ---