13f2304f8SSagi Grimberg // SPDX-License-Identifier: GPL-2.0 23f2304f8SSagi Grimberg /* 33f2304f8SSagi Grimberg * NVMe over Fabrics TCP host. 43f2304f8SSagi Grimberg * Copyright (c) 2018 Lightbits Labs. All rights reserved. 53f2304f8SSagi Grimberg */ 63f2304f8SSagi Grimberg #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 73f2304f8SSagi Grimberg #include <linux/module.h> 83f2304f8SSagi Grimberg #include <linux/init.h> 93f2304f8SSagi Grimberg #include <linux/slab.h> 103f2304f8SSagi Grimberg #include <linux/err.h> 113f2304f8SSagi Grimberg #include <linux/nvme-tcp.h> 123f2304f8SSagi Grimberg #include <net/sock.h> 133f2304f8SSagi Grimberg #include <net/tcp.h> 143f2304f8SSagi Grimberg #include <linux/blk-mq.h> 153f2304f8SSagi Grimberg #include <crypto/hash.h> 161a9460ceSSagi Grimberg #include <net/busy_poll.h> 173f2304f8SSagi Grimberg 183f2304f8SSagi Grimberg #include "nvme.h" 193f2304f8SSagi Grimberg #include "fabrics.h" 203f2304f8SSagi Grimberg 213f2304f8SSagi Grimberg struct nvme_tcp_queue; 223f2304f8SSagi Grimberg 233f2304f8SSagi Grimberg enum nvme_tcp_send_state { 243f2304f8SSagi Grimberg NVME_TCP_SEND_CMD_PDU = 0, 253f2304f8SSagi Grimberg NVME_TCP_SEND_H2C_PDU, 263f2304f8SSagi Grimberg NVME_TCP_SEND_DATA, 273f2304f8SSagi Grimberg NVME_TCP_SEND_DDGST, 283f2304f8SSagi Grimberg }; 293f2304f8SSagi Grimberg 303f2304f8SSagi Grimberg struct nvme_tcp_request { 313f2304f8SSagi Grimberg struct nvme_request req; 323f2304f8SSagi Grimberg void *pdu; 333f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 343f2304f8SSagi Grimberg u32 data_len; 353f2304f8SSagi Grimberg u32 pdu_len; 363f2304f8SSagi Grimberg u32 pdu_sent; 373f2304f8SSagi Grimberg u16 ttag; 383f2304f8SSagi Grimberg struct list_head entry; 39a7273d40SChristoph Hellwig __le32 ddgst; 403f2304f8SSagi Grimberg 413f2304f8SSagi Grimberg struct bio *curr_bio; 423f2304f8SSagi Grimberg struct iov_iter iter; 433f2304f8SSagi Grimberg 443f2304f8SSagi Grimberg /* send state */ 453f2304f8SSagi Grimberg size_t offset; 463f2304f8SSagi Grimberg size_t data_sent; 473f2304f8SSagi Grimberg enum nvme_tcp_send_state state; 483f2304f8SSagi Grimberg }; 493f2304f8SSagi Grimberg 503f2304f8SSagi Grimberg enum nvme_tcp_queue_flags { 513f2304f8SSagi Grimberg NVME_TCP_Q_ALLOCATED = 0, 523f2304f8SSagi Grimberg NVME_TCP_Q_LIVE = 1, 533f2304f8SSagi Grimberg }; 543f2304f8SSagi Grimberg 553f2304f8SSagi Grimberg enum nvme_tcp_recv_state { 563f2304f8SSagi Grimberg NVME_TCP_RECV_PDU = 0, 573f2304f8SSagi Grimberg NVME_TCP_RECV_DATA, 583f2304f8SSagi Grimberg NVME_TCP_RECV_DDGST, 593f2304f8SSagi Grimberg }; 603f2304f8SSagi Grimberg 613f2304f8SSagi Grimberg struct nvme_tcp_ctrl; 623f2304f8SSagi Grimberg struct nvme_tcp_queue { 633f2304f8SSagi Grimberg struct socket *sock; 643f2304f8SSagi Grimberg struct work_struct io_work; 653f2304f8SSagi Grimberg int io_cpu; 663f2304f8SSagi Grimberg 673f2304f8SSagi Grimberg spinlock_t lock; 683f2304f8SSagi Grimberg struct list_head send_list; 693f2304f8SSagi Grimberg 703f2304f8SSagi Grimberg /* recv state */ 713f2304f8SSagi Grimberg void *pdu; 723f2304f8SSagi Grimberg int pdu_remaining; 733f2304f8SSagi Grimberg int pdu_offset; 743f2304f8SSagi Grimberg size_t data_remaining; 753f2304f8SSagi Grimberg size_t ddgst_remaining; 761a9460ceSSagi Grimberg unsigned int nr_cqe; 773f2304f8SSagi Grimberg 783f2304f8SSagi Grimberg /* send state */ 793f2304f8SSagi Grimberg struct nvme_tcp_request *request; 803f2304f8SSagi Grimberg 813f2304f8SSagi Grimberg int queue_size; 823f2304f8SSagi Grimberg size_t cmnd_capsule_len; 833f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 843f2304f8SSagi Grimberg unsigned long flags; 853f2304f8SSagi Grimberg bool rd_enabled; 863f2304f8SSagi Grimberg 873f2304f8SSagi Grimberg bool hdr_digest; 883f2304f8SSagi Grimberg bool data_digest; 893f2304f8SSagi Grimberg struct ahash_request *rcv_hash; 903f2304f8SSagi Grimberg struct ahash_request *snd_hash; 913f2304f8SSagi Grimberg __le32 exp_ddgst; 923f2304f8SSagi Grimberg __le32 recv_ddgst; 933f2304f8SSagi Grimberg 943f2304f8SSagi Grimberg struct page_frag_cache pf_cache; 953f2304f8SSagi Grimberg 963f2304f8SSagi Grimberg void (*state_change)(struct sock *); 973f2304f8SSagi Grimberg void (*data_ready)(struct sock *); 983f2304f8SSagi Grimberg void (*write_space)(struct sock *); 993f2304f8SSagi Grimberg }; 1003f2304f8SSagi Grimberg 1013f2304f8SSagi Grimberg struct nvme_tcp_ctrl { 1023f2304f8SSagi Grimberg /* read only in the hot path */ 1033f2304f8SSagi Grimberg struct nvme_tcp_queue *queues; 1043f2304f8SSagi Grimberg struct blk_mq_tag_set tag_set; 1053f2304f8SSagi Grimberg 1063f2304f8SSagi Grimberg /* other member variables */ 1073f2304f8SSagi Grimberg struct list_head list; 1083f2304f8SSagi Grimberg struct blk_mq_tag_set admin_tag_set; 1093f2304f8SSagi Grimberg struct sockaddr_storage addr; 1103f2304f8SSagi Grimberg struct sockaddr_storage src_addr; 1113f2304f8SSagi Grimberg struct nvme_ctrl ctrl; 1123f2304f8SSagi Grimberg 1133f2304f8SSagi Grimberg struct work_struct err_work; 1143f2304f8SSagi Grimberg struct delayed_work connect_work; 1153f2304f8SSagi Grimberg struct nvme_tcp_request async_req; 11664861993SSagi Grimberg u32 io_queues[HCTX_MAX_TYPES]; 1173f2304f8SSagi Grimberg }; 1183f2304f8SSagi Grimberg 1193f2304f8SSagi Grimberg static LIST_HEAD(nvme_tcp_ctrl_list); 1203f2304f8SSagi Grimberg static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 1213f2304f8SSagi Grimberg static struct workqueue_struct *nvme_tcp_wq; 1223f2304f8SSagi Grimberg static struct blk_mq_ops nvme_tcp_mq_ops; 1233f2304f8SSagi Grimberg static struct blk_mq_ops nvme_tcp_admin_mq_ops; 1243f2304f8SSagi Grimberg 1253f2304f8SSagi Grimberg static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 1263f2304f8SSagi Grimberg { 1273f2304f8SSagi Grimberg return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 1283f2304f8SSagi Grimberg } 1293f2304f8SSagi Grimberg 1303f2304f8SSagi Grimberg static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 1313f2304f8SSagi Grimberg { 1323f2304f8SSagi Grimberg return queue - queue->ctrl->queues; 1333f2304f8SSagi Grimberg } 1343f2304f8SSagi Grimberg 1353f2304f8SSagi Grimberg static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 1363f2304f8SSagi Grimberg { 1373f2304f8SSagi Grimberg u32 queue_idx = nvme_tcp_queue_id(queue); 1383f2304f8SSagi Grimberg 1393f2304f8SSagi Grimberg if (queue_idx == 0) 1403f2304f8SSagi Grimberg return queue->ctrl->admin_tag_set.tags[queue_idx]; 1413f2304f8SSagi Grimberg return queue->ctrl->tag_set.tags[queue_idx - 1]; 1423f2304f8SSagi Grimberg } 1433f2304f8SSagi Grimberg 1443f2304f8SSagi Grimberg static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 1453f2304f8SSagi Grimberg { 1463f2304f8SSagi Grimberg return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 1473f2304f8SSagi Grimberg } 1483f2304f8SSagi Grimberg 1493f2304f8SSagi Grimberg static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 1503f2304f8SSagi Grimberg { 1513f2304f8SSagi Grimberg return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 1523f2304f8SSagi Grimberg } 1533f2304f8SSagi Grimberg 1543f2304f8SSagi Grimberg static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue) 1553f2304f8SSagi Grimberg { 1563f2304f8SSagi Grimberg return queue->cmnd_capsule_len - sizeof(struct nvme_command); 1573f2304f8SSagi Grimberg } 1583f2304f8SSagi Grimberg 1593f2304f8SSagi Grimberg static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 1603f2304f8SSagi Grimberg { 1613f2304f8SSagi Grimberg return req == &req->queue->ctrl->async_req; 1623f2304f8SSagi Grimberg } 1633f2304f8SSagi Grimberg 1643f2304f8SSagi Grimberg static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 1653f2304f8SSagi Grimberg { 1663f2304f8SSagi Grimberg struct request *rq; 1673f2304f8SSagi Grimberg unsigned int bytes; 1683f2304f8SSagi Grimberg 1693f2304f8SSagi Grimberg if (unlikely(nvme_tcp_async_req(req))) 1703f2304f8SSagi Grimberg return false; /* async events don't have a request */ 1713f2304f8SSagi Grimberg 1723f2304f8SSagi Grimberg rq = blk_mq_rq_from_pdu(req); 1733f2304f8SSagi Grimberg bytes = blk_rq_payload_bytes(rq); 1743f2304f8SSagi Grimberg 1753f2304f8SSagi Grimberg return rq_data_dir(rq) == WRITE && bytes && 1763f2304f8SSagi Grimberg bytes <= nvme_tcp_inline_data_size(req->queue); 1773f2304f8SSagi Grimberg } 1783f2304f8SSagi Grimberg 1793f2304f8SSagi Grimberg static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 1803f2304f8SSagi Grimberg { 1813f2304f8SSagi Grimberg return req->iter.bvec->bv_page; 1823f2304f8SSagi Grimberg } 1833f2304f8SSagi Grimberg 1843f2304f8SSagi Grimberg static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 1853f2304f8SSagi Grimberg { 1863f2304f8SSagi Grimberg return req->iter.bvec->bv_offset + req->iter.iov_offset; 1873f2304f8SSagi Grimberg } 1883f2304f8SSagi Grimberg 1893f2304f8SSagi Grimberg static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 1903f2304f8SSagi Grimberg { 1913f2304f8SSagi Grimberg return min_t(size_t, req->iter.bvec->bv_len - req->iter.iov_offset, 1923f2304f8SSagi Grimberg req->pdu_len - req->pdu_sent); 1933f2304f8SSagi Grimberg } 1943f2304f8SSagi Grimberg 1953f2304f8SSagi Grimberg static inline size_t nvme_tcp_req_offset(struct nvme_tcp_request *req) 1963f2304f8SSagi Grimberg { 1973f2304f8SSagi Grimberg return req->iter.iov_offset; 1983f2304f8SSagi Grimberg } 1993f2304f8SSagi Grimberg 2003f2304f8SSagi Grimberg static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 2013f2304f8SSagi Grimberg { 2023f2304f8SSagi Grimberg return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 2033f2304f8SSagi Grimberg req->pdu_len - req->pdu_sent : 0; 2043f2304f8SSagi Grimberg } 2053f2304f8SSagi Grimberg 2063f2304f8SSagi Grimberg static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 2073f2304f8SSagi Grimberg int len) 2083f2304f8SSagi Grimberg { 2093f2304f8SSagi Grimberg return nvme_tcp_pdu_data_left(req) <= len; 2103f2304f8SSagi Grimberg } 2113f2304f8SSagi Grimberg 2123f2304f8SSagi Grimberg static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 2133f2304f8SSagi Grimberg unsigned int dir) 2143f2304f8SSagi Grimberg { 2153f2304f8SSagi Grimberg struct request *rq = blk_mq_rq_from_pdu(req); 2163f2304f8SSagi Grimberg struct bio_vec *vec; 2173f2304f8SSagi Grimberg unsigned int size; 2183f2304f8SSagi Grimberg int nsegs; 2193f2304f8SSagi Grimberg size_t offset; 2203f2304f8SSagi Grimberg 2213f2304f8SSagi Grimberg if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 2223f2304f8SSagi Grimberg vec = &rq->special_vec; 2233f2304f8SSagi Grimberg nsegs = 1; 2243f2304f8SSagi Grimberg size = blk_rq_payload_bytes(rq); 2253f2304f8SSagi Grimberg offset = 0; 2263f2304f8SSagi Grimberg } else { 2273f2304f8SSagi Grimberg struct bio *bio = req->curr_bio; 2283f2304f8SSagi Grimberg 2293f2304f8SSagi Grimberg vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 2303f2304f8SSagi Grimberg nsegs = bio_segments(bio); 2313f2304f8SSagi Grimberg size = bio->bi_iter.bi_size; 2323f2304f8SSagi Grimberg offset = bio->bi_iter.bi_bvec_done; 2333f2304f8SSagi Grimberg } 2343f2304f8SSagi Grimberg 2353f2304f8SSagi Grimberg iov_iter_bvec(&req->iter, dir, vec, nsegs, size); 2363f2304f8SSagi Grimberg req->iter.iov_offset = offset; 2373f2304f8SSagi Grimberg } 2383f2304f8SSagi Grimberg 2393f2304f8SSagi Grimberg static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 2403f2304f8SSagi Grimberg int len) 2413f2304f8SSagi Grimberg { 2423f2304f8SSagi Grimberg req->data_sent += len; 2433f2304f8SSagi Grimberg req->pdu_sent += len; 2443f2304f8SSagi Grimberg iov_iter_advance(&req->iter, len); 2453f2304f8SSagi Grimberg if (!iov_iter_count(&req->iter) && 2463f2304f8SSagi Grimberg req->data_sent < req->data_len) { 2473f2304f8SSagi Grimberg req->curr_bio = req->curr_bio->bi_next; 2483f2304f8SSagi Grimberg nvme_tcp_init_iter(req, WRITE); 2493f2304f8SSagi Grimberg } 2503f2304f8SSagi Grimberg } 2513f2304f8SSagi Grimberg 2523f2304f8SSagi Grimberg static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req) 2533f2304f8SSagi Grimberg { 2543f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 2553f2304f8SSagi Grimberg 2563f2304f8SSagi Grimberg spin_lock(&queue->lock); 2573f2304f8SSagi Grimberg list_add_tail(&req->entry, &queue->send_list); 2583f2304f8SSagi Grimberg spin_unlock(&queue->lock); 2593f2304f8SSagi Grimberg 2603f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2613f2304f8SSagi Grimberg } 2623f2304f8SSagi Grimberg 2633f2304f8SSagi Grimberg static inline struct nvme_tcp_request * 2643f2304f8SSagi Grimberg nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 2653f2304f8SSagi Grimberg { 2663f2304f8SSagi Grimberg struct nvme_tcp_request *req; 2673f2304f8SSagi Grimberg 2683f2304f8SSagi Grimberg spin_lock(&queue->lock); 2693f2304f8SSagi Grimberg req = list_first_entry_or_null(&queue->send_list, 2703f2304f8SSagi Grimberg struct nvme_tcp_request, entry); 2713f2304f8SSagi Grimberg if (req) 2723f2304f8SSagi Grimberg list_del(&req->entry); 2733f2304f8SSagi Grimberg spin_unlock(&queue->lock); 2743f2304f8SSagi Grimberg 2753f2304f8SSagi Grimberg return req; 2763f2304f8SSagi Grimberg } 2773f2304f8SSagi Grimberg 278a7273d40SChristoph Hellwig static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 279a7273d40SChristoph Hellwig __le32 *dgst) 2803f2304f8SSagi Grimberg { 2813f2304f8SSagi Grimberg ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 2823f2304f8SSagi Grimberg crypto_ahash_final(hash); 2833f2304f8SSagi Grimberg } 2843f2304f8SSagi Grimberg 2853f2304f8SSagi Grimberg static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 2863f2304f8SSagi Grimberg struct page *page, off_t off, size_t len) 2873f2304f8SSagi Grimberg { 2883f2304f8SSagi Grimberg struct scatterlist sg; 2893f2304f8SSagi Grimberg 2903f2304f8SSagi Grimberg sg_init_marker(&sg, 1); 2913f2304f8SSagi Grimberg sg_set_page(&sg, page, len, off); 2923f2304f8SSagi Grimberg ahash_request_set_crypt(hash, &sg, NULL, len); 2933f2304f8SSagi Grimberg crypto_ahash_update(hash); 2943f2304f8SSagi Grimberg } 2953f2304f8SSagi Grimberg 2963f2304f8SSagi Grimberg static inline void nvme_tcp_hdgst(struct ahash_request *hash, 2973f2304f8SSagi Grimberg void *pdu, size_t len) 2983f2304f8SSagi Grimberg { 2993f2304f8SSagi Grimberg struct scatterlist sg; 3003f2304f8SSagi Grimberg 3013f2304f8SSagi Grimberg sg_init_one(&sg, pdu, len); 3023f2304f8SSagi Grimberg ahash_request_set_crypt(hash, &sg, pdu + len, len); 3033f2304f8SSagi Grimberg crypto_ahash_digest(hash); 3043f2304f8SSagi Grimberg } 3053f2304f8SSagi Grimberg 3063f2304f8SSagi Grimberg static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 3073f2304f8SSagi Grimberg void *pdu, size_t pdu_len) 3083f2304f8SSagi Grimberg { 3093f2304f8SSagi Grimberg struct nvme_tcp_hdr *hdr = pdu; 3103f2304f8SSagi Grimberg __le32 recv_digest; 3113f2304f8SSagi Grimberg __le32 exp_digest; 3123f2304f8SSagi Grimberg 3133f2304f8SSagi Grimberg if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 3143f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 3153f2304f8SSagi Grimberg "queue %d: header digest flag is cleared\n", 3163f2304f8SSagi Grimberg nvme_tcp_queue_id(queue)); 3173f2304f8SSagi Grimberg return -EPROTO; 3183f2304f8SSagi Grimberg } 3193f2304f8SSagi Grimberg 3203f2304f8SSagi Grimberg recv_digest = *(__le32 *)(pdu + hdr->hlen); 3213f2304f8SSagi Grimberg nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 3223f2304f8SSagi Grimberg exp_digest = *(__le32 *)(pdu + hdr->hlen); 3233f2304f8SSagi Grimberg if (recv_digest != exp_digest) { 3243f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 3253f2304f8SSagi Grimberg "header digest error: recv %#x expected %#x\n", 3263f2304f8SSagi Grimberg le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 3273f2304f8SSagi Grimberg return -EIO; 3283f2304f8SSagi Grimberg } 3293f2304f8SSagi Grimberg 3303f2304f8SSagi Grimberg return 0; 3313f2304f8SSagi Grimberg } 3323f2304f8SSagi Grimberg 3333f2304f8SSagi Grimberg static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 3343f2304f8SSagi Grimberg { 3353f2304f8SSagi Grimberg struct nvme_tcp_hdr *hdr = pdu; 3363f2304f8SSagi Grimberg u8 digest_len = nvme_tcp_hdgst_len(queue); 3373f2304f8SSagi Grimberg u32 len; 3383f2304f8SSagi Grimberg 3393f2304f8SSagi Grimberg len = le32_to_cpu(hdr->plen) - hdr->hlen - 3403f2304f8SSagi Grimberg ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 3413f2304f8SSagi Grimberg 3423f2304f8SSagi Grimberg if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 3433f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 3443f2304f8SSagi Grimberg "queue %d: data digest flag is cleared\n", 3453f2304f8SSagi Grimberg nvme_tcp_queue_id(queue)); 3463f2304f8SSagi Grimberg return -EPROTO; 3473f2304f8SSagi Grimberg } 3483f2304f8SSagi Grimberg crypto_ahash_init(queue->rcv_hash); 3493f2304f8SSagi Grimberg 3503f2304f8SSagi Grimberg return 0; 3513f2304f8SSagi Grimberg } 3523f2304f8SSagi Grimberg 3533f2304f8SSagi Grimberg static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 3543f2304f8SSagi Grimberg struct request *rq, unsigned int hctx_idx) 3553f2304f8SSagi Grimberg { 3563f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 3573f2304f8SSagi Grimberg 3583f2304f8SSagi Grimberg page_frag_free(req->pdu); 3593f2304f8SSagi Grimberg } 3603f2304f8SSagi Grimberg 3613f2304f8SSagi Grimberg static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 3623f2304f8SSagi Grimberg struct request *rq, unsigned int hctx_idx, 3633f2304f8SSagi Grimberg unsigned int numa_node) 3643f2304f8SSagi Grimberg { 3653f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = set->driver_data; 3663f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 3673f2304f8SSagi Grimberg int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 3683f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 3693f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 3703f2304f8SSagi Grimberg 3713f2304f8SSagi Grimberg req->pdu = page_frag_alloc(&queue->pf_cache, 3723f2304f8SSagi Grimberg sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 3733f2304f8SSagi Grimberg GFP_KERNEL | __GFP_ZERO); 3743f2304f8SSagi Grimberg if (!req->pdu) 3753f2304f8SSagi Grimberg return -ENOMEM; 3763f2304f8SSagi Grimberg 3773f2304f8SSagi Grimberg req->queue = queue; 3783f2304f8SSagi Grimberg nvme_req(rq)->ctrl = &ctrl->ctrl; 3793f2304f8SSagi Grimberg 3803f2304f8SSagi Grimberg return 0; 3813f2304f8SSagi Grimberg } 3823f2304f8SSagi Grimberg 3833f2304f8SSagi Grimberg static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 3843f2304f8SSagi Grimberg unsigned int hctx_idx) 3853f2304f8SSagi Grimberg { 3863f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = data; 3873f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 3883f2304f8SSagi Grimberg 3893f2304f8SSagi Grimberg hctx->driver_data = queue; 3903f2304f8SSagi Grimberg return 0; 3913f2304f8SSagi Grimberg } 3923f2304f8SSagi Grimberg 3933f2304f8SSagi Grimberg static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 3943f2304f8SSagi Grimberg unsigned int hctx_idx) 3953f2304f8SSagi Grimberg { 3963f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = data; 3973f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[0]; 3983f2304f8SSagi Grimberg 3993f2304f8SSagi Grimberg hctx->driver_data = queue; 4003f2304f8SSagi Grimberg return 0; 4013f2304f8SSagi Grimberg } 4023f2304f8SSagi Grimberg 4033f2304f8SSagi Grimberg static enum nvme_tcp_recv_state 4043f2304f8SSagi Grimberg nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 4053f2304f8SSagi Grimberg { 4063f2304f8SSagi Grimberg return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 4073f2304f8SSagi Grimberg (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 4083f2304f8SSagi Grimberg NVME_TCP_RECV_DATA; 4093f2304f8SSagi Grimberg } 4103f2304f8SSagi Grimberg 4113f2304f8SSagi Grimberg static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 4123f2304f8SSagi Grimberg { 4133f2304f8SSagi Grimberg queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 4143f2304f8SSagi Grimberg nvme_tcp_hdgst_len(queue); 4153f2304f8SSagi Grimberg queue->pdu_offset = 0; 4163f2304f8SSagi Grimberg queue->data_remaining = -1; 4173f2304f8SSagi Grimberg queue->ddgst_remaining = 0; 4183f2304f8SSagi Grimberg } 4193f2304f8SSagi Grimberg 4203f2304f8SSagi Grimberg static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 4213f2304f8SSagi Grimberg { 4223f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 4233f2304f8SSagi Grimberg return; 4243f2304f8SSagi Grimberg 42597b2512aSNigel Kirkland queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 4263f2304f8SSagi Grimberg } 4273f2304f8SSagi Grimberg 4283f2304f8SSagi Grimberg static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 4293f2304f8SSagi Grimberg struct nvme_completion *cqe) 4303f2304f8SSagi Grimberg { 4313f2304f8SSagi Grimberg struct request *rq; 4323f2304f8SSagi Grimberg 4333f2304f8SSagi Grimberg rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), cqe->command_id); 4343f2304f8SSagi Grimberg if (!rq) { 4353f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 4363f2304f8SSagi Grimberg "queue %d tag 0x%x not found\n", 4373f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), cqe->command_id); 4383f2304f8SSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 4393f2304f8SSagi Grimberg return -EINVAL; 4403f2304f8SSagi Grimberg } 4413f2304f8SSagi Grimberg 4423f2304f8SSagi Grimberg nvme_end_request(rq, cqe->status, cqe->result); 4431a9460ceSSagi Grimberg queue->nr_cqe++; 4443f2304f8SSagi Grimberg 4453f2304f8SSagi Grimberg return 0; 4463f2304f8SSagi Grimberg } 4473f2304f8SSagi Grimberg 4483f2304f8SSagi Grimberg static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 4493f2304f8SSagi Grimberg struct nvme_tcp_data_pdu *pdu) 4503f2304f8SSagi Grimberg { 4513f2304f8SSagi Grimberg struct request *rq; 4523f2304f8SSagi Grimberg 4533f2304f8SSagi Grimberg rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 4543f2304f8SSagi Grimberg if (!rq) { 4553f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 4563f2304f8SSagi Grimberg "queue %d tag %#x not found\n", 4573f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), pdu->command_id); 4583f2304f8SSagi Grimberg return -ENOENT; 4593f2304f8SSagi Grimberg } 4603f2304f8SSagi Grimberg 4613f2304f8SSagi Grimberg if (!blk_rq_payload_bytes(rq)) { 4623f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 4633f2304f8SSagi Grimberg "queue %d tag %#x unexpected data\n", 4643f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 4653f2304f8SSagi Grimberg return -EIO; 4663f2304f8SSagi Grimberg } 4673f2304f8SSagi Grimberg 4683f2304f8SSagi Grimberg queue->data_remaining = le32_to_cpu(pdu->data_length); 4693f2304f8SSagi Grimberg 470602d674cSSagi Grimberg if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 471602d674cSSagi Grimberg unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 472602d674cSSagi Grimberg dev_err(queue->ctrl->ctrl.device, 473602d674cSSagi Grimberg "queue %d tag %#x SUCCESS set but not last PDU\n", 474602d674cSSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 475602d674cSSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 476602d674cSSagi Grimberg return -EPROTO; 477602d674cSSagi Grimberg } 478602d674cSSagi Grimberg 4793f2304f8SSagi Grimberg return 0; 4803f2304f8SSagi Grimberg } 4813f2304f8SSagi Grimberg 4823f2304f8SSagi Grimberg static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 4833f2304f8SSagi Grimberg struct nvme_tcp_rsp_pdu *pdu) 4843f2304f8SSagi Grimberg { 4853f2304f8SSagi Grimberg struct nvme_completion *cqe = &pdu->cqe; 4863f2304f8SSagi Grimberg int ret = 0; 4873f2304f8SSagi Grimberg 4883f2304f8SSagi Grimberg /* 4893f2304f8SSagi Grimberg * AEN requests are special as they don't time out and can 4903f2304f8SSagi Grimberg * survive any kind of queue freeze and often don't respond to 4913f2304f8SSagi Grimberg * aborts. We don't even bother to allocate a struct request 4923f2304f8SSagi Grimberg * for them but rather special case them here. 4933f2304f8SSagi Grimberg */ 49458a8df67SIsrael Rukshin if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 49558a8df67SIsrael Rukshin cqe->command_id))) 4963f2304f8SSagi Grimberg nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 4973f2304f8SSagi Grimberg &cqe->result); 4983f2304f8SSagi Grimberg else 4993f2304f8SSagi Grimberg ret = nvme_tcp_process_nvme_cqe(queue, cqe); 5003f2304f8SSagi Grimberg 5013f2304f8SSagi Grimberg return ret; 5023f2304f8SSagi Grimberg } 5033f2304f8SSagi Grimberg 5043f2304f8SSagi Grimberg static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req, 5053f2304f8SSagi Grimberg struct nvme_tcp_r2t_pdu *pdu) 5063f2304f8SSagi Grimberg { 5073f2304f8SSagi Grimberg struct nvme_tcp_data_pdu *data = req->pdu; 5083f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 5093f2304f8SSagi Grimberg struct request *rq = blk_mq_rq_from_pdu(req); 5103f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 5113f2304f8SSagi Grimberg u8 ddgst = nvme_tcp_ddgst_len(queue); 5123f2304f8SSagi Grimberg 5133f2304f8SSagi Grimberg req->pdu_len = le32_to_cpu(pdu->r2t_length); 5143f2304f8SSagi Grimberg req->pdu_sent = 0; 5153f2304f8SSagi Grimberg 5163f2304f8SSagi Grimberg if (unlikely(req->data_sent + req->pdu_len > req->data_len)) { 5173f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 5183f2304f8SSagi Grimberg "req %d r2t len %u exceeded data len %u (%zu sent)\n", 5193f2304f8SSagi Grimberg rq->tag, req->pdu_len, req->data_len, 5203f2304f8SSagi Grimberg req->data_sent); 5213f2304f8SSagi Grimberg return -EPROTO; 5223f2304f8SSagi Grimberg } 5233f2304f8SSagi Grimberg 5243f2304f8SSagi Grimberg if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) { 5253f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 5263f2304f8SSagi Grimberg "req %d unexpected r2t offset %u (expected %zu)\n", 5273f2304f8SSagi Grimberg rq->tag, le32_to_cpu(pdu->r2t_offset), 5283f2304f8SSagi Grimberg req->data_sent); 5293f2304f8SSagi Grimberg return -EPROTO; 5303f2304f8SSagi Grimberg } 5313f2304f8SSagi Grimberg 5323f2304f8SSagi Grimberg memset(data, 0, sizeof(*data)); 5333f2304f8SSagi Grimberg data->hdr.type = nvme_tcp_h2c_data; 5343f2304f8SSagi Grimberg data->hdr.flags = NVME_TCP_F_DATA_LAST; 5353f2304f8SSagi Grimberg if (queue->hdr_digest) 5363f2304f8SSagi Grimberg data->hdr.flags |= NVME_TCP_F_HDGST; 5373f2304f8SSagi Grimberg if (queue->data_digest) 5383f2304f8SSagi Grimberg data->hdr.flags |= NVME_TCP_F_DDGST; 5393f2304f8SSagi Grimberg data->hdr.hlen = sizeof(*data); 5403f2304f8SSagi Grimberg data->hdr.pdo = data->hdr.hlen + hdgst; 5413f2304f8SSagi Grimberg data->hdr.plen = 5423f2304f8SSagi Grimberg cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 5433f2304f8SSagi Grimberg data->ttag = pdu->ttag; 5443f2304f8SSagi Grimberg data->command_id = rq->tag; 5453f2304f8SSagi Grimberg data->data_offset = cpu_to_le32(req->data_sent); 5463f2304f8SSagi Grimberg data->data_length = cpu_to_le32(req->pdu_len); 5473f2304f8SSagi Grimberg return 0; 5483f2304f8SSagi Grimberg } 5493f2304f8SSagi Grimberg 5503f2304f8SSagi Grimberg static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 5513f2304f8SSagi Grimberg struct nvme_tcp_r2t_pdu *pdu) 5523f2304f8SSagi Grimberg { 5533f2304f8SSagi Grimberg struct nvme_tcp_request *req; 5543f2304f8SSagi Grimberg struct request *rq; 5553f2304f8SSagi Grimberg int ret; 5563f2304f8SSagi Grimberg 5573f2304f8SSagi Grimberg rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 5583f2304f8SSagi Grimberg if (!rq) { 5593f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 5603f2304f8SSagi Grimberg "queue %d tag %#x not found\n", 5613f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), pdu->command_id); 5623f2304f8SSagi Grimberg return -ENOENT; 5633f2304f8SSagi Grimberg } 5643f2304f8SSagi Grimberg req = blk_mq_rq_to_pdu(rq); 5653f2304f8SSagi Grimberg 5663f2304f8SSagi Grimberg ret = nvme_tcp_setup_h2c_data_pdu(req, pdu); 5673f2304f8SSagi Grimberg if (unlikely(ret)) 5683f2304f8SSagi Grimberg return ret; 5693f2304f8SSagi Grimberg 5703f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_H2C_PDU; 5713f2304f8SSagi Grimberg req->offset = 0; 5723f2304f8SSagi Grimberg 5733f2304f8SSagi Grimberg nvme_tcp_queue_request(req); 5743f2304f8SSagi Grimberg 5753f2304f8SSagi Grimberg return 0; 5763f2304f8SSagi Grimberg } 5773f2304f8SSagi Grimberg 5783f2304f8SSagi Grimberg static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 5793f2304f8SSagi Grimberg unsigned int *offset, size_t *len) 5803f2304f8SSagi Grimberg { 5813f2304f8SSagi Grimberg struct nvme_tcp_hdr *hdr; 5823f2304f8SSagi Grimberg char *pdu = queue->pdu; 5833f2304f8SSagi Grimberg size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 5843f2304f8SSagi Grimberg int ret; 5853f2304f8SSagi Grimberg 5863f2304f8SSagi Grimberg ret = skb_copy_bits(skb, *offset, 5873f2304f8SSagi Grimberg &pdu[queue->pdu_offset], rcv_len); 5883f2304f8SSagi Grimberg if (unlikely(ret)) 5893f2304f8SSagi Grimberg return ret; 5903f2304f8SSagi Grimberg 5913f2304f8SSagi Grimberg queue->pdu_remaining -= rcv_len; 5923f2304f8SSagi Grimberg queue->pdu_offset += rcv_len; 5933f2304f8SSagi Grimberg *offset += rcv_len; 5943f2304f8SSagi Grimberg *len -= rcv_len; 5953f2304f8SSagi Grimberg if (queue->pdu_remaining) 5963f2304f8SSagi Grimberg return 0; 5973f2304f8SSagi Grimberg 5983f2304f8SSagi Grimberg hdr = queue->pdu; 5993f2304f8SSagi Grimberg if (queue->hdr_digest) { 6003f2304f8SSagi Grimberg ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 6013f2304f8SSagi Grimberg if (unlikely(ret)) 6023f2304f8SSagi Grimberg return ret; 6033f2304f8SSagi Grimberg } 6043f2304f8SSagi Grimberg 6053f2304f8SSagi Grimberg 6063f2304f8SSagi Grimberg if (queue->data_digest) { 6073f2304f8SSagi Grimberg ret = nvme_tcp_check_ddgst(queue, queue->pdu); 6083f2304f8SSagi Grimberg if (unlikely(ret)) 6093f2304f8SSagi Grimberg return ret; 6103f2304f8SSagi Grimberg } 6113f2304f8SSagi Grimberg 6123f2304f8SSagi Grimberg switch (hdr->type) { 6133f2304f8SSagi Grimberg case nvme_tcp_c2h_data: 6146be18260SSagi Grimberg return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 6153f2304f8SSagi Grimberg case nvme_tcp_rsp: 6163f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 6176be18260SSagi Grimberg return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 6183f2304f8SSagi Grimberg case nvme_tcp_r2t: 6193f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 6206be18260SSagi Grimberg return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 6213f2304f8SSagi Grimberg default: 6223f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 6233f2304f8SSagi Grimberg "unsupported pdu type (%d)\n", hdr->type); 6243f2304f8SSagi Grimberg return -EINVAL; 6253f2304f8SSagi Grimberg } 6263f2304f8SSagi Grimberg } 6273f2304f8SSagi Grimberg 628988aef9eSChristoph Hellwig static inline void nvme_tcp_end_request(struct request *rq, u16 status) 629602d674cSSagi Grimberg { 630602d674cSSagi Grimberg union nvme_result res = {}; 631602d674cSSagi Grimberg 632602d674cSSagi Grimberg nvme_end_request(rq, cpu_to_le16(status << 1), res); 633602d674cSSagi Grimberg } 634602d674cSSagi Grimberg 6353f2304f8SSagi Grimberg static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 6363f2304f8SSagi Grimberg unsigned int *offset, size_t *len) 6373f2304f8SSagi Grimberg { 6383f2304f8SSagi Grimberg struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 6393f2304f8SSagi Grimberg struct nvme_tcp_request *req; 6403f2304f8SSagi Grimberg struct request *rq; 6413f2304f8SSagi Grimberg 6423f2304f8SSagi Grimberg rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 6433f2304f8SSagi Grimberg if (!rq) { 6443f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 6453f2304f8SSagi Grimberg "queue %d tag %#x not found\n", 6463f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), pdu->command_id); 6473f2304f8SSagi Grimberg return -ENOENT; 6483f2304f8SSagi Grimberg } 6493f2304f8SSagi Grimberg req = blk_mq_rq_to_pdu(rq); 6503f2304f8SSagi Grimberg 6513f2304f8SSagi Grimberg while (true) { 6523f2304f8SSagi Grimberg int recv_len, ret; 6533f2304f8SSagi Grimberg 6543f2304f8SSagi Grimberg recv_len = min_t(size_t, *len, queue->data_remaining); 6553f2304f8SSagi Grimberg if (!recv_len) 6563f2304f8SSagi Grimberg break; 6573f2304f8SSagi Grimberg 6583f2304f8SSagi Grimberg if (!iov_iter_count(&req->iter)) { 6593f2304f8SSagi Grimberg req->curr_bio = req->curr_bio->bi_next; 6603f2304f8SSagi Grimberg 6613f2304f8SSagi Grimberg /* 6623f2304f8SSagi Grimberg * If we don`t have any bios it means that controller 6633f2304f8SSagi Grimberg * sent more data than we requested, hence error 6643f2304f8SSagi Grimberg */ 6653f2304f8SSagi Grimberg if (!req->curr_bio) { 6663f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 6673f2304f8SSagi Grimberg "queue %d no space in request %#x", 6683f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 6693f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 6703f2304f8SSagi Grimberg return -EIO; 6713f2304f8SSagi Grimberg } 6723f2304f8SSagi Grimberg nvme_tcp_init_iter(req, READ); 6733f2304f8SSagi Grimberg } 6743f2304f8SSagi Grimberg 6753f2304f8SSagi Grimberg /* we can read only from what is left in this bio */ 6763f2304f8SSagi Grimberg recv_len = min_t(size_t, recv_len, 6773f2304f8SSagi Grimberg iov_iter_count(&req->iter)); 6783f2304f8SSagi Grimberg 6793f2304f8SSagi Grimberg if (queue->data_digest) 6803f2304f8SSagi Grimberg ret = skb_copy_and_hash_datagram_iter(skb, *offset, 6813f2304f8SSagi Grimberg &req->iter, recv_len, queue->rcv_hash); 6823f2304f8SSagi Grimberg else 6833f2304f8SSagi Grimberg ret = skb_copy_datagram_iter(skb, *offset, 6843f2304f8SSagi Grimberg &req->iter, recv_len); 6853f2304f8SSagi Grimberg if (ret) { 6863f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 6873f2304f8SSagi Grimberg "queue %d failed to copy request %#x data", 6883f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 6893f2304f8SSagi Grimberg return ret; 6903f2304f8SSagi Grimberg } 6913f2304f8SSagi Grimberg 6923f2304f8SSagi Grimberg *len -= recv_len; 6933f2304f8SSagi Grimberg *offset += recv_len; 6943f2304f8SSagi Grimberg queue->data_remaining -= recv_len; 6953f2304f8SSagi Grimberg } 6963f2304f8SSagi Grimberg 6973f2304f8SSagi Grimberg if (!queue->data_remaining) { 6983f2304f8SSagi Grimberg if (queue->data_digest) { 6993f2304f8SSagi Grimberg nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 7003f2304f8SSagi Grimberg queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 7013f2304f8SSagi Grimberg } else { 7021a9460ceSSagi Grimberg if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 703602d674cSSagi Grimberg nvme_tcp_end_request(rq, NVME_SC_SUCCESS); 7041a9460ceSSagi Grimberg queue->nr_cqe++; 7051a9460ceSSagi Grimberg } 7063f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 7073f2304f8SSagi Grimberg } 7083f2304f8SSagi Grimberg } 7093f2304f8SSagi Grimberg 7103f2304f8SSagi Grimberg return 0; 7113f2304f8SSagi Grimberg } 7123f2304f8SSagi Grimberg 7133f2304f8SSagi Grimberg static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 7143f2304f8SSagi Grimberg struct sk_buff *skb, unsigned int *offset, size_t *len) 7153f2304f8SSagi Grimberg { 716602d674cSSagi Grimberg struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 7173f2304f8SSagi Grimberg char *ddgst = (char *)&queue->recv_ddgst; 7183f2304f8SSagi Grimberg size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 7193f2304f8SSagi Grimberg off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 7203f2304f8SSagi Grimberg int ret; 7213f2304f8SSagi Grimberg 7223f2304f8SSagi Grimberg ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 7233f2304f8SSagi Grimberg if (unlikely(ret)) 7243f2304f8SSagi Grimberg return ret; 7253f2304f8SSagi Grimberg 7263f2304f8SSagi Grimberg queue->ddgst_remaining -= recv_len; 7273f2304f8SSagi Grimberg *offset += recv_len; 7283f2304f8SSagi Grimberg *len -= recv_len; 7293f2304f8SSagi Grimberg if (queue->ddgst_remaining) 7303f2304f8SSagi Grimberg return 0; 7313f2304f8SSagi Grimberg 7323f2304f8SSagi Grimberg if (queue->recv_ddgst != queue->exp_ddgst) { 7333f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 7343f2304f8SSagi Grimberg "data digest error: recv %#x expected %#x\n", 7353f2304f8SSagi Grimberg le32_to_cpu(queue->recv_ddgst), 7363f2304f8SSagi Grimberg le32_to_cpu(queue->exp_ddgst)); 7373f2304f8SSagi Grimberg return -EIO; 7383f2304f8SSagi Grimberg } 7393f2304f8SSagi Grimberg 740602d674cSSagi Grimberg if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 741602d674cSSagi Grimberg struct request *rq = blk_mq_tag_to_rq(nvme_tcp_tagset(queue), 742602d674cSSagi Grimberg pdu->command_id); 743602d674cSSagi Grimberg 744602d674cSSagi Grimberg nvme_tcp_end_request(rq, NVME_SC_SUCCESS); 7451a9460ceSSagi Grimberg queue->nr_cqe++; 746602d674cSSagi Grimberg } 747602d674cSSagi Grimberg 7483f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 7493f2304f8SSagi Grimberg return 0; 7503f2304f8SSagi Grimberg } 7513f2304f8SSagi Grimberg 7523f2304f8SSagi Grimberg static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 7533f2304f8SSagi Grimberg unsigned int offset, size_t len) 7543f2304f8SSagi Grimberg { 7553f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = desc->arg.data; 7563f2304f8SSagi Grimberg size_t consumed = len; 7573f2304f8SSagi Grimberg int result; 7583f2304f8SSagi Grimberg 7593f2304f8SSagi Grimberg while (len) { 7603f2304f8SSagi Grimberg switch (nvme_tcp_recv_state(queue)) { 7613f2304f8SSagi Grimberg case NVME_TCP_RECV_PDU: 7623f2304f8SSagi Grimberg result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 7633f2304f8SSagi Grimberg break; 7643f2304f8SSagi Grimberg case NVME_TCP_RECV_DATA: 7653f2304f8SSagi Grimberg result = nvme_tcp_recv_data(queue, skb, &offset, &len); 7663f2304f8SSagi Grimberg break; 7673f2304f8SSagi Grimberg case NVME_TCP_RECV_DDGST: 7683f2304f8SSagi Grimberg result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 7693f2304f8SSagi Grimberg break; 7703f2304f8SSagi Grimberg default: 7713f2304f8SSagi Grimberg result = -EFAULT; 7723f2304f8SSagi Grimberg } 7733f2304f8SSagi Grimberg if (result) { 7743f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 7753f2304f8SSagi Grimberg "receive failed: %d\n", result); 7763f2304f8SSagi Grimberg queue->rd_enabled = false; 7773f2304f8SSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 7783f2304f8SSagi Grimberg return result; 7793f2304f8SSagi Grimberg } 7803f2304f8SSagi Grimberg } 7813f2304f8SSagi Grimberg 7823f2304f8SSagi Grimberg return consumed; 7833f2304f8SSagi Grimberg } 7843f2304f8SSagi Grimberg 7853f2304f8SSagi Grimberg static void nvme_tcp_data_ready(struct sock *sk) 7863f2304f8SSagi Grimberg { 7873f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 7883f2304f8SSagi Grimberg 7893f2304f8SSagi Grimberg read_lock(&sk->sk_callback_lock); 7903f2304f8SSagi Grimberg queue = sk->sk_user_data; 7913f2304f8SSagi Grimberg if (likely(queue && queue->rd_enabled)) 7923f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 7933f2304f8SSagi Grimberg read_unlock(&sk->sk_callback_lock); 7943f2304f8SSagi Grimberg } 7953f2304f8SSagi Grimberg 7963f2304f8SSagi Grimberg static void nvme_tcp_write_space(struct sock *sk) 7973f2304f8SSagi Grimberg { 7983f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 7993f2304f8SSagi Grimberg 8003f2304f8SSagi Grimberg read_lock_bh(&sk->sk_callback_lock); 8013f2304f8SSagi Grimberg queue = sk->sk_user_data; 8023f2304f8SSagi Grimberg if (likely(queue && sk_stream_is_writeable(sk))) { 8033f2304f8SSagi Grimberg clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 8043f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 8053f2304f8SSagi Grimberg } 8063f2304f8SSagi Grimberg read_unlock_bh(&sk->sk_callback_lock); 8073f2304f8SSagi Grimberg } 8083f2304f8SSagi Grimberg 8093f2304f8SSagi Grimberg static void nvme_tcp_state_change(struct sock *sk) 8103f2304f8SSagi Grimberg { 8113f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 8123f2304f8SSagi Grimberg 8133f2304f8SSagi Grimberg read_lock(&sk->sk_callback_lock); 8143f2304f8SSagi Grimberg queue = sk->sk_user_data; 8153f2304f8SSagi Grimberg if (!queue) 8163f2304f8SSagi Grimberg goto done; 8173f2304f8SSagi Grimberg 8183f2304f8SSagi Grimberg switch (sk->sk_state) { 8193f2304f8SSagi Grimberg case TCP_CLOSE: 8203f2304f8SSagi Grimberg case TCP_CLOSE_WAIT: 8213f2304f8SSagi Grimberg case TCP_LAST_ACK: 8223f2304f8SSagi Grimberg case TCP_FIN_WAIT1: 8233f2304f8SSagi Grimberg case TCP_FIN_WAIT2: 8243f2304f8SSagi Grimberg /* fallthrough */ 8253f2304f8SSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 8263f2304f8SSagi Grimberg break; 8273f2304f8SSagi Grimberg default: 8283f2304f8SSagi Grimberg dev_info(queue->ctrl->ctrl.device, 8293f2304f8SSagi Grimberg "queue %d socket state %d\n", 8303f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), sk->sk_state); 8313f2304f8SSagi Grimberg } 8323f2304f8SSagi Grimberg 8333f2304f8SSagi Grimberg queue->state_change(sk); 8343f2304f8SSagi Grimberg done: 8353f2304f8SSagi Grimberg read_unlock(&sk->sk_callback_lock); 8363f2304f8SSagi Grimberg } 8373f2304f8SSagi Grimberg 8383f2304f8SSagi Grimberg static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 8393f2304f8SSagi Grimberg { 8403f2304f8SSagi Grimberg queue->request = NULL; 8413f2304f8SSagi Grimberg } 8423f2304f8SSagi Grimberg 8433f2304f8SSagi Grimberg static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 8443f2304f8SSagi Grimberg { 84516686010SSagi Grimberg nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_HOST_PATH_ERROR); 8463f2304f8SSagi Grimberg } 8473f2304f8SSagi Grimberg 8483f2304f8SSagi Grimberg static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 8493f2304f8SSagi Grimberg { 8503f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 8513f2304f8SSagi Grimberg 8523f2304f8SSagi Grimberg while (true) { 8533f2304f8SSagi Grimberg struct page *page = nvme_tcp_req_cur_page(req); 8543f2304f8SSagi Grimberg size_t offset = nvme_tcp_req_cur_offset(req); 8553f2304f8SSagi Grimberg size_t len = nvme_tcp_req_cur_length(req); 8563f2304f8SSagi Grimberg bool last = nvme_tcp_pdu_last_send(req, len); 8573f2304f8SSagi Grimberg int ret, flags = MSG_DONTWAIT; 8583f2304f8SSagi Grimberg 8593f2304f8SSagi Grimberg if (last && !queue->data_digest) 8603f2304f8SSagi Grimberg flags |= MSG_EOR; 8613f2304f8SSagi Grimberg else 8623f2304f8SSagi Grimberg flags |= MSG_MORE; 8633f2304f8SSagi Grimberg 86437c15219SMikhail Skorzhinskii /* can't zcopy slab pages */ 86537c15219SMikhail Skorzhinskii if (unlikely(PageSlab(page))) { 86637c15219SMikhail Skorzhinskii ret = sock_no_sendpage(queue->sock, page, offset, len, 86737c15219SMikhail Skorzhinskii flags); 86837c15219SMikhail Skorzhinskii } else { 86937c15219SMikhail Skorzhinskii ret = kernel_sendpage(queue->sock, page, offset, len, 87037c15219SMikhail Skorzhinskii flags); 87137c15219SMikhail Skorzhinskii } 8723f2304f8SSagi Grimberg if (ret <= 0) 8733f2304f8SSagi Grimberg return ret; 8743f2304f8SSagi Grimberg 8753f2304f8SSagi Grimberg nvme_tcp_advance_req(req, ret); 8763f2304f8SSagi Grimberg if (queue->data_digest) 8773f2304f8SSagi Grimberg nvme_tcp_ddgst_update(queue->snd_hash, page, 8783f2304f8SSagi Grimberg offset, ret); 8793f2304f8SSagi Grimberg 8803f2304f8SSagi Grimberg /* fully successful last write*/ 8813f2304f8SSagi Grimberg if (last && ret == len) { 8823f2304f8SSagi Grimberg if (queue->data_digest) { 8833f2304f8SSagi Grimberg nvme_tcp_ddgst_final(queue->snd_hash, 8843f2304f8SSagi Grimberg &req->ddgst); 8853f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_DDGST; 8863f2304f8SSagi Grimberg req->offset = 0; 8873f2304f8SSagi Grimberg } else { 8883f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 8893f2304f8SSagi Grimberg } 8903f2304f8SSagi Grimberg return 1; 8913f2304f8SSagi Grimberg } 8923f2304f8SSagi Grimberg } 8933f2304f8SSagi Grimberg return -EAGAIN; 8943f2304f8SSagi Grimberg } 8953f2304f8SSagi Grimberg 8963f2304f8SSagi Grimberg static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 8973f2304f8SSagi Grimberg { 8983f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 8993f2304f8SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = req->pdu; 9003f2304f8SSagi Grimberg bool inline_data = nvme_tcp_has_inline_data(req); 9013f2304f8SSagi Grimberg int flags = MSG_DONTWAIT | (inline_data ? MSG_MORE : MSG_EOR); 9023f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 9033f2304f8SSagi Grimberg int len = sizeof(*pdu) + hdgst - req->offset; 9043f2304f8SSagi Grimberg int ret; 9053f2304f8SSagi Grimberg 9063f2304f8SSagi Grimberg if (queue->hdr_digest && !req->offset) 9073f2304f8SSagi Grimberg nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 9083f2304f8SSagi Grimberg 9093f2304f8SSagi Grimberg ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 9103f2304f8SSagi Grimberg offset_in_page(pdu) + req->offset, len, flags); 9113f2304f8SSagi Grimberg if (unlikely(ret <= 0)) 9123f2304f8SSagi Grimberg return ret; 9133f2304f8SSagi Grimberg 9143f2304f8SSagi Grimberg len -= ret; 9153f2304f8SSagi Grimberg if (!len) { 9163f2304f8SSagi Grimberg if (inline_data) { 9173f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_DATA; 9183f2304f8SSagi Grimberg if (queue->data_digest) 9193f2304f8SSagi Grimberg crypto_ahash_init(queue->snd_hash); 9203f2304f8SSagi Grimberg nvme_tcp_init_iter(req, WRITE); 9213f2304f8SSagi Grimberg } else { 9223f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 9233f2304f8SSagi Grimberg } 9243f2304f8SSagi Grimberg return 1; 9253f2304f8SSagi Grimberg } 9263f2304f8SSagi Grimberg req->offset += ret; 9273f2304f8SSagi Grimberg 9283f2304f8SSagi Grimberg return -EAGAIN; 9293f2304f8SSagi Grimberg } 9303f2304f8SSagi Grimberg 9313f2304f8SSagi Grimberg static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 9323f2304f8SSagi Grimberg { 9333f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 9343f2304f8SSagi Grimberg struct nvme_tcp_data_pdu *pdu = req->pdu; 9353f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 9363f2304f8SSagi Grimberg int len = sizeof(*pdu) - req->offset + hdgst; 9373f2304f8SSagi Grimberg int ret; 9383f2304f8SSagi Grimberg 9393f2304f8SSagi Grimberg if (queue->hdr_digest && !req->offset) 9403f2304f8SSagi Grimberg nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 9413f2304f8SSagi Grimberg 9423f2304f8SSagi Grimberg ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 9433f2304f8SSagi Grimberg offset_in_page(pdu) + req->offset, len, 9443f2304f8SSagi Grimberg MSG_DONTWAIT | MSG_MORE); 9453f2304f8SSagi Grimberg if (unlikely(ret <= 0)) 9463f2304f8SSagi Grimberg return ret; 9473f2304f8SSagi Grimberg 9483f2304f8SSagi Grimberg len -= ret; 9493f2304f8SSagi Grimberg if (!len) { 9503f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_DATA; 9513f2304f8SSagi Grimberg if (queue->data_digest) 9523f2304f8SSagi Grimberg crypto_ahash_init(queue->snd_hash); 9533f2304f8SSagi Grimberg if (!req->data_sent) 9543f2304f8SSagi Grimberg nvme_tcp_init_iter(req, WRITE); 9553f2304f8SSagi Grimberg return 1; 9563f2304f8SSagi Grimberg } 9573f2304f8SSagi Grimberg req->offset += ret; 9583f2304f8SSagi Grimberg 9593f2304f8SSagi Grimberg return -EAGAIN; 9603f2304f8SSagi Grimberg } 9613f2304f8SSagi Grimberg 9623f2304f8SSagi Grimberg static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 9633f2304f8SSagi Grimberg { 9643f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 9653f2304f8SSagi Grimberg int ret; 9663f2304f8SSagi Grimberg struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR }; 9673f2304f8SSagi Grimberg struct kvec iov = { 9683f2304f8SSagi Grimberg .iov_base = &req->ddgst + req->offset, 9693f2304f8SSagi Grimberg .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 9703f2304f8SSagi Grimberg }; 9713f2304f8SSagi Grimberg 9723f2304f8SSagi Grimberg ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 9733f2304f8SSagi Grimberg if (unlikely(ret <= 0)) 9743f2304f8SSagi Grimberg return ret; 9753f2304f8SSagi Grimberg 9763f2304f8SSagi Grimberg if (req->offset + ret == NVME_TCP_DIGEST_LENGTH) { 9773f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 9783f2304f8SSagi Grimberg return 1; 9793f2304f8SSagi Grimberg } 9803f2304f8SSagi Grimberg 9813f2304f8SSagi Grimberg req->offset += ret; 9823f2304f8SSagi Grimberg return -EAGAIN; 9833f2304f8SSagi Grimberg } 9843f2304f8SSagi Grimberg 9853f2304f8SSagi Grimberg static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 9863f2304f8SSagi Grimberg { 9873f2304f8SSagi Grimberg struct nvme_tcp_request *req; 9883f2304f8SSagi Grimberg int ret = 1; 9893f2304f8SSagi Grimberg 9903f2304f8SSagi Grimberg if (!queue->request) { 9913f2304f8SSagi Grimberg queue->request = nvme_tcp_fetch_request(queue); 9923f2304f8SSagi Grimberg if (!queue->request) 9933f2304f8SSagi Grimberg return 0; 9943f2304f8SSagi Grimberg } 9953f2304f8SSagi Grimberg req = queue->request; 9963f2304f8SSagi Grimberg 9973f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_CMD_PDU) { 9983f2304f8SSagi Grimberg ret = nvme_tcp_try_send_cmd_pdu(req); 9993f2304f8SSagi Grimberg if (ret <= 0) 10003f2304f8SSagi Grimberg goto done; 10013f2304f8SSagi Grimberg if (!nvme_tcp_has_inline_data(req)) 10023f2304f8SSagi Grimberg return ret; 10033f2304f8SSagi Grimberg } 10043f2304f8SSagi Grimberg 10053f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_H2C_PDU) { 10063f2304f8SSagi Grimberg ret = nvme_tcp_try_send_data_pdu(req); 10073f2304f8SSagi Grimberg if (ret <= 0) 10083f2304f8SSagi Grimberg goto done; 10093f2304f8SSagi Grimberg } 10103f2304f8SSagi Grimberg 10113f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_DATA) { 10123f2304f8SSagi Grimberg ret = nvme_tcp_try_send_data(req); 10133f2304f8SSagi Grimberg if (ret <= 0) 10143f2304f8SSagi Grimberg goto done; 10153f2304f8SSagi Grimberg } 10163f2304f8SSagi Grimberg 10173f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_DDGST) 10183f2304f8SSagi Grimberg ret = nvme_tcp_try_send_ddgst(req); 10193f2304f8SSagi Grimberg done: 10203f2304f8SSagi Grimberg if (ret == -EAGAIN) 10213f2304f8SSagi Grimberg ret = 0; 10223f2304f8SSagi Grimberg return ret; 10233f2304f8SSagi Grimberg } 10243f2304f8SSagi Grimberg 10253f2304f8SSagi Grimberg static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 10263f2304f8SSagi Grimberg { 102710407ec9SPotnuri Bharat Teja struct socket *sock = queue->sock; 102810407ec9SPotnuri Bharat Teja struct sock *sk = sock->sk; 10293f2304f8SSagi Grimberg read_descriptor_t rd_desc; 10303f2304f8SSagi Grimberg int consumed; 10313f2304f8SSagi Grimberg 10323f2304f8SSagi Grimberg rd_desc.arg.data = queue; 10333f2304f8SSagi Grimberg rd_desc.count = 1; 10343f2304f8SSagi Grimberg lock_sock(sk); 10351a9460ceSSagi Grimberg queue->nr_cqe = 0; 103610407ec9SPotnuri Bharat Teja consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 10373f2304f8SSagi Grimberg release_sock(sk); 10383f2304f8SSagi Grimberg return consumed; 10393f2304f8SSagi Grimberg } 10403f2304f8SSagi Grimberg 10413f2304f8SSagi Grimberg static void nvme_tcp_io_work(struct work_struct *w) 10423f2304f8SSagi Grimberg { 10433f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = 10443f2304f8SSagi Grimberg container_of(w, struct nvme_tcp_queue, io_work); 1045ddef2957SWunderlich, Mark unsigned long deadline = jiffies + msecs_to_jiffies(1); 10463f2304f8SSagi Grimberg 10473f2304f8SSagi Grimberg do { 10483f2304f8SSagi Grimberg bool pending = false; 10493f2304f8SSagi Grimberg int result; 10503f2304f8SSagi Grimberg 10513f2304f8SSagi Grimberg result = nvme_tcp_try_send(queue); 10523f2304f8SSagi Grimberg if (result > 0) { 10533f2304f8SSagi Grimberg pending = true; 10543f2304f8SSagi Grimberg } else if (unlikely(result < 0)) { 10553f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 10563f2304f8SSagi Grimberg "failed to send request %d\n", result); 10572d570a7cSAnton Eidelman 10582d570a7cSAnton Eidelman /* 10592d570a7cSAnton Eidelman * Fail the request unless peer closed the connection, 10602d570a7cSAnton Eidelman * in which case error recovery flow will complete all. 10612d570a7cSAnton Eidelman */ 10622d570a7cSAnton Eidelman if ((result != -EPIPE) && (result != -ECONNRESET)) 10633f2304f8SSagi Grimberg nvme_tcp_fail_request(queue->request); 10643f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 10653f2304f8SSagi Grimberg return; 10663f2304f8SSagi Grimberg } 10673f2304f8SSagi Grimberg 10683f2304f8SSagi Grimberg result = nvme_tcp_try_recv(queue); 10693f2304f8SSagi Grimberg if (result > 0) 10703f2304f8SSagi Grimberg pending = true; 10713f2304f8SSagi Grimberg 10723f2304f8SSagi Grimberg if (!pending) 10733f2304f8SSagi Grimberg return; 10743f2304f8SSagi Grimberg 1075ddef2957SWunderlich, Mark } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 10763f2304f8SSagi Grimberg 10773f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 10783f2304f8SSagi Grimberg } 10793f2304f8SSagi Grimberg 10803f2304f8SSagi Grimberg static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 10813f2304f8SSagi Grimberg { 10823f2304f8SSagi Grimberg struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 10833f2304f8SSagi Grimberg 10843f2304f8SSagi Grimberg ahash_request_free(queue->rcv_hash); 10853f2304f8SSagi Grimberg ahash_request_free(queue->snd_hash); 10863f2304f8SSagi Grimberg crypto_free_ahash(tfm); 10873f2304f8SSagi Grimberg } 10883f2304f8SSagi Grimberg 10893f2304f8SSagi Grimberg static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 10903f2304f8SSagi Grimberg { 10913f2304f8SSagi Grimberg struct crypto_ahash *tfm; 10923f2304f8SSagi Grimberg 10933f2304f8SSagi Grimberg tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 10943f2304f8SSagi Grimberg if (IS_ERR(tfm)) 10953f2304f8SSagi Grimberg return PTR_ERR(tfm); 10963f2304f8SSagi Grimberg 10973f2304f8SSagi Grimberg queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 10983f2304f8SSagi Grimberg if (!queue->snd_hash) 10993f2304f8SSagi Grimberg goto free_tfm; 11003f2304f8SSagi Grimberg ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 11013f2304f8SSagi Grimberg 11023f2304f8SSagi Grimberg queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 11033f2304f8SSagi Grimberg if (!queue->rcv_hash) 11043f2304f8SSagi Grimberg goto free_snd_hash; 11053f2304f8SSagi Grimberg ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 11063f2304f8SSagi Grimberg 11073f2304f8SSagi Grimberg return 0; 11083f2304f8SSagi Grimberg free_snd_hash: 11093f2304f8SSagi Grimberg ahash_request_free(queue->snd_hash); 11103f2304f8SSagi Grimberg free_tfm: 11113f2304f8SSagi Grimberg crypto_free_ahash(tfm); 11123f2304f8SSagi Grimberg return -ENOMEM; 11133f2304f8SSagi Grimberg } 11143f2304f8SSagi Grimberg 11153f2304f8SSagi Grimberg static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 11163f2304f8SSagi Grimberg { 11173f2304f8SSagi Grimberg struct nvme_tcp_request *async = &ctrl->async_req; 11183f2304f8SSagi Grimberg 11193f2304f8SSagi Grimberg page_frag_free(async->pdu); 11203f2304f8SSagi Grimberg } 11213f2304f8SSagi Grimberg 11223f2304f8SSagi Grimberg static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 11233f2304f8SSagi Grimberg { 11243f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[0]; 11253f2304f8SSagi Grimberg struct nvme_tcp_request *async = &ctrl->async_req; 11263f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 11273f2304f8SSagi Grimberg 11283f2304f8SSagi Grimberg async->pdu = page_frag_alloc(&queue->pf_cache, 11293f2304f8SSagi Grimberg sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 11303f2304f8SSagi Grimberg GFP_KERNEL | __GFP_ZERO); 11313f2304f8SSagi Grimberg if (!async->pdu) 11323f2304f8SSagi Grimberg return -ENOMEM; 11333f2304f8SSagi Grimberg 11343f2304f8SSagi Grimberg async->queue = &ctrl->queues[0]; 11353f2304f8SSagi Grimberg return 0; 11363f2304f8SSagi Grimberg } 11373f2304f8SSagi Grimberg 11383f2304f8SSagi Grimberg static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 11393f2304f8SSagi Grimberg { 11403f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 11413f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 11423f2304f8SSagi Grimberg 11433f2304f8SSagi Grimberg if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 11443f2304f8SSagi Grimberg return; 11453f2304f8SSagi Grimberg 11463f2304f8SSagi Grimberg if (queue->hdr_digest || queue->data_digest) 11473f2304f8SSagi Grimberg nvme_tcp_free_crypto(queue); 11483f2304f8SSagi Grimberg 11493f2304f8SSagi Grimberg sock_release(queue->sock); 11503f2304f8SSagi Grimberg kfree(queue->pdu); 11513f2304f8SSagi Grimberg } 11523f2304f8SSagi Grimberg 11533f2304f8SSagi Grimberg static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 11543f2304f8SSagi Grimberg { 11553f2304f8SSagi Grimberg struct nvme_tcp_icreq_pdu *icreq; 11563f2304f8SSagi Grimberg struct nvme_tcp_icresp_pdu *icresp; 11573f2304f8SSagi Grimberg struct msghdr msg = {}; 11583f2304f8SSagi Grimberg struct kvec iov; 11593f2304f8SSagi Grimberg bool ctrl_hdgst, ctrl_ddgst; 11603f2304f8SSagi Grimberg int ret; 11613f2304f8SSagi Grimberg 11623f2304f8SSagi Grimberg icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 11633f2304f8SSagi Grimberg if (!icreq) 11643f2304f8SSagi Grimberg return -ENOMEM; 11653f2304f8SSagi Grimberg 11663f2304f8SSagi Grimberg icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 11673f2304f8SSagi Grimberg if (!icresp) { 11683f2304f8SSagi Grimberg ret = -ENOMEM; 11693f2304f8SSagi Grimberg goto free_icreq; 11703f2304f8SSagi Grimberg } 11713f2304f8SSagi Grimberg 11723f2304f8SSagi Grimberg icreq->hdr.type = nvme_tcp_icreq; 11733f2304f8SSagi Grimberg icreq->hdr.hlen = sizeof(*icreq); 11743f2304f8SSagi Grimberg icreq->hdr.pdo = 0; 11753f2304f8SSagi Grimberg icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 11763f2304f8SSagi Grimberg icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 11773f2304f8SSagi Grimberg icreq->maxr2t = 0; /* single inflight r2t supported */ 11783f2304f8SSagi Grimberg icreq->hpda = 0; /* no alignment constraint */ 11793f2304f8SSagi Grimberg if (queue->hdr_digest) 11803f2304f8SSagi Grimberg icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 11813f2304f8SSagi Grimberg if (queue->data_digest) 11823f2304f8SSagi Grimberg icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 11833f2304f8SSagi Grimberg 11843f2304f8SSagi Grimberg iov.iov_base = icreq; 11853f2304f8SSagi Grimberg iov.iov_len = sizeof(*icreq); 11863f2304f8SSagi Grimberg ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 11873f2304f8SSagi Grimberg if (ret < 0) 11883f2304f8SSagi Grimberg goto free_icresp; 11893f2304f8SSagi Grimberg 11903f2304f8SSagi Grimberg memset(&msg, 0, sizeof(msg)); 11913f2304f8SSagi Grimberg iov.iov_base = icresp; 11923f2304f8SSagi Grimberg iov.iov_len = sizeof(*icresp); 11933f2304f8SSagi Grimberg ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 11943f2304f8SSagi Grimberg iov.iov_len, msg.msg_flags); 11953f2304f8SSagi Grimberg if (ret < 0) 11963f2304f8SSagi Grimberg goto free_icresp; 11973f2304f8SSagi Grimberg 11983f2304f8SSagi Grimberg ret = -EINVAL; 11993f2304f8SSagi Grimberg if (icresp->hdr.type != nvme_tcp_icresp) { 12003f2304f8SSagi Grimberg pr_err("queue %d: bad type returned %d\n", 12013f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->hdr.type); 12023f2304f8SSagi Grimberg goto free_icresp; 12033f2304f8SSagi Grimberg } 12043f2304f8SSagi Grimberg 12053f2304f8SSagi Grimberg if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 12063f2304f8SSagi Grimberg pr_err("queue %d: bad pdu length returned %d\n", 12073f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->hdr.plen); 12083f2304f8SSagi Grimberg goto free_icresp; 12093f2304f8SSagi Grimberg } 12103f2304f8SSagi Grimberg 12113f2304f8SSagi Grimberg if (icresp->pfv != NVME_TCP_PFV_1_0) { 12123f2304f8SSagi Grimberg pr_err("queue %d: bad pfv returned %d\n", 12133f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->pfv); 12143f2304f8SSagi Grimberg goto free_icresp; 12153f2304f8SSagi Grimberg } 12163f2304f8SSagi Grimberg 12173f2304f8SSagi Grimberg ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 12183f2304f8SSagi Grimberg if ((queue->data_digest && !ctrl_ddgst) || 12193f2304f8SSagi Grimberg (!queue->data_digest && ctrl_ddgst)) { 12203f2304f8SSagi Grimberg pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 12213f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), 12223f2304f8SSagi Grimberg queue->data_digest ? "enabled" : "disabled", 12233f2304f8SSagi Grimberg ctrl_ddgst ? "enabled" : "disabled"); 12243f2304f8SSagi Grimberg goto free_icresp; 12253f2304f8SSagi Grimberg } 12263f2304f8SSagi Grimberg 12273f2304f8SSagi Grimberg ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 12283f2304f8SSagi Grimberg if ((queue->hdr_digest && !ctrl_hdgst) || 12293f2304f8SSagi Grimberg (!queue->hdr_digest && ctrl_hdgst)) { 12303f2304f8SSagi Grimberg pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 12313f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), 12323f2304f8SSagi Grimberg queue->hdr_digest ? "enabled" : "disabled", 12333f2304f8SSagi Grimberg ctrl_hdgst ? "enabled" : "disabled"); 12343f2304f8SSagi Grimberg goto free_icresp; 12353f2304f8SSagi Grimberg } 12363f2304f8SSagi Grimberg 12373f2304f8SSagi Grimberg if (icresp->cpda != 0) { 12383f2304f8SSagi Grimberg pr_err("queue %d: unsupported cpda returned %d\n", 12393f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->cpda); 12403f2304f8SSagi Grimberg goto free_icresp; 12413f2304f8SSagi Grimberg } 12423f2304f8SSagi Grimberg 12433f2304f8SSagi Grimberg ret = 0; 12443f2304f8SSagi Grimberg free_icresp: 12453f2304f8SSagi Grimberg kfree(icresp); 12463f2304f8SSagi Grimberg free_icreq: 12473f2304f8SSagi Grimberg kfree(icreq); 12483f2304f8SSagi Grimberg return ret; 12493f2304f8SSagi Grimberg } 12503f2304f8SSagi Grimberg 12513f2304f8SSagi Grimberg static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, 12523f2304f8SSagi Grimberg int qid, size_t queue_size) 12533f2304f8SSagi Grimberg { 12543f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 12553f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 12563f2304f8SSagi Grimberg struct linger sol = { .l_onoff = 1, .l_linger = 0 }; 1257873946f4SSagi Grimberg int ret, opt, rcv_pdu_size, n; 12583f2304f8SSagi Grimberg 12593f2304f8SSagi Grimberg queue->ctrl = ctrl; 12603f2304f8SSagi Grimberg INIT_LIST_HEAD(&queue->send_list); 12613f2304f8SSagi Grimberg spin_lock_init(&queue->lock); 12623f2304f8SSagi Grimberg INIT_WORK(&queue->io_work, nvme_tcp_io_work); 12633f2304f8SSagi Grimberg queue->queue_size = queue_size; 12643f2304f8SSagi Grimberg 12653f2304f8SSagi Grimberg if (qid > 0) 12669924b030SIsrael Rukshin queue->cmnd_capsule_len = nctrl->ioccsz * 16; 12673f2304f8SSagi Grimberg else 12683f2304f8SSagi Grimberg queue->cmnd_capsule_len = sizeof(struct nvme_command) + 12693f2304f8SSagi Grimberg NVME_TCP_ADMIN_CCSZ; 12703f2304f8SSagi Grimberg 12713f2304f8SSagi Grimberg ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 12723f2304f8SSagi Grimberg IPPROTO_TCP, &queue->sock); 12733f2304f8SSagi Grimberg if (ret) { 12749924b030SIsrael Rukshin dev_err(nctrl->device, 12753f2304f8SSagi Grimberg "failed to create socket: %d\n", ret); 12763f2304f8SSagi Grimberg return ret; 12773f2304f8SSagi Grimberg } 12783f2304f8SSagi Grimberg 12793f2304f8SSagi Grimberg /* Single syn retry */ 12803f2304f8SSagi Grimberg opt = 1; 12813f2304f8SSagi Grimberg ret = kernel_setsockopt(queue->sock, IPPROTO_TCP, TCP_SYNCNT, 12823f2304f8SSagi Grimberg (char *)&opt, sizeof(opt)); 12833f2304f8SSagi Grimberg if (ret) { 12849924b030SIsrael Rukshin dev_err(nctrl->device, 12853f2304f8SSagi Grimberg "failed to set TCP_SYNCNT sock opt %d\n", ret); 12863f2304f8SSagi Grimberg goto err_sock; 12873f2304f8SSagi Grimberg } 12883f2304f8SSagi Grimberg 12893f2304f8SSagi Grimberg /* Set TCP no delay */ 12903f2304f8SSagi Grimberg opt = 1; 12913f2304f8SSagi Grimberg ret = kernel_setsockopt(queue->sock, IPPROTO_TCP, 12923f2304f8SSagi Grimberg TCP_NODELAY, (char *)&opt, sizeof(opt)); 12933f2304f8SSagi Grimberg if (ret) { 12949924b030SIsrael Rukshin dev_err(nctrl->device, 12953f2304f8SSagi Grimberg "failed to set TCP_NODELAY sock opt %d\n", ret); 12963f2304f8SSagi Grimberg goto err_sock; 12973f2304f8SSagi Grimberg } 12983f2304f8SSagi Grimberg 12993f2304f8SSagi Grimberg /* 13003f2304f8SSagi Grimberg * Cleanup whatever is sitting in the TCP transmit queue on socket 13013f2304f8SSagi Grimberg * close. This is done to prevent stale data from being sent should 13023f2304f8SSagi Grimberg * the network connection be restored before TCP times out. 13033f2304f8SSagi Grimberg */ 13043f2304f8SSagi Grimberg ret = kernel_setsockopt(queue->sock, SOL_SOCKET, SO_LINGER, 13053f2304f8SSagi Grimberg (char *)&sol, sizeof(sol)); 13063f2304f8SSagi Grimberg if (ret) { 13079924b030SIsrael Rukshin dev_err(nctrl->device, 13083f2304f8SSagi Grimberg "failed to set SO_LINGER sock opt %d\n", ret); 13093f2304f8SSagi Grimberg goto err_sock; 13103f2304f8SSagi Grimberg } 13113f2304f8SSagi Grimberg 1312bb13985dSIsrael Rukshin /* Set socket type of service */ 1313bb13985dSIsrael Rukshin if (nctrl->opts->tos >= 0) { 1314bb13985dSIsrael Rukshin opt = nctrl->opts->tos; 1315bb13985dSIsrael Rukshin ret = kernel_setsockopt(queue->sock, SOL_IP, IP_TOS, 1316bb13985dSIsrael Rukshin (char *)&opt, sizeof(opt)); 1317bb13985dSIsrael Rukshin if (ret) { 1318bb13985dSIsrael Rukshin dev_err(nctrl->device, 1319bb13985dSIsrael Rukshin "failed to set IP_TOS sock opt %d\n", ret); 1320bb13985dSIsrael Rukshin goto err_sock; 1321bb13985dSIsrael Rukshin } 1322bb13985dSIsrael Rukshin } 1323bb13985dSIsrael Rukshin 13243f2304f8SSagi Grimberg queue->sock->sk->sk_allocation = GFP_ATOMIC; 1325873946f4SSagi Grimberg if (!qid) 1326873946f4SSagi Grimberg n = 0; 1327873946f4SSagi Grimberg else 1328873946f4SSagi Grimberg n = (qid - 1) % num_online_cpus(); 1329873946f4SSagi Grimberg queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 13303f2304f8SSagi Grimberg queue->request = NULL; 13313f2304f8SSagi Grimberg queue->data_remaining = 0; 13323f2304f8SSagi Grimberg queue->ddgst_remaining = 0; 13333f2304f8SSagi Grimberg queue->pdu_remaining = 0; 13343f2304f8SSagi Grimberg queue->pdu_offset = 0; 13353f2304f8SSagi Grimberg sk_set_memalloc(queue->sock->sk); 13363f2304f8SSagi Grimberg 13379924b030SIsrael Rukshin if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 13383f2304f8SSagi Grimberg ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 13393f2304f8SSagi Grimberg sizeof(ctrl->src_addr)); 13403f2304f8SSagi Grimberg if (ret) { 13419924b030SIsrael Rukshin dev_err(nctrl->device, 13423f2304f8SSagi Grimberg "failed to bind queue %d socket %d\n", 13433f2304f8SSagi Grimberg qid, ret); 13443f2304f8SSagi Grimberg goto err_sock; 13453f2304f8SSagi Grimberg } 13463f2304f8SSagi Grimberg } 13473f2304f8SSagi Grimberg 13483f2304f8SSagi Grimberg queue->hdr_digest = nctrl->opts->hdr_digest; 13493f2304f8SSagi Grimberg queue->data_digest = nctrl->opts->data_digest; 13503f2304f8SSagi Grimberg if (queue->hdr_digest || queue->data_digest) { 13513f2304f8SSagi Grimberg ret = nvme_tcp_alloc_crypto(queue); 13523f2304f8SSagi Grimberg if (ret) { 13539924b030SIsrael Rukshin dev_err(nctrl->device, 13543f2304f8SSagi Grimberg "failed to allocate queue %d crypto\n", qid); 13553f2304f8SSagi Grimberg goto err_sock; 13563f2304f8SSagi Grimberg } 13573f2304f8SSagi Grimberg } 13583f2304f8SSagi Grimberg 13593f2304f8SSagi Grimberg rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 13603f2304f8SSagi Grimberg nvme_tcp_hdgst_len(queue); 13613f2304f8SSagi Grimberg queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 13623f2304f8SSagi Grimberg if (!queue->pdu) { 13633f2304f8SSagi Grimberg ret = -ENOMEM; 13643f2304f8SSagi Grimberg goto err_crypto; 13653f2304f8SSagi Grimberg } 13663f2304f8SSagi Grimberg 13679924b030SIsrael Rukshin dev_dbg(nctrl->device, "connecting queue %d\n", 13683f2304f8SSagi Grimberg nvme_tcp_queue_id(queue)); 13693f2304f8SSagi Grimberg 13703f2304f8SSagi Grimberg ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 13713f2304f8SSagi Grimberg sizeof(ctrl->addr), 0); 13723f2304f8SSagi Grimberg if (ret) { 13739924b030SIsrael Rukshin dev_err(nctrl->device, 13743f2304f8SSagi Grimberg "failed to connect socket: %d\n", ret); 13753f2304f8SSagi Grimberg goto err_rcv_pdu; 13763f2304f8SSagi Grimberg } 13773f2304f8SSagi Grimberg 13783f2304f8SSagi Grimberg ret = nvme_tcp_init_connection(queue); 13793f2304f8SSagi Grimberg if (ret) 13803f2304f8SSagi Grimberg goto err_init_connect; 13813f2304f8SSagi Grimberg 13823f2304f8SSagi Grimberg queue->rd_enabled = true; 13833f2304f8SSagi Grimberg set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 13843f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 13853f2304f8SSagi Grimberg 13863f2304f8SSagi Grimberg write_lock_bh(&queue->sock->sk->sk_callback_lock); 13873f2304f8SSagi Grimberg queue->sock->sk->sk_user_data = queue; 13883f2304f8SSagi Grimberg queue->state_change = queue->sock->sk->sk_state_change; 13893f2304f8SSagi Grimberg queue->data_ready = queue->sock->sk->sk_data_ready; 13903f2304f8SSagi Grimberg queue->write_space = queue->sock->sk->sk_write_space; 13913f2304f8SSagi Grimberg queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 13923f2304f8SSagi Grimberg queue->sock->sk->sk_state_change = nvme_tcp_state_change; 13933f2304f8SSagi Grimberg queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1394ac1c4e18SSebastian Andrzej Siewior #ifdef CONFIG_NET_RX_BUSY_POLL 13951a9460ceSSagi Grimberg queue->sock->sk->sk_ll_usec = 1; 1396ac1c4e18SSebastian Andrzej Siewior #endif 13973f2304f8SSagi Grimberg write_unlock_bh(&queue->sock->sk->sk_callback_lock); 13983f2304f8SSagi Grimberg 13993f2304f8SSagi Grimberg return 0; 14003f2304f8SSagi Grimberg 14013f2304f8SSagi Grimberg err_init_connect: 14023f2304f8SSagi Grimberg kernel_sock_shutdown(queue->sock, SHUT_RDWR); 14033f2304f8SSagi Grimberg err_rcv_pdu: 14043f2304f8SSagi Grimberg kfree(queue->pdu); 14053f2304f8SSagi Grimberg err_crypto: 14063f2304f8SSagi Grimberg if (queue->hdr_digest || queue->data_digest) 14073f2304f8SSagi Grimberg nvme_tcp_free_crypto(queue); 14083f2304f8SSagi Grimberg err_sock: 14093f2304f8SSagi Grimberg sock_release(queue->sock); 14103f2304f8SSagi Grimberg queue->sock = NULL; 14113f2304f8SSagi Grimberg return ret; 14123f2304f8SSagi Grimberg } 14133f2304f8SSagi Grimberg 14143f2304f8SSagi Grimberg static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue) 14153f2304f8SSagi Grimberg { 14163f2304f8SSagi Grimberg struct socket *sock = queue->sock; 14173f2304f8SSagi Grimberg 14183f2304f8SSagi Grimberg write_lock_bh(&sock->sk->sk_callback_lock); 14193f2304f8SSagi Grimberg sock->sk->sk_user_data = NULL; 14203f2304f8SSagi Grimberg sock->sk->sk_data_ready = queue->data_ready; 14213f2304f8SSagi Grimberg sock->sk->sk_state_change = queue->state_change; 14223f2304f8SSagi Grimberg sock->sk->sk_write_space = queue->write_space; 14233f2304f8SSagi Grimberg write_unlock_bh(&sock->sk->sk_callback_lock); 14243f2304f8SSagi Grimberg } 14253f2304f8SSagi Grimberg 14263f2304f8SSagi Grimberg static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 14273f2304f8SSagi Grimberg { 14283f2304f8SSagi Grimberg kernel_sock_shutdown(queue->sock, SHUT_RDWR); 14293f2304f8SSagi Grimberg nvme_tcp_restore_sock_calls(queue); 14303f2304f8SSagi Grimberg cancel_work_sync(&queue->io_work); 14313f2304f8SSagi Grimberg } 14323f2304f8SSagi Grimberg 14333f2304f8SSagi Grimberg static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 14343f2304f8SSagi Grimberg { 14353f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 14363f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 14373f2304f8SSagi Grimberg 14383f2304f8SSagi Grimberg if (!test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 14393f2304f8SSagi Grimberg return; 14403f2304f8SSagi Grimberg 14413f2304f8SSagi Grimberg __nvme_tcp_stop_queue(queue); 14423f2304f8SSagi Grimberg } 14433f2304f8SSagi Grimberg 14443f2304f8SSagi Grimberg static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 14453f2304f8SSagi Grimberg { 14463f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 14473f2304f8SSagi Grimberg int ret; 14483f2304f8SSagi Grimberg 14493f2304f8SSagi Grimberg if (idx) 145026c68227SSagi Grimberg ret = nvmf_connect_io_queue(nctrl, idx, false); 14513f2304f8SSagi Grimberg else 14523f2304f8SSagi Grimberg ret = nvmf_connect_admin_queue(nctrl); 14533f2304f8SSagi Grimberg 14543f2304f8SSagi Grimberg if (!ret) { 14553f2304f8SSagi Grimberg set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags); 14563f2304f8SSagi Grimberg } else { 1457f34e2589SSagi Grimberg if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags)) 14583f2304f8SSagi Grimberg __nvme_tcp_stop_queue(&ctrl->queues[idx]); 14593f2304f8SSagi Grimberg dev_err(nctrl->device, 14603f2304f8SSagi Grimberg "failed to connect queue: %d ret=%d\n", idx, ret); 14613f2304f8SSagi Grimberg } 14623f2304f8SSagi Grimberg return ret; 14633f2304f8SSagi Grimberg } 14643f2304f8SSagi Grimberg 14653f2304f8SSagi Grimberg static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl, 14663f2304f8SSagi Grimberg bool admin) 14673f2304f8SSagi Grimberg { 14683f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 14693f2304f8SSagi Grimberg struct blk_mq_tag_set *set; 14703f2304f8SSagi Grimberg int ret; 14713f2304f8SSagi Grimberg 14723f2304f8SSagi Grimberg if (admin) { 14733f2304f8SSagi Grimberg set = &ctrl->admin_tag_set; 14743f2304f8SSagi Grimberg memset(set, 0, sizeof(*set)); 14753f2304f8SSagi Grimberg set->ops = &nvme_tcp_admin_mq_ops; 14763f2304f8SSagi Grimberg set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; 14773f2304f8SSagi Grimberg set->reserved_tags = 2; /* connect + keep-alive */ 14783f2304f8SSagi Grimberg set->numa_node = NUMA_NO_NODE; 14793f2304f8SSagi Grimberg set->cmd_size = sizeof(struct nvme_tcp_request); 14803f2304f8SSagi Grimberg set->driver_data = ctrl; 14813f2304f8SSagi Grimberg set->nr_hw_queues = 1; 14823f2304f8SSagi Grimberg set->timeout = ADMIN_TIMEOUT; 14833f2304f8SSagi Grimberg } else { 14843f2304f8SSagi Grimberg set = &ctrl->tag_set; 14853f2304f8SSagi Grimberg memset(set, 0, sizeof(*set)); 14863f2304f8SSagi Grimberg set->ops = &nvme_tcp_mq_ops; 14873f2304f8SSagi Grimberg set->queue_depth = nctrl->sqsize + 1; 14883f2304f8SSagi Grimberg set->reserved_tags = 1; /* fabric connect */ 14893f2304f8SSagi Grimberg set->numa_node = NUMA_NO_NODE; 14903f2304f8SSagi Grimberg set->flags = BLK_MQ_F_SHOULD_MERGE; 14913f2304f8SSagi Grimberg set->cmd_size = sizeof(struct nvme_tcp_request); 14923f2304f8SSagi Grimberg set->driver_data = ctrl; 14933f2304f8SSagi Grimberg set->nr_hw_queues = nctrl->queue_count - 1; 14943f2304f8SSagi Grimberg set->timeout = NVME_IO_TIMEOUT; 14951a9460ceSSagi Grimberg set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; 14963f2304f8SSagi Grimberg } 14973f2304f8SSagi Grimberg 14983f2304f8SSagi Grimberg ret = blk_mq_alloc_tag_set(set); 14993f2304f8SSagi Grimberg if (ret) 15003f2304f8SSagi Grimberg return ERR_PTR(ret); 15013f2304f8SSagi Grimberg 15023f2304f8SSagi Grimberg return set; 15033f2304f8SSagi Grimberg } 15043f2304f8SSagi Grimberg 15053f2304f8SSagi Grimberg static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 15063f2304f8SSagi Grimberg { 15073f2304f8SSagi Grimberg if (to_tcp_ctrl(ctrl)->async_req.pdu) { 15083f2304f8SSagi Grimberg nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 15093f2304f8SSagi Grimberg to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 15103f2304f8SSagi Grimberg } 15113f2304f8SSagi Grimberg 15123f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, 0); 15133f2304f8SSagi Grimberg } 15143f2304f8SSagi Grimberg 15153f2304f8SSagi Grimberg static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 15163f2304f8SSagi Grimberg { 15173f2304f8SSagi Grimberg int i; 15183f2304f8SSagi Grimberg 15193f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) 15203f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, i); 15213f2304f8SSagi Grimberg } 15223f2304f8SSagi Grimberg 15233f2304f8SSagi Grimberg static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 15243f2304f8SSagi Grimberg { 15253f2304f8SSagi Grimberg int i; 15263f2304f8SSagi Grimberg 15273f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) 15283f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, i); 15293f2304f8SSagi Grimberg } 15303f2304f8SSagi Grimberg 15313f2304f8SSagi Grimberg static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) 15323f2304f8SSagi Grimberg { 15333f2304f8SSagi Grimberg int i, ret = 0; 15343f2304f8SSagi Grimberg 15353f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) { 15363f2304f8SSagi Grimberg ret = nvme_tcp_start_queue(ctrl, i); 15373f2304f8SSagi Grimberg if (ret) 15383f2304f8SSagi Grimberg goto out_stop_queues; 15393f2304f8SSagi Grimberg } 15403f2304f8SSagi Grimberg 15413f2304f8SSagi Grimberg return 0; 15423f2304f8SSagi Grimberg 15433f2304f8SSagi Grimberg out_stop_queues: 15443f2304f8SSagi Grimberg for (i--; i >= 1; i--) 15453f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, i); 15463f2304f8SSagi Grimberg return ret; 15473f2304f8SSagi Grimberg } 15483f2304f8SSagi Grimberg 15493f2304f8SSagi Grimberg static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 15503f2304f8SSagi Grimberg { 15513f2304f8SSagi Grimberg int ret; 15523f2304f8SSagi Grimberg 15533f2304f8SSagi Grimberg ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); 15543f2304f8SSagi Grimberg if (ret) 15553f2304f8SSagi Grimberg return ret; 15563f2304f8SSagi Grimberg 15573f2304f8SSagi Grimberg ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 15583f2304f8SSagi Grimberg if (ret) 15593f2304f8SSagi Grimberg goto out_free_queue; 15603f2304f8SSagi Grimberg 15613f2304f8SSagi Grimberg return 0; 15623f2304f8SSagi Grimberg 15633f2304f8SSagi Grimberg out_free_queue: 15643f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, 0); 15653f2304f8SSagi Grimberg return ret; 15663f2304f8SSagi Grimberg } 15673f2304f8SSagi Grimberg 1568efb973b1SSagi Grimberg static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 15693f2304f8SSagi Grimberg { 15703f2304f8SSagi Grimberg int i, ret; 15713f2304f8SSagi Grimberg 15723f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) { 15733f2304f8SSagi Grimberg ret = nvme_tcp_alloc_queue(ctrl, i, 15743f2304f8SSagi Grimberg ctrl->sqsize + 1); 15753f2304f8SSagi Grimberg if (ret) 15763f2304f8SSagi Grimberg goto out_free_queues; 15773f2304f8SSagi Grimberg } 15783f2304f8SSagi Grimberg 15793f2304f8SSagi Grimberg return 0; 15803f2304f8SSagi Grimberg 15813f2304f8SSagi Grimberg out_free_queues: 15823f2304f8SSagi Grimberg for (i--; i >= 1; i--) 15833f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, i); 15843f2304f8SSagi Grimberg 15853f2304f8SSagi Grimberg return ret; 15863f2304f8SSagi Grimberg } 15873f2304f8SSagi Grimberg 15883f2304f8SSagi Grimberg static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) 15893f2304f8SSagi Grimberg { 1590873946f4SSagi Grimberg unsigned int nr_io_queues; 1591873946f4SSagi Grimberg 1592873946f4SSagi Grimberg nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); 1593873946f4SSagi Grimberg nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); 15941a9460ceSSagi Grimberg nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); 1595873946f4SSagi Grimberg 1596873946f4SSagi Grimberg return nr_io_queues; 15973f2304f8SSagi Grimberg } 15983f2304f8SSagi Grimberg 159964861993SSagi Grimberg static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl, 160064861993SSagi Grimberg unsigned int nr_io_queues) 160164861993SSagi Grimberg { 160264861993SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 160364861993SSagi Grimberg struct nvmf_ctrl_options *opts = nctrl->opts; 160464861993SSagi Grimberg 160564861993SSagi Grimberg if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { 160664861993SSagi Grimberg /* 160764861993SSagi Grimberg * separate read/write queues 160864861993SSagi Grimberg * hand out dedicated default queues only after we have 160964861993SSagi Grimberg * sufficient read queues. 161064861993SSagi Grimberg */ 161164861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; 161264861993SSagi Grimberg nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; 161364861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT] = 161464861993SSagi Grimberg min(opts->nr_write_queues, nr_io_queues); 161564861993SSagi Grimberg nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 161664861993SSagi Grimberg } else { 161764861993SSagi Grimberg /* 161864861993SSagi Grimberg * shared read/write queues 161964861993SSagi Grimberg * either no write queues were requested, or we don't have 162064861993SSagi Grimberg * sufficient queue count to have dedicated default queues. 162164861993SSagi Grimberg */ 162264861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT] = 162364861993SSagi Grimberg min(opts->nr_io_queues, nr_io_queues); 162464861993SSagi Grimberg nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 162564861993SSagi Grimberg } 16261a9460ceSSagi Grimberg 16271a9460ceSSagi Grimberg if (opts->nr_poll_queues && nr_io_queues) { 16281a9460ceSSagi Grimberg /* map dedicated poll queues only if we have queues left */ 16291a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL] = 16301a9460ceSSagi Grimberg min(opts->nr_poll_queues, nr_io_queues); 16311a9460ceSSagi Grimberg } 163264861993SSagi Grimberg } 163364861993SSagi Grimberg 1634efb973b1SSagi Grimberg static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 16353f2304f8SSagi Grimberg { 16363f2304f8SSagi Grimberg unsigned int nr_io_queues; 16373f2304f8SSagi Grimberg int ret; 16383f2304f8SSagi Grimberg 16393f2304f8SSagi Grimberg nr_io_queues = nvme_tcp_nr_io_queues(ctrl); 16403f2304f8SSagi Grimberg ret = nvme_set_queue_count(ctrl, &nr_io_queues); 16413f2304f8SSagi Grimberg if (ret) 16423f2304f8SSagi Grimberg return ret; 16433f2304f8SSagi Grimberg 16443f2304f8SSagi Grimberg ctrl->queue_count = nr_io_queues + 1; 16453f2304f8SSagi Grimberg if (ctrl->queue_count < 2) 16463f2304f8SSagi Grimberg return 0; 16473f2304f8SSagi Grimberg 16483f2304f8SSagi Grimberg dev_info(ctrl->device, 16493f2304f8SSagi Grimberg "creating %d I/O queues.\n", nr_io_queues); 16503f2304f8SSagi Grimberg 165164861993SSagi Grimberg nvme_tcp_set_io_queues(ctrl, nr_io_queues); 165264861993SSagi Grimberg 1653efb973b1SSagi Grimberg return __nvme_tcp_alloc_io_queues(ctrl); 16543f2304f8SSagi Grimberg } 16553f2304f8SSagi Grimberg 16563f2304f8SSagi Grimberg static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 16573f2304f8SSagi Grimberg { 16583f2304f8SSagi Grimberg nvme_tcp_stop_io_queues(ctrl); 16593f2304f8SSagi Grimberg if (remove) { 16603f2304f8SSagi Grimberg blk_cleanup_queue(ctrl->connect_q); 16613f2304f8SSagi Grimberg blk_mq_free_tag_set(ctrl->tagset); 16623f2304f8SSagi Grimberg } 16633f2304f8SSagi Grimberg nvme_tcp_free_io_queues(ctrl); 16643f2304f8SSagi Grimberg } 16653f2304f8SSagi Grimberg 16663f2304f8SSagi Grimberg static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 16673f2304f8SSagi Grimberg { 16683f2304f8SSagi Grimberg int ret; 16693f2304f8SSagi Grimberg 1670efb973b1SSagi Grimberg ret = nvme_tcp_alloc_io_queues(ctrl); 16713f2304f8SSagi Grimberg if (ret) 16723f2304f8SSagi Grimberg return ret; 16733f2304f8SSagi Grimberg 16743f2304f8SSagi Grimberg if (new) { 16753f2304f8SSagi Grimberg ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false); 16763f2304f8SSagi Grimberg if (IS_ERR(ctrl->tagset)) { 16773f2304f8SSagi Grimberg ret = PTR_ERR(ctrl->tagset); 16783f2304f8SSagi Grimberg goto out_free_io_queues; 16793f2304f8SSagi Grimberg } 16803f2304f8SSagi Grimberg 16813f2304f8SSagi Grimberg ctrl->connect_q = blk_mq_init_queue(ctrl->tagset); 16823f2304f8SSagi Grimberg if (IS_ERR(ctrl->connect_q)) { 16833f2304f8SSagi Grimberg ret = PTR_ERR(ctrl->connect_q); 16843f2304f8SSagi Grimberg goto out_free_tag_set; 16853f2304f8SSagi Grimberg } 16863f2304f8SSagi Grimberg } else { 16873f2304f8SSagi Grimberg blk_mq_update_nr_hw_queues(ctrl->tagset, 16883f2304f8SSagi Grimberg ctrl->queue_count - 1); 16893f2304f8SSagi Grimberg } 16903f2304f8SSagi Grimberg 16913f2304f8SSagi Grimberg ret = nvme_tcp_start_io_queues(ctrl); 16923f2304f8SSagi Grimberg if (ret) 16933f2304f8SSagi Grimberg goto out_cleanup_connect_q; 16943f2304f8SSagi Grimberg 16953f2304f8SSagi Grimberg return 0; 16963f2304f8SSagi Grimberg 16973f2304f8SSagi Grimberg out_cleanup_connect_q: 1698e85037a2SSagi Grimberg if (new) 16993f2304f8SSagi Grimberg blk_cleanup_queue(ctrl->connect_q); 17003f2304f8SSagi Grimberg out_free_tag_set: 17013f2304f8SSagi Grimberg if (new) 17023f2304f8SSagi Grimberg blk_mq_free_tag_set(ctrl->tagset); 17033f2304f8SSagi Grimberg out_free_io_queues: 17043f2304f8SSagi Grimberg nvme_tcp_free_io_queues(ctrl); 17053f2304f8SSagi Grimberg return ret; 17063f2304f8SSagi Grimberg } 17073f2304f8SSagi Grimberg 17083f2304f8SSagi Grimberg static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 17093f2304f8SSagi Grimberg { 17103f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 17113f2304f8SSagi Grimberg if (remove) { 17123f2304f8SSagi Grimberg blk_cleanup_queue(ctrl->admin_q); 1713e7832cb4SSagi Grimberg blk_cleanup_queue(ctrl->fabrics_q); 17143f2304f8SSagi Grimberg blk_mq_free_tag_set(ctrl->admin_tagset); 17153f2304f8SSagi Grimberg } 17163f2304f8SSagi Grimberg nvme_tcp_free_admin_queue(ctrl); 17173f2304f8SSagi Grimberg } 17183f2304f8SSagi Grimberg 17193f2304f8SSagi Grimberg static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 17203f2304f8SSagi Grimberg { 17213f2304f8SSagi Grimberg int error; 17223f2304f8SSagi Grimberg 17233f2304f8SSagi Grimberg error = nvme_tcp_alloc_admin_queue(ctrl); 17243f2304f8SSagi Grimberg if (error) 17253f2304f8SSagi Grimberg return error; 17263f2304f8SSagi Grimberg 17273f2304f8SSagi Grimberg if (new) { 17283f2304f8SSagi Grimberg ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true); 17293f2304f8SSagi Grimberg if (IS_ERR(ctrl->admin_tagset)) { 17303f2304f8SSagi Grimberg error = PTR_ERR(ctrl->admin_tagset); 17313f2304f8SSagi Grimberg goto out_free_queue; 17323f2304f8SSagi Grimberg } 17333f2304f8SSagi Grimberg 1734e7832cb4SSagi Grimberg ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset); 1735e7832cb4SSagi Grimberg if (IS_ERR(ctrl->fabrics_q)) { 1736e7832cb4SSagi Grimberg error = PTR_ERR(ctrl->fabrics_q); 1737e7832cb4SSagi Grimberg goto out_free_tagset; 1738e7832cb4SSagi Grimberg } 1739e7832cb4SSagi Grimberg 17403f2304f8SSagi Grimberg ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset); 17413f2304f8SSagi Grimberg if (IS_ERR(ctrl->admin_q)) { 17423f2304f8SSagi Grimberg error = PTR_ERR(ctrl->admin_q); 1743e7832cb4SSagi Grimberg goto out_cleanup_fabrics_q; 17443f2304f8SSagi Grimberg } 17453f2304f8SSagi Grimberg } 17463f2304f8SSagi Grimberg 17473f2304f8SSagi Grimberg error = nvme_tcp_start_queue(ctrl, 0); 17483f2304f8SSagi Grimberg if (error) 17493f2304f8SSagi Grimberg goto out_cleanup_queue; 17503f2304f8SSagi Grimberg 1751c0f2f45bSSagi Grimberg error = nvme_enable_ctrl(ctrl); 17523f2304f8SSagi Grimberg if (error) 17533f2304f8SSagi Grimberg goto out_stop_queue; 17543f2304f8SSagi Grimberg 1755e7832cb4SSagi Grimberg blk_mq_unquiesce_queue(ctrl->admin_q); 1756e7832cb4SSagi Grimberg 17573f2304f8SSagi Grimberg error = nvme_init_identify(ctrl); 17583f2304f8SSagi Grimberg if (error) 17593f2304f8SSagi Grimberg goto out_stop_queue; 17603f2304f8SSagi Grimberg 17613f2304f8SSagi Grimberg return 0; 17623f2304f8SSagi Grimberg 17633f2304f8SSagi Grimberg out_stop_queue: 17643f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 17653f2304f8SSagi Grimberg out_cleanup_queue: 17663f2304f8SSagi Grimberg if (new) 17673f2304f8SSagi Grimberg blk_cleanup_queue(ctrl->admin_q); 1768e7832cb4SSagi Grimberg out_cleanup_fabrics_q: 1769e7832cb4SSagi Grimberg if (new) 1770e7832cb4SSagi Grimberg blk_cleanup_queue(ctrl->fabrics_q); 17713f2304f8SSagi Grimberg out_free_tagset: 17723f2304f8SSagi Grimberg if (new) 17733f2304f8SSagi Grimberg blk_mq_free_tag_set(ctrl->admin_tagset); 17743f2304f8SSagi Grimberg out_free_queue: 17753f2304f8SSagi Grimberg nvme_tcp_free_admin_queue(ctrl); 17763f2304f8SSagi Grimberg return error; 17773f2304f8SSagi Grimberg } 17783f2304f8SSagi Grimberg 17793f2304f8SSagi Grimberg static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 17803f2304f8SSagi Grimberg bool remove) 17813f2304f8SSagi Grimberg { 17823f2304f8SSagi Grimberg blk_mq_quiesce_queue(ctrl->admin_q); 17833f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 1784622b8b68SMing Lei if (ctrl->admin_tagset) { 17857a425896SSagi Grimberg blk_mq_tagset_busy_iter(ctrl->admin_tagset, 17867a425896SSagi Grimberg nvme_cancel_request, ctrl); 1787622b8b68SMing Lei blk_mq_tagset_wait_completed_request(ctrl->admin_tagset); 1788622b8b68SMing Lei } 1789e7832cb4SSagi Grimberg if (remove) 17903f2304f8SSagi Grimberg blk_mq_unquiesce_queue(ctrl->admin_q); 17913f2304f8SSagi Grimberg nvme_tcp_destroy_admin_queue(ctrl, remove); 17923f2304f8SSagi Grimberg } 17933f2304f8SSagi Grimberg 17943f2304f8SSagi Grimberg static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 17953f2304f8SSagi Grimberg bool remove) 17963f2304f8SSagi Grimberg { 17973f2304f8SSagi Grimberg if (ctrl->queue_count <= 1) 17983f2304f8SSagi Grimberg return; 17993f2304f8SSagi Grimberg nvme_stop_queues(ctrl); 18003f2304f8SSagi Grimberg nvme_tcp_stop_io_queues(ctrl); 1801622b8b68SMing Lei if (ctrl->tagset) { 18027a425896SSagi Grimberg blk_mq_tagset_busy_iter(ctrl->tagset, 18037a425896SSagi Grimberg nvme_cancel_request, ctrl); 1804622b8b68SMing Lei blk_mq_tagset_wait_completed_request(ctrl->tagset); 1805622b8b68SMing Lei } 18063f2304f8SSagi Grimberg if (remove) 18073f2304f8SSagi Grimberg nvme_start_queues(ctrl); 18083f2304f8SSagi Grimberg nvme_tcp_destroy_io_queues(ctrl, remove); 18093f2304f8SSagi Grimberg } 18103f2304f8SSagi Grimberg 18113f2304f8SSagi Grimberg static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 18123f2304f8SSagi Grimberg { 18133f2304f8SSagi Grimberg /* If we are resetting/deleting then do nothing */ 18143f2304f8SSagi Grimberg if (ctrl->state != NVME_CTRL_CONNECTING) { 18153f2304f8SSagi Grimberg WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 18163f2304f8SSagi Grimberg ctrl->state == NVME_CTRL_LIVE); 18173f2304f8SSagi Grimberg return; 18183f2304f8SSagi Grimberg } 18193f2304f8SSagi Grimberg 18203f2304f8SSagi Grimberg if (nvmf_should_reconnect(ctrl)) { 18213f2304f8SSagi Grimberg dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 18223f2304f8SSagi Grimberg ctrl->opts->reconnect_delay); 18233f2304f8SSagi Grimberg queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 18243f2304f8SSagi Grimberg ctrl->opts->reconnect_delay * HZ); 18253f2304f8SSagi Grimberg } else { 18263f2304f8SSagi Grimberg dev_info(ctrl->device, "Removing controller...\n"); 18273f2304f8SSagi Grimberg nvme_delete_ctrl(ctrl); 18283f2304f8SSagi Grimberg } 18293f2304f8SSagi Grimberg } 18303f2304f8SSagi Grimberg 18313f2304f8SSagi Grimberg static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 18323f2304f8SSagi Grimberg { 18333f2304f8SSagi Grimberg struct nvmf_ctrl_options *opts = ctrl->opts; 1834312910f4SColin Ian King int ret; 18353f2304f8SSagi Grimberg 18363f2304f8SSagi Grimberg ret = nvme_tcp_configure_admin_queue(ctrl, new); 18373f2304f8SSagi Grimberg if (ret) 18383f2304f8SSagi Grimberg return ret; 18393f2304f8SSagi Grimberg 18403f2304f8SSagi Grimberg if (ctrl->icdoff) { 18413f2304f8SSagi Grimberg dev_err(ctrl->device, "icdoff is not supported!\n"); 18423f2304f8SSagi Grimberg goto destroy_admin; 18433f2304f8SSagi Grimberg } 18443f2304f8SSagi Grimberg 18453f2304f8SSagi Grimberg if (opts->queue_size > ctrl->sqsize + 1) 18463f2304f8SSagi Grimberg dev_warn(ctrl->device, 18473f2304f8SSagi Grimberg "queue_size %zu > ctrl sqsize %u, clamping down\n", 18483f2304f8SSagi Grimberg opts->queue_size, ctrl->sqsize + 1); 18493f2304f8SSagi Grimberg 18503f2304f8SSagi Grimberg if (ctrl->sqsize + 1 > ctrl->maxcmd) { 18513f2304f8SSagi Grimberg dev_warn(ctrl->device, 18523f2304f8SSagi Grimberg "sqsize %u > ctrl maxcmd %u, clamping down\n", 18533f2304f8SSagi Grimberg ctrl->sqsize + 1, ctrl->maxcmd); 18543f2304f8SSagi Grimberg ctrl->sqsize = ctrl->maxcmd - 1; 18553f2304f8SSagi Grimberg } 18563f2304f8SSagi Grimberg 18573f2304f8SSagi Grimberg if (ctrl->queue_count > 1) { 18583f2304f8SSagi Grimberg ret = nvme_tcp_configure_io_queues(ctrl, new); 18593f2304f8SSagi Grimberg if (ret) 18603f2304f8SSagi Grimberg goto destroy_admin; 18613f2304f8SSagi Grimberg } 18623f2304f8SSagi Grimberg 18633f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 18643f2304f8SSagi Grimberg /* state change failure is ok if we're in DELETING state */ 18653f2304f8SSagi Grimberg WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING); 18663f2304f8SSagi Grimberg ret = -EINVAL; 18673f2304f8SSagi Grimberg goto destroy_io; 18683f2304f8SSagi Grimberg } 18693f2304f8SSagi Grimberg 18703f2304f8SSagi Grimberg nvme_start_ctrl(ctrl); 18713f2304f8SSagi Grimberg return 0; 18723f2304f8SSagi Grimberg 18733f2304f8SSagi Grimberg destroy_io: 18743f2304f8SSagi Grimberg if (ctrl->queue_count > 1) 18753f2304f8SSagi Grimberg nvme_tcp_destroy_io_queues(ctrl, new); 18763f2304f8SSagi Grimberg destroy_admin: 18773f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 18783f2304f8SSagi Grimberg nvme_tcp_destroy_admin_queue(ctrl, new); 18793f2304f8SSagi Grimberg return ret; 18803f2304f8SSagi Grimberg } 18813f2304f8SSagi Grimberg 18823f2304f8SSagi Grimberg static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 18833f2304f8SSagi Grimberg { 18843f2304f8SSagi Grimberg struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 18853f2304f8SSagi Grimberg struct nvme_tcp_ctrl, connect_work); 18863f2304f8SSagi Grimberg struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 18873f2304f8SSagi Grimberg 18883f2304f8SSagi Grimberg ++ctrl->nr_reconnects; 18893f2304f8SSagi Grimberg 18903f2304f8SSagi Grimberg if (nvme_tcp_setup_ctrl(ctrl, false)) 18913f2304f8SSagi Grimberg goto requeue; 18923f2304f8SSagi Grimberg 189356a77d26SColin Ian King dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 18943f2304f8SSagi Grimberg ctrl->nr_reconnects); 18953f2304f8SSagi Grimberg 18963f2304f8SSagi Grimberg ctrl->nr_reconnects = 0; 18973f2304f8SSagi Grimberg 18983f2304f8SSagi Grimberg return; 18993f2304f8SSagi Grimberg 19003f2304f8SSagi Grimberg requeue: 19013f2304f8SSagi Grimberg dev_info(ctrl->device, "Failed reconnect attempt %d\n", 19023f2304f8SSagi Grimberg ctrl->nr_reconnects); 19033f2304f8SSagi Grimberg nvme_tcp_reconnect_or_remove(ctrl); 19043f2304f8SSagi Grimberg } 19053f2304f8SSagi Grimberg 19063f2304f8SSagi Grimberg static void nvme_tcp_error_recovery_work(struct work_struct *work) 19073f2304f8SSagi Grimberg { 19083f2304f8SSagi Grimberg struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 19093f2304f8SSagi Grimberg struct nvme_tcp_ctrl, err_work); 19103f2304f8SSagi Grimberg struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 19113f2304f8SSagi Grimberg 19123f2304f8SSagi Grimberg nvme_stop_keep_alive(ctrl); 19133f2304f8SSagi Grimberg nvme_tcp_teardown_io_queues(ctrl, false); 19143f2304f8SSagi Grimberg /* unquiesce to fail fast pending requests */ 19153f2304f8SSagi Grimberg nvme_start_queues(ctrl); 19163f2304f8SSagi Grimberg nvme_tcp_teardown_admin_queue(ctrl, false); 1917e7832cb4SSagi Grimberg blk_mq_unquiesce_queue(ctrl->admin_q); 19183f2304f8SSagi Grimberg 19193f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 19203f2304f8SSagi Grimberg /* state change failure is ok if we're in DELETING state */ 19213f2304f8SSagi Grimberg WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING); 19223f2304f8SSagi Grimberg return; 19233f2304f8SSagi Grimberg } 19243f2304f8SSagi Grimberg 19253f2304f8SSagi Grimberg nvme_tcp_reconnect_or_remove(ctrl); 19263f2304f8SSagi Grimberg } 19273f2304f8SSagi Grimberg 19283f2304f8SSagi Grimberg static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 19293f2304f8SSagi Grimberg { 1930794a4cb3SSagi Grimberg cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work); 1931794a4cb3SSagi Grimberg cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 1932794a4cb3SSagi Grimberg 19333f2304f8SSagi Grimberg nvme_tcp_teardown_io_queues(ctrl, shutdown); 1934e7832cb4SSagi Grimberg blk_mq_quiesce_queue(ctrl->admin_q); 19353f2304f8SSagi Grimberg if (shutdown) 19363f2304f8SSagi Grimberg nvme_shutdown_ctrl(ctrl); 19373f2304f8SSagi Grimberg else 1938b5b05048SSagi Grimberg nvme_disable_ctrl(ctrl); 19393f2304f8SSagi Grimberg nvme_tcp_teardown_admin_queue(ctrl, shutdown); 19403f2304f8SSagi Grimberg } 19413f2304f8SSagi Grimberg 19423f2304f8SSagi Grimberg static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 19433f2304f8SSagi Grimberg { 19443f2304f8SSagi Grimberg nvme_tcp_teardown_ctrl(ctrl, true); 19453f2304f8SSagi Grimberg } 19463f2304f8SSagi Grimberg 19473f2304f8SSagi Grimberg static void nvme_reset_ctrl_work(struct work_struct *work) 19483f2304f8SSagi Grimberg { 19493f2304f8SSagi Grimberg struct nvme_ctrl *ctrl = 19503f2304f8SSagi Grimberg container_of(work, struct nvme_ctrl, reset_work); 19513f2304f8SSagi Grimberg 19523f2304f8SSagi Grimberg nvme_stop_ctrl(ctrl); 19533f2304f8SSagi Grimberg nvme_tcp_teardown_ctrl(ctrl, false); 19543f2304f8SSagi Grimberg 19553f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 19563f2304f8SSagi Grimberg /* state change failure is ok if we're in DELETING state */ 19573f2304f8SSagi Grimberg WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING); 19583f2304f8SSagi Grimberg return; 19593f2304f8SSagi Grimberg } 19603f2304f8SSagi Grimberg 19613f2304f8SSagi Grimberg if (nvme_tcp_setup_ctrl(ctrl, false)) 19623f2304f8SSagi Grimberg goto out_fail; 19633f2304f8SSagi Grimberg 19643f2304f8SSagi Grimberg return; 19653f2304f8SSagi Grimberg 19663f2304f8SSagi Grimberg out_fail: 19673f2304f8SSagi Grimberg ++ctrl->nr_reconnects; 19683f2304f8SSagi Grimberg nvme_tcp_reconnect_or_remove(ctrl); 19693f2304f8SSagi Grimberg } 19703f2304f8SSagi Grimberg 19713f2304f8SSagi Grimberg static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 19723f2304f8SSagi Grimberg { 19733f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 19743f2304f8SSagi Grimberg 19753f2304f8SSagi Grimberg if (list_empty(&ctrl->list)) 19763f2304f8SSagi Grimberg goto free_ctrl; 19773f2304f8SSagi Grimberg 19783f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 19793f2304f8SSagi Grimberg list_del(&ctrl->list); 19803f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 19813f2304f8SSagi Grimberg 19823f2304f8SSagi Grimberg nvmf_free_options(nctrl->opts); 19833f2304f8SSagi Grimberg free_ctrl: 19843f2304f8SSagi Grimberg kfree(ctrl->queues); 19853f2304f8SSagi Grimberg kfree(ctrl); 19863f2304f8SSagi Grimberg } 19873f2304f8SSagi Grimberg 19883f2304f8SSagi Grimberg static void nvme_tcp_set_sg_null(struct nvme_command *c) 19893f2304f8SSagi Grimberg { 19903f2304f8SSagi Grimberg struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 19913f2304f8SSagi Grimberg 19923f2304f8SSagi Grimberg sg->addr = 0; 19933f2304f8SSagi Grimberg sg->length = 0; 19943f2304f8SSagi Grimberg sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 19953f2304f8SSagi Grimberg NVME_SGL_FMT_TRANSPORT_A; 19963f2304f8SSagi Grimberg } 19973f2304f8SSagi Grimberg 19983f2304f8SSagi Grimberg static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 19993f2304f8SSagi Grimberg struct nvme_command *c, u32 data_len) 20003f2304f8SSagi Grimberg { 20013f2304f8SSagi Grimberg struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 20023f2304f8SSagi Grimberg 20033f2304f8SSagi Grimberg sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 20043f2304f8SSagi Grimberg sg->length = cpu_to_le32(data_len); 20053f2304f8SSagi Grimberg sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 20063f2304f8SSagi Grimberg } 20073f2304f8SSagi Grimberg 20083f2304f8SSagi Grimberg static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 20093f2304f8SSagi Grimberg u32 data_len) 20103f2304f8SSagi Grimberg { 20113f2304f8SSagi Grimberg struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 20123f2304f8SSagi Grimberg 20133f2304f8SSagi Grimberg sg->addr = 0; 20143f2304f8SSagi Grimberg sg->length = cpu_to_le32(data_len); 20153f2304f8SSagi Grimberg sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 20163f2304f8SSagi Grimberg NVME_SGL_FMT_TRANSPORT_A; 20173f2304f8SSagi Grimberg } 20183f2304f8SSagi Grimberg 20193f2304f8SSagi Grimberg static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 20203f2304f8SSagi Grimberg { 20213f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 20223f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[0]; 20233f2304f8SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 20243f2304f8SSagi Grimberg struct nvme_command *cmd = &pdu->cmd; 20253f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 20263f2304f8SSagi Grimberg 20273f2304f8SSagi Grimberg memset(pdu, 0, sizeof(*pdu)); 20283f2304f8SSagi Grimberg pdu->hdr.type = nvme_tcp_cmd; 20293f2304f8SSagi Grimberg if (queue->hdr_digest) 20303f2304f8SSagi Grimberg pdu->hdr.flags |= NVME_TCP_F_HDGST; 20313f2304f8SSagi Grimberg pdu->hdr.hlen = sizeof(*pdu); 20323f2304f8SSagi Grimberg pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 20333f2304f8SSagi Grimberg 20343f2304f8SSagi Grimberg cmd->common.opcode = nvme_admin_async_event; 20353f2304f8SSagi Grimberg cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 20363f2304f8SSagi Grimberg cmd->common.flags |= NVME_CMD_SGL_METABUF; 20373f2304f8SSagi Grimberg nvme_tcp_set_sg_null(cmd); 20383f2304f8SSagi Grimberg 20393f2304f8SSagi Grimberg ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 20403f2304f8SSagi Grimberg ctrl->async_req.offset = 0; 20413f2304f8SSagi Grimberg ctrl->async_req.curr_bio = NULL; 20423f2304f8SSagi Grimberg ctrl->async_req.data_len = 0; 20433f2304f8SSagi Grimberg 20443f2304f8SSagi Grimberg nvme_tcp_queue_request(&ctrl->async_req); 20453f2304f8SSagi Grimberg } 20463f2304f8SSagi Grimberg 20473f2304f8SSagi Grimberg static enum blk_eh_timer_return 20483f2304f8SSagi Grimberg nvme_tcp_timeout(struct request *rq, bool reserved) 20493f2304f8SSagi Grimberg { 20503f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 20513f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = req->queue->ctrl; 20523f2304f8SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = req->pdu; 20533f2304f8SSagi Grimberg 205492b98e88SKeith Busch /* 205592b98e88SKeith Busch * Restart the timer if a controller reset is already scheduled. Any 205692b98e88SKeith Busch * timed out commands would be handled before entering the connecting 205792b98e88SKeith Busch * state. 205892b98e88SKeith Busch */ 205992b98e88SKeith Busch if (ctrl->ctrl.state == NVME_CTRL_RESETTING) 206092b98e88SKeith Busch return BLK_EH_RESET_TIMER; 206192b98e88SKeith Busch 206239d57757SSagi Grimberg dev_warn(ctrl->ctrl.device, 20633f2304f8SSagi Grimberg "queue %d: timeout request %#x type %d\n", 206439d57757SSagi Grimberg nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type); 20653f2304f8SSagi Grimberg 20663f2304f8SSagi Grimberg if (ctrl->ctrl.state != NVME_CTRL_LIVE) { 206739d57757SSagi Grimberg /* 206839d57757SSagi Grimberg * Teardown immediately if controller times out while starting 206939d57757SSagi Grimberg * or we are already started error recovery. all outstanding 207039d57757SSagi Grimberg * requests are completed on shutdown, so we return BLK_EH_DONE. 207139d57757SSagi Grimberg */ 207239d57757SSagi Grimberg flush_work(&ctrl->err_work); 207339d57757SSagi Grimberg nvme_tcp_teardown_io_queues(&ctrl->ctrl, false); 207439d57757SSagi Grimberg nvme_tcp_teardown_admin_queue(&ctrl->ctrl, false); 20753f2304f8SSagi Grimberg return BLK_EH_DONE; 20763f2304f8SSagi Grimberg } 20773f2304f8SSagi Grimberg 207839d57757SSagi Grimberg dev_warn(ctrl->ctrl.device, "starting error recovery\n"); 20793f2304f8SSagi Grimberg nvme_tcp_error_recovery(&ctrl->ctrl); 20803f2304f8SSagi Grimberg 20813f2304f8SSagi Grimberg return BLK_EH_RESET_TIMER; 20823f2304f8SSagi Grimberg } 20833f2304f8SSagi Grimberg 20843f2304f8SSagi Grimberg static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 20853f2304f8SSagi Grimberg struct request *rq) 20863f2304f8SSagi Grimberg { 20873f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 20883f2304f8SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = req->pdu; 20893f2304f8SSagi Grimberg struct nvme_command *c = &pdu->cmd; 20903f2304f8SSagi Grimberg 20913f2304f8SSagi Grimberg c->common.flags |= NVME_CMD_SGL_METABUF; 20923f2304f8SSagi Grimberg 20933f2304f8SSagi Grimberg if (rq_data_dir(rq) == WRITE && req->data_len && 20943f2304f8SSagi Grimberg req->data_len <= nvme_tcp_inline_data_size(queue)) 20953f2304f8SSagi Grimberg nvme_tcp_set_sg_inline(queue, c, req->data_len); 20963f2304f8SSagi Grimberg else 20973f2304f8SSagi Grimberg nvme_tcp_set_sg_host_data(c, req->data_len); 20983f2304f8SSagi Grimberg 20993f2304f8SSagi Grimberg return 0; 21003f2304f8SSagi Grimberg } 21013f2304f8SSagi Grimberg 21023f2304f8SSagi Grimberg static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 21033f2304f8SSagi Grimberg struct request *rq) 21043f2304f8SSagi Grimberg { 21053f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 21063f2304f8SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = req->pdu; 21073f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 21083f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 21093f2304f8SSagi Grimberg blk_status_t ret; 21103f2304f8SSagi Grimberg 21113f2304f8SSagi Grimberg ret = nvme_setup_cmd(ns, rq, &pdu->cmd); 21123f2304f8SSagi Grimberg if (ret) 21133f2304f8SSagi Grimberg return ret; 21143f2304f8SSagi Grimberg 21153f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_CMD_PDU; 21163f2304f8SSagi Grimberg req->offset = 0; 21173f2304f8SSagi Grimberg req->data_sent = 0; 21183f2304f8SSagi Grimberg req->pdu_len = 0; 21193f2304f8SSagi Grimberg req->pdu_sent = 0; 21203f2304f8SSagi Grimberg req->data_len = blk_rq_payload_bytes(rq); 21213f2304f8SSagi Grimberg req->curr_bio = rq->bio; 21223f2304f8SSagi Grimberg 21233f2304f8SSagi Grimberg if (rq_data_dir(rq) == WRITE && 21243f2304f8SSagi Grimberg req->data_len <= nvme_tcp_inline_data_size(queue)) 21253f2304f8SSagi Grimberg req->pdu_len = req->data_len; 21263f2304f8SSagi Grimberg else if (req->curr_bio) 21273f2304f8SSagi Grimberg nvme_tcp_init_iter(req, READ); 21283f2304f8SSagi Grimberg 21293f2304f8SSagi Grimberg pdu->hdr.type = nvme_tcp_cmd; 21303f2304f8SSagi Grimberg pdu->hdr.flags = 0; 21313f2304f8SSagi Grimberg if (queue->hdr_digest) 21323f2304f8SSagi Grimberg pdu->hdr.flags |= NVME_TCP_F_HDGST; 21333f2304f8SSagi Grimberg if (queue->data_digest && req->pdu_len) { 21343f2304f8SSagi Grimberg pdu->hdr.flags |= NVME_TCP_F_DDGST; 21353f2304f8SSagi Grimberg ddgst = nvme_tcp_ddgst_len(queue); 21363f2304f8SSagi Grimberg } 21373f2304f8SSagi Grimberg pdu->hdr.hlen = sizeof(*pdu); 21383f2304f8SSagi Grimberg pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 21393f2304f8SSagi Grimberg pdu->hdr.plen = 21403f2304f8SSagi Grimberg cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 21413f2304f8SSagi Grimberg 21423f2304f8SSagi Grimberg ret = nvme_tcp_map_data(queue, rq); 21433f2304f8SSagi Grimberg if (unlikely(ret)) { 214428a4cac4SMax Gurtovoy nvme_cleanup_cmd(rq); 21453f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 21463f2304f8SSagi Grimberg "Failed to map data (%d)\n", ret); 21473f2304f8SSagi Grimberg return ret; 21483f2304f8SSagi Grimberg } 21493f2304f8SSagi Grimberg 21503f2304f8SSagi Grimberg return 0; 21513f2304f8SSagi Grimberg } 21523f2304f8SSagi Grimberg 21533f2304f8SSagi Grimberg static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 21543f2304f8SSagi Grimberg const struct blk_mq_queue_data *bd) 21553f2304f8SSagi Grimberg { 21563f2304f8SSagi Grimberg struct nvme_ns *ns = hctx->queue->queuedata; 21573f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = hctx->driver_data; 21583f2304f8SSagi Grimberg struct request *rq = bd->rq; 21593f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 21603f2304f8SSagi Grimberg bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 21613f2304f8SSagi Grimberg blk_status_t ret; 21623f2304f8SSagi Grimberg 21633f2304f8SSagi Grimberg if (!nvmf_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 21643f2304f8SSagi Grimberg return nvmf_fail_nonready_command(&queue->ctrl->ctrl, rq); 21653f2304f8SSagi Grimberg 21663f2304f8SSagi Grimberg ret = nvme_tcp_setup_cmd_pdu(ns, rq); 21673f2304f8SSagi Grimberg if (unlikely(ret)) 21683f2304f8SSagi Grimberg return ret; 21693f2304f8SSagi Grimberg 21703f2304f8SSagi Grimberg blk_mq_start_request(rq); 21713f2304f8SSagi Grimberg 21723f2304f8SSagi Grimberg nvme_tcp_queue_request(req); 21733f2304f8SSagi Grimberg 21743f2304f8SSagi Grimberg return BLK_STS_OK; 21753f2304f8SSagi Grimberg } 21763f2304f8SSagi Grimberg 2177873946f4SSagi Grimberg static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2178873946f4SSagi Grimberg { 2179873946f4SSagi Grimberg struct nvme_tcp_ctrl *ctrl = set->driver_data; 218064861993SSagi Grimberg struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2181873946f4SSagi Grimberg 218264861993SSagi Grimberg if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { 2183873946f4SSagi Grimberg /* separate read/write queues */ 2184873946f4SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].nr_queues = 218564861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 218664861993SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 218764861993SSagi Grimberg set->map[HCTX_TYPE_READ].nr_queues = 218864861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ]; 2189873946f4SSagi Grimberg set->map[HCTX_TYPE_READ].queue_offset = 219064861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2191873946f4SSagi Grimberg } else { 219264861993SSagi Grimberg /* shared read/write queues */ 2193873946f4SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].nr_queues = 219464861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 219564861993SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 219664861993SSagi Grimberg set->map[HCTX_TYPE_READ].nr_queues = 219764861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2198873946f4SSagi Grimberg set->map[HCTX_TYPE_READ].queue_offset = 0; 2199873946f4SSagi Grimberg } 2200873946f4SSagi Grimberg blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 2201873946f4SSagi Grimberg blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 220264861993SSagi Grimberg 22031a9460ceSSagi Grimberg if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { 22041a9460ceSSagi Grimberg /* map dedicated poll queues only if we have queues left */ 22051a9460ceSSagi Grimberg set->map[HCTX_TYPE_POLL].nr_queues = 22061a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL]; 22071a9460ceSSagi Grimberg set->map[HCTX_TYPE_POLL].queue_offset = 22081a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT] + 22091a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ]; 22101a9460ceSSagi Grimberg blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 22111a9460ceSSagi Grimberg } 22121a9460ceSSagi Grimberg 221364861993SSagi Grimberg dev_info(ctrl->ctrl.device, 22141a9460ceSSagi Grimberg "mapped %d/%d/%d default/read/poll queues.\n", 221564861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT], 22161a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ], 22171a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL]); 221864861993SSagi Grimberg 2219873946f4SSagi Grimberg return 0; 2220873946f4SSagi Grimberg } 2221873946f4SSagi Grimberg 22221a9460ceSSagi Grimberg static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) 22231a9460ceSSagi Grimberg { 22241a9460ceSSagi Grimberg struct nvme_tcp_queue *queue = hctx->driver_data; 22251a9460ceSSagi Grimberg struct sock *sk = queue->sock->sk; 22261a9460ceSSagi Grimberg 22273f926af3SEric Dumazet if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 22281a9460ceSSagi Grimberg sk_busy_loop(sk, true); 22291a9460ceSSagi Grimberg nvme_tcp_try_recv(queue); 22301a9460ceSSagi Grimberg return queue->nr_cqe; 22311a9460ceSSagi Grimberg } 22321a9460ceSSagi Grimberg 22333f2304f8SSagi Grimberg static struct blk_mq_ops nvme_tcp_mq_ops = { 22343f2304f8SSagi Grimberg .queue_rq = nvme_tcp_queue_rq, 22353f2304f8SSagi Grimberg .complete = nvme_complete_rq, 22363f2304f8SSagi Grimberg .init_request = nvme_tcp_init_request, 22373f2304f8SSagi Grimberg .exit_request = nvme_tcp_exit_request, 22383f2304f8SSagi Grimberg .init_hctx = nvme_tcp_init_hctx, 22393f2304f8SSagi Grimberg .timeout = nvme_tcp_timeout, 2240873946f4SSagi Grimberg .map_queues = nvme_tcp_map_queues, 22411a9460ceSSagi Grimberg .poll = nvme_tcp_poll, 22423f2304f8SSagi Grimberg }; 22433f2304f8SSagi Grimberg 22443f2304f8SSagi Grimberg static struct blk_mq_ops nvme_tcp_admin_mq_ops = { 22453f2304f8SSagi Grimberg .queue_rq = nvme_tcp_queue_rq, 22463f2304f8SSagi Grimberg .complete = nvme_complete_rq, 22473f2304f8SSagi Grimberg .init_request = nvme_tcp_init_request, 22483f2304f8SSagi Grimberg .exit_request = nvme_tcp_exit_request, 22493f2304f8SSagi Grimberg .init_hctx = nvme_tcp_init_admin_hctx, 22503f2304f8SSagi Grimberg .timeout = nvme_tcp_timeout, 22513f2304f8SSagi Grimberg }; 22523f2304f8SSagi Grimberg 22533f2304f8SSagi Grimberg static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 22543f2304f8SSagi Grimberg .name = "tcp", 22553f2304f8SSagi Grimberg .module = THIS_MODULE, 22563f2304f8SSagi Grimberg .flags = NVME_F_FABRICS, 22573f2304f8SSagi Grimberg .reg_read32 = nvmf_reg_read32, 22583f2304f8SSagi Grimberg .reg_read64 = nvmf_reg_read64, 22593f2304f8SSagi Grimberg .reg_write32 = nvmf_reg_write32, 22603f2304f8SSagi Grimberg .free_ctrl = nvme_tcp_free_ctrl, 22613f2304f8SSagi Grimberg .submit_async_event = nvme_tcp_submit_async_event, 22623f2304f8SSagi Grimberg .delete_ctrl = nvme_tcp_delete_ctrl, 22633f2304f8SSagi Grimberg .get_address = nvmf_get_address, 22643f2304f8SSagi Grimberg }; 22653f2304f8SSagi Grimberg 22663f2304f8SSagi Grimberg static bool 22673f2304f8SSagi Grimberg nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 22683f2304f8SSagi Grimberg { 22693f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 22703f2304f8SSagi Grimberg bool found = false; 22713f2304f8SSagi Grimberg 22723f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 22733f2304f8SSagi Grimberg list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 22743f2304f8SSagi Grimberg found = nvmf_ip_options_match(&ctrl->ctrl, opts); 22753f2304f8SSagi Grimberg if (found) 22763f2304f8SSagi Grimberg break; 22773f2304f8SSagi Grimberg } 22783f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 22793f2304f8SSagi Grimberg 22803f2304f8SSagi Grimberg return found; 22813f2304f8SSagi Grimberg } 22823f2304f8SSagi Grimberg 22833f2304f8SSagi Grimberg static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 22843f2304f8SSagi Grimberg struct nvmf_ctrl_options *opts) 22853f2304f8SSagi Grimberg { 22863f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 22873f2304f8SSagi Grimberg int ret; 22883f2304f8SSagi Grimberg 22893f2304f8SSagi Grimberg ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 22903f2304f8SSagi Grimberg if (!ctrl) 22913f2304f8SSagi Grimberg return ERR_PTR(-ENOMEM); 22923f2304f8SSagi Grimberg 22933f2304f8SSagi Grimberg INIT_LIST_HEAD(&ctrl->list); 22943f2304f8SSagi Grimberg ctrl->ctrl.opts = opts; 22951a9460ceSSagi Grimberg ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 22961a9460ceSSagi Grimberg opts->nr_poll_queues + 1; 22973f2304f8SSagi Grimberg ctrl->ctrl.sqsize = opts->queue_size - 1; 22983f2304f8SSagi Grimberg ctrl->ctrl.kato = opts->kato; 22993f2304f8SSagi Grimberg 23003f2304f8SSagi Grimberg INIT_DELAYED_WORK(&ctrl->connect_work, 23013f2304f8SSagi Grimberg nvme_tcp_reconnect_ctrl_work); 23023f2304f8SSagi Grimberg INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 23033f2304f8SSagi Grimberg INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 23043f2304f8SSagi Grimberg 23053f2304f8SSagi Grimberg if (!(opts->mask & NVMF_OPT_TRSVCID)) { 23063f2304f8SSagi Grimberg opts->trsvcid = 23073f2304f8SSagi Grimberg kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 23083f2304f8SSagi Grimberg if (!opts->trsvcid) { 23093f2304f8SSagi Grimberg ret = -ENOMEM; 23103f2304f8SSagi Grimberg goto out_free_ctrl; 23113f2304f8SSagi Grimberg } 23123f2304f8SSagi Grimberg opts->mask |= NVMF_OPT_TRSVCID; 23133f2304f8SSagi Grimberg } 23143f2304f8SSagi Grimberg 23153f2304f8SSagi Grimberg ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 23163f2304f8SSagi Grimberg opts->traddr, opts->trsvcid, &ctrl->addr); 23173f2304f8SSagi Grimberg if (ret) { 23183f2304f8SSagi Grimberg pr_err("malformed address passed: %s:%s\n", 23193f2304f8SSagi Grimberg opts->traddr, opts->trsvcid); 23203f2304f8SSagi Grimberg goto out_free_ctrl; 23213f2304f8SSagi Grimberg } 23223f2304f8SSagi Grimberg 23233f2304f8SSagi Grimberg if (opts->mask & NVMF_OPT_HOST_TRADDR) { 23243f2304f8SSagi Grimberg ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 23253f2304f8SSagi Grimberg opts->host_traddr, NULL, &ctrl->src_addr); 23263f2304f8SSagi Grimberg if (ret) { 23273f2304f8SSagi Grimberg pr_err("malformed src address passed: %s\n", 23283f2304f8SSagi Grimberg opts->host_traddr); 23293f2304f8SSagi Grimberg goto out_free_ctrl; 23303f2304f8SSagi Grimberg } 23313f2304f8SSagi Grimberg } 23323f2304f8SSagi Grimberg 23333f2304f8SSagi Grimberg if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 23343f2304f8SSagi Grimberg ret = -EALREADY; 23353f2304f8SSagi Grimberg goto out_free_ctrl; 23363f2304f8SSagi Grimberg } 23373f2304f8SSagi Grimberg 2338873946f4SSagi Grimberg ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 23393f2304f8SSagi Grimberg GFP_KERNEL); 23403f2304f8SSagi Grimberg if (!ctrl->queues) { 23413f2304f8SSagi Grimberg ret = -ENOMEM; 23423f2304f8SSagi Grimberg goto out_free_ctrl; 23433f2304f8SSagi Grimberg } 23443f2304f8SSagi Grimberg 23453f2304f8SSagi Grimberg ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 23463f2304f8SSagi Grimberg if (ret) 23473f2304f8SSagi Grimberg goto out_kfree_queues; 23483f2304f8SSagi Grimberg 23493f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 23503f2304f8SSagi Grimberg WARN_ON_ONCE(1); 23513f2304f8SSagi Grimberg ret = -EINTR; 23523f2304f8SSagi Grimberg goto out_uninit_ctrl; 23533f2304f8SSagi Grimberg } 23543f2304f8SSagi Grimberg 23553f2304f8SSagi Grimberg ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 23563f2304f8SSagi Grimberg if (ret) 23573f2304f8SSagi Grimberg goto out_uninit_ctrl; 23583f2304f8SSagi Grimberg 23593f2304f8SSagi Grimberg dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 23603f2304f8SSagi Grimberg ctrl->ctrl.opts->subsysnqn, &ctrl->addr); 23613f2304f8SSagi Grimberg 23623f2304f8SSagi Grimberg nvme_get_ctrl(&ctrl->ctrl); 23633f2304f8SSagi Grimberg 23643f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 23653f2304f8SSagi Grimberg list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 23663f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 23673f2304f8SSagi Grimberg 23683f2304f8SSagi Grimberg return &ctrl->ctrl; 23693f2304f8SSagi Grimberg 23703f2304f8SSagi Grimberg out_uninit_ctrl: 23713f2304f8SSagi Grimberg nvme_uninit_ctrl(&ctrl->ctrl); 23723f2304f8SSagi Grimberg nvme_put_ctrl(&ctrl->ctrl); 23733f2304f8SSagi Grimberg if (ret > 0) 23743f2304f8SSagi Grimberg ret = -EIO; 23753f2304f8SSagi Grimberg return ERR_PTR(ret); 23763f2304f8SSagi Grimberg out_kfree_queues: 23773f2304f8SSagi Grimberg kfree(ctrl->queues); 23783f2304f8SSagi Grimberg out_free_ctrl: 23793f2304f8SSagi Grimberg kfree(ctrl); 23803f2304f8SSagi Grimberg return ERR_PTR(ret); 23813f2304f8SSagi Grimberg } 23823f2304f8SSagi Grimberg 23833f2304f8SSagi Grimberg static struct nvmf_transport_ops nvme_tcp_transport = { 23843f2304f8SSagi Grimberg .name = "tcp", 23853f2304f8SSagi Grimberg .module = THIS_MODULE, 23863f2304f8SSagi Grimberg .required_opts = NVMF_OPT_TRADDR, 23873f2304f8SSagi Grimberg .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 23883f2304f8SSagi Grimberg NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2389873946f4SSagi Grimberg NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2390bb13985dSIsrael Rukshin NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 2391bb13985dSIsrael Rukshin NVMF_OPT_TOS, 23923f2304f8SSagi Grimberg .create_ctrl = nvme_tcp_create_ctrl, 23933f2304f8SSagi Grimberg }; 23943f2304f8SSagi Grimberg 23953f2304f8SSagi Grimberg static int __init nvme_tcp_init_module(void) 23963f2304f8SSagi Grimberg { 23973f2304f8SSagi Grimberg nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 23983f2304f8SSagi Grimberg WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 23993f2304f8SSagi Grimberg if (!nvme_tcp_wq) 24003f2304f8SSagi Grimberg return -ENOMEM; 24013f2304f8SSagi Grimberg 24023f2304f8SSagi Grimberg nvmf_register_transport(&nvme_tcp_transport); 24033f2304f8SSagi Grimberg return 0; 24043f2304f8SSagi Grimberg } 24053f2304f8SSagi Grimberg 24063f2304f8SSagi Grimberg static void __exit nvme_tcp_cleanup_module(void) 24073f2304f8SSagi Grimberg { 24083f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 24093f2304f8SSagi Grimberg 24103f2304f8SSagi Grimberg nvmf_unregister_transport(&nvme_tcp_transport); 24113f2304f8SSagi Grimberg 24123f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 24133f2304f8SSagi Grimberg list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 24143f2304f8SSagi Grimberg nvme_delete_ctrl(&ctrl->ctrl); 24153f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 24163f2304f8SSagi Grimberg flush_workqueue(nvme_delete_wq); 24173f2304f8SSagi Grimberg 24183f2304f8SSagi Grimberg destroy_workqueue(nvme_tcp_wq); 24193f2304f8SSagi Grimberg } 24203f2304f8SSagi Grimberg 24213f2304f8SSagi Grimberg module_init(nvme_tcp_init_module); 24223f2304f8SSagi Grimberg module_exit(nvme_tcp_cleanup_module); 24233f2304f8SSagi Grimberg 24243f2304f8SSagi Grimberg MODULE_LICENSE("GPL v2"); 2425