13f2304f8SSagi Grimberg // SPDX-License-Identifier: GPL-2.0 23f2304f8SSagi Grimberg /* 33f2304f8SSagi Grimberg * NVMe over Fabrics TCP host. 43f2304f8SSagi Grimberg * Copyright (c) 2018 Lightbits Labs. All rights reserved. 53f2304f8SSagi Grimberg */ 63f2304f8SSagi Grimberg #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 73f2304f8SSagi Grimberg #include <linux/module.h> 83f2304f8SSagi Grimberg #include <linux/init.h> 93f2304f8SSagi Grimberg #include <linux/slab.h> 103f2304f8SSagi Grimberg #include <linux/err.h> 113f2304f8SSagi Grimberg #include <linux/nvme-tcp.h> 123f2304f8SSagi Grimberg #include <net/sock.h> 133f2304f8SSagi Grimberg #include <net/tcp.h> 143f2304f8SSagi Grimberg #include <linux/blk-mq.h> 153f2304f8SSagi Grimberg #include <crypto/hash.h> 161a9460ceSSagi Grimberg #include <net/busy_poll.h> 1740e0b090SPeilin Ye #include <trace/events/sock.h> 183f2304f8SSagi Grimberg 193f2304f8SSagi Grimberg #include "nvme.h" 203f2304f8SSagi Grimberg #include "fabrics.h" 213f2304f8SSagi Grimberg 223f2304f8SSagi Grimberg struct nvme_tcp_queue; 233f2304f8SSagi Grimberg 249912ade3SWunderlich, Mark /* Define the socket priority to use for connections were it is desirable 259912ade3SWunderlich, Mark * that the NIC consider performing optimized packet processing or filtering. 269912ade3SWunderlich, Mark * A non-zero value being sufficient to indicate general consideration of any 279912ade3SWunderlich, Mark * possible optimization. Making it a module param allows for alternative 289912ade3SWunderlich, Mark * values that may be unique for some NIC implementations. 299912ade3SWunderlich, Mark */ 309912ade3SWunderlich, Mark static int so_priority; 319912ade3SWunderlich, Mark module_param(so_priority, int, 0644); 329912ade3SWunderlich, Mark MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 339912ade3SWunderlich, Mark 34841aee4dSChris Leech #ifdef CONFIG_DEBUG_LOCK_ALLOC 35841aee4dSChris Leech /* lockdep can detect a circular dependency of the form 36841aee4dSChris Leech * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock 37841aee4dSChris Leech * because dependencies are tracked for both nvme-tcp and user contexts. Using 38841aee4dSChris Leech * a separate class prevents lockdep from conflating nvme-tcp socket use with 39841aee4dSChris Leech * user-space socket API use. 40841aee4dSChris Leech */ 41841aee4dSChris Leech static struct lock_class_key nvme_tcp_sk_key[2]; 42841aee4dSChris Leech static struct lock_class_key nvme_tcp_slock_key[2]; 43841aee4dSChris Leech 44841aee4dSChris Leech static void nvme_tcp_reclassify_socket(struct socket *sock) 45841aee4dSChris Leech { 46841aee4dSChris Leech struct sock *sk = sock->sk; 47841aee4dSChris Leech 48841aee4dSChris Leech if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 49841aee4dSChris Leech return; 50841aee4dSChris Leech 51841aee4dSChris Leech switch (sk->sk_family) { 52841aee4dSChris Leech case AF_INET: 53841aee4dSChris Leech sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", 54841aee4dSChris Leech &nvme_tcp_slock_key[0], 55841aee4dSChris Leech "sk_lock-AF_INET-NVME", 56841aee4dSChris Leech &nvme_tcp_sk_key[0]); 57841aee4dSChris Leech break; 58841aee4dSChris Leech case AF_INET6: 59841aee4dSChris Leech sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", 60841aee4dSChris Leech &nvme_tcp_slock_key[1], 61841aee4dSChris Leech "sk_lock-AF_INET6-NVME", 62841aee4dSChris Leech &nvme_tcp_sk_key[1]); 63841aee4dSChris Leech break; 64841aee4dSChris Leech default: 65841aee4dSChris Leech WARN_ON_ONCE(1); 66841aee4dSChris Leech } 67841aee4dSChris Leech } 68841aee4dSChris Leech #else 69841aee4dSChris Leech static void nvme_tcp_reclassify_socket(struct socket *sock) { } 70841aee4dSChris Leech #endif 71841aee4dSChris Leech 723f2304f8SSagi Grimberg enum nvme_tcp_send_state { 733f2304f8SSagi Grimberg NVME_TCP_SEND_CMD_PDU = 0, 743f2304f8SSagi Grimberg NVME_TCP_SEND_H2C_PDU, 753f2304f8SSagi Grimberg NVME_TCP_SEND_DATA, 763f2304f8SSagi Grimberg NVME_TCP_SEND_DDGST, 773f2304f8SSagi Grimberg }; 783f2304f8SSagi Grimberg 793f2304f8SSagi Grimberg struct nvme_tcp_request { 803f2304f8SSagi Grimberg struct nvme_request req; 813f2304f8SSagi Grimberg void *pdu; 823f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 833f2304f8SSagi Grimberg u32 data_len; 843f2304f8SSagi Grimberg u32 pdu_len; 853f2304f8SSagi Grimberg u32 pdu_sent; 86c2700d28SVarun Prakash u32 h2cdata_left; 87c2700d28SVarun Prakash u32 h2cdata_offset; 883f2304f8SSagi Grimberg u16 ttag; 891ba2e507SDaniel Wagner __le16 status; 903f2304f8SSagi Grimberg struct list_head entry; 9115ec928aSSagi Grimberg struct llist_node lentry; 92a7273d40SChristoph Hellwig __le32 ddgst; 933f2304f8SSagi Grimberg 943f2304f8SSagi Grimberg struct bio *curr_bio; 953f2304f8SSagi Grimberg struct iov_iter iter; 963f2304f8SSagi Grimberg 973f2304f8SSagi Grimberg /* send state */ 983f2304f8SSagi Grimberg size_t offset; 993f2304f8SSagi Grimberg size_t data_sent; 1003f2304f8SSagi Grimberg enum nvme_tcp_send_state state; 1013f2304f8SSagi Grimberg }; 1023f2304f8SSagi Grimberg 1033f2304f8SSagi Grimberg enum nvme_tcp_queue_flags { 1043f2304f8SSagi Grimberg NVME_TCP_Q_ALLOCATED = 0, 1053f2304f8SSagi Grimberg NVME_TCP_Q_LIVE = 1, 10672e5d757SSagi Grimberg NVME_TCP_Q_POLLING = 2, 1073f2304f8SSagi Grimberg }; 1083f2304f8SSagi Grimberg 1093f2304f8SSagi Grimberg enum nvme_tcp_recv_state { 1103f2304f8SSagi Grimberg NVME_TCP_RECV_PDU = 0, 1113f2304f8SSagi Grimberg NVME_TCP_RECV_DATA, 1123f2304f8SSagi Grimberg NVME_TCP_RECV_DDGST, 1133f2304f8SSagi Grimberg }; 1143f2304f8SSagi Grimberg 1153f2304f8SSagi Grimberg struct nvme_tcp_ctrl; 1163f2304f8SSagi Grimberg struct nvme_tcp_queue { 1173f2304f8SSagi Grimberg struct socket *sock; 1183f2304f8SSagi Grimberg struct work_struct io_work; 1193f2304f8SSagi Grimberg int io_cpu; 1203f2304f8SSagi Grimberg 1219ebbfe49SChao Leng struct mutex queue_lock; 122db5ad6b7SSagi Grimberg struct mutex send_mutex; 12315ec928aSSagi Grimberg struct llist_head req_list; 1243f2304f8SSagi Grimberg struct list_head send_list; 1253f2304f8SSagi Grimberg 1263f2304f8SSagi Grimberg /* recv state */ 1273f2304f8SSagi Grimberg void *pdu; 1283f2304f8SSagi Grimberg int pdu_remaining; 1293f2304f8SSagi Grimberg int pdu_offset; 1303f2304f8SSagi Grimberg size_t data_remaining; 1313f2304f8SSagi Grimberg size_t ddgst_remaining; 1321a9460ceSSagi Grimberg unsigned int nr_cqe; 1333f2304f8SSagi Grimberg 1343f2304f8SSagi Grimberg /* send state */ 1353f2304f8SSagi Grimberg struct nvme_tcp_request *request; 1363f2304f8SSagi Grimberg 137c2700d28SVarun Prakash u32 maxh2cdata; 1383f2304f8SSagi Grimberg size_t cmnd_capsule_len; 1393f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 1403f2304f8SSagi Grimberg unsigned long flags; 1413f2304f8SSagi Grimberg bool rd_enabled; 1423f2304f8SSagi Grimberg 1433f2304f8SSagi Grimberg bool hdr_digest; 1443f2304f8SSagi Grimberg bool data_digest; 1453f2304f8SSagi Grimberg struct ahash_request *rcv_hash; 1463f2304f8SSagi Grimberg struct ahash_request *snd_hash; 1473f2304f8SSagi Grimberg __le32 exp_ddgst; 1483f2304f8SSagi Grimberg __le32 recv_ddgst; 1493f2304f8SSagi Grimberg 1503f2304f8SSagi Grimberg struct page_frag_cache pf_cache; 1513f2304f8SSagi Grimberg 1523f2304f8SSagi Grimberg void (*state_change)(struct sock *); 1533f2304f8SSagi Grimberg void (*data_ready)(struct sock *); 1543f2304f8SSagi Grimberg void (*write_space)(struct sock *); 1553f2304f8SSagi Grimberg }; 1563f2304f8SSagi Grimberg 1573f2304f8SSagi Grimberg struct nvme_tcp_ctrl { 1583f2304f8SSagi Grimberg /* read only in the hot path */ 1593f2304f8SSagi Grimberg struct nvme_tcp_queue *queues; 1603f2304f8SSagi Grimberg struct blk_mq_tag_set tag_set; 1613f2304f8SSagi Grimberg 1623f2304f8SSagi Grimberg /* other member variables */ 1633f2304f8SSagi Grimberg struct list_head list; 1643f2304f8SSagi Grimberg struct blk_mq_tag_set admin_tag_set; 1653f2304f8SSagi Grimberg struct sockaddr_storage addr; 1663f2304f8SSagi Grimberg struct sockaddr_storage src_addr; 1673f2304f8SSagi Grimberg struct nvme_ctrl ctrl; 1683f2304f8SSagi Grimberg 1693f2304f8SSagi Grimberg struct work_struct err_work; 1703f2304f8SSagi Grimberg struct delayed_work connect_work; 1713f2304f8SSagi Grimberg struct nvme_tcp_request async_req; 17264861993SSagi Grimberg u32 io_queues[HCTX_MAX_TYPES]; 1733f2304f8SSagi Grimberg }; 1743f2304f8SSagi Grimberg 1753f2304f8SSagi Grimberg static LIST_HEAD(nvme_tcp_ctrl_list); 1763f2304f8SSagi Grimberg static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 1773f2304f8SSagi Grimberg static struct workqueue_struct *nvme_tcp_wq; 1786acbd961SRikard Falkeborn static const struct blk_mq_ops nvme_tcp_mq_ops; 1796acbd961SRikard Falkeborn static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 180db5ad6b7SSagi Grimberg static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 1813f2304f8SSagi Grimberg 1823f2304f8SSagi Grimberg static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 1833f2304f8SSagi Grimberg { 1843f2304f8SSagi Grimberg return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 1853f2304f8SSagi Grimberg } 1863f2304f8SSagi Grimberg 1873f2304f8SSagi Grimberg static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 1883f2304f8SSagi Grimberg { 1893f2304f8SSagi Grimberg return queue - queue->ctrl->queues; 1903f2304f8SSagi Grimberg } 1913f2304f8SSagi Grimberg 1923f2304f8SSagi Grimberg static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 1933f2304f8SSagi Grimberg { 1943f2304f8SSagi Grimberg u32 queue_idx = nvme_tcp_queue_id(queue); 1953f2304f8SSagi Grimberg 1963f2304f8SSagi Grimberg if (queue_idx == 0) 1973f2304f8SSagi Grimberg return queue->ctrl->admin_tag_set.tags[queue_idx]; 1983f2304f8SSagi Grimberg return queue->ctrl->tag_set.tags[queue_idx - 1]; 1993f2304f8SSagi Grimberg } 2003f2304f8SSagi Grimberg 2013f2304f8SSagi Grimberg static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 2023f2304f8SSagi Grimberg { 2033f2304f8SSagi Grimberg return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 2043f2304f8SSagi Grimberg } 2053f2304f8SSagi Grimberg 2063f2304f8SSagi Grimberg static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 2073f2304f8SSagi Grimberg { 2083f2304f8SSagi Grimberg return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 2093f2304f8SSagi Grimberg } 2103f2304f8SSagi Grimberg 211a3406352SSagi Grimberg static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req) 212a3406352SSagi Grimberg { 213a3406352SSagi Grimberg return req->pdu; 214a3406352SSagi Grimberg } 215a3406352SSagi Grimberg 216a3406352SSagi Grimberg static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req) 217a3406352SSagi Grimberg { 218a3406352SSagi Grimberg /* use the pdu space in the back for the data pdu */ 219a3406352SSagi Grimberg return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) - 220a3406352SSagi Grimberg sizeof(struct nvme_tcp_data_pdu); 221a3406352SSagi Grimberg } 222a3406352SSagi Grimberg 22353ee9e29SCaleb Sander static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req) 2243f2304f8SSagi Grimberg { 22553ee9e29SCaleb Sander if (nvme_is_fabrics(req->req.cmd)) 22653ee9e29SCaleb Sander return NVME_TCP_ADMIN_CCSZ; 22753ee9e29SCaleb Sander return req->queue->cmnd_capsule_len - sizeof(struct nvme_command); 2283f2304f8SSagi Grimberg } 2293f2304f8SSagi Grimberg 2303f2304f8SSagi Grimberg static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 2313f2304f8SSagi Grimberg { 2323f2304f8SSagi Grimberg return req == &req->queue->ctrl->async_req; 2333f2304f8SSagi Grimberg } 2343f2304f8SSagi Grimberg 2353f2304f8SSagi Grimberg static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 2363f2304f8SSagi Grimberg { 2373f2304f8SSagi Grimberg struct request *rq; 2383f2304f8SSagi Grimberg 2393f2304f8SSagi Grimberg if (unlikely(nvme_tcp_async_req(req))) 2403f2304f8SSagi Grimberg return false; /* async events don't have a request */ 2413f2304f8SSagi Grimberg 2423f2304f8SSagi Grimberg rq = blk_mq_rq_from_pdu(req); 2433f2304f8SSagi Grimberg 24425e5cb78SSagi Grimberg return rq_data_dir(rq) == WRITE && req->data_len && 24553ee9e29SCaleb Sander req->data_len <= nvme_tcp_inline_data_size(req); 2463f2304f8SSagi Grimberg } 2473f2304f8SSagi Grimberg 2483f2304f8SSagi Grimberg static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 2493f2304f8SSagi Grimberg { 2503f2304f8SSagi Grimberg return req->iter.bvec->bv_page; 2513f2304f8SSagi Grimberg } 2523f2304f8SSagi Grimberg 2533f2304f8SSagi Grimberg static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 2543f2304f8SSagi Grimberg { 2553f2304f8SSagi Grimberg return req->iter.bvec->bv_offset + req->iter.iov_offset; 2563f2304f8SSagi Grimberg } 2573f2304f8SSagi Grimberg 2583f2304f8SSagi Grimberg static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 2593f2304f8SSagi Grimberg { 260ca1ff67dSSagi Grimberg return min_t(size_t, iov_iter_single_seg_count(&req->iter), 2613f2304f8SSagi Grimberg req->pdu_len - req->pdu_sent); 2623f2304f8SSagi Grimberg } 2633f2304f8SSagi Grimberg 2643f2304f8SSagi Grimberg static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 2653f2304f8SSagi Grimberg { 2663f2304f8SSagi Grimberg return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 2673f2304f8SSagi Grimberg req->pdu_len - req->pdu_sent : 0; 2683f2304f8SSagi Grimberg } 2693f2304f8SSagi Grimberg 2703f2304f8SSagi Grimberg static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 2713f2304f8SSagi Grimberg int len) 2723f2304f8SSagi Grimberg { 2733f2304f8SSagi Grimberg return nvme_tcp_pdu_data_left(req) <= len; 2743f2304f8SSagi Grimberg } 2753f2304f8SSagi Grimberg 2763f2304f8SSagi Grimberg static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 2773f2304f8SSagi Grimberg unsigned int dir) 2783f2304f8SSagi Grimberg { 2793f2304f8SSagi Grimberg struct request *rq = blk_mq_rq_from_pdu(req); 2803f2304f8SSagi Grimberg struct bio_vec *vec; 2813f2304f8SSagi Grimberg unsigned int size; 2820dc9edafSSagi Grimberg int nr_bvec; 2833f2304f8SSagi Grimberg size_t offset; 2843f2304f8SSagi Grimberg 2853f2304f8SSagi Grimberg if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 2863f2304f8SSagi Grimberg vec = &rq->special_vec; 2870dc9edafSSagi Grimberg nr_bvec = 1; 2883f2304f8SSagi Grimberg size = blk_rq_payload_bytes(rq); 2893f2304f8SSagi Grimberg offset = 0; 2903f2304f8SSagi Grimberg } else { 2913f2304f8SSagi Grimberg struct bio *bio = req->curr_bio; 2920dc9edafSSagi Grimberg struct bvec_iter bi; 2930dc9edafSSagi Grimberg struct bio_vec bv; 2943f2304f8SSagi Grimberg 2953f2304f8SSagi Grimberg vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 2960dc9edafSSagi Grimberg nr_bvec = 0; 2970dc9edafSSagi Grimberg bio_for_each_bvec(bv, bio, bi) { 2980dc9edafSSagi Grimberg nr_bvec++; 2990dc9edafSSagi Grimberg } 3003f2304f8SSagi Grimberg size = bio->bi_iter.bi_size; 3013f2304f8SSagi Grimberg offset = bio->bi_iter.bi_bvec_done; 3023f2304f8SSagi Grimberg } 3033f2304f8SSagi Grimberg 3040dc9edafSSagi Grimberg iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); 3053f2304f8SSagi Grimberg req->iter.iov_offset = offset; 3063f2304f8SSagi Grimberg } 3073f2304f8SSagi Grimberg 3083f2304f8SSagi Grimberg static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 3093f2304f8SSagi Grimberg int len) 3103f2304f8SSagi Grimberg { 3113f2304f8SSagi Grimberg req->data_sent += len; 3123f2304f8SSagi Grimberg req->pdu_sent += len; 3133f2304f8SSagi Grimberg iov_iter_advance(&req->iter, len); 3143f2304f8SSagi Grimberg if (!iov_iter_count(&req->iter) && 3153f2304f8SSagi Grimberg req->data_sent < req->data_len) { 3163f2304f8SSagi Grimberg req->curr_bio = req->curr_bio->bi_next; 317de4eda9dSAl Viro nvme_tcp_init_iter(req, ITER_SOURCE); 3183f2304f8SSagi Grimberg } 3193f2304f8SSagi Grimberg } 3203f2304f8SSagi Grimberg 3215c11f7d9SSagi Grimberg static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 3225c11f7d9SSagi Grimberg { 3235c11f7d9SSagi Grimberg int ret; 3245c11f7d9SSagi Grimberg 3255c11f7d9SSagi Grimberg /* drain the send queue as much as we can... */ 3265c11f7d9SSagi Grimberg do { 3275c11f7d9SSagi Grimberg ret = nvme_tcp_try_send(queue); 3285c11f7d9SSagi Grimberg } while (ret > 0); 3295c11f7d9SSagi Grimberg } 3305c11f7d9SSagi Grimberg 33170f437fbSKeith Busch static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 33270f437fbSKeith Busch { 33370f437fbSKeith Busch return !list_empty(&queue->send_list) || 3343770a42bSSagi Grimberg !llist_empty(&queue->req_list); 33570f437fbSKeith Busch } 33670f437fbSKeith Busch 337db5ad6b7SSagi Grimberg static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 33886f0348aSSagi Grimberg bool sync, bool last) 3393f2304f8SSagi Grimberg { 3403f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 341db5ad6b7SSagi Grimberg bool empty; 3423f2304f8SSagi Grimberg 34315ec928aSSagi Grimberg empty = llist_add(&req->lentry, &queue->req_list) && 34415ec928aSSagi Grimberg list_empty(&queue->send_list) && !queue->request; 3453f2304f8SSagi Grimberg 346db5ad6b7SSagi Grimberg /* 347db5ad6b7SSagi Grimberg * if we're the first on the send_list and we can try to send 348db5ad6b7SSagi Grimberg * directly, otherwise queue io_work. Also, only do that if we 349db5ad6b7SSagi Grimberg * are on the same cpu, so we don't introduce contention. 350db5ad6b7SSagi Grimberg */ 351bb833370SSagi Grimberg if (queue->io_cpu == raw_smp_processor_id() && 352db5ad6b7SSagi Grimberg sync && empty && mutex_trylock(&queue->send_mutex)) { 3535c11f7d9SSagi Grimberg nvme_tcp_send_all(queue); 354db5ad6b7SSagi Grimberg mutex_unlock(&queue->send_mutex); 3553f2304f8SSagi Grimberg } 35670f437fbSKeith Busch 35770f437fbSKeith Busch if (last && nvme_tcp_queue_more(queue)) 35870f437fbSKeith Busch queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 359db5ad6b7SSagi Grimberg } 3603f2304f8SSagi Grimberg 36115ec928aSSagi Grimberg static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 36215ec928aSSagi Grimberg { 36315ec928aSSagi Grimberg struct nvme_tcp_request *req; 36415ec928aSSagi Grimberg struct llist_node *node; 36515ec928aSSagi Grimberg 36615ec928aSSagi Grimberg for (node = llist_del_all(&queue->req_list); node; node = node->next) { 36715ec928aSSagi Grimberg req = llist_entry(node, struct nvme_tcp_request, lentry); 36815ec928aSSagi Grimberg list_add(&req->entry, &queue->send_list); 36915ec928aSSagi Grimberg } 37015ec928aSSagi Grimberg } 37115ec928aSSagi Grimberg 3723f2304f8SSagi Grimberg static inline struct nvme_tcp_request * 3733f2304f8SSagi Grimberg nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 3743f2304f8SSagi Grimberg { 3753f2304f8SSagi Grimberg struct nvme_tcp_request *req; 3763f2304f8SSagi Grimberg 3773f2304f8SSagi Grimberg req = list_first_entry_or_null(&queue->send_list, 3783f2304f8SSagi Grimberg struct nvme_tcp_request, entry); 37915ec928aSSagi Grimberg if (!req) { 38015ec928aSSagi Grimberg nvme_tcp_process_req_list(queue); 38115ec928aSSagi Grimberg req = list_first_entry_or_null(&queue->send_list, 38215ec928aSSagi Grimberg struct nvme_tcp_request, entry); 38315ec928aSSagi Grimberg if (unlikely(!req)) 38415ec928aSSagi Grimberg return NULL; 38515ec928aSSagi Grimberg } 3863f2304f8SSagi Grimberg 38715ec928aSSagi Grimberg list_del(&req->entry); 3883f2304f8SSagi Grimberg return req; 3893f2304f8SSagi Grimberg } 3903f2304f8SSagi Grimberg 391a7273d40SChristoph Hellwig static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 392a7273d40SChristoph Hellwig __le32 *dgst) 3933f2304f8SSagi Grimberg { 3943f2304f8SSagi Grimberg ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 3953f2304f8SSagi Grimberg crypto_ahash_final(hash); 3963f2304f8SSagi Grimberg } 3973f2304f8SSagi Grimberg 3983f2304f8SSagi Grimberg static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 3993f2304f8SSagi Grimberg struct page *page, off_t off, size_t len) 4003f2304f8SSagi Grimberg { 4013f2304f8SSagi Grimberg struct scatterlist sg; 4023f2304f8SSagi Grimberg 4035fa9add6SNam Cao sg_init_table(&sg, 1); 4043f2304f8SSagi Grimberg sg_set_page(&sg, page, len, off); 4053f2304f8SSagi Grimberg ahash_request_set_crypt(hash, &sg, NULL, len); 4063f2304f8SSagi Grimberg crypto_ahash_update(hash); 4073f2304f8SSagi Grimberg } 4083f2304f8SSagi Grimberg 4093f2304f8SSagi Grimberg static inline void nvme_tcp_hdgst(struct ahash_request *hash, 4103f2304f8SSagi Grimberg void *pdu, size_t len) 4113f2304f8SSagi Grimberg { 4123f2304f8SSagi Grimberg struct scatterlist sg; 4133f2304f8SSagi Grimberg 4143f2304f8SSagi Grimberg sg_init_one(&sg, pdu, len); 4153f2304f8SSagi Grimberg ahash_request_set_crypt(hash, &sg, pdu + len, len); 4163f2304f8SSagi Grimberg crypto_ahash_digest(hash); 4173f2304f8SSagi Grimberg } 4183f2304f8SSagi Grimberg 4193f2304f8SSagi Grimberg static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 4203f2304f8SSagi Grimberg void *pdu, size_t pdu_len) 4213f2304f8SSagi Grimberg { 4223f2304f8SSagi Grimberg struct nvme_tcp_hdr *hdr = pdu; 4233f2304f8SSagi Grimberg __le32 recv_digest; 4243f2304f8SSagi Grimberg __le32 exp_digest; 4253f2304f8SSagi Grimberg 4263f2304f8SSagi Grimberg if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 4273f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 4283f2304f8SSagi Grimberg "queue %d: header digest flag is cleared\n", 4293f2304f8SSagi Grimberg nvme_tcp_queue_id(queue)); 4303f2304f8SSagi Grimberg return -EPROTO; 4313f2304f8SSagi Grimberg } 4323f2304f8SSagi Grimberg 4333f2304f8SSagi Grimberg recv_digest = *(__le32 *)(pdu + hdr->hlen); 4343f2304f8SSagi Grimberg nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 4353f2304f8SSagi Grimberg exp_digest = *(__le32 *)(pdu + hdr->hlen); 4363f2304f8SSagi Grimberg if (recv_digest != exp_digest) { 4373f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 4383f2304f8SSagi Grimberg "header digest error: recv %#x expected %#x\n", 4393f2304f8SSagi Grimberg le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 4403f2304f8SSagi Grimberg return -EIO; 4413f2304f8SSagi Grimberg } 4423f2304f8SSagi Grimberg 4433f2304f8SSagi Grimberg return 0; 4443f2304f8SSagi Grimberg } 4453f2304f8SSagi Grimberg 4463f2304f8SSagi Grimberg static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 4473f2304f8SSagi Grimberg { 4483f2304f8SSagi Grimberg struct nvme_tcp_hdr *hdr = pdu; 4493f2304f8SSagi Grimberg u8 digest_len = nvme_tcp_hdgst_len(queue); 4503f2304f8SSagi Grimberg u32 len; 4513f2304f8SSagi Grimberg 4523f2304f8SSagi Grimberg len = le32_to_cpu(hdr->plen) - hdr->hlen - 4533f2304f8SSagi Grimberg ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 4543f2304f8SSagi Grimberg 4553f2304f8SSagi Grimberg if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 4563f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 4573f2304f8SSagi Grimberg "queue %d: data digest flag is cleared\n", 4583f2304f8SSagi Grimberg nvme_tcp_queue_id(queue)); 4593f2304f8SSagi Grimberg return -EPROTO; 4603f2304f8SSagi Grimberg } 4613f2304f8SSagi Grimberg crypto_ahash_init(queue->rcv_hash); 4623f2304f8SSagi Grimberg 4633f2304f8SSagi Grimberg return 0; 4643f2304f8SSagi Grimberg } 4653f2304f8SSagi Grimberg 4663f2304f8SSagi Grimberg static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 4673f2304f8SSagi Grimberg struct request *rq, unsigned int hctx_idx) 4683f2304f8SSagi Grimberg { 4693f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 4703f2304f8SSagi Grimberg 4713f2304f8SSagi Grimberg page_frag_free(req->pdu); 4723f2304f8SSagi Grimberg } 4733f2304f8SSagi Grimberg 4743f2304f8SSagi Grimberg static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 4753f2304f8SSagi Grimberg struct request *rq, unsigned int hctx_idx, 4763f2304f8SSagi Grimberg unsigned int numa_node) 4773f2304f8SSagi Grimberg { 47806427ca0SChristoph Hellwig struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 4793f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 480f4b9e6c9SKeith Busch struct nvme_tcp_cmd_pdu *pdu; 4813f2304f8SSagi Grimberg int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 4823f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 4833f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 4843f2304f8SSagi Grimberg 4853f2304f8SSagi Grimberg req->pdu = page_frag_alloc(&queue->pf_cache, 4863f2304f8SSagi Grimberg sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 4873f2304f8SSagi Grimberg GFP_KERNEL | __GFP_ZERO); 4883f2304f8SSagi Grimberg if (!req->pdu) 4893f2304f8SSagi Grimberg return -ENOMEM; 4903f2304f8SSagi Grimberg 491f4b9e6c9SKeith Busch pdu = req->pdu; 4923f2304f8SSagi Grimberg req->queue = queue; 4933f2304f8SSagi Grimberg nvme_req(rq)->ctrl = &ctrl->ctrl; 494f4b9e6c9SKeith Busch nvme_req(rq)->cmd = &pdu->cmd; 4953f2304f8SSagi Grimberg 4963f2304f8SSagi Grimberg return 0; 4973f2304f8SSagi Grimberg } 4983f2304f8SSagi Grimberg 4993f2304f8SSagi Grimberg static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 5003f2304f8SSagi Grimberg unsigned int hctx_idx) 5013f2304f8SSagi Grimberg { 50206427ca0SChristoph Hellwig struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 5033f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 5043f2304f8SSagi Grimberg 5053f2304f8SSagi Grimberg hctx->driver_data = queue; 5063f2304f8SSagi Grimberg return 0; 5073f2304f8SSagi Grimberg } 5083f2304f8SSagi Grimberg 5093f2304f8SSagi Grimberg static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 5103f2304f8SSagi Grimberg unsigned int hctx_idx) 5113f2304f8SSagi Grimberg { 51206427ca0SChristoph Hellwig struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data); 5133f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[0]; 5143f2304f8SSagi Grimberg 5153f2304f8SSagi Grimberg hctx->driver_data = queue; 5163f2304f8SSagi Grimberg return 0; 5173f2304f8SSagi Grimberg } 5183f2304f8SSagi Grimberg 5193f2304f8SSagi Grimberg static enum nvme_tcp_recv_state 5203f2304f8SSagi Grimberg nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 5213f2304f8SSagi Grimberg { 5223f2304f8SSagi Grimberg return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 5233f2304f8SSagi Grimberg (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 5243f2304f8SSagi Grimberg NVME_TCP_RECV_DATA; 5253f2304f8SSagi Grimberg } 5263f2304f8SSagi Grimberg 5273f2304f8SSagi Grimberg static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 5283f2304f8SSagi Grimberg { 5293f2304f8SSagi Grimberg queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 5303f2304f8SSagi Grimberg nvme_tcp_hdgst_len(queue); 5313f2304f8SSagi Grimberg queue->pdu_offset = 0; 5323f2304f8SSagi Grimberg queue->data_remaining = -1; 5333f2304f8SSagi Grimberg queue->ddgst_remaining = 0; 5343f2304f8SSagi Grimberg } 5353f2304f8SSagi Grimberg 5363f2304f8SSagi Grimberg static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 5373f2304f8SSagi Grimberg { 5383f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 5393f2304f8SSagi Grimberg return; 5403f2304f8SSagi Grimberg 541236187c4SSagi Grimberg dev_warn(ctrl->device, "starting error recovery\n"); 54297b2512aSNigel Kirkland queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 5433f2304f8SSagi Grimberg } 5443f2304f8SSagi Grimberg 5453f2304f8SSagi Grimberg static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 5463f2304f8SSagi Grimberg struct nvme_completion *cqe) 5473f2304f8SSagi Grimberg { 5481ba2e507SDaniel Wagner struct nvme_tcp_request *req; 5493f2304f8SSagi Grimberg struct request *rq; 5503f2304f8SSagi Grimberg 551e7006de6SSagi Grimberg rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 5523f2304f8SSagi Grimberg if (!rq) { 5533f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 554e7006de6SSagi Grimberg "got bad cqe.command_id %#x on queue %d\n", 555e7006de6SSagi Grimberg cqe->command_id, nvme_tcp_queue_id(queue)); 5563f2304f8SSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 5573f2304f8SSagi Grimberg return -EINVAL; 5583f2304f8SSagi Grimberg } 5593f2304f8SSagi Grimberg 5601ba2e507SDaniel Wagner req = blk_mq_rq_to_pdu(rq); 5611ba2e507SDaniel Wagner if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) 5621ba2e507SDaniel Wagner req->status = cqe->status; 5631ba2e507SDaniel Wagner 5641ba2e507SDaniel Wagner if (!nvme_try_complete_req(rq, req->status, cqe->result)) 565ff029451SChristoph Hellwig nvme_complete_rq(rq); 5661a9460ceSSagi Grimberg queue->nr_cqe++; 5673f2304f8SSagi Grimberg 5683f2304f8SSagi Grimberg return 0; 5693f2304f8SSagi Grimberg } 5703f2304f8SSagi Grimberg 5713f2304f8SSagi Grimberg static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 5723f2304f8SSagi Grimberg struct nvme_tcp_data_pdu *pdu) 5733f2304f8SSagi Grimberg { 5743f2304f8SSagi Grimberg struct request *rq; 5753f2304f8SSagi Grimberg 576e7006de6SSagi Grimberg rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 5773f2304f8SSagi Grimberg if (!rq) { 5783f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 579e7006de6SSagi Grimberg "got bad c2hdata.command_id %#x on queue %d\n", 580e7006de6SSagi Grimberg pdu->command_id, nvme_tcp_queue_id(queue)); 5813f2304f8SSagi Grimberg return -ENOENT; 5823f2304f8SSagi Grimberg } 5833f2304f8SSagi Grimberg 5843f2304f8SSagi Grimberg if (!blk_rq_payload_bytes(rq)) { 5853f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 5863f2304f8SSagi Grimberg "queue %d tag %#x unexpected data\n", 5873f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 5883f2304f8SSagi Grimberg return -EIO; 5893f2304f8SSagi Grimberg } 5903f2304f8SSagi Grimberg 5913f2304f8SSagi Grimberg queue->data_remaining = le32_to_cpu(pdu->data_length); 5923f2304f8SSagi Grimberg 593602d674cSSagi Grimberg if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 594602d674cSSagi Grimberg unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 595602d674cSSagi Grimberg dev_err(queue->ctrl->ctrl.device, 596602d674cSSagi Grimberg "queue %d tag %#x SUCCESS set but not last PDU\n", 597602d674cSSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 598602d674cSSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 599602d674cSSagi Grimberg return -EPROTO; 600602d674cSSagi Grimberg } 601602d674cSSagi Grimberg 6023f2304f8SSagi Grimberg return 0; 6033f2304f8SSagi Grimberg } 6043f2304f8SSagi Grimberg 6053f2304f8SSagi Grimberg static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 6063f2304f8SSagi Grimberg struct nvme_tcp_rsp_pdu *pdu) 6073f2304f8SSagi Grimberg { 6083f2304f8SSagi Grimberg struct nvme_completion *cqe = &pdu->cqe; 6093f2304f8SSagi Grimberg int ret = 0; 6103f2304f8SSagi Grimberg 6113f2304f8SSagi Grimberg /* 6123f2304f8SSagi Grimberg * AEN requests are special as they don't time out and can 6133f2304f8SSagi Grimberg * survive any kind of queue freeze and often don't respond to 6143f2304f8SSagi Grimberg * aborts. We don't even bother to allocate a struct request 6153f2304f8SSagi Grimberg * for them but rather special case them here. 6163f2304f8SSagi Grimberg */ 61758a8df67SIsrael Rukshin if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 61858a8df67SIsrael Rukshin cqe->command_id))) 6193f2304f8SSagi Grimberg nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 6203f2304f8SSagi Grimberg &cqe->result); 6213f2304f8SSagi Grimberg else 6223f2304f8SSagi Grimberg ret = nvme_tcp_process_nvme_cqe(queue, cqe); 6233f2304f8SSagi Grimberg 6243f2304f8SSagi Grimberg return ret; 6253f2304f8SSagi Grimberg } 6263f2304f8SSagi Grimberg 627c2700d28SVarun Prakash static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req) 6283f2304f8SSagi Grimberg { 629a3406352SSagi Grimberg struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req); 6303f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 6313f2304f8SSagi Grimberg struct request *rq = blk_mq_rq_from_pdu(req); 632c2700d28SVarun Prakash u32 h2cdata_sent = req->pdu_len; 6333f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 6343f2304f8SSagi Grimberg u8 ddgst = nvme_tcp_ddgst_len(queue); 6353f2304f8SSagi Grimberg 6361d3ef9c3SVarun Prakash req->state = NVME_TCP_SEND_H2C_PDU; 6371d3ef9c3SVarun Prakash req->offset = 0; 638c2700d28SVarun Prakash req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata); 6393f2304f8SSagi Grimberg req->pdu_sent = 0; 640c2700d28SVarun Prakash req->h2cdata_left -= req->pdu_len; 641c2700d28SVarun Prakash req->h2cdata_offset += h2cdata_sent; 6423f2304f8SSagi Grimberg 6433f2304f8SSagi Grimberg memset(data, 0, sizeof(*data)); 6443f2304f8SSagi Grimberg data->hdr.type = nvme_tcp_h2c_data; 645c2700d28SVarun Prakash if (!req->h2cdata_left) 6463f2304f8SSagi Grimberg data->hdr.flags = NVME_TCP_F_DATA_LAST; 6473f2304f8SSagi Grimberg if (queue->hdr_digest) 6483f2304f8SSagi Grimberg data->hdr.flags |= NVME_TCP_F_HDGST; 6493f2304f8SSagi Grimberg if (queue->data_digest) 6503f2304f8SSagi Grimberg data->hdr.flags |= NVME_TCP_F_DDGST; 6513f2304f8SSagi Grimberg data->hdr.hlen = sizeof(*data); 6523f2304f8SSagi Grimberg data->hdr.pdo = data->hdr.hlen + hdgst; 6533f2304f8SSagi Grimberg data->hdr.plen = 6543f2304f8SSagi Grimberg cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 655c2700d28SVarun Prakash data->ttag = req->ttag; 656e7006de6SSagi Grimberg data->command_id = nvme_cid(rq); 657c2700d28SVarun Prakash data->data_offset = cpu_to_le32(req->h2cdata_offset); 6583f2304f8SSagi Grimberg data->data_length = cpu_to_le32(req->pdu_len); 6593f2304f8SSagi Grimberg } 6603f2304f8SSagi Grimberg 6613f2304f8SSagi Grimberg static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 6623f2304f8SSagi Grimberg struct nvme_tcp_r2t_pdu *pdu) 6633f2304f8SSagi Grimberg { 6643f2304f8SSagi Grimberg struct nvme_tcp_request *req; 6653f2304f8SSagi Grimberg struct request *rq; 6661d3ef9c3SVarun Prakash u32 r2t_length = le32_to_cpu(pdu->r2t_length); 667c2700d28SVarun Prakash u32 r2t_offset = le32_to_cpu(pdu->r2t_offset); 6683f2304f8SSagi Grimberg 669e7006de6SSagi Grimberg rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 6703f2304f8SSagi Grimberg if (!rq) { 6713f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 672e7006de6SSagi Grimberg "got bad r2t.command_id %#x on queue %d\n", 673e7006de6SSagi Grimberg pdu->command_id, nvme_tcp_queue_id(queue)); 6743f2304f8SSagi Grimberg return -ENOENT; 6753f2304f8SSagi Grimberg } 6763f2304f8SSagi Grimberg req = blk_mq_rq_to_pdu(rq); 6773f2304f8SSagi Grimberg 6781d3ef9c3SVarun Prakash if (unlikely(!r2t_length)) { 6791d3ef9c3SVarun Prakash dev_err(queue->ctrl->ctrl.device, 6801d3ef9c3SVarun Prakash "req %d r2t len is %u, probably a bug...\n", 6811d3ef9c3SVarun Prakash rq->tag, r2t_length); 6821d3ef9c3SVarun Prakash return -EPROTO; 6831d3ef9c3SVarun Prakash } 6843f2304f8SSagi Grimberg 6851d3ef9c3SVarun Prakash if (unlikely(req->data_sent + r2t_length > req->data_len)) { 6861d3ef9c3SVarun Prakash dev_err(queue->ctrl->ctrl.device, 6871d3ef9c3SVarun Prakash "req %d r2t len %u exceeded data len %u (%zu sent)\n", 6881d3ef9c3SVarun Prakash rq->tag, r2t_length, req->data_len, req->data_sent); 6891d3ef9c3SVarun Prakash return -EPROTO; 6901d3ef9c3SVarun Prakash } 6913f2304f8SSagi Grimberg 692c2700d28SVarun Prakash if (unlikely(r2t_offset < req->data_sent)) { 6931d3ef9c3SVarun Prakash dev_err(queue->ctrl->ctrl.device, 6941d3ef9c3SVarun Prakash "req %d unexpected r2t offset %u (expected %zu)\n", 695c2700d28SVarun Prakash rq->tag, r2t_offset, req->data_sent); 6961d3ef9c3SVarun Prakash return -EPROTO; 6971d3ef9c3SVarun Prakash } 6981d3ef9c3SVarun Prakash 699c2700d28SVarun Prakash req->pdu_len = 0; 700c2700d28SVarun Prakash req->h2cdata_left = r2t_length; 701c2700d28SVarun Prakash req->h2cdata_offset = r2t_offset; 702c2700d28SVarun Prakash req->ttag = pdu->ttag; 703c2700d28SVarun Prakash 704c2700d28SVarun Prakash nvme_tcp_setup_h2c_data_pdu(req); 70586f0348aSSagi Grimberg nvme_tcp_queue_request(req, false, true); 7063f2304f8SSagi Grimberg 7073f2304f8SSagi Grimberg return 0; 7083f2304f8SSagi Grimberg } 7093f2304f8SSagi Grimberg 7103f2304f8SSagi Grimberg static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 7113f2304f8SSagi Grimberg unsigned int *offset, size_t *len) 7123f2304f8SSagi Grimberg { 7133f2304f8SSagi Grimberg struct nvme_tcp_hdr *hdr; 7143f2304f8SSagi Grimberg char *pdu = queue->pdu; 7153f2304f8SSagi Grimberg size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 7163f2304f8SSagi Grimberg int ret; 7173f2304f8SSagi Grimberg 7183f2304f8SSagi Grimberg ret = skb_copy_bits(skb, *offset, 7193f2304f8SSagi Grimberg &pdu[queue->pdu_offset], rcv_len); 7203f2304f8SSagi Grimberg if (unlikely(ret)) 7213f2304f8SSagi Grimberg return ret; 7223f2304f8SSagi Grimberg 7233f2304f8SSagi Grimberg queue->pdu_remaining -= rcv_len; 7243f2304f8SSagi Grimberg queue->pdu_offset += rcv_len; 7253f2304f8SSagi Grimberg *offset += rcv_len; 7263f2304f8SSagi Grimberg *len -= rcv_len; 7273f2304f8SSagi Grimberg if (queue->pdu_remaining) 7283f2304f8SSagi Grimberg return 0; 7293f2304f8SSagi Grimberg 7303f2304f8SSagi Grimberg hdr = queue->pdu; 7313f2304f8SSagi Grimberg if (queue->hdr_digest) { 7323f2304f8SSagi Grimberg ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 7333f2304f8SSagi Grimberg if (unlikely(ret)) 7343f2304f8SSagi Grimberg return ret; 7353f2304f8SSagi Grimberg } 7363f2304f8SSagi Grimberg 7373f2304f8SSagi Grimberg 7383f2304f8SSagi Grimberg if (queue->data_digest) { 7393f2304f8SSagi Grimberg ret = nvme_tcp_check_ddgst(queue, queue->pdu); 7403f2304f8SSagi Grimberg if (unlikely(ret)) 7413f2304f8SSagi Grimberg return ret; 7423f2304f8SSagi Grimberg } 7433f2304f8SSagi Grimberg 7443f2304f8SSagi Grimberg switch (hdr->type) { 7453f2304f8SSagi Grimberg case nvme_tcp_c2h_data: 7466be18260SSagi Grimberg return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 7473f2304f8SSagi Grimberg case nvme_tcp_rsp: 7483f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 7496be18260SSagi Grimberg return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 7503f2304f8SSagi Grimberg case nvme_tcp_r2t: 7513f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 7526be18260SSagi Grimberg return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 7533f2304f8SSagi Grimberg default: 7543f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 7553f2304f8SSagi Grimberg "unsupported pdu type (%d)\n", hdr->type); 7563f2304f8SSagi Grimberg return -EINVAL; 7573f2304f8SSagi Grimberg } 7583f2304f8SSagi Grimberg } 7593f2304f8SSagi Grimberg 760988aef9eSChristoph Hellwig static inline void nvme_tcp_end_request(struct request *rq, u16 status) 761602d674cSSagi Grimberg { 762602d674cSSagi Grimberg union nvme_result res = {}; 763602d674cSSagi Grimberg 7642eb81a33SChristoph Hellwig if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 765ff029451SChristoph Hellwig nvme_complete_rq(rq); 766602d674cSSagi Grimberg } 767602d674cSSagi Grimberg 7683f2304f8SSagi Grimberg static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 7693f2304f8SSagi Grimberg unsigned int *offset, size_t *len) 7703f2304f8SSagi Grimberg { 7713f2304f8SSagi Grimberg struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 7723b01a9d0SSagi Grimberg struct request *rq = 773e7006de6SSagi Grimberg nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 7743b01a9d0SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 7753f2304f8SSagi Grimberg 7763f2304f8SSagi Grimberg while (true) { 7773f2304f8SSagi Grimberg int recv_len, ret; 7783f2304f8SSagi Grimberg 7793f2304f8SSagi Grimberg recv_len = min_t(size_t, *len, queue->data_remaining); 7803f2304f8SSagi Grimberg if (!recv_len) 7813f2304f8SSagi Grimberg break; 7823f2304f8SSagi Grimberg 7833f2304f8SSagi Grimberg if (!iov_iter_count(&req->iter)) { 7843f2304f8SSagi Grimberg req->curr_bio = req->curr_bio->bi_next; 7853f2304f8SSagi Grimberg 7863f2304f8SSagi Grimberg /* 7873f2304f8SSagi Grimberg * If we don`t have any bios it means that controller 7883f2304f8SSagi Grimberg * sent more data than we requested, hence error 7893f2304f8SSagi Grimberg */ 7903f2304f8SSagi Grimberg if (!req->curr_bio) { 7913f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 7923f2304f8SSagi Grimberg "queue %d no space in request %#x", 7933f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 7943f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 7953f2304f8SSagi Grimberg return -EIO; 7963f2304f8SSagi Grimberg } 797de4eda9dSAl Viro nvme_tcp_init_iter(req, ITER_DEST); 7983f2304f8SSagi Grimberg } 7993f2304f8SSagi Grimberg 8003f2304f8SSagi Grimberg /* we can read only from what is left in this bio */ 8013f2304f8SSagi Grimberg recv_len = min_t(size_t, recv_len, 8023f2304f8SSagi Grimberg iov_iter_count(&req->iter)); 8033f2304f8SSagi Grimberg 8043f2304f8SSagi Grimberg if (queue->data_digest) 8053f2304f8SSagi Grimberg ret = skb_copy_and_hash_datagram_iter(skb, *offset, 8063f2304f8SSagi Grimberg &req->iter, recv_len, queue->rcv_hash); 8073f2304f8SSagi Grimberg else 8083f2304f8SSagi Grimberg ret = skb_copy_datagram_iter(skb, *offset, 8093f2304f8SSagi Grimberg &req->iter, recv_len); 8103f2304f8SSagi Grimberg if (ret) { 8113f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 8123f2304f8SSagi Grimberg "queue %d failed to copy request %#x data", 8133f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), rq->tag); 8143f2304f8SSagi Grimberg return ret; 8153f2304f8SSagi Grimberg } 8163f2304f8SSagi Grimberg 8173f2304f8SSagi Grimberg *len -= recv_len; 8183f2304f8SSagi Grimberg *offset += recv_len; 8193f2304f8SSagi Grimberg queue->data_remaining -= recv_len; 8203f2304f8SSagi Grimberg } 8213f2304f8SSagi Grimberg 8223f2304f8SSagi Grimberg if (!queue->data_remaining) { 8233f2304f8SSagi Grimberg if (queue->data_digest) { 8243f2304f8SSagi Grimberg nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 8253f2304f8SSagi Grimberg queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 8263f2304f8SSagi Grimberg } else { 8271a9460ceSSagi Grimberg if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 8281ba2e507SDaniel Wagner nvme_tcp_end_request(rq, 8291ba2e507SDaniel Wagner le16_to_cpu(req->status)); 8301a9460ceSSagi Grimberg queue->nr_cqe++; 8311a9460ceSSagi Grimberg } 8323f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 8333f2304f8SSagi Grimberg } 8343f2304f8SSagi Grimberg } 8353f2304f8SSagi Grimberg 8363f2304f8SSagi Grimberg return 0; 8373f2304f8SSagi Grimberg } 8383f2304f8SSagi Grimberg 8393f2304f8SSagi Grimberg static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 8403f2304f8SSagi Grimberg struct sk_buff *skb, unsigned int *offset, size_t *len) 8413f2304f8SSagi Grimberg { 842602d674cSSagi Grimberg struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 8433f2304f8SSagi Grimberg char *ddgst = (char *)&queue->recv_ddgst; 8443f2304f8SSagi Grimberg size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 8453f2304f8SSagi Grimberg off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 8463f2304f8SSagi Grimberg int ret; 8473f2304f8SSagi Grimberg 8483f2304f8SSagi Grimberg ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 8493f2304f8SSagi Grimberg if (unlikely(ret)) 8503f2304f8SSagi Grimberg return ret; 8513f2304f8SSagi Grimberg 8523f2304f8SSagi Grimberg queue->ddgst_remaining -= recv_len; 8533f2304f8SSagi Grimberg *offset += recv_len; 8543f2304f8SSagi Grimberg *len -= recv_len; 8553f2304f8SSagi Grimberg if (queue->ddgst_remaining) 8563f2304f8SSagi Grimberg return 0; 8573f2304f8SSagi Grimberg 8583f2304f8SSagi Grimberg if (queue->recv_ddgst != queue->exp_ddgst) { 8591ba2e507SDaniel Wagner struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 8601ba2e507SDaniel Wagner pdu->command_id); 8611ba2e507SDaniel Wagner struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 8621ba2e507SDaniel Wagner 8631ba2e507SDaniel Wagner req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR); 8641ba2e507SDaniel Wagner 8653f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 8663f2304f8SSagi Grimberg "data digest error: recv %#x expected %#x\n", 8673f2304f8SSagi Grimberg le32_to_cpu(queue->recv_ddgst), 8683f2304f8SSagi Grimberg le32_to_cpu(queue->exp_ddgst)); 8693f2304f8SSagi Grimberg } 8703f2304f8SSagi Grimberg 871602d674cSSagi Grimberg if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 872e7006de6SSagi Grimberg struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 873602d674cSSagi Grimberg pdu->command_id); 8741ba2e507SDaniel Wagner struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 875602d674cSSagi Grimberg 8761ba2e507SDaniel Wagner nvme_tcp_end_request(rq, le16_to_cpu(req->status)); 8771a9460ceSSagi Grimberg queue->nr_cqe++; 878602d674cSSagi Grimberg } 879602d674cSSagi Grimberg 8803f2304f8SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 8813f2304f8SSagi Grimberg return 0; 8823f2304f8SSagi Grimberg } 8833f2304f8SSagi Grimberg 8843f2304f8SSagi Grimberg static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 8853f2304f8SSagi Grimberg unsigned int offset, size_t len) 8863f2304f8SSagi Grimberg { 8873f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = desc->arg.data; 8883f2304f8SSagi Grimberg size_t consumed = len; 8893f2304f8SSagi Grimberg int result; 8903f2304f8SSagi Grimberg 891aeacfcefSChris Leech if (unlikely(!queue->rd_enabled)) 892aeacfcefSChris Leech return -EFAULT; 893aeacfcefSChris Leech 8943f2304f8SSagi Grimberg while (len) { 8953f2304f8SSagi Grimberg switch (nvme_tcp_recv_state(queue)) { 8963f2304f8SSagi Grimberg case NVME_TCP_RECV_PDU: 8973f2304f8SSagi Grimberg result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 8983f2304f8SSagi Grimberg break; 8993f2304f8SSagi Grimberg case NVME_TCP_RECV_DATA: 9003f2304f8SSagi Grimberg result = nvme_tcp_recv_data(queue, skb, &offset, &len); 9013f2304f8SSagi Grimberg break; 9023f2304f8SSagi Grimberg case NVME_TCP_RECV_DDGST: 9033f2304f8SSagi Grimberg result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 9043f2304f8SSagi Grimberg break; 9053f2304f8SSagi Grimberg default: 9063f2304f8SSagi Grimberg result = -EFAULT; 9073f2304f8SSagi Grimberg } 9083f2304f8SSagi Grimberg if (result) { 9093f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 9103f2304f8SSagi Grimberg "receive failed: %d\n", result); 9113f2304f8SSagi Grimberg queue->rd_enabled = false; 9123f2304f8SSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 9133f2304f8SSagi Grimberg return result; 9143f2304f8SSagi Grimberg } 9153f2304f8SSagi Grimberg } 9163f2304f8SSagi Grimberg 9173f2304f8SSagi Grimberg return consumed; 9183f2304f8SSagi Grimberg } 9193f2304f8SSagi Grimberg 9203f2304f8SSagi Grimberg static void nvme_tcp_data_ready(struct sock *sk) 9213f2304f8SSagi Grimberg { 9223f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 9233f2304f8SSagi Grimberg 92440e0b090SPeilin Ye trace_sk_data_ready(sk); 92540e0b090SPeilin Ye 926386e5e6eSSagi Grimberg read_lock_bh(&sk->sk_callback_lock); 9273f2304f8SSagi Grimberg queue = sk->sk_user_data; 92872e5d757SSagi Grimberg if (likely(queue && queue->rd_enabled) && 92972e5d757SSagi Grimberg !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 9303f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 931386e5e6eSSagi Grimberg read_unlock_bh(&sk->sk_callback_lock); 9323f2304f8SSagi Grimberg } 9333f2304f8SSagi Grimberg 9343f2304f8SSagi Grimberg static void nvme_tcp_write_space(struct sock *sk) 9353f2304f8SSagi Grimberg { 9363f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 9373f2304f8SSagi Grimberg 9383f2304f8SSagi Grimberg read_lock_bh(&sk->sk_callback_lock); 9393f2304f8SSagi Grimberg queue = sk->sk_user_data; 9403f2304f8SSagi Grimberg if (likely(queue && sk_stream_is_writeable(sk))) { 9413f2304f8SSagi Grimberg clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 9423f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 9433f2304f8SSagi Grimberg } 9443f2304f8SSagi Grimberg read_unlock_bh(&sk->sk_callback_lock); 9453f2304f8SSagi Grimberg } 9463f2304f8SSagi Grimberg 9473f2304f8SSagi Grimberg static void nvme_tcp_state_change(struct sock *sk) 9483f2304f8SSagi Grimberg { 9493f2304f8SSagi Grimberg struct nvme_tcp_queue *queue; 9503f2304f8SSagi Grimberg 9518b73b45dSSagi Grimberg read_lock_bh(&sk->sk_callback_lock); 9523f2304f8SSagi Grimberg queue = sk->sk_user_data; 9533f2304f8SSagi Grimberg if (!queue) 9543f2304f8SSagi Grimberg goto done; 9553f2304f8SSagi Grimberg 9563f2304f8SSagi Grimberg switch (sk->sk_state) { 9573f2304f8SSagi Grimberg case TCP_CLOSE: 9583f2304f8SSagi Grimberg case TCP_CLOSE_WAIT: 9593f2304f8SSagi Grimberg case TCP_LAST_ACK: 9603f2304f8SSagi Grimberg case TCP_FIN_WAIT1: 9613f2304f8SSagi Grimberg case TCP_FIN_WAIT2: 9623f2304f8SSagi Grimberg nvme_tcp_error_recovery(&queue->ctrl->ctrl); 9633f2304f8SSagi Grimberg break; 9643f2304f8SSagi Grimberg default: 9653f2304f8SSagi Grimberg dev_info(queue->ctrl->ctrl.device, 9663f2304f8SSagi Grimberg "queue %d socket state %d\n", 9673f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), sk->sk_state); 9683f2304f8SSagi Grimberg } 9693f2304f8SSagi Grimberg 9703f2304f8SSagi Grimberg queue->state_change(sk); 9713f2304f8SSagi Grimberg done: 9728b73b45dSSagi Grimberg read_unlock_bh(&sk->sk_callback_lock); 9733f2304f8SSagi Grimberg } 9743f2304f8SSagi Grimberg 9753f2304f8SSagi Grimberg static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 9763f2304f8SSagi Grimberg { 9773f2304f8SSagi Grimberg queue->request = NULL; 9783f2304f8SSagi Grimberg } 9793f2304f8SSagi Grimberg 9803f2304f8SSagi Grimberg static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 9813f2304f8SSagi Grimberg { 98263573807SSagi Grimberg if (nvme_tcp_async_req(req)) { 98363573807SSagi Grimberg union nvme_result res = {}; 98463573807SSagi Grimberg 98563573807SSagi Grimberg nvme_complete_async_event(&req->queue->ctrl->ctrl, 98663573807SSagi Grimberg cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res); 98763573807SSagi Grimberg } else { 98863573807SSagi Grimberg nvme_tcp_end_request(blk_mq_rq_from_pdu(req), 98963573807SSagi Grimberg NVME_SC_HOST_PATH_ERROR); 99063573807SSagi Grimberg } 9913f2304f8SSagi Grimberg } 9923f2304f8SSagi Grimberg 9933f2304f8SSagi Grimberg static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 9943f2304f8SSagi Grimberg { 9953f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 99625e1f67eSSagi Grimberg int req_data_len = req->data_len; 997c2700d28SVarun Prakash u32 h2cdata_left = req->h2cdata_left; 9983f2304f8SSagi Grimberg 9993f2304f8SSagi Grimberg while (true) { 1000*77698878SDavid Howells struct bio_vec bvec; 1001*77698878SDavid Howells struct msghdr msg = { 1002*77698878SDavid Howells .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, 1003*77698878SDavid Howells }; 10043f2304f8SSagi Grimberg struct page *page = nvme_tcp_req_cur_page(req); 10053f2304f8SSagi Grimberg size_t offset = nvme_tcp_req_cur_offset(req); 10063f2304f8SSagi Grimberg size_t len = nvme_tcp_req_cur_length(req); 10073f2304f8SSagi Grimberg bool last = nvme_tcp_pdu_last_send(req, len); 100825e1f67eSSagi Grimberg int req_data_sent = req->data_sent; 1009*77698878SDavid Howells int ret; 10103f2304f8SSagi Grimberg 1011122e5b9fSSagi Grimberg if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 1012*77698878SDavid Howells msg.msg_flags |= MSG_EOR; 10133f2304f8SSagi Grimberg else 1014*77698878SDavid Howells msg.msg_flags |= MSG_MORE; 10153f2304f8SSagi Grimberg 1016*77698878SDavid Howells if (!sendpage_ok(page)) 1017*77698878SDavid Howells msg.msg_flags &= ~MSG_SPLICE_PAGES, 1018*77698878SDavid Howells 1019*77698878SDavid Howells bvec_set_page(&bvec, page, len, offset); 1020*77698878SDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1021*77698878SDavid Howells ret = sock_sendmsg(queue->sock, &msg); 10223f2304f8SSagi Grimberg if (ret <= 0) 10233f2304f8SSagi Grimberg return ret; 10243f2304f8SSagi Grimberg 10253f2304f8SSagi Grimberg if (queue->data_digest) 10263f2304f8SSagi Grimberg nvme_tcp_ddgst_update(queue->snd_hash, page, 10273f2304f8SSagi Grimberg offset, ret); 10283f2304f8SSagi Grimberg 1029e371af03SSagi Grimberg /* 1030e371af03SSagi Grimberg * update the request iterator except for the last payload send 1031e371af03SSagi Grimberg * in the request where we don't want to modify it as we may 1032e371af03SSagi Grimberg * compete with the RX path completing the request. 1033e371af03SSagi Grimberg */ 103425e1f67eSSagi Grimberg if (req_data_sent + ret < req_data_len) 1035e371af03SSagi Grimberg nvme_tcp_advance_req(req, ret); 1036e371af03SSagi Grimberg 1037e371af03SSagi Grimberg /* fully successful last send in current PDU */ 10383f2304f8SSagi Grimberg if (last && ret == len) { 10393f2304f8SSagi Grimberg if (queue->data_digest) { 10403f2304f8SSagi Grimberg nvme_tcp_ddgst_final(queue->snd_hash, 10413f2304f8SSagi Grimberg &req->ddgst); 10423f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_DDGST; 10433f2304f8SSagi Grimberg req->offset = 0; 10443f2304f8SSagi Grimberg } else { 1045c2700d28SVarun Prakash if (h2cdata_left) 1046c2700d28SVarun Prakash nvme_tcp_setup_h2c_data_pdu(req); 1047c2700d28SVarun Prakash else 10483f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 10493f2304f8SSagi Grimberg } 10503f2304f8SSagi Grimberg return 1; 10513f2304f8SSagi Grimberg } 10523f2304f8SSagi Grimberg } 10533f2304f8SSagi Grimberg return -EAGAIN; 10543f2304f8SSagi Grimberg } 10553f2304f8SSagi Grimberg 10563f2304f8SSagi Grimberg static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 10573f2304f8SSagi Grimberg { 10583f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 1059a3406352SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 1060*77698878SDavid Howells struct bio_vec bvec; 1061*77698878SDavid Howells struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, }; 10623f2304f8SSagi Grimberg bool inline_data = nvme_tcp_has_inline_data(req); 10633f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 10643f2304f8SSagi Grimberg int len = sizeof(*pdu) + hdgst - req->offset; 10653f2304f8SSagi Grimberg int ret; 10663f2304f8SSagi Grimberg 1067122e5b9fSSagi Grimberg if (inline_data || nvme_tcp_queue_more(queue)) 1068*77698878SDavid Howells msg.msg_flags |= MSG_MORE; 10695bb052d7SSagi Grimberg else 1070*77698878SDavid Howells msg.msg_flags |= MSG_EOR; 10715bb052d7SSagi Grimberg 10723f2304f8SSagi Grimberg if (queue->hdr_digest && !req->offset) 10733f2304f8SSagi Grimberg nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 10743f2304f8SSagi Grimberg 1075*77698878SDavid Howells bvec_set_virt(&bvec, (void *)pdu + req->offset, len); 1076*77698878SDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1077*77698878SDavid Howells ret = sock_sendmsg(queue->sock, &msg); 10783f2304f8SSagi Grimberg if (unlikely(ret <= 0)) 10793f2304f8SSagi Grimberg return ret; 10803f2304f8SSagi Grimberg 10813f2304f8SSagi Grimberg len -= ret; 10823f2304f8SSagi Grimberg if (!len) { 10833f2304f8SSagi Grimberg if (inline_data) { 10843f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_DATA; 10853f2304f8SSagi Grimberg if (queue->data_digest) 10863f2304f8SSagi Grimberg crypto_ahash_init(queue->snd_hash); 10873f2304f8SSagi Grimberg } else { 10883f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 10893f2304f8SSagi Grimberg } 10903f2304f8SSagi Grimberg return 1; 10913f2304f8SSagi Grimberg } 10923f2304f8SSagi Grimberg req->offset += ret; 10933f2304f8SSagi Grimberg 10943f2304f8SSagi Grimberg return -EAGAIN; 10953f2304f8SSagi Grimberg } 10963f2304f8SSagi Grimberg 10973f2304f8SSagi Grimberg static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 10983f2304f8SSagi Grimberg { 10993f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 1100a3406352SSagi Grimberg struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req); 1101*77698878SDavid Howells struct bio_vec bvec; 1102*77698878SDavid Howells struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, }; 11033f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 11043f2304f8SSagi Grimberg int len = sizeof(*pdu) - req->offset + hdgst; 11053f2304f8SSagi Grimberg int ret; 11063f2304f8SSagi Grimberg 11073f2304f8SSagi Grimberg if (queue->hdr_digest && !req->offset) 11083f2304f8SSagi Grimberg nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 11093f2304f8SSagi Grimberg 1110c2700d28SVarun Prakash if (!req->h2cdata_left) 1111*77698878SDavid Howells msg.msg_flags |= MSG_SPLICE_PAGES; 1112*77698878SDavid Howells 1113*77698878SDavid Howells bvec_set_virt(&bvec, (void *)pdu + req->offset, len); 1114*77698878SDavid Howells iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len); 1115*77698878SDavid Howells ret = sock_sendmsg(queue->sock, &msg); 11163f2304f8SSagi Grimberg if (unlikely(ret <= 0)) 11173f2304f8SSagi Grimberg return ret; 11183f2304f8SSagi Grimberg 11193f2304f8SSagi Grimberg len -= ret; 11203f2304f8SSagi Grimberg if (!len) { 11213f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_DATA; 11223f2304f8SSagi Grimberg if (queue->data_digest) 11233f2304f8SSagi Grimberg crypto_ahash_init(queue->snd_hash); 11243f2304f8SSagi Grimberg return 1; 11253f2304f8SSagi Grimberg } 11263f2304f8SSagi Grimberg req->offset += ret; 11273f2304f8SSagi Grimberg 11283f2304f8SSagi Grimberg return -EAGAIN; 11293f2304f8SSagi Grimberg } 11303f2304f8SSagi Grimberg 11313f2304f8SSagi Grimberg static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 11323f2304f8SSagi Grimberg { 11333f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 1134ce7723e9SVarun Prakash size_t offset = req->offset; 1135c2700d28SVarun Prakash u32 h2cdata_left = req->h2cdata_left; 11363f2304f8SSagi Grimberg int ret; 1137122e5b9fSSagi Grimberg struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 11383f2304f8SSagi Grimberg struct kvec iov = { 1139d89b9f3bSVarun Prakash .iov_base = (u8 *)&req->ddgst + req->offset, 11403f2304f8SSagi Grimberg .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 11413f2304f8SSagi Grimberg }; 11423f2304f8SSagi Grimberg 1143122e5b9fSSagi Grimberg if (nvme_tcp_queue_more(queue)) 1144122e5b9fSSagi Grimberg msg.msg_flags |= MSG_MORE; 1145122e5b9fSSagi Grimberg else 1146122e5b9fSSagi Grimberg msg.msg_flags |= MSG_EOR; 1147122e5b9fSSagi Grimberg 11483f2304f8SSagi Grimberg ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 11493f2304f8SSagi Grimberg if (unlikely(ret <= 0)) 11503f2304f8SSagi Grimberg return ret; 11513f2304f8SSagi Grimberg 1152ce7723e9SVarun Prakash if (offset + ret == NVME_TCP_DIGEST_LENGTH) { 1153c2700d28SVarun Prakash if (h2cdata_left) 1154c2700d28SVarun Prakash nvme_tcp_setup_h2c_data_pdu(req); 1155c2700d28SVarun Prakash else 11563f2304f8SSagi Grimberg nvme_tcp_done_send_req(queue); 11573f2304f8SSagi Grimberg return 1; 11583f2304f8SSagi Grimberg } 11593f2304f8SSagi Grimberg 11603f2304f8SSagi Grimberg req->offset += ret; 11613f2304f8SSagi Grimberg return -EAGAIN; 11623f2304f8SSagi Grimberg } 11633f2304f8SSagi Grimberg 11643f2304f8SSagi Grimberg static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 11653f2304f8SSagi Grimberg { 11663f2304f8SSagi Grimberg struct nvme_tcp_request *req; 116783e1226bSSagi Grimberg unsigned int noreclaim_flag; 11683f2304f8SSagi Grimberg int ret = 1; 11693f2304f8SSagi Grimberg 11703f2304f8SSagi Grimberg if (!queue->request) { 11713f2304f8SSagi Grimberg queue->request = nvme_tcp_fetch_request(queue); 11723f2304f8SSagi Grimberg if (!queue->request) 11733f2304f8SSagi Grimberg return 0; 11743f2304f8SSagi Grimberg } 11753f2304f8SSagi Grimberg req = queue->request; 11763f2304f8SSagi Grimberg 117783e1226bSSagi Grimberg noreclaim_flag = memalloc_noreclaim_save(); 11783f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_CMD_PDU) { 11793f2304f8SSagi Grimberg ret = nvme_tcp_try_send_cmd_pdu(req); 11803f2304f8SSagi Grimberg if (ret <= 0) 11813f2304f8SSagi Grimberg goto done; 11823f2304f8SSagi Grimberg if (!nvme_tcp_has_inline_data(req)) 118383e1226bSSagi Grimberg goto out; 11843f2304f8SSagi Grimberg } 11853f2304f8SSagi Grimberg 11863f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_H2C_PDU) { 11873f2304f8SSagi Grimberg ret = nvme_tcp_try_send_data_pdu(req); 11883f2304f8SSagi Grimberg if (ret <= 0) 11893f2304f8SSagi Grimberg goto done; 11903f2304f8SSagi Grimberg } 11913f2304f8SSagi Grimberg 11923f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_DATA) { 11933f2304f8SSagi Grimberg ret = nvme_tcp_try_send_data(req); 11943f2304f8SSagi Grimberg if (ret <= 0) 11953f2304f8SSagi Grimberg goto done; 11963f2304f8SSagi Grimberg } 11973f2304f8SSagi Grimberg 11983f2304f8SSagi Grimberg if (req->state == NVME_TCP_SEND_DDGST) 11993f2304f8SSagi Grimberg ret = nvme_tcp_try_send_ddgst(req); 12003f2304f8SSagi Grimberg done: 12015ff4e112SSagi Grimberg if (ret == -EAGAIN) { 12023f2304f8SSagi Grimberg ret = 0; 12035ff4e112SSagi Grimberg } else if (ret < 0) { 12045ff4e112SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 12055ff4e112SSagi Grimberg "failed to send request %d\n", ret); 12065ff4e112SSagi Grimberg nvme_tcp_fail_request(queue->request); 12075ff4e112SSagi Grimberg nvme_tcp_done_send_req(queue); 12085ff4e112SSagi Grimberg } 120983e1226bSSagi Grimberg out: 121083e1226bSSagi Grimberg memalloc_noreclaim_restore(noreclaim_flag); 12113f2304f8SSagi Grimberg return ret; 12123f2304f8SSagi Grimberg } 12133f2304f8SSagi Grimberg 12143f2304f8SSagi Grimberg static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 12153f2304f8SSagi Grimberg { 121610407ec9SPotnuri Bharat Teja struct socket *sock = queue->sock; 121710407ec9SPotnuri Bharat Teja struct sock *sk = sock->sk; 12183f2304f8SSagi Grimberg read_descriptor_t rd_desc; 12193f2304f8SSagi Grimberg int consumed; 12203f2304f8SSagi Grimberg 12213f2304f8SSagi Grimberg rd_desc.arg.data = queue; 12223f2304f8SSagi Grimberg rd_desc.count = 1; 12233f2304f8SSagi Grimberg lock_sock(sk); 12241a9460ceSSagi Grimberg queue->nr_cqe = 0; 122510407ec9SPotnuri Bharat Teja consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 12263f2304f8SSagi Grimberg release_sock(sk); 12273f2304f8SSagi Grimberg return consumed; 12283f2304f8SSagi Grimberg } 12293f2304f8SSagi Grimberg 12303f2304f8SSagi Grimberg static void nvme_tcp_io_work(struct work_struct *w) 12313f2304f8SSagi Grimberg { 12323f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = 12333f2304f8SSagi Grimberg container_of(w, struct nvme_tcp_queue, io_work); 1234ddef2957SWunderlich, Mark unsigned long deadline = jiffies + msecs_to_jiffies(1); 12353f2304f8SSagi Grimberg 12363f2304f8SSagi Grimberg do { 12373f2304f8SSagi Grimberg bool pending = false; 12383f2304f8SSagi Grimberg int result; 12393f2304f8SSagi Grimberg 1240db5ad6b7SSagi Grimberg if (mutex_trylock(&queue->send_mutex)) { 12413f2304f8SSagi Grimberg result = nvme_tcp_try_send(queue); 1242db5ad6b7SSagi Grimberg mutex_unlock(&queue->send_mutex); 12435ff4e112SSagi Grimberg if (result > 0) 12443f2304f8SSagi Grimberg pending = true; 12455ff4e112SSagi Grimberg else if (unlikely(result < 0)) 12465ff4e112SSagi Grimberg break; 124770f437fbSKeith Busch } 12483f2304f8SSagi Grimberg 12493f2304f8SSagi Grimberg result = nvme_tcp_try_recv(queue); 12503f2304f8SSagi Grimberg if (result > 0) 12513f2304f8SSagi Grimberg pending = true; 1252761ad26cSSagi Grimberg else if (unlikely(result < 0)) 125339d06079SSagi Grimberg return; 12543f2304f8SSagi Grimberg 1255160f3549SSagi Grimberg if (!pending || !queue->rd_enabled) 12563f2304f8SSagi Grimberg return; 12573f2304f8SSagi Grimberg 1258ddef2957SWunderlich, Mark } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 12593f2304f8SSagi Grimberg 12603f2304f8SSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 12613f2304f8SSagi Grimberg } 12623f2304f8SSagi Grimberg 12633f2304f8SSagi Grimberg static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 12643f2304f8SSagi Grimberg { 12653f2304f8SSagi Grimberg struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 12663f2304f8SSagi Grimberg 12673f2304f8SSagi Grimberg ahash_request_free(queue->rcv_hash); 12683f2304f8SSagi Grimberg ahash_request_free(queue->snd_hash); 12693f2304f8SSagi Grimberg crypto_free_ahash(tfm); 12703f2304f8SSagi Grimberg } 12713f2304f8SSagi Grimberg 12723f2304f8SSagi Grimberg static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 12733f2304f8SSagi Grimberg { 12743f2304f8SSagi Grimberg struct crypto_ahash *tfm; 12753f2304f8SSagi Grimberg 12763f2304f8SSagi Grimberg tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 12773f2304f8SSagi Grimberg if (IS_ERR(tfm)) 12783f2304f8SSagi Grimberg return PTR_ERR(tfm); 12793f2304f8SSagi Grimberg 12803f2304f8SSagi Grimberg queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 12813f2304f8SSagi Grimberg if (!queue->snd_hash) 12823f2304f8SSagi Grimberg goto free_tfm; 12833f2304f8SSagi Grimberg ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 12843f2304f8SSagi Grimberg 12853f2304f8SSagi Grimberg queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 12863f2304f8SSagi Grimberg if (!queue->rcv_hash) 12873f2304f8SSagi Grimberg goto free_snd_hash; 12883f2304f8SSagi Grimberg ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 12893f2304f8SSagi Grimberg 12903f2304f8SSagi Grimberg return 0; 12913f2304f8SSagi Grimberg free_snd_hash: 12923f2304f8SSagi Grimberg ahash_request_free(queue->snd_hash); 12933f2304f8SSagi Grimberg free_tfm: 12943f2304f8SSagi Grimberg crypto_free_ahash(tfm); 12953f2304f8SSagi Grimberg return -ENOMEM; 12963f2304f8SSagi Grimberg } 12973f2304f8SSagi Grimberg 12983f2304f8SSagi Grimberg static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 12993f2304f8SSagi Grimberg { 13003f2304f8SSagi Grimberg struct nvme_tcp_request *async = &ctrl->async_req; 13013f2304f8SSagi Grimberg 13023f2304f8SSagi Grimberg page_frag_free(async->pdu); 13033f2304f8SSagi Grimberg } 13043f2304f8SSagi Grimberg 13053f2304f8SSagi Grimberg static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 13063f2304f8SSagi Grimberg { 13073f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[0]; 13083f2304f8SSagi Grimberg struct nvme_tcp_request *async = &ctrl->async_req; 13093f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 13103f2304f8SSagi Grimberg 13113f2304f8SSagi Grimberg async->pdu = page_frag_alloc(&queue->pf_cache, 13123f2304f8SSagi Grimberg sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 13133f2304f8SSagi Grimberg GFP_KERNEL | __GFP_ZERO); 13143f2304f8SSagi Grimberg if (!async->pdu) 13153f2304f8SSagi Grimberg return -ENOMEM; 13163f2304f8SSagi Grimberg 13173f2304f8SSagi Grimberg async->queue = &ctrl->queues[0]; 13183f2304f8SSagi Grimberg return 0; 13193f2304f8SSagi Grimberg } 13203f2304f8SSagi Grimberg 13213f2304f8SSagi Grimberg static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 13223f2304f8SSagi Grimberg { 1323a5053c92SMaurizio Lombardi struct page *page; 13243f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 13253f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 132683e1226bSSagi Grimberg unsigned int noreclaim_flag; 13273f2304f8SSagi Grimberg 13283f2304f8SSagi Grimberg if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 13293f2304f8SSagi Grimberg return; 13303f2304f8SSagi Grimberg 13313f2304f8SSagi Grimberg if (queue->hdr_digest || queue->data_digest) 13323f2304f8SSagi Grimberg nvme_tcp_free_crypto(queue); 13333f2304f8SSagi Grimberg 1334a5053c92SMaurizio Lombardi if (queue->pf_cache.va) { 1335a5053c92SMaurizio Lombardi page = virt_to_head_page(queue->pf_cache.va); 1336a5053c92SMaurizio Lombardi __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); 1337a5053c92SMaurizio Lombardi queue->pf_cache.va = NULL; 1338a5053c92SMaurizio Lombardi } 133983e1226bSSagi Grimberg 134083e1226bSSagi Grimberg noreclaim_flag = memalloc_noreclaim_save(); 13413f2304f8SSagi Grimberg sock_release(queue->sock); 134283e1226bSSagi Grimberg memalloc_noreclaim_restore(noreclaim_flag); 134383e1226bSSagi Grimberg 13443f2304f8SSagi Grimberg kfree(queue->pdu); 1345d48f92cdSKeith Busch mutex_destroy(&queue->send_mutex); 13469ebbfe49SChao Leng mutex_destroy(&queue->queue_lock); 13473f2304f8SSagi Grimberg } 13483f2304f8SSagi Grimberg 13493f2304f8SSagi Grimberg static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 13503f2304f8SSagi Grimberg { 13513f2304f8SSagi Grimberg struct nvme_tcp_icreq_pdu *icreq; 13523f2304f8SSagi Grimberg struct nvme_tcp_icresp_pdu *icresp; 13533f2304f8SSagi Grimberg struct msghdr msg = {}; 13543f2304f8SSagi Grimberg struct kvec iov; 13553f2304f8SSagi Grimberg bool ctrl_hdgst, ctrl_ddgst; 1356c2700d28SVarun Prakash u32 maxh2cdata; 13573f2304f8SSagi Grimberg int ret; 13583f2304f8SSagi Grimberg 13593f2304f8SSagi Grimberg icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 13603f2304f8SSagi Grimberg if (!icreq) 13613f2304f8SSagi Grimberg return -ENOMEM; 13623f2304f8SSagi Grimberg 13633f2304f8SSagi Grimberg icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 13643f2304f8SSagi Grimberg if (!icresp) { 13653f2304f8SSagi Grimberg ret = -ENOMEM; 13663f2304f8SSagi Grimberg goto free_icreq; 13673f2304f8SSagi Grimberg } 13683f2304f8SSagi Grimberg 13693f2304f8SSagi Grimberg icreq->hdr.type = nvme_tcp_icreq; 13703f2304f8SSagi Grimberg icreq->hdr.hlen = sizeof(*icreq); 13713f2304f8SSagi Grimberg icreq->hdr.pdo = 0; 13723f2304f8SSagi Grimberg icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 13733f2304f8SSagi Grimberg icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 13743f2304f8SSagi Grimberg icreq->maxr2t = 0; /* single inflight r2t supported */ 13753f2304f8SSagi Grimberg icreq->hpda = 0; /* no alignment constraint */ 13763f2304f8SSagi Grimberg if (queue->hdr_digest) 13773f2304f8SSagi Grimberg icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 13783f2304f8SSagi Grimberg if (queue->data_digest) 13793f2304f8SSagi Grimberg icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 13803f2304f8SSagi Grimberg 13813f2304f8SSagi Grimberg iov.iov_base = icreq; 13823f2304f8SSagi Grimberg iov.iov_len = sizeof(*icreq); 13833f2304f8SSagi Grimberg ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 13843f2304f8SSagi Grimberg if (ret < 0) 13853f2304f8SSagi Grimberg goto free_icresp; 13863f2304f8SSagi Grimberg 13873f2304f8SSagi Grimberg memset(&msg, 0, sizeof(msg)); 13883f2304f8SSagi Grimberg iov.iov_base = icresp; 13893f2304f8SSagi Grimberg iov.iov_len = sizeof(*icresp); 13903f2304f8SSagi Grimberg ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 13913f2304f8SSagi Grimberg iov.iov_len, msg.msg_flags); 13923f2304f8SSagi Grimberg if (ret < 0) 13933f2304f8SSagi Grimberg goto free_icresp; 13943f2304f8SSagi Grimberg 13953f2304f8SSagi Grimberg ret = -EINVAL; 13963f2304f8SSagi Grimberg if (icresp->hdr.type != nvme_tcp_icresp) { 13973f2304f8SSagi Grimberg pr_err("queue %d: bad type returned %d\n", 13983f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->hdr.type); 13993f2304f8SSagi Grimberg goto free_icresp; 14003f2304f8SSagi Grimberg } 14013f2304f8SSagi Grimberg 14023f2304f8SSagi Grimberg if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 14033f2304f8SSagi Grimberg pr_err("queue %d: bad pdu length returned %d\n", 14043f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->hdr.plen); 14053f2304f8SSagi Grimberg goto free_icresp; 14063f2304f8SSagi Grimberg } 14073f2304f8SSagi Grimberg 14083f2304f8SSagi Grimberg if (icresp->pfv != NVME_TCP_PFV_1_0) { 14093f2304f8SSagi Grimberg pr_err("queue %d: bad pfv returned %d\n", 14103f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->pfv); 14113f2304f8SSagi Grimberg goto free_icresp; 14123f2304f8SSagi Grimberg } 14133f2304f8SSagi Grimberg 14143f2304f8SSagi Grimberg ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 14153f2304f8SSagi Grimberg if ((queue->data_digest && !ctrl_ddgst) || 14163f2304f8SSagi Grimberg (!queue->data_digest && ctrl_ddgst)) { 14173f2304f8SSagi Grimberg pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 14183f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), 14193f2304f8SSagi Grimberg queue->data_digest ? "enabled" : "disabled", 14203f2304f8SSagi Grimberg ctrl_ddgst ? "enabled" : "disabled"); 14213f2304f8SSagi Grimberg goto free_icresp; 14223f2304f8SSagi Grimberg } 14233f2304f8SSagi Grimberg 14243f2304f8SSagi Grimberg ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 14253f2304f8SSagi Grimberg if ((queue->hdr_digest && !ctrl_hdgst) || 14263f2304f8SSagi Grimberg (!queue->hdr_digest && ctrl_hdgst)) { 14273f2304f8SSagi Grimberg pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 14283f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), 14293f2304f8SSagi Grimberg queue->hdr_digest ? "enabled" : "disabled", 14303f2304f8SSagi Grimberg ctrl_hdgst ? "enabled" : "disabled"); 14313f2304f8SSagi Grimberg goto free_icresp; 14323f2304f8SSagi Grimberg } 14333f2304f8SSagi Grimberg 14343f2304f8SSagi Grimberg if (icresp->cpda != 0) { 14353f2304f8SSagi Grimberg pr_err("queue %d: unsupported cpda returned %d\n", 14363f2304f8SSagi Grimberg nvme_tcp_queue_id(queue), icresp->cpda); 14373f2304f8SSagi Grimberg goto free_icresp; 14383f2304f8SSagi Grimberg } 14393f2304f8SSagi Grimberg 1440c2700d28SVarun Prakash maxh2cdata = le32_to_cpu(icresp->maxdata); 1441c2700d28SVarun Prakash if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) { 1442c2700d28SVarun Prakash pr_err("queue %d: invalid maxh2cdata returned %u\n", 1443c2700d28SVarun Prakash nvme_tcp_queue_id(queue), maxh2cdata); 1444c2700d28SVarun Prakash goto free_icresp; 1445c2700d28SVarun Prakash } 1446c2700d28SVarun Prakash queue->maxh2cdata = maxh2cdata; 1447c2700d28SVarun Prakash 14483f2304f8SSagi Grimberg ret = 0; 14493f2304f8SSagi Grimberg free_icresp: 14503f2304f8SSagi Grimberg kfree(icresp); 14513f2304f8SSagi Grimberg free_icreq: 14523f2304f8SSagi Grimberg kfree(icreq); 14533f2304f8SSagi Grimberg return ret; 14543f2304f8SSagi Grimberg } 14553f2304f8SSagi Grimberg 145640510a63SSagi Grimberg static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 145740510a63SSagi Grimberg { 145840510a63SSagi Grimberg return nvme_tcp_queue_id(queue) == 0; 145940510a63SSagi Grimberg } 146040510a63SSagi Grimberg 146140510a63SSagi Grimberg static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 146240510a63SSagi Grimberg { 146340510a63SSagi Grimberg struct nvme_tcp_ctrl *ctrl = queue->ctrl; 146440510a63SSagi Grimberg int qid = nvme_tcp_queue_id(queue); 146540510a63SSagi Grimberg 146640510a63SSagi Grimberg return !nvme_tcp_admin_queue(queue) && 146740510a63SSagi Grimberg qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 146840510a63SSagi Grimberg } 146940510a63SSagi Grimberg 147040510a63SSagi Grimberg static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 147140510a63SSagi Grimberg { 147240510a63SSagi Grimberg struct nvme_tcp_ctrl *ctrl = queue->ctrl; 147340510a63SSagi Grimberg int qid = nvme_tcp_queue_id(queue); 147440510a63SSagi Grimberg 147540510a63SSagi Grimberg return !nvme_tcp_admin_queue(queue) && 147640510a63SSagi Grimberg !nvme_tcp_default_queue(queue) && 147740510a63SSagi Grimberg qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 147840510a63SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ]; 147940510a63SSagi Grimberg } 148040510a63SSagi Grimberg 148140510a63SSagi Grimberg static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 148240510a63SSagi Grimberg { 148340510a63SSagi Grimberg struct nvme_tcp_ctrl *ctrl = queue->ctrl; 148440510a63SSagi Grimberg int qid = nvme_tcp_queue_id(queue); 148540510a63SSagi Grimberg 148640510a63SSagi Grimberg return !nvme_tcp_admin_queue(queue) && 148740510a63SSagi Grimberg !nvme_tcp_default_queue(queue) && 148840510a63SSagi Grimberg !nvme_tcp_read_queue(queue) && 148940510a63SSagi Grimberg qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 149040510a63SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ] + 149140510a63SSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL]; 149240510a63SSagi Grimberg } 149340510a63SSagi Grimberg 149440510a63SSagi Grimberg static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 149540510a63SSagi Grimberg { 149640510a63SSagi Grimberg struct nvme_tcp_ctrl *ctrl = queue->ctrl; 149740510a63SSagi Grimberg int qid = nvme_tcp_queue_id(queue); 149840510a63SSagi Grimberg int n = 0; 149940510a63SSagi Grimberg 150040510a63SSagi Grimberg if (nvme_tcp_default_queue(queue)) 150140510a63SSagi Grimberg n = qid - 1; 150240510a63SSagi Grimberg else if (nvme_tcp_read_queue(queue)) 150340510a63SSagi Grimberg n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; 150440510a63SSagi Grimberg else if (nvme_tcp_poll_queue(queue)) 150540510a63SSagi Grimberg n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 150640510a63SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ] - 1; 150740510a63SSagi Grimberg queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 150840510a63SSagi Grimberg } 150940510a63SSagi Grimberg 1510fb8745d0SChristoph Hellwig static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid) 15113f2304f8SSagi Grimberg { 15123f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 15133f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 15146ebf71baSChristoph Hellwig int ret, rcv_pdu_size; 15153f2304f8SSagi Grimberg 15169ebbfe49SChao Leng mutex_init(&queue->queue_lock); 15173f2304f8SSagi Grimberg queue->ctrl = ctrl; 151815ec928aSSagi Grimberg init_llist_head(&queue->req_list); 15193f2304f8SSagi Grimberg INIT_LIST_HEAD(&queue->send_list); 1520db5ad6b7SSagi Grimberg mutex_init(&queue->send_mutex); 15213f2304f8SSagi Grimberg INIT_WORK(&queue->io_work, nvme_tcp_io_work); 15223f2304f8SSagi Grimberg 15233f2304f8SSagi Grimberg if (qid > 0) 15249924b030SIsrael Rukshin queue->cmnd_capsule_len = nctrl->ioccsz * 16; 15253f2304f8SSagi Grimberg else 15263f2304f8SSagi Grimberg queue->cmnd_capsule_len = sizeof(struct nvme_command) + 15273f2304f8SSagi Grimberg NVME_TCP_ADMIN_CCSZ; 15283f2304f8SSagi Grimberg 15293f2304f8SSagi Grimberg ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 15303f2304f8SSagi Grimberg IPPROTO_TCP, &queue->sock); 15313f2304f8SSagi Grimberg if (ret) { 15329924b030SIsrael Rukshin dev_err(nctrl->device, 15333f2304f8SSagi Grimberg "failed to create socket: %d\n", ret); 15349ebbfe49SChao Leng goto err_destroy_mutex; 15353f2304f8SSagi Grimberg } 15363f2304f8SSagi Grimberg 1537841aee4dSChris Leech nvme_tcp_reclassify_socket(queue->sock); 1538841aee4dSChris Leech 15393f2304f8SSagi Grimberg /* Single syn retry */ 1540557eadfcSChristoph Hellwig tcp_sock_set_syncnt(queue->sock->sk, 1); 15413f2304f8SSagi Grimberg 15423f2304f8SSagi Grimberg /* Set TCP no delay */ 154312abc5eeSChristoph Hellwig tcp_sock_set_nodelay(queue->sock->sk); 15443f2304f8SSagi Grimberg 15453f2304f8SSagi Grimberg /* 15463f2304f8SSagi Grimberg * Cleanup whatever is sitting in the TCP transmit queue on socket 15473f2304f8SSagi Grimberg * close. This is done to prevent stale data from being sent should 15483f2304f8SSagi Grimberg * the network connection be restored before TCP times out. 15493f2304f8SSagi Grimberg */ 1550c433594cSChristoph Hellwig sock_no_linger(queue->sock->sk); 15513f2304f8SSagi Grimberg 15526e434967SChristoph Hellwig if (so_priority > 0) 15536e434967SChristoph Hellwig sock_set_priority(queue->sock->sk, so_priority); 15549912ade3SWunderlich, Mark 1555bb13985dSIsrael Rukshin /* Set socket type of service */ 15566ebf71baSChristoph Hellwig if (nctrl->opts->tos >= 0) 15576ebf71baSChristoph Hellwig ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1558bb13985dSIsrael Rukshin 1559adc99fd3SSagi Grimberg /* Set 10 seconds timeout for icresp recvmsg */ 1560adc99fd3SSagi Grimberg queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1561adc99fd3SSagi Grimberg 15623f2304f8SSagi Grimberg queue->sock->sk->sk_allocation = GFP_ATOMIC; 156398123866SBenjamin Coddington queue->sock->sk->sk_use_task_frag = false; 156440510a63SSagi Grimberg nvme_tcp_set_queue_io_cpu(queue); 15653f2304f8SSagi Grimberg queue->request = NULL; 15663f2304f8SSagi Grimberg queue->data_remaining = 0; 15673f2304f8SSagi Grimberg queue->ddgst_remaining = 0; 15683f2304f8SSagi Grimberg queue->pdu_remaining = 0; 15693f2304f8SSagi Grimberg queue->pdu_offset = 0; 15703f2304f8SSagi Grimberg sk_set_memalloc(queue->sock->sk); 15713f2304f8SSagi Grimberg 15729924b030SIsrael Rukshin if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 15733f2304f8SSagi Grimberg ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 15743f2304f8SSagi Grimberg sizeof(ctrl->src_addr)); 15753f2304f8SSagi Grimberg if (ret) { 15769924b030SIsrael Rukshin dev_err(nctrl->device, 15773f2304f8SSagi Grimberg "failed to bind queue %d socket %d\n", 15783f2304f8SSagi Grimberg qid, ret); 15793f2304f8SSagi Grimberg goto err_sock; 15803f2304f8SSagi Grimberg } 15813f2304f8SSagi Grimberg } 15823f2304f8SSagi Grimberg 15833ede8f72SMartin Belanger if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) { 15843ede8f72SMartin Belanger char *iface = nctrl->opts->host_iface; 15853ede8f72SMartin Belanger sockptr_t optval = KERNEL_SOCKPTR(iface); 15863ede8f72SMartin Belanger 15873ede8f72SMartin Belanger ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE, 15883ede8f72SMartin Belanger optval, strlen(iface)); 15893ede8f72SMartin Belanger if (ret) { 15903ede8f72SMartin Belanger dev_err(nctrl->device, 15913ede8f72SMartin Belanger "failed to bind to interface %s queue %d err %d\n", 15923ede8f72SMartin Belanger iface, qid, ret); 15933ede8f72SMartin Belanger goto err_sock; 15943ede8f72SMartin Belanger } 15953ede8f72SMartin Belanger } 15963ede8f72SMartin Belanger 15973f2304f8SSagi Grimberg queue->hdr_digest = nctrl->opts->hdr_digest; 15983f2304f8SSagi Grimberg queue->data_digest = nctrl->opts->data_digest; 15993f2304f8SSagi Grimberg if (queue->hdr_digest || queue->data_digest) { 16003f2304f8SSagi Grimberg ret = nvme_tcp_alloc_crypto(queue); 16013f2304f8SSagi Grimberg if (ret) { 16029924b030SIsrael Rukshin dev_err(nctrl->device, 16033f2304f8SSagi Grimberg "failed to allocate queue %d crypto\n", qid); 16043f2304f8SSagi Grimberg goto err_sock; 16053f2304f8SSagi Grimberg } 16063f2304f8SSagi Grimberg } 16073f2304f8SSagi Grimberg 16083f2304f8SSagi Grimberg rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 16093f2304f8SSagi Grimberg nvme_tcp_hdgst_len(queue); 16103f2304f8SSagi Grimberg queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 16113f2304f8SSagi Grimberg if (!queue->pdu) { 16123f2304f8SSagi Grimberg ret = -ENOMEM; 16133f2304f8SSagi Grimberg goto err_crypto; 16143f2304f8SSagi Grimberg } 16153f2304f8SSagi Grimberg 16169924b030SIsrael Rukshin dev_dbg(nctrl->device, "connecting queue %d\n", 16173f2304f8SSagi Grimberg nvme_tcp_queue_id(queue)); 16183f2304f8SSagi Grimberg 16193f2304f8SSagi Grimberg ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 16203f2304f8SSagi Grimberg sizeof(ctrl->addr), 0); 16213f2304f8SSagi Grimberg if (ret) { 16229924b030SIsrael Rukshin dev_err(nctrl->device, 16233f2304f8SSagi Grimberg "failed to connect socket: %d\n", ret); 16243f2304f8SSagi Grimberg goto err_rcv_pdu; 16253f2304f8SSagi Grimberg } 16263f2304f8SSagi Grimberg 16273f2304f8SSagi Grimberg ret = nvme_tcp_init_connection(queue); 16283f2304f8SSagi Grimberg if (ret) 16293f2304f8SSagi Grimberg goto err_init_connect; 16303f2304f8SSagi Grimberg 16313f2304f8SSagi Grimberg set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 16323f2304f8SSagi Grimberg 16333f2304f8SSagi Grimberg return 0; 16343f2304f8SSagi Grimberg 16353f2304f8SSagi Grimberg err_init_connect: 16363f2304f8SSagi Grimberg kernel_sock_shutdown(queue->sock, SHUT_RDWR); 16373f2304f8SSagi Grimberg err_rcv_pdu: 16383f2304f8SSagi Grimberg kfree(queue->pdu); 16393f2304f8SSagi Grimberg err_crypto: 16403f2304f8SSagi Grimberg if (queue->hdr_digest || queue->data_digest) 16413f2304f8SSagi Grimberg nvme_tcp_free_crypto(queue); 16423f2304f8SSagi Grimberg err_sock: 16433f2304f8SSagi Grimberg sock_release(queue->sock); 16443f2304f8SSagi Grimberg queue->sock = NULL; 16459ebbfe49SChao Leng err_destroy_mutex: 1646d48f92cdSKeith Busch mutex_destroy(&queue->send_mutex); 16479ebbfe49SChao Leng mutex_destroy(&queue->queue_lock); 16483f2304f8SSagi Grimberg return ret; 16493f2304f8SSagi Grimberg } 16503f2304f8SSagi Grimberg 165188eaba80SSagi Grimberg static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue) 16523f2304f8SSagi Grimberg { 16533f2304f8SSagi Grimberg struct socket *sock = queue->sock; 16543f2304f8SSagi Grimberg 16553f2304f8SSagi Grimberg write_lock_bh(&sock->sk->sk_callback_lock); 16563f2304f8SSagi Grimberg sock->sk->sk_user_data = NULL; 16573f2304f8SSagi Grimberg sock->sk->sk_data_ready = queue->data_ready; 16583f2304f8SSagi Grimberg sock->sk->sk_state_change = queue->state_change; 16593f2304f8SSagi Grimberg sock->sk->sk_write_space = queue->write_space; 16603f2304f8SSagi Grimberg write_unlock_bh(&sock->sk->sk_callback_lock); 16613f2304f8SSagi Grimberg } 16623f2304f8SSagi Grimberg 16633f2304f8SSagi Grimberg static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 16643f2304f8SSagi Grimberg { 16653f2304f8SSagi Grimberg kernel_sock_shutdown(queue->sock, SHUT_RDWR); 166688eaba80SSagi Grimberg nvme_tcp_restore_sock_ops(queue); 16673f2304f8SSagi Grimberg cancel_work_sync(&queue->io_work); 16683f2304f8SSagi Grimberg } 16693f2304f8SSagi Grimberg 16703f2304f8SSagi Grimberg static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 16713f2304f8SSagi Grimberg { 16723f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 16733f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 16743f2304f8SSagi Grimberg 16752bff487fSMaurizio Lombardi if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 16762bff487fSMaurizio Lombardi return; 16772bff487fSMaurizio Lombardi 16789ebbfe49SChao Leng mutex_lock(&queue->queue_lock); 16799ebbfe49SChao Leng if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 16803f2304f8SSagi Grimberg __nvme_tcp_stop_queue(queue); 16819ebbfe49SChao Leng mutex_unlock(&queue->queue_lock); 16823f2304f8SSagi Grimberg } 16833f2304f8SSagi Grimberg 168488eaba80SSagi Grimberg static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue) 168588eaba80SSagi Grimberg { 168688eaba80SSagi Grimberg write_lock_bh(&queue->sock->sk->sk_callback_lock); 168788eaba80SSagi Grimberg queue->sock->sk->sk_user_data = queue; 168888eaba80SSagi Grimberg queue->state_change = queue->sock->sk->sk_state_change; 168988eaba80SSagi Grimberg queue->data_ready = queue->sock->sk->sk_data_ready; 169088eaba80SSagi Grimberg queue->write_space = queue->sock->sk->sk_write_space; 169188eaba80SSagi Grimberg queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 169288eaba80SSagi Grimberg queue->sock->sk->sk_state_change = nvme_tcp_state_change; 169388eaba80SSagi Grimberg queue->sock->sk->sk_write_space = nvme_tcp_write_space; 169488eaba80SSagi Grimberg #ifdef CONFIG_NET_RX_BUSY_POLL 169588eaba80SSagi Grimberg queue->sock->sk->sk_ll_usec = 1; 169688eaba80SSagi Grimberg #endif 169788eaba80SSagi Grimberg write_unlock_bh(&queue->sock->sk->sk_callback_lock); 169888eaba80SSagi Grimberg } 169988eaba80SSagi Grimberg 17003f2304f8SSagi Grimberg static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 17013f2304f8SSagi Grimberg { 17023f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 170388eaba80SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[idx]; 17043f2304f8SSagi Grimberg int ret; 17053f2304f8SSagi Grimberg 170688eaba80SSagi Grimberg queue->rd_enabled = true; 170788eaba80SSagi Grimberg nvme_tcp_init_recv_ctx(queue); 170888eaba80SSagi Grimberg nvme_tcp_setup_sock_ops(queue); 170988eaba80SSagi Grimberg 17103f2304f8SSagi Grimberg if (idx) 1711be42a33bSKeith Busch ret = nvmf_connect_io_queue(nctrl, idx); 17123f2304f8SSagi Grimberg else 17133f2304f8SSagi Grimberg ret = nvmf_connect_admin_queue(nctrl); 17143f2304f8SSagi Grimberg 17153f2304f8SSagi Grimberg if (!ret) { 171688eaba80SSagi Grimberg set_bit(NVME_TCP_Q_LIVE, &queue->flags); 17173f2304f8SSagi Grimberg } else { 171888eaba80SSagi Grimberg if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 171988eaba80SSagi Grimberg __nvme_tcp_stop_queue(queue); 17203f2304f8SSagi Grimberg dev_err(nctrl->device, 17213f2304f8SSagi Grimberg "failed to connect queue: %d ret=%d\n", idx, ret); 17223f2304f8SSagi Grimberg } 17233f2304f8SSagi Grimberg return ret; 17243f2304f8SSagi Grimberg } 17253f2304f8SSagi Grimberg 17263f2304f8SSagi Grimberg static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 17273f2304f8SSagi Grimberg { 17283f2304f8SSagi Grimberg if (to_tcp_ctrl(ctrl)->async_req.pdu) { 1729ceb1e087SDavid Milburn cancel_work_sync(&ctrl->async_event_work); 17303f2304f8SSagi Grimberg nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 17313f2304f8SSagi Grimberg to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 17323f2304f8SSagi Grimberg } 17333f2304f8SSagi Grimberg 17343f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, 0); 17353f2304f8SSagi Grimberg } 17363f2304f8SSagi Grimberg 17373f2304f8SSagi Grimberg static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 17383f2304f8SSagi Grimberg { 17393f2304f8SSagi Grimberg int i; 17403f2304f8SSagi Grimberg 17413f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) 17423f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, i); 17433f2304f8SSagi Grimberg } 17443f2304f8SSagi Grimberg 17453f2304f8SSagi Grimberg static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 17463f2304f8SSagi Grimberg { 17473f2304f8SSagi Grimberg int i; 17483f2304f8SSagi Grimberg 17493f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) 17503f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, i); 17513f2304f8SSagi Grimberg } 17523f2304f8SSagi Grimberg 175309035f86SDaniel Wagner static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl, 175409035f86SDaniel Wagner int first, int last) 17553f2304f8SSagi Grimberg { 1756462b8b2dSChaitanya Kulkarni int i, ret; 17573f2304f8SSagi Grimberg 175809035f86SDaniel Wagner for (i = first; i < last; i++) { 17593f2304f8SSagi Grimberg ret = nvme_tcp_start_queue(ctrl, i); 17603f2304f8SSagi Grimberg if (ret) 17613f2304f8SSagi Grimberg goto out_stop_queues; 17623f2304f8SSagi Grimberg } 17633f2304f8SSagi Grimberg 17643f2304f8SSagi Grimberg return 0; 17653f2304f8SSagi Grimberg 17663f2304f8SSagi Grimberg out_stop_queues: 176709035f86SDaniel Wagner for (i--; i >= first; i--) 17683f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, i); 17693f2304f8SSagi Grimberg return ret; 17703f2304f8SSagi Grimberg } 17713f2304f8SSagi Grimberg 17723f2304f8SSagi Grimberg static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 17733f2304f8SSagi Grimberg { 17743f2304f8SSagi Grimberg int ret; 17753f2304f8SSagi Grimberg 1776fb8745d0SChristoph Hellwig ret = nvme_tcp_alloc_queue(ctrl, 0); 17773f2304f8SSagi Grimberg if (ret) 17783f2304f8SSagi Grimberg return ret; 17793f2304f8SSagi Grimberg 17803f2304f8SSagi Grimberg ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 17813f2304f8SSagi Grimberg if (ret) 17823f2304f8SSagi Grimberg goto out_free_queue; 17833f2304f8SSagi Grimberg 17843f2304f8SSagi Grimberg return 0; 17853f2304f8SSagi Grimberg 17863f2304f8SSagi Grimberg out_free_queue: 17873f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, 0); 17883f2304f8SSagi Grimberg return ret; 17893f2304f8SSagi Grimberg } 17903f2304f8SSagi Grimberg 1791efb973b1SSagi Grimberg static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 17923f2304f8SSagi Grimberg { 17933f2304f8SSagi Grimberg int i, ret; 17943f2304f8SSagi Grimberg 17953f2304f8SSagi Grimberg for (i = 1; i < ctrl->queue_count; i++) { 1796fb8745d0SChristoph Hellwig ret = nvme_tcp_alloc_queue(ctrl, i); 17973f2304f8SSagi Grimberg if (ret) 17983f2304f8SSagi Grimberg goto out_free_queues; 17993f2304f8SSagi Grimberg } 18003f2304f8SSagi Grimberg 18013f2304f8SSagi Grimberg return 0; 18023f2304f8SSagi Grimberg 18033f2304f8SSagi Grimberg out_free_queues: 18043f2304f8SSagi Grimberg for (i--; i >= 1; i--) 18053f2304f8SSagi Grimberg nvme_tcp_free_queue(ctrl, i); 18063f2304f8SSagi Grimberg 18073f2304f8SSagi Grimberg return ret; 18083f2304f8SSagi Grimberg } 18093f2304f8SSagi Grimberg 18103f2304f8SSagi Grimberg static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) 18113f2304f8SSagi Grimberg { 1812873946f4SSagi Grimberg unsigned int nr_io_queues; 1813873946f4SSagi Grimberg 1814873946f4SSagi Grimberg nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); 1815873946f4SSagi Grimberg nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); 18161a9460ceSSagi Grimberg nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); 1817873946f4SSagi Grimberg 1818873946f4SSagi Grimberg return nr_io_queues; 18193f2304f8SSagi Grimberg } 18203f2304f8SSagi Grimberg 182164861993SSagi Grimberg static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl, 182264861993SSagi Grimberg unsigned int nr_io_queues) 182364861993SSagi Grimberg { 182464861993SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 182564861993SSagi Grimberg struct nvmf_ctrl_options *opts = nctrl->opts; 182664861993SSagi Grimberg 182764861993SSagi Grimberg if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { 182864861993SSagi Grimberg /* 182964861993SSagi Grimberg * separate read/write queues 183064861993SSagi Grimberg * hand out dedicated default queues only after we have 183164861993SSagi Grimberg * sufficient read queues. 183264861993SSagi Grimberg */ 183364861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; 183464861993SSagi Grimberg nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; 183564861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT] = 183664861993SSagi Grimberg min(opts->nr_write_queues, nr_io_queues); 183764861993SSagi Grimberg nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 183864861993SSagi Grimberg } else { 183964861993SSagi Grimberg /* 184064861993SSagi Grimberg * shared read/write queues 184164861993SSagi Grimberg * either no write queues were requested, or we don't have 184264861993SSagi Grimberg * sufficient queue count to have dedicated default queues. 184364861993SSagi Grimberg */ 184464861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT] = 184564861993SSagi Grimberg min(opts->nr_io_queues, nr_io_queues); 184664861993SSagi Grimberg nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 184764861993SSagi Grimberg } 18481a9460ceSSagi Grimberg 18491a9460ceSSagi Grimberg if (opts->nr_poll_queues && nr_io_queues) { 18501a9460ceSSagi Grimberg /* map dedicated poll queues only if we have queues left */ 18511a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL] = 18521a9460ceSSagi Grimberg min(opts->nr_poll_queues, nr_io_queues); 18531a9460ceSSagi Grimberg } 185464861993SSagi Grimberg } 185564861993SSagi Grimberg 1856efb973b1SSagi Grimberg static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 18573f2304f8SSagi Grimberg { 18583f2304f8SSagi Grimberg unsigned int nr_io_queues; 18593f2304f8SSagi Grimberg int ret; 18603f2304f8SSagi Grimberg 18613f2304f8SSagi Grimberg nr_io_queues = nvme_tcp_nr_io_queues(ctrl); 18623f2304f8SSagi Grimberg ret = nvme_set_queue_count(ctrl, &nr_io_queues); 18633f2304f8SSagi Grimberg if (ret) 18643f2304f8SSagi Grimberg return ret; 18653f2304f8SSagi Grimberg 1866664227fdSRuozhu Li if (nr_io_queues == 0) { 186772f57242SSagi Grimberg dev_err(ctrl->device, 186872f57242SSagi Grimberg "unable to set any I/O queues\n"); 186972f57242SSagi Grimberg return -ENOMEM; 187072f57242SSagi Grimberg } 18713f2304f8SSagi Grimberg 1872664227fdSRuozhu Li ctrl->queue_count = nr_io_queues + 1; 18733f2304f8SSagi Grimberg dev_info(ctrl->device, 18743f2304f8SSagi Grimberg "creating %d I/O queues.\n", nr_io_queues); 18753f2304f8SSagi Grimberg 187664861993SSagi Grimberg nvme_tcp_set_io_queues(ctrl, nr_io_queues); 187764861993SSagi Grimberg 1878efb973b1SSagi Grimberg return __nvme_tcp_alloc_io_queues(ctrl); 18793f2304f8SSagi Grimberg } 18803f2304f8SSagi Grimberg 18813f2304f8SSagi Grimberg static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 18823f2304f8SSagi Grimberg { 18833f2304f8SSagi Grimberg nvme_tcp_stop_io_queues(ctrl); 1884de777825SChristoph Hellwig if (remove) 1885de777825SChristoph Hellwig nvme_remove_io_tag_set(ctrl); 18863f2304f8SSagi Grimberg nvme_tcp_free_io_queues(ctrl); 18873f2304f8SSagi Grimberg } 18883f2304f8SSagi Grimberg 18893f2304f8SSagi Grimberg static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 18903f2304f8SSagi Grimberg { 189109035f86SDaniel Wagner int ret, nr_queues; 18923f2304f8SSagi Grimberg 1893efb973b1SSagi Grimberg ret = nvme_tcp_alloc_io_queues(ctrl); 18943f2304f8SSagi Grimberg if (ret) 18953f2304f8SSagi Grimberg return ret; 18963f2304f8SSagi Grimberg 18973f2304f8SSagi Grimberg if (new) { 1898de777825SChristoph Hellwig ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set, 1899de777825SChristoph Hellwig &nvme_tcp_mq_ops, 1900dcef7727SChristoph Hellwig ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2, 1901de777825SChristoph Hellwig sizeof(struct nvme_tcp_request)); 19022f7a7e5dSChristoph Hellwig if (ret) 19033f2304f8SSagi Grimberg goto out_free_io_queues; 19043f2304f8SSagi Grimberg } 19053f2304f8SSagi Grimberg 190609035f86SDaniel Wagner /* 190709035f86SDaniel Wagner * Only start IO queues for which we have allocated the tagset 190809035f86SDaniel Wagner * and limitted it to the available queues. On reconnects, the 190909035f86SDaniel Wagner * queue number might have changed. 191009035f86SDaniel Wagner */ 191109035f86SDaniel Wagner nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count); 191209035f86SDaniel Wagner ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues); 19133f2304f8SSagi Grimberg if (ret) 19143f2304f8SSagi Grimberg goto out_cleanup_connect_q; 19153f2304f8SSagi Grimberg 19162875b0aeSSagi Grimberg if (!new) { 19179f27bd70SChristoph Hellwig nvme_unquiesce_io_queues(ctrl); 1918e5c01f4fSSagi Grimberg if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 1919e5c01f4fSSagi Grimberg /* 1920e5c01f4fSSagi Grimberg * If we timed out waiting for freeze we are likely to 1921e5c01f4fSSagi Grimberg * be stuck. Fail the controller initialization just 1922e5c01f4fSSagi Grimberg * to be safe. 1923e5c01f4fSSagi Grimberg */ 1924e5c01f4fSSagi Grimberg ret = -ENODEV; 1925e5c01f4fSSagi Grimberg goto out_wait_freeze_timed_out; 1926e5c01f4fSSagi Grimberg } 19272875b0aeSSagi Grimberg blk_mq_update_nr_hw_queues(ctrl->tagset, 19282875b0aeSSagi Grimberg ctrl->queue_count - 1); 19292875b0aeSSagi Grimberg nvme_unfreeze(ctrl); 19302875b0aeSSagi Grimberg } 19312875b0aeSSagi Grimberg 193209035f86SDaniel Wagner /* 193309035f86SDaniel Wagner * If the number of queues has increased (reconnect case) 193409035f86SDaniel Wagner * start all new queues now. 193509035f86SDaniel Wagner */ 193609035f86SDaniel Wagner ret = nvme_tcp_start_io_queues(ctrl, nr_queues, 193709035f86SDaniel Wagner ctrl->tagset->nr_hw_queues + 1); 193809035f86SDaniel Wagner if (ret) 193909035f86SDaniel Wagner goto out_wait_freeze_timed_out; 194009035f86SDaniel Wagner 19413f2304f8SSagi Grimberg return 0; 19423f2304f8SSagi Grimberg 1943e5c01f4fSSagi Grimberg out_wait_freeze_timed_out: 19449f27bd70SChristoph Hellwig nvme_quiesce_io_queues(ctrl); 194570a99574SChao Leng nvme_sync_io_queues(ctrl); 1946e5c01f4fSSagi Grimberg nvme_tcp_stop_io_queues(ctrl); 19473f2304f8SSagi Grimberg out_cleanup_connect_q: 194870a99574SChao Leng nvme_cancel_tagset(ctrl); 1949e85037a2SSagi Grimberg if (new) 1950de777825SChristoph Hellwig nvme_remove_io_tag_set(ctrl); 19513f2304f8SSagi Grimberg out_free_io_queues: 19523f2304f8SSagi Grimberg nvme_tcp_free_io_queues(ctrl); 19533f2304f8SSagi Grimberg return ret; 19543f2304f8SSagi Grimberg } 19553f2304f8SSagi Grimberg 19563f2304f8SSagi Grimberg static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 19573f2304f8SSagi Grimberg { 19583f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 1959de777825SChristoph Hellwig if (remove) 1960de777825SChristoph Hellwig nvme_remove_admin_tag_set(ctrl); 19613f2304f8SSagi Grimberg nvme_tcp_free_admin_queue(ctrl); 19623f2304f8SSagi Grimberg } 19633f2304f8SSagi Grimberg 19643f2304f8SSagi Grimberg static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 19653f2304f8SSagi Grimberg { 19663f2304f8SSagi Grimberg int error; 19673f2304f8SSagi Grimberg 19683f2304f8SSagi Grimberg error = nvme_tcp_alloc_admin_queue(ctrl); 19693f2304f8SSagi Grimberg if (error) 19703f2304f8SSagi Grimberg return error; 19713f2304f8SSagi Grimberg 19723f2304f8SSagi Grimberg if (new) { 1973de777825SChristoph Hellwig error = nvme_alloc_admin_tag_set(ctrl, 1974de777825SChristoph Hellwig &to_tcp_ctrl(ctrl)->admin_tag_set, 1975db45e1a5SChristoph Hellwig &nvme_tcp_admin_mq_ops, 1976de777825SChristoph Hellwig sizeof(struct nvme_tcp_request)); 19772f7a7e5dSChristoph Hellwig if (error) 19783f2304f8SSagi Grimberg goto out_free_queue; 19793f2304f8SSagi Grimberg } 19803f2304f8SSagi Grimberg 19813f2304f8SSagi Grimberg error = nvme_tcp_start_queue(ctrl, 0); 19823f2304f8SSagi Grimberg if (error) 1983de777825SChristoph Hellwig goto out_cleanup_tagset; 19843f2304f8SSagi Grimberg 1985c0f2f45bSSagi Grimberg error = nvme_enable_ctrl(ctrl); 19863f2304f8SSagi Grimberg if (error) 19873f2304f8SSagi Grimberg goto out_stop_queue; 19883f2304f8SSagi Grimberg 19899f27bd70SChristoph Hellwig nvme_unquiesce_admin_queue(ctrl); 1990e7832cb4SSagi Grimberg 199194cc781fSChristoph Hellwig error = nvme_init_ctrl_finish(ctrl, false); 19923f2304f8SSagi Grimberg if (error) 199370a99574SChao Leng goto out_quiesce_queue; 19943f2304f8SSagi Grimberg 19953f2304f8SSagi Grimberg return 0; 19963f2304f8SSagi Grimberg 199770a99574SChao Leng out_quiesce_queue: 19989f27bd70SChristoph Hellwig nvme_quiesce_admin_queue(ctrl); 199970a99574SChao Leng blk_sync_queue(ctrl->admin_q); 20003f2304f8SSagi Grimberg out_stop_queue: 20013f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 200270a99574SChao Leng nvme_cancel_admin_tagset(ctrl); 2003de777825SChristoph Hellwig out_cleanup_tagset: 20043f2304f8SSagi Grimberg if (new) 2005de777825SChristoph Hellwig nvme_remove_admin_tag_set(ctrl); 20063f2304f8SSagi Grimberg out_free_queue: 20073f2304f8SSagi Grimberg nvme_tcp_free_admin_queue(ctrl); 20083f2304f8SSagi Grimberg return error; 20093f2304f8SSagi Grimberg } 20103f2304f8SSagi Grimberg 20113f2304f8SSagi Grimberg static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 20123f2304f8SSagi Grimberg bool remove) 20133f2304f8SSagi Grimberg { 20149f27bd70SChristoph Hellwig nvme_quiesce_admin_queue(ctrl); 2015d6f66210SChao Leng blk_sync_queue(ctrl->admin_q); 20163f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 2017563c8158SChao Leng nvme_cancel_admin_tagset(ctrl); 2018e7832cb4SSagi Grimberg if (remove) 20199f27bd70SChristoph Hellwig nvme_unquiesce_admin_queue(ctrl); 20203f2304f8SSagi Grimberg nvme_tcp_destroy_admin_queue(ctrl, remove); 20213f2304f8SSagi Grimberg } 20223f2304f8SSagi Grimberg 20233f2304f8SSagi Grimberg static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 20243f2304f8SSagi Grimberg bool remove) 20253f2304f8SSagi Grimberg { 20263f2304f8SSagi Grimberg if (ctrl->queue_count <= 1) 2027d6f66210SChao Leng return; 20289f27bd70SChristoph Hellwig nvme_quiesce_admin_queue(ctrl); 20292875b0aeSSagi Grimberg nvme_start_freeze(ctrl); 20309f27bd70SChristoph Hellwig nvme_quiesce_io_queues(ctrl); 2031d6f66210SChao Leng nvme_sync_io_queues(ctrl); 20323f2304f8SSagi Grimberg nvme_tcp_stop_io_queues(ctrl); 2033563c8158SChao Leng nvme_cancel_tagset(ctrl); 20343f2304f8SSagi Grimberg if (remove) 20359f27bd70SChristoph Hellwig nvme_unquiesce_io_queues(ctrl); 20363f2304f8SSagi Grimberg nvme_tcp_destroy_io_queues(ctrl, remove); 20373f2304f8SSagi Grimberg } 20383f2304f8SSagi Grimberg 20393f2304f8SSagi Grimberg static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 20403f2304f8SSagi Grimberg { 20413f2304f8SSagi Grimberg /* If we are resetting/deleting then do nothing */ 20423f2304f8SSagi Grimberg if (ctrl->state != NVME_CTRL_CONNECTING) { 20433f2304f8SSagi Grimberg WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 20443f2304f8SSagi Grimberg ctrl->state == NVME_CTRL_LIVE); 20453f2304f8SSagi Grimberg return; 20463f2304f8SSagi Grimberg } 20473f2304f8SSagi Grimberg 20483f2304f8SSagi Grimberg if (nvmf_should_reconnect(ctrl)) { 20493f2304f8SSagi Grimberg dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 20503f2304f8SSagi Grimberg ctrl->opts->reconnect_delay); 20513f2304f8SSagi Grimberg queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 20523f2304f8SSagi Grimberg ctrl->opts->reconnect_delay * HZ); 20533f2304f8SSagi Grimberg } else { 20543f2304f8SSagi Grimberg dev_info(ctrl->device, "Removing controller...\n"); 20553f2304f8SSagi Grimberg nvme_delete_ctrl(ctrl); 20563f2304f8SSagi Grimberg } 20573f2304f8SSagi Grimberg } 20583f2304f8SSagi Grimberg 20593f2304f8SSagi Grimberg static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 20603f2304f8SSagi Grimberg { 20613f2304f8SSagi Grimberg struct nvmf_ctrl_options *opts = ctrl->opts; 2062312910f4SColin Ian King int ret; 20633f2304f8SSagi Grimberg 20643f2304f8SSagi Grimberg ret = nvme_tcp_configure_admin_queue(ctrl, new); 20653f2304f8SSagi Grimberg if (ret) 20663f2304f8SSagi Grimberg return ret; 20673f2304f8SSagi Grimberg 20683f2304f8SSagi Grimberg if (ctrl->icdoff) { 2069522af60cSDan Carpenter ret = -EOPNOTSUPP; 20703f2304f8SSagi Grimberg dev_err(ctrl->device, "icdoff is not supported!\n"); 20713f2304f8SSagi Grimberg goto destroy_admin; 20723f2304f8SSagi Grimberg } 20733f2304f8SSagi Grimberg 20743b54064fSChaitanya Kulkarni if (!nvme_ctrl_sgl_supported(ctrl)) { 2075522af60cSDan Carpenter ret = -EOPNOTSUPP; 207673ffcefcSMax Gurtovoy dev_err(ctrl->device, "Mandatory sgls are not supported!\n"); 207773ffcefcSMax Gurtovoy goto destroy_admin; 207873ffcefcSMax Gurtovoy } 207973ffcefcSMax Gurtovoy 20803f2304f8SSagi Grimberg if (opts->queue_size > ctrl->sqsize + 1) 20813f2304f8SSagi Grimberg dev_warn(ctrl->device, 20823f2304f8SSagi Grimberg "queue_size %zu > ctrl sqsize %u, clamping down\n", 20833f2304f8SSagi Grimberg opts->queue_size, ctrl->sqsize + 1); 20843f2304f8SSagi Grimberg 20853f2304f8SSagi Grimberg if (ctrl->sqsize + 1 > ctrl->maxcmd) { 20863f2304f8SSagi Grimberg dev_warn(ctrl->device, 20873f2304f8SSagi Grimberg "sqsize %u > ctrl maxcmd %u, clamping down\n", 20883f2304f8SSagi Grimberg ctrl->sqsize + 1, ctrl->maxcmd); 20893f2304f8SSagi Grimberg ctrl->sqsize = ctrl->maxcmd - 1; 20903f2304f8SSagi Grimberg } 20913f2304f8SSagi Grimberg 20923f2304f8SSagi Grimberg if (ctrl->queue_count > 1) { 20933f2304f8SSagi Grimberg ret = nvme_tcp_configure_io_queues(ctrl, new); 20943f2304f8SSagi Grimberg if (ret) 20953f2304f8SSagi Grimberg goto destroy_admin; 20963f2304f8SSagi Grimberg } 20973f2304f8SSagi Grimberg 20983f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2099bea54ef5SIsrael Rukshin /* 2100ecca390eSSagi Grimberg * state change failure is ok if we started ctrl delete, 2101bea54ef5SIsrael Rukshin * unless we're during creation of a new controller to 2102bea54ef5SIsrael Rukshin * avoid races with teardown flow. 2103bea54ef5SIsrael Rukshin */ 2104ecca390eSSagi Grimberg WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2105ecca390eSSagi Grimberg ctrl->state != NVME_CTRL_DELETING_NOIO); 2106bea54ef5SIsrael Rukshin WARN_ON_ONCE(new); 21073f2304f8SSagi Grimberg ret = -EINVAL; 21083f2304f8SSagi Grimberg goto destroy_io; 21093f2304f8SSagi Grimberg } 21103f2304f8SSagi Grimberg 21113f2304f8SSagi Grimberg nvme_start_ctrl(ctrl); 21123f2304f8SSagi Grimberg return 0; 21133f2304f8SSagi Grimberg 21143f2304f8SSagi Grimberg destroy_io: 211570a99574SChao Leng if (ctrl->queue_count > 1) { 21169f27bd70SChristoph Hellwig nvme_quiesce_io_queues(ctrl); 211770a99574SChao Leng nvme_sync_io_queues(ctrl); 211870a99574SChao Leng nvme_tcp_stop_io_queues(ctrl); 211970a99574SChao Leng nvme_cancel_tagset(ctrl); 21203f2304f8SSagi Grimberg nvme_tcp_destroy_io_queues(ctrl, new); 212170a99574SChao Leng } 21223f2304f8SSagi Grimberg destroy_admin: 21239f27bd70SChristoph Hellwig nvme_quiesce_admin_queue(ctrl); 212470a99574SChao Leng blk_sync_queue(ctrl->admin_q); 21253f2304f8SSagi Grimberg nvme_tcp_stop_queue(ctrl, 0); 212670a99574SChao Leng nvme_cancel_admin_tagset(ctrl); 21273f2304f8SSagi Grimberg nvme_tcp_destroy_admin_queue(ctrl, new); 21283f2304f8SSagi Grimberg return ret; 21293f2304f8SSagi Grimberg } 21303f2304f8SSagi Grimberg 21313f2304f8SSagi Grimberg static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 21323f2304f8SSagi Grimberg { 21333f2304f8SSagi Grimberg struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 21343f2304f8SSagi Grimberg struct nvme_tcp_ctrl, connect_work); 21353f2304f8SSagi Grimberg struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 21363f2304f8SSagi Grimberg 21373f2304f8SSagi Grimberg ++ctrl->nr_reconnects; 21383f2304f8SSagi Grimberg 21393f2304f8SSagi Grimberg if (nvme_tcp_setup_ctrl(ctrl, false)) 21403f2304f8SSagi Grimberg goto requeue; 21413f2304f8SSagi Grimberg 214256a77d26SColin Ian King dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 21433f2304f8SSagi Grimberg ctrl->nr_reconnects); 21443f2304f8SSagi Grimberg 21453f2304f8SSagi Grimberg ctrl->nr_reconnects = 0; 21463f2304f8SSagi Grimberg 21473f2304f8SSagi Grimberg return; 21483f2304f8SSagi Grimberg 21493f2304f8SSagi Grimberg requeue: 21503f2304f8SSagi Grimberg dev_info(ctrl->device, "Failed reconnect attempt %d\n", 21513f2304f8SSagi Grimberg ctrl->nr_reconnects); 21523f2304f8SSagi Grimberg nvme_tcp_reconnect_or_remove(ctrl); 21533f2304f8SSagi Grimberg } 21543f2304f8SSagi Grimberg 21553f2304f8SSagi Grimberg static void nvme_tcp_error_recovery_work(struct work_struct *work) 21563f2304f8SSagi Grimberg { 21573f2304f8SSagi Grimberg struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 21583f2304f8SSagi Grimberg struct nvme_tcp_ctrl, err_work); 21593f2304f8SSagi Grimberg struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 21603f2304f8SSagi Grimberg 21613f2304f8SSagi Grimberg nvme_stop_keep_alive(ctrl); 2162ff9fc7ebSSagi Grimberg flush_work(&ctrl->async_event_work); 21633f2304f8SSagi Grimberg nvme_tcp_teardown_io_queues(ctrl, false); 21643f2304f8SSagi Grimberg /* unquiesce to fail fast pending requests */ 21659f27bd70SChristoph Hellwig nvme_unquiesce_io_queues(ctrl); 21663f2304f8SSagi Grimberg nvme_tcp_teardown_admin_queue(ctrl, false); 21679f27bd70SChristoph Hellwig nvme_unquiesce_admin_queue(ctrl); 21681f1a4f89SSagi Grimberg nvme_auth_stop(ctrl); 21693f2304f8SSagi Grimberg 21703f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2171ecca390eSSagi Grimberg /* state change failure is ok if we started ctrl delete */ 2172ecca390eSSagi Grimberg WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2173ecca390eSSagi Grimberg ctrl->state != NVME_CTRL_DELETING_NOIO); 21743f2304f8SSagi Grimberg return; 21753f2304f8SSagi Grimberg } 21763f2304f8SSagi Grimberg 21773f2304f8SSagi Grimberg nvme_tcp_reconnect_or_remove(ctrl); 21783f2304f8SSagi Grimberg } 21793f2304f8SSagi Grimberg 21803f2304f8SSagi Grimberg static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 21813f2304f8SSagi Grimberg { 21823f2304f8SSagi Grimberg nvme_tcp_teardown_io_queues(ctrl, shutdown); 21839f27bd70SChristoph Hellwig nvme_quiesce_admin_queue(ctrl); 2184285b6e9bSChristoph Hellwig nvme_disable_ctrl(ctrl, shutdown); 21853f2304f8SSagi Grimberg nvme_tcp_teardown_admin_queue(ctrl, shutdown); 21863f2304f8SSagi Grimberg } 21873f2304f8SSagi Grimberg 21883f2304f8SSagi Grimberg static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 21893f2304f8SSagi Grimberg { 21903f2304f8SSagi Grimberg nvme_tcp_teardown_ctrl(ctrl, true); 21913f2304f8SSagi Grimberg } 21923f2304f8SSagi Grimberg 21933f2304f8SSagi Grimberg static void nvme_reset_ctrl_work(struct work_struct *work) 21943f2304f8SSagi Grimberg { 21953f2304f8SSagi Grimberg struct nvme_ctrl *ctrl = 21963f2304f8SSagi Grimberg container_of(work, struct nvme_ctrl, reset_work); 21973f2304f8SSagi Grimberg 21983f2304f8SSagi Grimberg nvme_stop_ctrl(ctrl); 21993f2304f8SSagi Grimberg nvme_tcp_teardown_ctrl(ctrl, false); 22003f2304f8SSagi Grimberg 22013f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2202ecca390eSSagi Grimberg /* state change failure is ok if we started ctrl delete */ 2203ecca390eSSagi Grimberg WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2204ecca390eSSagi Grimberg ctrl->state != NVME_CTRL_DELETING_NOIO); 22053f2304f8SSagi Grimberg return; 22063f2304f8SSagi Grimberg } 22073f2304f8SSagi Grimberg 22083f2304f8SSagi Grimberg if (nvme_tcp_setup_ctrl(ctrl, false)) 22093f2304f8SSagi Grimberg goto out_fail; 22103f2304f8SSagi Grimberg 22113f2304f8SSagi Grimberg return; 22123f2304f8SSagi Grimberg 22133f2304f8SSagi Grimberg out_fail: 22143f2304f8SSagi Grimberg ++ctrl->nr_reconnects; 22153f2304f8SSagi Grimberg nvme_tcp_reconnect_or_remove(ctrl); 22163f2304f8SSagi Grimberg } 22173f2304f8SSagi Grimberg 2218f7f70f4aSRuozhu Li static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) 2219f7f70f4aSRuozhu Li { 2220c4abd875SSagi Grimberg flush_work(&to_tcp_ctrl(ctrl)->err_work); 2221f7f70f4aSRuozhu Li cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2222f7f70f4aSRuozhu Li } 2223f7f70f4aSRuozhu Li 22243f2304f8SSagi Grimberg static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 22253f2304f8SSagi Grimberg { 22263f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 22273f2304f8SSagi Grimberg 22283f2304f8SSagi Grimberg if (list_empty(&ctrl->list)) 22293f2304f8SSagi Grimberg goto free_ctrl; 22303f2304f8SSagi Grimberg 22313f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 22323f2304f8SSagi Grimberg list_del(&ctrl->list); 22333f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 22343f2304f8SSagi Grimberg 22353f2304f8SSagi Grimberg nvmf_free_options(nctrl->opts); 22363f2304f8SSagi Grimberg free_ctrl: 22373f2304f8SSagi Grimberg kfree(ctrl->queues); 22383f2304f8SSagi Grimberg kfree(ctrl); 22393f2304f8SSagi Grimberg } 22403f2304f8SSagi Grimberg 22413f2304f8SSagi Grimberg static void nvme_tcp_set_sg_null(struct nvme_command *c) 22423f2304f8SSagi Grimberg { 22433f2304f8SSagi Grimberg struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 22443f2304f8SSagi Grimberg 22453f2304f8SSagi Grimberg sg->addr = 0; 22463f2304f8SSagi Grimberg sg->length = 0; 22473f2304f8SSagi Grimberg sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 22483f2304f8SSagi Grimberg NVME_SGL_FMT_TRANSPORT_A; 22493f2304f8SSagi Grimberg } 22503f2304f8SSagi Grimberg 22513f2304f8SSagi Grimberg static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 22523f2304f8SSagi Grimberg struct nvme_command *c, u32 data_len) 22533f2304f8SSagi Grimberg { 22543f2304f8SSagi Grimberg struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 22553f2304f8SSagi Grimberg 22563f2304f8SSagi Grimberg sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 22573f2304f8SSagi Grimberg sg->length = cpu_to_le32(data_len); 22583f2304f8SSagi Grimberg sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 22593f2304f8SSagi Grimberg } 22603f2304f8SSagi Grimberg 22613f2304f8SSagi Grimberg static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 22623f2304f8SSagi Grimberg u32 data_len) 22633f2304f8SSagi Grimberg { 22643f2304f8SSagi Grimberg struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 22653f2304f8SSagi Grimberg 22663f2304f8SSagi Grimberg sg->addr = 0; 22673f2304f8SSagi Grimberg sg->length = cpu_to_le32(data_len); 22683f2304f8SSagi Grimberg sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 22693f2304f8SSagi Grimberg NVME_SGL_FMT_TRANSPORT_A; 22703f2304f8SSagi Grimberg } 22713f2304f8SSagi Grimberg 22723f2304f8SSagi Grimberg static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 22733f2304f8SSagi Grimberg { 22743f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 22753f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = &ctrl->queues[0]; 22763f2304f8SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 22773f2304f8SSagi Grimberg struct nvme_command *cmd = &pdu->cmd; 22783f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue); 22793f2304f8SSagi Grimberg 22803f2304f8SSagi Grimberg memset(pdu, 0, sizeof(*pdu)); 22813f2304f8SSagi Grimberg pdu->hdr.type = nvme_tcp_cmd; 22823f2304f8SSagi Grimberg if (queue->hdr_digest) 22833f2304f8SSagi Grimberg pdu->hdr.flags |= NVME_TCP_F_HDGST; 22843f2304f8SSagi Grimberg pdu->hdr.hlen = sizeof(*pdu); 22853f2304f8SSagi Grimberg pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 22863f2304f8SSagi Grimberg 22873f2304f8SSagi Grimberg cmd->common.opcode = nvme_admin_async_event; 22883f2304f8SSagi Grimberg cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 22893f2304f8SSagi Grimberg cmd->common.flags |= NVME_CMD_SGL_METABUF; 22903f2304f8SSagi Grimberg nvme_tcp_set_sg_null(cmd); 22913f2304f8SSagi Grimberg 22923f2304f8SSagi Grimberg ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 22933f2304f8SSagi Grimberg ctrl->async_req.offset = 0; 22943f2304f8SSagi Grimberg ctrl->async_req.curr_bio = NULL; 22953f2304f8SSagi Grimberg ctrl->async_req.data_len = 0; 22963f2304f8SSagi Grimberg 229786f0348aSSagi Grimberg nvme_tcp_queue_request(&ctrl->async_req, true, true); 22983f2304f8SSagi Grimberg } 22993f2304f8SSagi Grimberg 2300236187c4SSagi Grimberg static void nvme_tcp_complete_timed_out(struct request *rq) 2301236187c4SSagi Grimberg { 2302236187c4SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2303236187c4SSagi Grimberg struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2304236187c4SSagi Grimberg 2305236187c4SSagi Grimberg nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 230693ba75c9SChaitanya Kulkarni nvmf_complete_timed_out_request(rq); 2307236187c4SSagi Grimberg } 2308236187c4SSagi Grimberg 23099bdb4833SJohn Garry static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) 23103f2304f8SSagi Grimberg { 23113f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2312236187c4SSagi Grimberg struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2313a3406352SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 231499607843SAmit Engel u8 opc = pdu->cmd.common.opcode, fctype = pdu->cmd.fabrics.fctype; 231599607843SAmit Engel int qid = nvme_tcp_queue_id(req->queue); 23163f2304f8SSagi Grimberg 2317236187c4SSagi Grimberg dev_warn(ctrl->device, 231899607843SAmit Engel "queue %d: timeout cid %#x type %d opcode %#x (%s)\n", 231999607843SAmit Engel nvme_tcp_queue_id(req->queue), nvme_cid(rq), pdu->hdr.type, 232099607843SAmit Engel opc, nvme_opcode_str(qid, opc, fctype)); 23213f2304f8SSagi Grimberg 2322236187c4SSagi Grimberg if (ctrl->state != NVME_CTRL_LIVE) { 232339d57757SSagi Grimberg /* 2324236187c4SSagi Grimberg * If we are resetting, connecting or deleting we should 2325236187c4SSagi Grimberg * complete immediately because we may block controller 2326236187c4SSagi Grimberg * teardown or setup sequence 2327236187c4SSagi Grimberg * - ctrl disable/shutdown fabrics requests 2328236187c4SSagi Grimberg * - connect requests 2329236187c4SSagi Grimberg * - initialization admin requests 2330236187c4SSagi Grimberg * - I/O requests that entered after unquiescing and 2331236187c4SSagi Grimberg * the controller stopped responding 2332236187c4SSagi Grimberg * 2333236187c4SSagi Grimberg * All other requests should be cancelled by the error 2334236187c4SSagi Grimberg * recovery work, so it's fine that we fail it here. 233539d57757SSagi Grimberg */ 2336236187c4SSagi Grimberg nvme_tcp_complete_timed_out(rq); 23373f2304f8SSagi Grimberg return BLK_EH_DONE; 23383f2304f8SSagi Grimberg } 23393f2304f8SSagi Grimberg 2340236187c4SSagi Grimberg /* 2341236187c4SSagi Grimberg * LIVE state should trigger the normal error recovery which will 2342236187c4SSagi Grimberg * handle completing this request. 2343236187c4SSagi Grimberg */ 2344236187c4SSagi Grimberg nvme_tcp_error_recovery(ctrl); 23453f2304f8SSagi Grimberg return BLK_EH_RESET_TIMER; 23463f2304f8SSagi Grimberg } 23473f2304f8SSagi Grimberg 23483f2304f8SSagi Grimberg static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 23493f2304f8SSagi Grimberg struct request *rq) 23503f2304f8SSagi Grimberg { 23513f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2352a3406352SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 23533f2304f8SSagi Grimberg struct nvme_command *c = &pdu->cmd; 23543f2304f8SSagi Grimberg 23553f2304f8SSagi Grimberg c->common.flags |= NVME_CMD_SGL_METABUF; 23563f2304f8SSagi Grimberg 235725e5cb78SSagi Grimberg if (!blk_rq_nr_phys_segments(rq)) 235825e5cb78SSagi Grimberg nvme_tcp_set_sg_null(c); 235925e5cb78SSagi Grimberg else if (rq_data_dir(rq) == WRITE && 236053ee9e29SCaleb Sander req->data_len <= nvme_tcp_inline_data_size(req)) 23613f2304f8SSagi Grimberg nvme_tcp_set_sg_inline(queue, c, req->data_len); 23623f2304f8SSagi Grimberg else 23633f2304f8SSagi Grimberg nvme_tcp_set_sg_host_data(c, req->data_len); 23643f2304f8SSagi Grimberg 23653f2304f8SSagi Grimberg return 0; 23663f2304f8SSagi Grimberg } 23673f2304f8SSagi Grimberg 23683f2304f8SSagi Grimberg static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 23693f2304f8SSagi Grimberg struct request *rq) 23703f2304f8SSagi Grimberg { 23713f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2372a3406352SSagi Grimberg struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req); 23733f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = req->queue; 23743f2304f8SSagi Grimberg u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 23753f2304f8SSagi Grimberg blk_status_t ret; 23763f2304f8SSagi Grimberg 2377f4b9e6c9SKeith Busch ret = nvme_setup_cmd(ns, rq); 23783f2304f8SSagi Grimberg if (ret) 23793f2304f8SSagi Grimberg return ret; 23803f2304f8SSagi Grimberg 23813f2304f8SSagi Grimberg req->state = NVME_TCP_SEND_CMD_PDU; 23821ba2e507SDaniel Wagner req->status = cpu_to_le16(NVME_SC_SUCCESS); 23833f2304f8SSagi Grimberg req->offset = 0; 23843f2304f8SSagi Grimberg req->data_sent = 0; 23853f2304f8SSagi Grimberg req->pdu_len = 0; 23863f2304f8SSagi Grimberg req->pdu_sent = 0; 2387c2700d28SVarun Prakash req->h2cdata_left = 0; 238825e5cb78SSagi Grimberg req->data_len = blk_rq_nr_phys_segments(rq) ? 238925e5cb78SSagi Grimberg blk_rq_payload_bytes(rq) : 0; 23903f2304f8SSagi Grimberg req->curr_bio = rq->bio; 2391e11e5116SSagi Grimberg if (req->curr_bio && req->data_len) 2392cb9b870fSSagi Grimberg nvme_tcp_init_iter(req, rq_data_dir(rq)); 23933f2304f8SSagi Grimberg 23943f2304f8SSagi Grimberg if (rq_data_dir(rq) == WRITE && 239553ee9e29SCaleb Sander req->data_len <= nvme_tcp_inline_data_size(req)) 23963f2304f8SSagi Grimberg req->pdu_len = req->data_len; 23973f2304f8SSagi Grimberg 23983f2304f8SSagi Grimberg pdu->hdr.type = nvme_tcp_cmd; 23993f2304f8SSagi Grimberg pdu->hdr.flags = 0; 24003f2304f8SSagi Grimberg if (queue->hdr_digest) 24013f2304f8SSagi Grimberg pdu->hdr.flags |= NVME_TCP_F_HDGST; 24023f2304f8SSagi Grimberg if (queue->data_digest && req->pdu_len) { 24033f2304f8SSagi Grimberg pdu->hdr.flags |= NVME_TCP_F_DDGST; 24043f2304f8SSagi Grimberg ddgst = nvme_tcp_ddgst_len(queue); 24053f2304f8SSagi Grimberg } 24063f2304f8SSagi Grimberg pdu->hdr.hlen = sizeof(*pdu); 24073f2304f8SSagi Grimberg pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 24083f2304f8SSagi Grimberg pdu->hdr.plen = 24093f2304f8SSagi Grimberg cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 24103f2304f8SSagi Grimberg 24113f2304f8SSagi Grimberg ret = nvme_tcp_map_data(queue, rq); 24123f2304f8SSagi Grimberg if (unlikely(ret)) { 241328a4cac4SMax Gurtovoy nvme_cleanup_cmd(rq); 24143f2304f8SSagi Grimberg dev_err(queue->ctrl->ctrl.device, 24153f2304f8SSagi Grimberg "Failed to map data (%d)\n", ret); 24163f2304f8SSagi Grimberg return ret; 24173f2304f8SSagi Grimberg } 24183f2304f8SSagi Grimberg 24193f2304f8SSagi Grimberg return 0; 24203f2304f8SSagi Grimberg } 24213f2304f8SSagi Grimberg 242286f0348aSSagi Grimberg static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 242386f0348aSSagi Grimberg { 242486f0348aSSagi Grimberg struct nvme_tcp_queue *queue = hctx->driver_data; 242586f0348aSSagi Grimberg 242686f0348aSSagi Grimberg if (!llist_empty(&queue->req_list)) 242786f0348aSSagi Grimberg queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 242886f0348aSSagi Grimberg } 242986f0348aSSagi Grimberg 24303f2304f8SSagi Grimberg static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 24313f2304f8SSagi Grimberg const struct blk_mq_queue_data *bd) 24323f2304f8SSagi Grimberg { 24333f2304f8SSagi Grimberg struct nvme_ns *ns = hctx->queue->queuedata; 24343f2304f8SSagi Grimberg struct nvme_tcp_queue *queue = hctx->driver_data; 24353f2304f8SSagi Grimberg struct request *rq = bd->rq; 24363f2304f8SSagi Grimberg struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 24373f2304f8SSagi Grimberg bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 24383f2304f8SSagi Grimberg blk_status_t ret; 24393f2304f8SSagi Grimberg 2440a9715744STao Chiu if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2441a9715744STao Chiu return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 24423f2304f8SSagi Grimberg 24433f2304f8SSagi Grimberg ret = nvme_tcp_setup_cmd_pdu(ns, rq); 24443f2304f8SSagi Grimberg if (unlikely(ret)) 24453f2304f8SSagi Grimberg return ret; 24463f2304f8SSagi Grimberg 24476887fc64SSagi Grimberg nvme_start_request(rq); 24483f2304f8SSagi Grimberg 244986f0348aSSagi Grimberg nvme_tcp_queue_request(req, true, bd->last); 24503f2304f8SSagi Grimberg 24513f2304f8SSagi Grimberg return BLK_STS_OK; 24523f2304f8SSagi Grimberg } 24533f2304f8SSagi Grimberg 2454a4e1d0b7SBart Van Assche static void nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2455873946f4SSagi Grimberg { 245606427ca0SChristoph Hellwig struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data); 245764861993SSagi Grimberg struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2458873946f4SSagi Grimberg 245964861993SSagi Grimberg if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { 2460873946f4SSagi Grimberg /* separate read/write queues */ 2461873946f4SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].nr_queues = 246264861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 246364861993SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 246464861993SSagi Grimberg set->map[HCTX_TYPE_READ].nr_queues = 246564861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ]; 2466873946f4SSagi Grimberg set->map[HCTX_TYPE_READ].queue_offset = 246764861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2468873946f4SSagi Grimberg } else { 246964861993SSagi Grimberg /* shared read/write queues */ 2470873946f4SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].nr_queues = 247164861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 247264861993SSagi Grimberg set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 247364861993SSagi Grimberg set->map[HCTX_TYPE_READ].nr_queues = 247464861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2475873946f4SSagi Grimberg set->map[HCTX_TYPE_READ].queue_offset = 0; 2476873946f4SSagi Grimberg } 2477873946f4SSagi Grimberg blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 2478873946f4SSagi Grimberg blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 247964861993SSagi Grimberg 24801a9460ceSSagi Grimberg if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { 24811a9460ceSSagi Grimberg /* map dedicated poll queues only if we have queues left */ 24821a9460ceSSagi Grimberg set->map[HCTX_TYPE_POLL].nr_queues = 24831a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL]; 24841a9460ceSSagi Grimberg set->map[HCTX_TYPE_POLL].queue_offset = 24851a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT] + 24861a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ]; 24871a9460ceSSagi Grimberg blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 24881a9460ceSSagi Grimberg } 24891a9460ceSSagi Grimberg 249064861993SSagi Grimberg dev_info(ctrl->ctrl.device, 24911a9460ceSSagi Grimberg "mapped %d/%d/%d default/read/poll queues.\n", 249264861993SSagi Grimberg ctrl->io_queues[HCTX_TYPE_DEFAULT], 24931a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_READ], 24941a9460ceSSagi Grimberg ctrl->io_queues[HCTX_TYPE_POLL]); 2495873946f4SSagi Grimberg } 2496873946f4SSagi Grimberg 24975a72e899SJens Axboe static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 24981a9460ceSSagi Grimberg { 24991a9460ceSSagi Grimberg struct nvme_tcp_queue *queue = hctx->driver_data; 25001a9460ceSSagi Grimberg struct sock *sk = queue->sock->sk; 25011a9460ceSSagi Grimberg 2502f86e5bf8SSagi Grimberg if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2503f86e5bf8SSagi Grimberg return 0; 2504f86e5bf8SSagi Grimberg 250572e5d757SSagi Grimberg set_bit(NVME_TCP_Q_POLLING, &queue->flags); 25063f926af3SEric Dumazet if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 25071a9460ceSSagi Grimberg sk_busy_loop(sk, true); 25081a9460ceSSagi Grimberg nvme_tcp_try_recv(queue); 250972e5d757SSagi Grimberg clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 25101a9460ceSSagi Grimberg return queue->nr_cqe; 25111a9460ceSSagi Grimberg } 25121a9460ceSSagi Grimberg 251302c57a82SMartin Belanger static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size) 251402c57a82SMartin Belanger { 251502c57a82SMartin Belanger struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0]; 251602c57a82SMartin Belanger struct sockaddr_storage src_addr; 251702c57a82SMartin Belanger int ret, len; 251802c57a82SMartin Belanger 251902c57a82SMartin Belanger len = nvmf_get_address(ctrl, buf, size); 252002c57a82SMartin Belanger 252176d54bf2SAkinobu Mita mutex_lock(&queue->queue_lock); 252276d54bf2SAkinobu Mita 252376d54bf2SAkinobu Mita if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 252476d54bf2SAkinobu Mita goto done; 252502c57a82SMartin Belanger ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr); 252602c57a82SMartin Belanger if (ret > 0) { 252702c57a82SMartin Belanger if (len > 0) 252802c57a82SMartin Belanger len--; /* strip trailing newline */ 252902c57a82SMartin Belanger len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n", 253002c57a82SMartin Belanger (len) ? "," : "", &src_addr); 253102c57a82SMartin Belanger } 253276d54bf2SAkinobu Mita done: 253376d54bf2SAkinobu Mita mutex_unlock(&queue->queue_lock); 253402c57a82SMartin Belanger 253502c57a82SMartin Belanger return len; 253602c57a82SMartin Belanger } 253702c57a82SMartin Belanger 25386acbd961SRikard Falkeborn static const struct blk_mq_ops nvme_tcp_mq_ops = { 25393f2304f8SSagi Grimberg .queue_rq = nvme_tcp_queue_rq, 254086f0348aSSagi Grimberg .commit_rqs = nvme_tcp_commit_rqs, 25413f2304f8SSagi Grimberg .complete = nvme_complete_rq, 25423f2304f8SSagi Grimberg .init_request = nvme_tcp_init_request, 25433f2304f8SSagi Grimberg .exit_request = nvme_tcp_exit_request, 25443f2304f8SSagi Grimberg .init_hctx = nvme_tcp_init_hctx, 25453f2304f8SSagi Grimberg .timeout = nvme_tcp_timeout, 2546873946f4SSagi Grimberg .map_queues = nvme_tcp_map_queues, 25471a9460ceSSagi Grimberg .poll = nvme_tcp_poll, 25483f2304f8SSagi Grimberg }; 25493f2304f8SSagi Grimberg 25506acbd961SRikard Falkeborn static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 25513f2304f8SSagi Grimberg .queue_rq = nvme_tcp_queue_rq, 25523f2304f8SSagi Grimberg .complete = nvme_complete_rq, 25533f2304f8SSagi Grimberg .init_request = nvme_tcp_init_request, 25543f2304f8SSagi Grimberg .exit_request = nvme_tcp_exit_request, 25553f2304f8SSagi Grimberg .init_hctx = nvme_tcp_init_admin_hctx, 25563f2304f8SSagi Grimberg .timeout = nvme_tcp_timeout, 25573f2304f8SSagi Grimberg }; 25583f2304f8SSagi Grimberg 25593f2304f8SSagi Grimberg static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 25603f2304f8SSagi Grimberg .name = "tcp", 25613f2304f8SSagi Grimberg .module = THIS_MODULE, 2562db45e1a5SChristoph Hellwig .flags = NVME_F_FABRICS | NVME_F_BLOCKING, 25633f2304f8SSagi Grimberg .reg_read32 = nvmf_reg_read32, 25643f2304f8SSagi Grimberg .reg_read64 = nvmf_reg_read64, 25653f2304f8SSagi Grimberg .reg_write32 = nvmf_reg_write32, 25663f2304f8SSagi Grimberg .free_ctrl = nvme_tcp_free_ctrl, 25673f2304f8SSagi Grimberg .submit_async_event = nvme_tcp_submit_async_event, 25683f2304f8SSagi Grimberg .delete_ctrl = nvme_tcp_delete_ctrl, 256902c57a82SMartin Belanger .get_address = nvme_tcp_get_address, 2570f7f70f4aSRuozhu Li .stop_ctrl = nvme_tcp_stop_ctrl, 25713f2304f8SSagi Grimberg }; 25723f2304f8SSagi Grimberg 25733f2304f8SSagi Grimberg static bool 25743f2304f8SSagi Grimberg nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 25753f2304f8SSagi Grimberg { 25763f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 25773f2304f8SSagi Grimberg bool found = false; 25783f2304f8SSagi Grimberg 25793f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 25803f2304f8SSagi Grimberg list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 25813f2304f8SSagi Grimberg found = nvmf_ip_options_match(&ctrl->ctrl, opts); 25823f2304f8SSagi Grimberg if (found) 25833f2304f8SSagi Grimberg break; 25843f2304f8SSagi Grimberg } 25853f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 25863f2304f8SSagi Grimberg 25873f2304f8SSagi Grimberg return found; 25883f2304f8SSagi Grimberg } 25893f2304f8SSagi Grimberg 25903f2304f8SSagi Grimberg static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 25913f2304f8SSagi Grimberg struct nvmf_ctrl_options *opts) 25923f2304f8SSagi Grimberg { 25933f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 25943f2304f8SSagi Grimberg int ret; 25953f2304f8SSagi Grimberg 25963f2304f8SSagi Grimberg ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 25973f2304f8SSagi Grimberg if (!ctrl) 25983f2304f8SSagi Grimberg return ERR_PTR(-ENOMEM); 25993f2304f8SSagi Grimberg 26003f2304f8SSagi Grimberg INIT_LIST_HEAD(&ctrl->list); 26013f2304f8SSagi Grimberg ctrl->ctrl.opts = opts; 26021a9460ceSSagi Grimberg ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 26031a9460ceSSagi Grimberg opts->nr_poll_queues + 1; 26043f2304f8SSagi Grimberg ctrl->ctrl.sqsize = opts->queue_size - 1; 26053f2304f8SSagi Grimberg ctrl->ctrl.kato = opts->kato; 26063f2304f8SSagi Grimberg 26073f2304f8SSagi Grimberg INIT_DELAYED_WORK(&ctrl->connect_work, 26083f2304f8SSagi Grimberg nvme_tcp_reconnect_ctrl_work); 26093f2304f8SSagi Grimberg INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 26103f2304f8SSagi Grimberg INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 26113f2304f8SSagi Grimberg 26123f2304f8SSagi Grimberg if (!(opts->mask & NVMF_OPT_TRSVCID)) { 26133f2304f8SSagi Grimberg opts->trsvcid = 26143f2304f8SSagi Grimberg kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 26153f2304f8SSagi Grimberg if (!opts->trsvcid) { 26163f2304f8SSagi Grimberg ret = -ENOMEM; 26173f2304f8SSagi Grimberg goto out_free_ctrl; 26183f2304f8SSagi Grimberg } 26193f2304f8SSagi Grimberg opts->mask |= NVMF_OPT_TRSVCID; 26203f2304f8SSagi Grimberg } 26213f2304f8SSagi Grimberg 26223f2304f8SSagi Grimberg ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 26233f2304f8SSagi Grimberg opts->traddr, opts->trsvcid, &ctrl->addr); 26243f2304f8SSagi Grimberg if (ret) { 26253f2304f8SSagi Grimberg pr_err("malformed address passed: %s:%s\n", 26263f2304f8SSagi Grimberg opts->traddr, opts->trsvcid); 26273f2304f8SSagi Grimberg goto out_free_ctrl; 26283f2304f8SSagi Grimberg } 26293f2304f8SSagi Grimberg 26303f2304f8SSagi Grimberg if (opts->mask & NVMF_OPT_HOST_TRADDR) { 26313f2304f8SSagi Grimberg ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 26323f2304f8SSagi Grimberg opts->host_traddr, NULL, &ctrl->src_addr); 26333f2304f8SSagi Grimberg if (ret) { 26343f2304f8SSagi Grimberg pr_err("malformed src address passed: %s\n", 26353f2304f8SSagi Grimberg opts->host_traddr); 26363f2304f8SSagi Grimberg goto out_free_ctrl; 26373f2304f8SSagi Grimberg } 26383f2304f8SSagi Grimberg } 26393f2304f8SSagi Grimberg 26403ede8f72SMartin Belanger if (opts->mask & NVMF_OPT_HOST_IFACE) { 26418b43ced6SPrabhakar Kushwaha if (!__dev_get_by_name(&init_net, opts->host_iface)) { 26423ede8f72SMartin Belanger pr_err("invalid interface passed: %s\n", 26433ede8f72SMartin Belanger opts->host_iface); 26443ede8f72SMartin Belanger ret = -ENODEV; 26453ede8f72SMartin Belanger goto out_free_ctrl; 26463ede8f72SMartin Belanger } 26473ede8f72SMartin Belanger } 26483ede8f72SMartin Belanger 26493f2304f8SSagi Grimberg if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 26503f2304f8SSagi Grimberg ret = -EALREADY; 26513f2304f8SSagi Grimberg goto out_free_ctrl; 26523f2304f8SSagi Grimberg } 26533f2304f8SSagi Grimberg 2654873946f4SSagi Grimberg ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 26553f2304f8SSagi Grimberg GFP_KERNEL); 26563f2304f8SSagi Grimberg if (!ctrl->queues) { 26573f2304f8SSagi Grimberg ret = -ENOMEM; 26583f2304f8SSagi Grimberg goto out_free_ctrl; 26593f2304f8SSagi Grimberg } 26603f2304f8SSagi Grimberg 26613f2304f8SSagi Grimberg ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 26623f2304f8SSagi Grimberg if (ret) 26633f2304f8SSagi Grimberg goto out_kfree_queues; 26643f2304f8SSagi Grimberg 26653f2304f8SSagi Grimberg if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 26663f2304f8SSagi Grimberg WARN_ON_ONCE(1); 26673f2304f8SSagi Grimberg ret = -EINTR; 26683f2304f8SSagi Grimberg goto out_uninit_ctrl; 26693f2304f8SSagi Grimberg } 26703f2304f8SSagi Grimberg 26713f2304f8SSagi Grimberg ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 26723f2304f8SSagi Grimberg if (ret) 26733f2304f8SSagi Grimberg goto out_uninit_ctrl; 26743f2304f8SSagi Grimberg 26753f2304f8SSagi Grimberg dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 2676e5ea42faSHannes Reinecke nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr); 26773f2304f8SSagi Grimberg 26783f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 26793f2304f8SSagi Grimberg list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 26803f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 26813f2304f8SSagi Grimberg 26823f2304f8SSagi Grimberg return &ctrl->ctrl; 26833f2304f8SSagi Grimberg 26843f2304f8SSagi Grimberg out_uninit_ctrl: 26853f2304f8SSagi Grimberg nvme_uninit_ctrl(&ctrl->ctrl); 26863f2304f8SSagi Grimberg nvme_put_ctrl(&ctrl->ctrl); 26873f2304f8SSagi Grimberg if (ret > 0) 26883f2304f8SSagi Grimberg ret = -EIO; 26893f2304f8SSagi Grimberg return ERR_PTR(ret); 26903f2304f8SSagi Grimberg out_kfree_queues: 26913f2304f8SSagi Grimberg kfree(ctrl->queues); 26923f2304f8SSagi Grimberg out_free_ctrl: 26933f2304f8SSagi Grimberg kfree(ctrl); 26943f2304f8SSagi Grimberg return ERR_PTR(ret); 26953f2304f8SSagi Grimberg } 26963f2304f8SSagi Grimberg 26973f2304f8SSagi Grimberg static struct nvmf_transport_ops nvme_tcp_transport = { 26983f2304f8SSagi Grimberg .name = "tcp", 26993f2304f8SSagi Grimberg .module = THIS_MODULE, 27003f2304f8SSagi Grimberg .required_opts = NVMF_OPT_TRADDR, 27013f2304f8SSagi Grimberg .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 27023f2304f8SSagi Grimberg NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2703873946f4SSagi Grimberg NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2704bb13985dSIsrael Rukshin NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 27053ede8f72SMartin Belanger NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE, 27063f2304f8SSagi Grimberg .create_ctrl = nvme_tcp_create_ctrl, 27073f2304f8SSagi Grimberg }; 27083f2304f8SSagi Grimberg 27093f2304f8SSagi Grimberg static int __init nvme_tcp_init_module(void) 27103f2304f8SSagi Grimberg { 27117e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8); 27127e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72); 27137e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24); 27147e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24); 27157e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24); 27167e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128); 27177e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128); 27187e87965dSSagi Grimberg BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24); 27197e87965dSSagi Grimberg 27203f2304f8SSagi Grimberg nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 27213f2304f8SSagi Grimberg WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 27223f2304f8SSagi Grimberg if (!nvme_tcp_wq) 27233f2304f8SSagi Grimberg return -ENOMEM; 27243f2304f8SSagi Grimberg 27253f2304f8SSagi Grimberg nvmf_register_transport(&nvme_tcp_transport); 27263f2304f8SSagi Grimberg return 0; 27273f2304f8SSagi Grimberg } 27283f2304f8SSagi Grimberg 27293f2304f8SSagi Grimberg static void __exit nvme_tcp_cleanup_module(void) 27303f2304f8SSagi Grimberg { 27313f2304f8SSagi Grimberg struct nvme_tcp_ctrl *ctrl; 27323f2304f8SSagi Grimberg 27333f2304f8SSagi Grimberg nvmf_unregister_transport(&nvme_tcp_transport); 27343f2304f8SSagi Grimberg 27353f2304f8SSagi Grimberg mutex_lock(&nvme_tcp_ctrl_mutex); 27363f2304f8SSagi Grimberg list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 27373f2304f8SSagi Grimberg nvme_delete_ctrl(&ctrl->ctrl); 27383f2304f8SSagi Grimberg mutex_unlock(&nvme_tcp_ctrl_mutex); 27393f2304f8SSagi Grimberg flush_workqueue(nvme_delete_wq); 27403f2304f8SSagi Grimberg 27413f2304f8SSagi Grimberg destroy_workqueue(nvme_tcp_wq); 27423f2304f8SSagi Grimberg } 27433f2304f8SSagi Grimberg 27443f2304f8SSagi Grimberg module_init(nvme_tcp_init_module); 27453f2304f8SSagi Grimberg module_exit(nvme_tcp_cleanup_module); 27463f2304f8SSagi Grimberg 27473f2304f8SSagi Grimberg MODULE_LICENSE("GPL v2"); 2748