1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP host. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/err.h> 11 #include <linux/nvme-tcp.h> 12 #include <net/sock.h> 13 #include <net/tcp.h> 14 #include <linux/blk-mq.h> 15 #include <crypto/hash.h> 16 #include <net/busy_poll.h> 17 18 #include "nvme.h" 19 #include "fabrics.h" 20 21 struct nvme_tcp_queue; 22 23 /* Define the socket priority to use for connections were it is desirable 24 * that the NIC consider performing optimized packet processing or filtering. 25 * A non-zero value being sufficient to indicate general consideration of any 26 * possible optimization. Making it a module param allows for alternative 27 * values that may be unique for some NIC implementations. 28 */ 29 static int so_priority; 30 module_param(so_priority, int, 0644); 31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 32 33 #ifdef CONFIG_DEBUG_LOCK_ALLOC 34 /* lockdep can detect a circular dependency of the form 35 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock 36 * because dependencies are tracked for both nvme-tcp and user contexts. Using 37 * a separate class prevents lockdep from conflating nvme-tcp socket use with 38 * user-space socket API use. 39 */ 40 static struct lock_class_key nvme_tcp_sk_key[2]; 41 static struct lock_class_key nvme_tcp_slock_key[2]; 42 43 static void nvme_tcp_reclassify_socket(struct socket *sock) 44 { 45 struct sock *sk = sock->sk; 46 47 if (WARN_ON_ONCE(!sock_allow_reclassification(sk))) 48 return; 49 50 switch (sk->sk_family) { 51 case AF_INET: 52 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME", 53 &nvme_tcp_slock_key[0], 54 "sk_lock-AF_INET-NVME", 55 &nvme_tcp_sk_key[0]); 56 break; 57 case AF_INET6: 58 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME", 59 &nvme_tcp_slock_key[1], 60 "sk_lock-AF_INET6-NVME", 61 &nvme_tcp_sk_key[1]); 62 break; 63 default: 64 WARN_ON_ONCE(1); 65 } 66 } 67 #else 68 static void nvme_tcp_reclassify_socket(struct socket *sock) { } 69 #endif 70 71 enum nvme_tcp_send_state { 72 NVME_TCP_SEND_CMD_PDU = 0, 73 NVME_TCP_SEND_H2C_PDU, 74 NVME_TCP_SEND_DATA, 75 NVME_TCP_SEND_DDGST, 76 }; 77 78 struct nvme_tcp_request { 79 struct nvme_request req; 80 void *pdu; 81 struct nvme_tcp_queue *queue; 82 u32 data_len; 83 u32 pdu_len; 84 u32 pdu_sent; 85 u32 h2cdata_left; 86 u32 h2cdata_offset; 87 u16 ttag; 88 __le16 status; 89 struct list_head entry; 90 struct llist_node lentry; 91 __le32 ddgst; 92 93 struct bio *curr_bio; 94 struct iov_iter iter; 95 96 /* send state */ 97 size_t offset; 98 size_t data_sent; 99 enum nvme_tcp_send_state state; 100 }; 101 102 enum nvme_tcp_queue_flags { 103 NVME_TCP_Q_ALLOCATED = 0, 104 NVME_TCP_Q_LIVE = 1, 105 NVME_TCP_Q_POLLING = 2, 106 }; 107 108 enum nvme_tcp_recv_state { 109 NVME_TCP_RECV_PDU = 0, 110 NVME_TCP_RECV_DATA, 111 NVME_TCP_RECV_DDGST, 112 }; 113 114 struct nvme_tcp_ctrl; 115 struct nvme_tcp_queue { 116 struct socket *sock; 117 struct work_struct io_work; 118 int io_cpu; 119 120 struct mutex queue_lock; 121 struct mutex send_mutex; 122 struct llist_head req_list; 123 struct list_head send_list; 124 bool more_requests; 125 126 /* recv state */ 127 void *pdu; 128 int pdu_remaining; 129 int pdu_offset; 130 size_t data_remaining; 131 size_t ddgst_remaining; 132 unsigned int nr_cqe; 133 134 /* send state */ 135 struct nvme_tcp_request *request; 136 137 int queue_size; 138 u32 maxh2cdata; 139 size_t cmnd_capsule_len; 140 struct nvme_tcp_ctrl *ctrl; 141 unsigned long flags; 142 bool rd_enabled; 143 144 bool hdr_digest; 145 bool data_digest; 146 struct ahash_request *rcv_hash; 147 struct ahash_request *snd_hash; 148 __le32 exp_ddgst; 149 __le32 recv_ddgst; 150 151 struct page_frag_cache pf_cache; 152 153 void (*state_change)(struct sock *); 154 void (*data_ready)(struct sock *); 155 void (*write_space)(struct sock *); 156 }; 157 158 struct nvme_tcp_ctrl { 159 /* read only in the hot path */ 160 struct nvme_tcp_queue *queues; 161 struct blk_mq_tag_set tag_set; 162 163 /* other member variables */ 164 struct list_head list; 165 struct blk_mq_tag_set admin_tag_set; 166 struct sockaddr_storage addr; 167 struct sockaddr_storage src_addr; 168 struct nvme_ctrl ctrl; 169 170 struct work_struct err_work; 171 struct delayed_work connect_work; 172 struct nvme_tcp_request async_req; 173 u32 io_queues[HCTX_MAX_TYPES]; 174 }; 175 176 static LIST_HEAD(nvme_tcp_ctrl_list); 177 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 178 static struct workqueue_struct *nvme_tcp_wq; 179 static const struct blk_mq_ops nvme_tcp_mq_ops; 180 static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 181 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 182 183 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 184 { 185 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 186 } 187 188 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 189 { 190 return queue - queue->ctrl->queues; 191 } 192 193 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 194 { 195 u32 queue_idx = nvme_tcp_queue_id(queue); 196 197 if (queue_idx == 0) 198 return queue->ctrl->admin_tag_set.tags[queue_idx]; 199 return queue->ctrl->tag_set.tags[queue_idx - 1]; 200 } 201 202 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 203 { 204 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 205 } 206 207 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 208 { 209 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 210 } 211 212 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req) 213 { 214 if (nvme_is_fabrics(req->req.cmd)) 215 return NVME_TCP_ADMIN_CCSZ; 216 return req->queue->cmnd_capsule_len - sizeof(struct nvme_command); 217 } 218 219 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 220 { 221 return req == &req->queue->ctrl->async_req; 222 } 223 224 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 225 { 226 struct request *rq; 227 228 if (unlikely(nvme_tcp_async_req(req))) 229 return false; /* async events don't have a request */ 230 231 rq = blk_mq_rq_from_pdu(req); 232 233 return rq_data_dir(rq) == WRITE && req->data_len && 234 req->data_len <= nvme_tcp_inline_data_size(req); 235 } 236 237 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 238 { 239 return req->iter.bvec->bv_page; 240 } 241 242 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 243 { 244 return req->iter.bvec->bv_offset + req->iter.iov_offset; 245 } 246 247 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 248 { 249 return min_t(size_t, iov_iter_single_seg_count(&req->iter), 250 req->pdu_len - req->pdu_sent); 251 } 252 253 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 254 { 255 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 256 req->pdu_len - req->pdu_sent : 0; 257 } 258 259 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 260 int len) 261 { 262 return nvme_tcp_pdu_data_left(req) <= len; 263 } 264 265 static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 266 unsigned int dir) 267 { 268 struct request *rq = blk_mq_rq_from_pdu(req); 269 struct bio_vec *vec; 270 unsigned int size; 271 int nr_bvec; 272 size_t offset; 273 274 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 275 vec = &rq->special_vec; 276 nr_bvec = 1; 277 size = blk_rq_payload_bytes(rq); 278 offset = 0; 279 } else { 280 struct bio *bio = req->curr_bio; 281 struct bvec_iter bi; 282 struct bio_vec bv; 283 284 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 285 nr_bvec = 0; 286 bio_for_each_bvec(bv, bio, bi) { 287 nr_bvec++; 288 } 289 size = bio->bi_iter.bi_size; 290 offset = bio->bi_iter.bi_bvec_done; 291 } 292 293 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); 294 req->iter.iov_offset = offset; 295 } 296 297 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 298 int len) 299 { 300 req->data_sent += len; 301 req->pdu_sent += len; 302 iov_iter_advance(&req->iter, len); 303 if (!iov_iter_count(&req->iter) && 304 req->data_sent < req->data_len) { 305 req->curr_bio = req->curr_bio->bi_next; 306 nvme_tcp_init_iter(req, WRITE); 307 } 308 } 309 310 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 311 { 312 int ret; 313 314 /* drain the send queue as much as we can... */ 315 do { 316 ret = nvme_tcp_try_send(queue); 317 } while (ret > 0); 318 } 319 320 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 321 { 322 return !list_empty(&queue->send_list) || 323 !llist_empty(&queue->req_list) || queue->more_requests; 324 } 325 326 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 327 bool sync, bool last) 328 { 329 struct nvme_tcp_queue *queue = req->queue; 330 bool empty; 331 332 empty = llist_add(&req->lentry, &queue->req_list) && 333 list_empty(&queue->send_list) && !queue->request; 334 335 /* 336 * if we're the first on the send_list and we can try to send 337 * directly, otherwise queue io_work. Also, only do that if we 338 * are on the same cpu, so we don't introduce contention. 339 */ 340 if (queue->io_cpu == raw_smp_processor_id() && 341 sync && empty && mutex_trylock(&queue->send_mutex)) { 342 queue->more_requests = !last; 343 nvme_tcp_send_all(queue); 344 queue->more_requests = false; 345 mutex_unlock(&queue->send_mutex); 346 } 347 348 if (last && nvme_tcp_queue_more(queue)) 349 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 350 } 351 352 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 353 { 354 struct nvme_tcp_request *req; 355 struct llist_node *node; 356 357 for (node = llist_del_all(&queue->req_list); node; node = node->next) { 358 req = llist_entry(node, struct nvme_tcp_request, lentry); 359 list_add(&req->entry, &queue->send_list); 360 } 361 } 362 363 static inline struct nvme_tcp_request * 364 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 365 { 366 struct nvme_tcp_request *req; 367 368 req = list_first_entry_or_null(&queue->send_list, 369 struct nvme_tcp_request, entry); 370 if (!req) { 371 nvme_tcp_process_req_list(queue); 372 req = list_first_entry_or_null(&queue->send_list, 373 struct nvme_tcp_request, entry); 374 if (unlikely(!req)) 375 return NULL; 376 } 377 378 list_del(&req->entry); 379 return req; 380 } 381 382 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 383 __le32 *dgst) 384 { 385 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 386 crypto_ahash_final(hash); 387 } 388 389 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 390 struct page *page, off_t off, size_t len) 391 { 392 struct scatterlist sg; 393 394 sg_init_marker(&sg, 1); 395 sg_set_page(&sg, page, len, off); 396 ahash_request_set_crypt(hash, &sg, NULL, len); 397 crypto_ahash_update(hash); 398 } 399 400 static inline void nvme_tcp_hdgst(struct ahash_request *hash, 401 void *pdu, size_t len) 402 { 403 struct scatterlist sg; 404 405 sg_init_one(&sg, pdu, len); 406 ahash_request_set_crypt(hash, &sg, pdu + len, len); 407 crypto_ahash_digest(hash); 408 } 409 410 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 411 void *pdu, size_t pdu_len) 412 { 413 struct nvme_tcp_hdr *hdr = pdu; 414 __le32 recv_digest; 415 __le32 exp_digest; 416 417 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 418 dev_err(queue->ctrl->ctrl.device, 419 "queue %d: header digest flag is cleared\n", 420 nvme_tcp_queue_id(queue)); 421 return -EPROTO; 422 } 423 424 recv_digest = *(__le32 *)(pdu + hdr->hlen); 425 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 426 exp_digest = *(__le32 *)(pdu + hdr->hlen); 427 if (recv_digest != exp_digest) { 428 dev_err(queue->ctrl->ctrl.device, 429 "header digest error: recv %#x expected %#x\n", 430 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 431 return -EIO; 432 } 433 434 return 0; 435 } 436 437 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 438 { 439 struct nvme_tcp_hdr *hdr = pdu; 440 u8 digest_len = nvme_tcp_hdgst_len(queue); 441 u32 len; 442 443 len = le32_to_cpu(hdr->plen) - hdr->hlen - 444 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 445 446 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 447 dev_err(queue->ctrl->ctrl.device, 448 "queue %d: data digest flag is cleared\n", 449 nvme_tcp_queue_id(queue)); 450 return -EPROTO; 451 } 452 crypto_ahash_init(queue->rcv_hash); 453 454 return 0; 455 } 456 457 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 458 struct request *rq, unsigned int hctx_idx) 459 { 460 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 461 462 page_frag_free(req->pdu); 463 } 464 465 static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 466 struct request *rq, unsigned int hctx_idx, 467 unsigned int numa_node) 468 { 469 struct nvme_tcp_ctrl *ctrl = set->driver_data; 470 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 471 struct nvme_tcp_cmd_pdu *pdu; 472 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 473 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 474 u8 hdgst = nvme_tcp_hdgst_len(queue); 475 476 req->pdu = page_frag_alloc(&queue->pf_cache, 477 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 478 GFP_KERNEL | __GFP_ZERO); 479 if (!req->pdu) 480 return -ENOMEM; 481 482 pdu = req->pdu; 483 req->queue = queue; 484 nvme_req(rq)->ctrl = &ctrl->ctrl; 485 nvme_req(rq)->cmd = &pdu->cmd; 486 487 return 0; 488 } 489 490 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 491 unsigned int hctx_idx) 492 { 493 struct nvme_tcp_ctrl *ctrl = data; 494 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 495 496 hctx->driver_data = queue; 497 return 0; 498 } 499 500 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 501 unsigned int hctx_idx) 502 { 503 struct nvme_tcp_ctrl *ctrl = data; 504 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 505 506 hctx->driver_data = queue; 507 return 0; 508 } 509 510 static enum nvme_tcp_recv_state 511 nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 512 { 513 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 514 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 515 NVME_TCP_RECV_DATA; 516 } 517 518 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 519 { 520 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 521 nvme_tcp_hdgst_len(queue); 522 queue->pdu_offset = 0; 523 queue->data_remaining = -1; 524 queue->ddgst_remaining = 0; 525 } 526 527 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 528 { 529 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 530 return; 531 532 dev_warn(ctrl->device, "starting error recovery\n"); 533 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 534 } 535 536 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 537 struct nvme_completion *cqe) 538 { 539 struct nvme_tcp_request *req; 540 struct request *rq; 541 542 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 543 if (!rq) { 544 dev_err(queue->ctrl->ctrl.device, 545 "got bad cqe.command_id %#x on queue %d\n", 546 cqe->command_id, nvme_tcp_queue_id(queue)); 547 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 548 return -EINVAL; 549 } 550 551 req = blk_mq_rq_to_pdu(rq); 552 if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) 553 req->status = cqe->status; 554 555 if (!nvme_try_complete_req(rq, req->status, cqe->result)) 556 nvme_complete_rq(rq); 557 queue->nr_cqe++; 558 559 return 0; 560 } 561 562 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 563 struct nvme_tcp_data_pdu *pdu) 564 { 565 struct request *rq; 566 567 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 568 if (!rq) { 569 dev_err(queue->ctrl->ctrl.device, 570 "got bad c2hdata.command_id %#x on queue %d\n", 571 pdu->command_id, nvme_tcp_queue_id(queue)); 572 return -ENOENT; 573 } 574 575 if (!blk_rq_payload_bytes(rq)) { 576 dev_err(queue->ctrl->ctrl.device, 577 "queue %d tag %#x unexpected data\n", 578 nvme_tcp_queue_id(queue), rq->tag); 579 return -EIO; 580 } 581 582 queue->data_remaining = le32_to_cpu(pdu->data_length); 583 584 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 585 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 586 dev_err(queue->ctrl->ctrl.device, 587 "queue %d tag %#x SUCCESS set but not last PDU\n", 588 nvme_tcp_queue_id(queue), rq->tag); 589 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 590 return -EPROTO; 591 } 592 593 return 0; 594 } 595 596 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 597 struct nvme_tcp_rsp_pdu *pdu) 598 { 599 struct nvme_completion *cqe = &pdu->cqe; 600 int ret = 0; 601 602 /* 603 * AEN requests are special as they don't time out and can 604 * survive any kind of queue freeze and often don't respond to 605 * aborts. We don't even bother to allocate a struct request 606 * for them but rather special case them here. 607 */ 608 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 609 cqe->command_id))) 610 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 611 &cqe->result); 612 else 613 ret = nvme_tcp_process_nvme_cqe(queue, cqe); 614 615 return ret; 616 } 617 618 static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req) 619 { 620 struct nvme_tcp_data_pdu *data = req->pdu; 621 struct nvme_tcp_queue *queue = req->queue; 622 struct request *rq = blk_mq_rq_from_pdu(req); 623 u32 h2cdata_sent = req->pdu_len; 624 u8 hdgst = nvme_tcp_hdgst_len(queue); 625 u8 ddgst = nvme_tcp_ddgst_len(queue); 626 627 req->state = NVME_TCP_SEND_H2C_PDU; 628 req->offset = 0; 629 req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata); 630 req->pdu_sent = 0; 631 req->h2cdata_left -= req->pdu_len; 632 req->h2cdata_offset += h2cdata_sent; 633 634 memset(data, 0, sizeof(*data)); 635 data->hdr.type = nvme_tcp_h2c_data; 636 if (!req->h2cdata_left) 637 data->hdr.flags = NVME_TCP_F_DATA_LAST; 638 if (queue->hdr_digest) 639 data->hdr.flags |= NVME_TCP_F_HDGST; 640 if (queue->data_digest) 641 data->hdr.flags |= NVME_TCP_F_DDGST; 642 data->hdr.hlen = sizeof(*data); 643 data->hdr.pdo = data->hdr.hlen + hdgst; 644 data->hdr.plen = 645 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 646 data->ttag = req->ttag; 647 data->command_id = nvme_cid(rq); 648 data->data_offset = cpu_to_le32(req->h2cdata_offset); 649 data->data_length = cpu_to_le32(req->pdu_len); 650 } 651 652 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 653 struct nvme_tcp_r2t_pdu *pdu) 654 { 655 struct nvme_tcp_request *req; 656 struct request *rq; 657 u32 r2t_length = le32_to_cpu(pdu->r2t_length); 658 u32 r2t_offset = le32_to_cpu(pdu->r2t_offset); 659 660 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 661 if (!rq) { 662 dev_err(queue->ctrl->ctrl.device, 663 "got bad r2t.command_id %#x on queue %d\n", 664 pdu->command_id, nvme_tcp_queue_id(queue)); 665 return -ENOENT; 666 } 667 req = blk_mq_rq_to_pdu(rq); 668 669 if (unlikely(!r2t_length)) { 670 dev_err(queue->ctrl->ctrl.device, 671 "req %d r2t len is %u, probably a bug...\n", 672 rq->tag, r2t_length); 673 return -EPROTO; 674 } 675 676 if (unlikely(req->data_sent + r2t_length > req->data_len)) { 677 dev_err(queue->ctrl->ctrl.device, 678 "req %d r2t len %u exceeded data len %u (%zu sent)\n", 679 rq->tag, r2t_length, req->data_len, req->data_sent); 680 return -EPROTO; 681 } 682 683 if (unlikely(r2t_offset < req->data_sent)) { 684 dev_err(queue->ctrl->ctrl.device, 685 "req %d unexpected r2t offset %u (expected %zu)\n", 686 rq->tag, r2t_offset, req->data_sent); 687 return -EPROTO; 688 } 689 690 req->pdu_len = 0; 691 req->h2cdata_left = r2t_length; 692 req->h2cdata_offset = r2t_offset; 693 req->ttag = pdu->ttag; 694 695 nvme_tcp_setup_h2c_data_pdu(req); 696 nvme_tcp_queue_request(req, false, true); 697 698 return 0; 699 } 700 701 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 702 unsigned int *offset, size_t *len) 703 { 704 struct nvme_tcp_hdr *hdr; 705 char *pdu = queue->pdu; 706 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 707 int ret; 708 709 ret = skb_copy_bits(skb, *offset, 710 &pdu[queue->pdu_offset], rcv_len); 711 if (unlikely(ret)) 712 return ret; 713 714 queue->pdu_remaining -= rcv_len; 715 queue->pdu_offset += rcv_len; 716 *offset += rcv_len; 717 *len -= rcv_len; 718 if (queue->pdu_remaining) 719 return 0; 720 721 hdr = queue->pdu; 722 if (queue->hdr_digest) { 723 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 724 if (unlikely(ret)) 725 return ret; 726 } 727 728 729 if (queue->data_digest) { 730 ret = nvme_tcp_check_ddgst(queue, queue->pdu); 731 if (unlikely(ret)) 732 return ret; 733 } 734 735 switch (hdr->type) { 736 case nvme_tcp_c2h_data: 737 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 738 case nvme_tcp_rsp: 739 nvme_tcp_init_recv_ctx(queue); 740 return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 741 case nvme_tcp_r2t: 742 nvme_tcp_init_recv_ctx(queue); 743 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 744 default: 745 dev_err(queue->ctrl->ctrl.device, 746 "unsupported pdu type (%d)\n", hdr->type); 747 return -EINVAL; 748 } 749 } 750 751 static inline void nvme_tcp_end_request(struct request *rq, u16 status) 752 { 753 union nvme_result res = {}; 754 755 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 756 nvme_complete_rq(rq); 757 } 758 759 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 760 unsigned int *offset, size_t *len) 761 { 762 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 763 struct request *rq = 764 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 765 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 766 767 while (true) { 768 int recv_len, ret; 769 770 recv_len = min_t(size_t, *len, queue->data_remaining); 771 if (!recv_len) 772 break; 773 774 if (!iov_iter_count(&req->iter)) { 775 req->curr_bio = req->curr_bio->bi_next; 776 777 /* 778 * If we don`t have any bios it means that controller 779 * sent more data than we requested, hence error 780 */ 781 if (!req->curr_bio) { 782 dev_err(queue->ctrl->ctrl.device, 783 "queue %d no space in request %#x", 784 nvme_tcp_queue_id(queue), rq->tag); 785 nvme_tcp_init_recv_ctx(queue); 786 return -EIO; 787 } 788 nvme_tcp_init_iter(req, READ); 789 } 790 791 /* we can read only from what is left in this bio */ 792 recv_len = min_t(size_t, recv_len, 793 iov_iter_count(&req->iter)); 794 795 if (queue->data_digest) 796 ret = skb_copy_and_hash_datagram_iter(skb, *offset, 797 &req->iter, recv_len, queue->rcv_hash); 798 else 799 ret = skb_copy_datagram_iter(skb, *offset, 800 &req->iter, recv_len); 801 if (ret) { 802 dev_err(queue->ctrl->ctrl.device, 803 "queue %d failed to copy request %#x data", 804 nvme_tcp_queue_id(queue), rq->tag); 805 return ret; 806 } 807 808 *len -= recv_len; 809 *offset += recv_len; 810 queue->data_remaining -= recv_len; 811 } 812 813 if (!queue->data_remaining) { 814 if (queue->data_digest) { 815 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 816 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 817 } else { 818 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 819 nvme_tcp_end_request(rq, 820 le16_to_cpu(req->status)); 821 queue->nr_cqe++; 822 } 823 nvme_tcp_init_recv_ctx(queue); 824 } 825 } 826 827 return 0; 828 } 829 830 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 831 struct sk_buff *skb, unsigned int *offset, size_t *len) 832 { 833 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 834 char *ddgst = (char *)&queue->recv_ddgst; 835 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 836 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 837 int ret; 838 839 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 840 if (unlikely(ret)) 841 return ret; 842 843 queue->ddgst_remaining -= recv_len; 844 *offset += recv_len; 845 *len -= recv_len; 846 if (queue->ddgst_remaining) 847 return 0; 848 849 if (queue->recv_ddgst != queue->exp_ddgst) { 850 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 851 pdu->command_id); 852 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 853 854 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR); 855 856 dev_err(queue->ctrl->ctrl.device, 857 "data digest error: recv %#x expected %#x\n", 858 le32_to_cpu(queue->recv_ddgst), 859 le32_to_cpu(queue->exp_ddgst)); 860 } 861 862 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 863 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 864 pdu->command_id); 865 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 866 867 nvme_tcp_end_request(rq, le16_to_cpu(req->status)); 868 queue->nr_cqe++; 869 } 870 871 nvme_tcp_init_recv_ctx(queue); 872 return 0; 873 } 874 875 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 876 unsigned int offset, size_t len) 877 { 878 struct nvme_tcp_queue *queue = desc->arg.data; 879 size_t consumed = len; 880 int result; 881 882 while (len) { 883 switch (nvme_tcp_recv_state(queue)) { 884 case NVME_TCP_RECV_PDU: 885 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 886 break; 887 case NVME_TCP_RECV_DATA: 888 result = nvme_tcp_recv_data(queue, skb, &offset, &len); 889 break; 890 case NVME_TCP_RECV_DDGST: 891 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 892 break; 893 default: 894 result = -EFAULT; 895 } 896 if (result) { 897 dev_err(queue->ctrl->ctrl.device, 898 "receive failed: %d\n", result); 899 queue->rd_enabled = false; 900 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 901 return result; 902 } 903 } 904 905 return consumed; 906 } 907 908 static void nvme_tcp_data_ready(struct sock *sk) 909 { 910 struct nvme_tcp_queue *queue; 911 912 read_lock_bh(&sk->sk_callback_lock); 913 queue = sk->sk_user_data; 914 if (likely(queue && queue->rd_enabled) && 915 !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 916 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 917 read_unlock_bh(&sk->sk_callback_lock); 918 } 919 920 static void nvme_tcp_write_space(struct sock *sk) 921 { 922 struct nvme_tcp_queue *queue; 923 924 read_lock_bh(&sk->sk_callback_lock); 925 queue = sk->sk_user_data; 926 if (likely(queue && sk_stream_is_writeable(sk))) { 927 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 928 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 929 } 930 read_unlock_bh(&sk->sk_callback_lock); 931 } 932 933 static void nvme_tcp_state_change(struct sock *sk) 934 { 935 struct nvme_tcp_queue *queue; 936 937 read_lock_bh(&sk->sk_callback_lock); 938 queue = sk->sk_user_data; 939 if (!queue) 940 goto done; 941 942 switch (sk->sk_state) { 943 case TCP_CLOSE: 944 case TCP_CLOSE_WAIT: 945 case TCP_LAST_ACK: 946 case TCP_FIN_WAIT1: 947 case TCP_FIN_WAIT2: 948 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 949 break; 950 default: 951 dev_info(queue->ctrl->ctrl.device, 952 "queue %d socket state %d\n", 953 nvme_tcp_queue_id(queue), sk->sk_state); 954 } 955 956 queue->state_change(sk); 957 done: 958 read_unlock_bh(&sk->sk_callback_lock); 959 } 960 961 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 962 { 963 queue->request = NULL; 964 } 965 966 static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 967 { 968 if (nvme_tcp_async_req(req)) { 969 union nvme_result res = {}; 970 971 nvme_complete_async_event(&req->queue->ctrl->ctrl, 972 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res); 973 } else { 974 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), 975 NVME_SC_HOST_PATH_ERROR); 976 } 977 } 978 979 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 980 { 981 struct nvme_tcp_queue *queue = req->queue; 982 int req_data_len = req->data_len; 983 u32 h2cdata_left = req->h2cdata_left; 984 985 while (true) { 986 struct page *page = nvme_tcp_req_cur_page(req); 987 size_t offset = nvme_tcp_req_cur_offset(req); 988 size_t len = nvme_tcp_req_cur_length(req); 989 bool last = nvme_tcp_pdu_last_send(req, len); 990 int req_data_sent = req->data_sent; 991 int ret, flags = MSG_DONTWAIT; 992 993 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 994 flags |= MSG_EOR; 995 else 996 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 997 998 if (sendpage_ok(page)) { 999 ret = kernel_sendpage(queue->sock, page, offset, len, 1000 flags); 1001 } else { 1002 ret = sock_no_sendpage(queue->sock, page, offset, len, 1003 flags); 1004 } 1005 if (ret <= 0) 1006 return ret; 1007 1008 if (queue->data_digest) 1009 nvme_tcp_ddgst_update(queue->snd_hash, page, 1010 offset, ret); 1011 1012 /* 1013 * update the request iterator except for the last payload send 1014 * in the request where we don't want to modify it as we may 1015 * compete with the RX path completing the request. 1016 */ 1017 if (req_data_sent + ret < req_data_len) 1018 nvme_tcp_advance_req(req, ret); 1019 1020 /* fully successful last send in current PDU */ 1021 if (last && ret == len) { 1022 if (queue->data_digest) { 1023 nvme_tcp_ddgst_final(queue->snd_hash, 1024 &req->ddgst); 1025 req->state = NVME_TCP_SEND_DDGST; 1026 req->offset = 0; 1027 } else { 1028 if (h2cdata_left) 1029 nvme_tcp_setup_h2c_data_pdu(req); 1030 else 1031 nvme_tcp_done_send_req(queue); 1032 } 1033 return 1; 1034 } 1035 } 1036 return -EAGAIN; 1037 } 1038 1039 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 1040 { 1041 struct nvme_tcp_queue *queue = req->queue; 1042 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 1043 bool inline_data = nvme_tcp_has_inline_data(req); 1044 u8 hdgst = nvme_tcp_hdgst_len(queue); 1045 int len = sizeof(*pdu) + hdgst - req->offset; 1046 int flags = MSG_DONTWAIT; 1047 int ret; 1048 1049 if (inline_data || nvme_tcp_queue_more(queue)) 1050 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 1051 else 1052 flags |= MSG_EOR; 1053 1054 if (queue->hdr_digest && !req->offset) 1055 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1056 1057 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1058 offset_in_page(pdu) + req->offset, len, flags); 1059 if (unlikely(ret <= 0)) 1060 return ret; 1061 1062 len -= ret; 1063 if (!len) { 1064 if (inline_data) { 1065 req->state = NVME_TCP_SEND_DATA; 1066 if (queue->data_digest) 1067 crypto_ahash_init(queue->snd_hash); 1068 } else { 1069 nvme_tcp_done_send_req(queue); 1070 } 1071 return 1; 1072 } 1073 req->offset += ret; 1074 1075 return -EAGAIN; 1076 } 1077 1078 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 1079 { 1080 struct nvme_tcp_queue *queue = req->queue; 1081 struct nvme_tcp_data_pdu *pdu = req->pdu; 1082 u8 hdgst = nvme_tcp_hdgst_len(queue); 1083 int len = sizeof(*pdu) - req->offset + hdgst; 1084 int ret; 1085 1086 if (queue->hdr_digest && !req->offset) 1087 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1088 1089 if (!req->h2cdata_left) 1090 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1091 offset_in_page(pdu) + req->offset, len, 1092 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST); 1093 else 1094 ret = sock_no_sendpage(queue->sock, virt_to_page(pdu), 1095 offset_in_page(pdu) + req->offset, len, 1096 MSG_DONTWAIT | MSG_MORE); 1097 if (unlikely(ret <= 0)) 1098 return ret; 1099 1100 len -= ret; 1101 if (!len) { 1102 req->state = NVME_TCP_SEND_DATA; 1103 if (queue->data_digest) 1104 crypto_ahash_init(queue->snd_hash); 1105 return 1; 1106 } 1107 req->offset += ret; 1108 1109 return -EAGAIN; 1110 } 1111 1112 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 1113 { 1114 struct nvme_tcp_queue *queue = req->queue; 1115 size_t offset = req->offset; 1116 u32 h2cdata_left = req->h2cdata_left; 1117 int ret; 1118 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1119 struct kvec iov = { 1120 .iov_base = (u8 *)&req->ddgst + req->offset, 1121 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 1122 }; 1123 1124 if (nvme_tcp_queue_more(queue)) 1125 msg.msg_flags |= MSG_MORE; 1126 else 1127 msg.msg_flags |= MSG_EOR; 1128 1129 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1130 if (unlikely(ret <= 0)) 1131 return ret; 1132 1133 if (offset + ret == NVME_TCP_DIGEST_LENGTH) { 1134 if (h2cdata_left) 1135 nvme_tcp_setup_h2c_data_pdu(req); 1136 else 1137 nvme_tcp_done_send_req(queue); 1138 return 1; 1139 } 1140 1141 req->offset += ret; 1142 return -EAGAIN; 1143 } 1144 1145 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 1146 { 1147 struct nvme_tcp_request *req; 1148 int ret = 1; 1149 1150 if (!queue->request) { 1151 queue->request = nvme_tcp_fetch_request(queue); 1152 if (!queue->request) 1153 return 0; 1154 } 1155 req = queue->request; 1156 1157 if (req->state == NVME_TCP_SEND_CMD_PDU) { 1158 ret = nvme_tcp_try_send_cmd_pdu(req); 1159 if (ret <= 0) 1160 goto done; 1161 if (!nvme_tcp_has_inline_data(req)) 1162 return ret; 1163 } 1164 1165 if (req->state == NVME_TCP_SEND_H2C_PDU) { 1166 ret = nvme_tcp_try_send_data_pdu(req); 1167 if (ret <= 0) 1168 goto done; 1169 } 1170 1171 if (req->state == NVME_TCP_SEND_DATA) { 1172 ret = nvme_tcp_try_send_data(req); 1173 if (ret <= 0) 1174 goto done; 1175 } 1176 1177 if (req->state == NVME_TCP_SEND_DDGST) 1178 ret = nvme_tcp_try_send_ddgst(req); 1179 done: 1180 if (ret == -EAGAIN) { 1181 ret = 0; 1182 } else if (ret < 0) { 1183 dev_err(queue->ctrl->ctrl.device, 1184 "failed to send request %d\n", ret); 1185 nvme_tcp_fail_request(queue->request); 1186 nvme_tcp_done_send_req(queue); 1187 } 1188 return ret; 1189 } 1190 1191 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 1192 { 1193 struct socket *sock = queue->sock; 1194 struct sock *sk = sock->sk; 1195 read_descriptor_t rd_desc; 1196 int consumed; 1197 1198 rd_desc.arg.data = queue; 1199 rd_desc.count = 1; 1200 lock_sock(sk); 1201 queue->nr_cqe = 0; 1202 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 1203 release_sock(sk); 1204 return consumed; 1205 } 1206 1207 static void nvme_tcp_io_work(struct work_struct *w) 1208 { 1209 struct nvme_tcp_queue *queue = 1210 container_of(w, struct nvme_tcp_queue, io_work); 1211 unsigned long deadline = jiffies + msecs_to_jiffies(1); 1212 1213 do { 1214 bool pending = false; 1215 int result; 1216 1217 if (mutex_trylock(&queue->send_mutex)) { 1218 result = nvme_tcp_try_send(queue); 1219 mutex_unlock(&queue->send_mutex); 1220 if (result > 0) 1221 pending = true; 1222 else if (unlikely(result < 0)) 1223 break; 1224 } 1225 1226 result = nvme_tcp_try_recv(queue); 1227 if (result > 0) 1228 pending = true; 1229 else if (unlikely(result < 0)) 1230 return; 1231 1232 if (!pending) 1233 return; 1234 1235 } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 1236 1237 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1238 } 1239 1240 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 1241 { 1242 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 1243 1244 ahash_request_free(queue->rcv_hash); 1245 ahash_request_free(queue->snd_hash); 1246 crypto_free_ahash(tfm); 1247 } 1248 1249 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 1250 { 1251 struct crypto_ahash *tfm; 1252 1253 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 1254 if (IS_ERR(tfm)) 1255 return PTR_ERR(tfm); 1256 1257 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1258 if (!queue->snd_hash) 1259 goto free_tfm; 1260 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 1261 1262 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1263 if (!queue->rcv_hash) 1264 goto free_snd_hash; 1265 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 1266 1267 return 0; 1268 free_snd_hash: 1269 ahash_request_free(queue->snd_hash); 1270 free_tfm: 1271 crypto_free_ahash(tfm); 1272 return -ENOMEM; 1273 } 1274 1275 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 1276 { 1277 struct nvme_tcp_request *async = &ctrl->async_req; 1278 1279 page_frag_free(async->pdu); 1280 } 1281 1282 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 1283 { 1284 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 1285 struct nvme_tcp_request *async = &ctrl->async_req; 1286 u8 hdgst = nvme_tcp_hdgst_len(queue); 1287 1288 async->pdu = page_frag_alloc(&queue->pf_cache, 1289 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 1290 GFP_KERNEL | __GFP_ZERO); 1291 if (!async->pdu) 1292 return -ENOMEM; 1293 1294 async->queue = &ctrl->queues[0]; 1295 return 0; 1296 } 1297 1298 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 1299 { 1300 struct page *page; 1301 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1302 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1303 1304 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1305 return; 1306 1307 if (queue->hdr_digest || queue->data_digest) 1308 nvme_tcp_free_crypto(queue); 1309 1310 if (queue->pf_cache.va) { 1311 page = virt_to_head_page(queue->pf_cache.va); 1312 __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias); 1313 queue->pf_cache.va = NULL; 1314 } 1315 sock_release(queue->sock); 1316 kfree(queue->pdu); 1317 mutex_destroy(&queue->send_mutex); 1318 mutex_destroy(&queue->queue_lock); 1319 } 1320 1321 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 1322 { 1323 struct nvme_tcp_icreq_pdu *icreq; 1324 struct nvme_tcp_icresp_pdu *icresp; 1325 struct msghdr msg = {}; 1326 struct kvec iov; 1327 bool ctrl_hdgst, ctrl_ddgst; 1328 u32 maxh2cdata; 1329 int ret; 1330 1331 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 1332 if (!icreq) 1333 return -ENOMEM; 1334 1335 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 1336 if (!icresp) { 1337 ret = -ENOMEM; 1338 goto free_icreq; 1339 } 1340 1341 icreq->hdr.type = nvme_tcp_icreq; 1342 icreq->hdr.hlen = sizeof(*icreq); 1343 icreq->hdr.pdo = 0; 1344 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 1345 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 1346 icreq->maxr2t = 0; /* single inflight r2t supported */ 1347 icreq->hpda = 0; /* no alignment constraint */ 1348 if (queue->hdr_digest) 1349 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 1350 if (queue->data_digest) 1351 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 1352 1353 iov.iov_base = icreq; 1354 iov.iov_len = sizeof(*icreq); 1355 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1356 if (ret < 0) 1357 goto free_icresp; 1358 1359 memset(&msg, 0, sizeof(msg)); 1360 iov.iov_base = icresp; 1361 iov.iov_len = sizeof(*icresp); 1362 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1363 iov.iov_len, msg.msg_flags); 1364 if (ret < 0) 1365 goto free_icresp; 1366 1367 ret = -EINVAL; 1368 if (icresp->hdr.type != nvme_tcp_icresp) { 1369 pr_err("queue %d: bad type returned %d\n", 1370 nvme_tcp_queue_id(queue), icresp->hdr.type); 1371 goto free_icresp; 1372 } 1373 1374 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 1375 pr_err("queue %d: bad pdu length returned %d\n", 1376 nvme_tcp_queue_id(queue), icresp->hdr.plen); 1377 goto free_icresp; 1378 } 1379 1380 if (icresp->pfv != NVME_TCP_PFV_1_0) { 1381 pr_err("queue %d: bad pfv returned %d\n", 1382 nvme_tcp_queue_id(queue), icresp->pfv); 1383 goto free_icresp; 1384 } 1385 1386 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 1387 if ((queue->data_digest && !ctrl_ddgst) || 1388 (!queue->data_digest && ctrl_ddgst)) { 1389 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 1390 nvme_tcp_queue_id(queue), 1391 queue->data_digest ? "enabled" : "disabled", 1392 ctrl_ddgst ? "enabled" : "disabled"); 1393 goto free_icresp; 1394 } 1395 1396 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 1397 if ((queue->hdr_digest && !ctrl_hdgst) || 1398 (!queue->hdr_digest && ctrl_hdgst)) { 1399 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 1400 nvme_tcp_queue_id(queue), 1401 queue->hdr_digest ? "enabled" : "disabled", 1402 ctrl_hdgst ? "enabled" : "disabled"); 1403 goto free_icresp; 1404 } 1405 1406 if (icresp->cpda != 0) { 1407 pr_err("queue %d: unsupported cpda returned %d\n", 1408 nvme_tcp_queue_id(queue), icresp->cpda); 1409 goto free_icresp; 1410 } 1411 1412 maxh2cdata = le32_to_cpu(icresp->maxdata); 1413 if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) { 1414 pr_err("queue %d: invalid maxh2cdata returned %u\n", 1415 nvme_tcp_queue_id(queue), maxh2cdata); 1416 goto free_icresp; 1417 } 1418 queue->maxh2cdata = maxh2cdata; 1419 1420 ret = 0; 1421 free_icresp: 1422 kfree(icresp); 1423 free_icreq: 1424 kfree(icreq); 1425 return ret; 1426 } 1427 1428 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 1429 { 1430 return nvme_tcp_queue_id(queue) == 0; 1431 } 1432 1433 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 1434 { 1435 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1436 int qid = nvme_tcp_queue_id(queue); 1437 1438 return !nvme_tcp_admin_queue(queue) && 1439 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1440 } 1441 1442 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 1443 { 1444 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1445 int qid = nvme_tcp_queue_id(queue); 1446 1447 return !nvme_tcp_admin_queue(queue) && 1448 !nvme_tcp_default_queue(queue) && 1449 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1450 ctrl->io_queues[HCTX_TYPE_READ]; 1451 } 1452 1453 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 1454 { 1455 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1456 int qid = nvme_tcp_queue_id(queue); 1457 1458 return !nvme_tcp_admin_queue(queue) && 1459 !nvme_tcp_default_queue(queue) && 1460 !nvme_tcp_read_queue(queue) && 1461 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1462 ctrl->io_queues[HCTX_TYPE_READ] + 1463 ctrl->io_queues[HCTX_TYPE_POLL]; 1464 } 1465 1466 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 1467 { 1468 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1469 int qid = nvme_tcp_queue_id(queue); 1470 int n = 0; 1471 1472 if (nvme_tcp_default_queue(queue)) 1473 n = qid - 1; 1474 else if (nvme_tcp_read_queue(queue)) 1475 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; 1476 else if (nvme_tcp_poll_queue(queue)) 1477 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1478 ctrl->io_queues[HCTX_TYPE_READ] - 1; 1479 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 1480 } 1481 1482 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, 1483 int qid, size_t queue_size) 1484 { 1485 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1486 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1487 int ret, rcv_pdu_size; 1488 1489 mutex_init(&queue->queue_lock); 1490 queue->ctrl = ctrl; 1491 init_llist_head(&queue->req_list); 1492 INIT_LIST_HEAD(&queue->send_list); 1493 mutex_init(&queue->send_mutex); 1494 INIT_WORK(&queue->io_work, nvme_tcp_io_work); 1495 queue->queue_size = queue_size; 1496 1497 if (qid > 0) 1498 queue->cmnd_capsule_len = nctrl->ioccsz * 16; 1499 else 1500 queue->cmnd_capsule_len = sizeof(struct nvme_command) + 1501 NVME_TCP_ADMIN_CCSZ; 1502 1503 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 1504 IPPROTO_TCP, &queue->sock); 1505 if (ret) { 1506 dev_err(nctrl->device, 1507 "failed to create socket: %d\n", ret); 1508 goto err_destroy_mutex; 1509 } 1510 1511 nvme_tcp_reclassify_socket(queue->sock); 1512 1513 /* Single syn retry */ 1514 tcp_sock_set_syncnt(queue->sock->sk, 1); 1515 1516 /* Set TCP no delay */ 1517 tcp_sock_set_nodelay(queue->sock->sk); 1518 1519 /* 1520 * Cleanup whatever is sitting in the TCP transmit queue on socket 1521 * close. This is done to prevent stale data from being sent should 1522 * the network connection be restored before TCP times out. 1523 */ 1524 sock_no_linger(queue->sock->sk); 1525 1526 if (so_priority > 0) 1527 sock_set_priority(queue->sock->sk, so_priority); 1528 1529 /* Set socket type of service */ 1530 if (nctrl->opts->tos >= 0) 1531 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1532 1533 /* Set 10 seconds timeout for icresp recvmsg */ 1534 queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1535 1536 queue->sock->sk->sk_allocation = GFP_ATOMIC; 1537 nvme_tcp_set_queue_io_cpu(queue); 1538 queue->request = NULL; 1539 queue->data_remaining = 0; 1540 queue->ddgst_remaining = 0; 1541 queue->pdu_remaining = 0; 1542 queue->pdu_offset = 0; 1543 sk_set_memalloc(queue->sock->sk); 1544 1545 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 1546 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 1547 sizeof(ctrl->src_addr)); 1548 if (ret) { 1549 dev_err(nctrl->device, 1550 "failed to bind queue %d socket %d\n", 1551 qid, ret); 1552 goto err_sock; 1553 } 1554 } 1555 1556 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) { 1557 char *iface = nctrl->opts->host_iface; 1558 sockptr_t optval = KERNEL_SOCKPTR(iface); 1559 1560 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE, 1561 optval, strlen(iface)); 1562 if (ret) { 1563 dev_err(nctrl->device, 1564 "failed to bind to interface %s queue %d err %d\n", 1565 iface, qid, ret); 1566 goto err_sock; 1567 } 1568 } 1569 1570 queue->hdr_digest = nctrl->opts->hdr_digest; 1571 queue->data_digest = nctrl->opts->data_digest; 1572 if (queue->hdr_digest || queue->data_digest) { 1573 ret = nvme_tcp_alloc_crypto(queue); 1574 if (ret) { 1575 dev_err(nctrl->device, 1576 "failed to allocate queue %d crypto\n", qid); 1577 goto err_sock; 1578 } 1579 } 1580 1581 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 1582 nvme_tcp_hdgst_len(queue); 1583 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 1584 if (!queue->pdu) { 1585 ret = -ENOMEM; 1586 goto err_crypto; 1587 } 1588 1589 dev_dbg(nctrl->device, "connecting queue %d\n", 1590 nvme_tcp_queue_id(queue)); 1591 1592 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 1593 sizeof(ctrl->addr), 0); 1594 if (ret) { 1595 dev_err(nctrl->device, 1596 "failed to connect socket: %d\n", ret); 1597 goto err_rcv_pdu; 1598 } 1599 1600 ret = nvme_tcp_init_connection(queue); 1601 if (ret) 1602 goto err_init_connect; 1603 1604 queue->rd_enabled = true; 1605 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 1606 nvme_tcp_init_recv_ctx(queue); 1607 1608 write_lock_bh(&queue->sock->sk->sk_callback_lock); 1609 queue->sock->sk->sk_user_data = queue; 1610 queue->state_change = queue->sock->sk->sk_state_change; 1611 queue->data_ready = queue->sock->sk->sk_data_ready; 1612 queue->write_space = queue->sock->sk->sk_write_space; 1613 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 1614 queue->sock->sk->sk_state_change = nvme_tcp_state_change; 1615 queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1616 #ifdef CONFIG_NET_RX_BUSY_POLL 1617 queue->sock->sk->sk_ll_usec = 1; 1618 #endif 1619 write_unlock_bh(&queue->sock->sk->sk_callback_lock); 1620 1621 return 0; 1622 1623 err_init_connect: 1624 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1625 err_rcv_pdu: 1626 kfree(queue->pdu); 1627 err_crypto: 1628 if (queue->hdr_digest || queue->data_digest) 1629 nvme_tcp_free_crypto(queue); 1630 err_sock: 1631 sock_release(queue->sock); 1632 queue->sock = NULL; 1633 err_destroy_mutex: 1634 mutex_destroy(&queue->send_mutex); 1635 mutex_destroy(&queue->queue_lock); 1636 return ret; 1637 } 1638 1639 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue) 1640 { 1641 struct socket *sock = queue->sock; 1642 1643 write_lock_bh(&sock->sk->sk_callback_lock); 1644 sock->sk->sk_user_data = NULL; 1645 sock->sk->sk_data_ready = queue->data_ready; 1646 sock->sk->sk_state_change = queue->state_change; 1647 sock->sk->sk_write_space = queue->write_space; 1648 write_unlock_bh(&sock->sk->sk_callback_lock); 1649 } 1650 1651 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 1652 { 1653 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1654 nvme_tcp_restore_sock_calls(queue); 1655 cancel_work_sync(&queue->io_work); 1656 } 1657 1658 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 1659 { 1660 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1661 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1662 1663 if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1664 return; 1665 1666 mutex_lock(&queue->queue_lock); 1667 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 1668 __nvme_tcp_stop_queue(queue); 1669 mutex_unlock(&queue->queue_lock); 1670 } 1671 1672 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 1673 { 1674 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1675 int ret; 1676 1677 if (idx) 1678 ret = nvmf_connect_io_queue(nctrl, idx); 1679 else 1680 ret = nvmf_connect_admin_queue(nctrl); 1681 1682 if (!ret) { 1683 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags); 1684 } else { 1685 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags)) 1686 __nvme_tcp_stop_queue(&ctrl->queues[idx]); 1687 dev_err(nctrl->device, 1688 "failed to connect queue: %d ret=%d\n", idx, ret); 1689 } 1690 return ret; 1691 } 1692 1693 static int nvme_tcp_alloc_admin_tag_set(struct nvme_ctrl *nctrl) 1694 { 1695 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1696 struct blk_mq_tag_set *set = &ctrl->admin_tag_set; 1697 int ret; 1698 1699 memset(set, 0, sizeof(*set)); 1700 set->ops = &nvme_tcp_admin_mq_ops; 1701 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; 1702 set->reserved_tags = NVMF_RESERVED_TAGS; 1703 set->numa_node = nctrl->numa_node; 1704 set->flags = BLK_MQ_F_BLOCKING; 1705 set->cmd_size = sizeof(struct nvme_tcp_request); 1706 set->driver_data = ctrl; 1707 set->nr_hw_queues = 1; 1708 set->timeout = NVME_ADMIN_TIMEOUT; 1709 ret = blk_mq_alloc_tag_set(set); 1710 if (!ret) 1711 nctrl->admin_tagset = set; 1712 return ret; 1713 } 1714 1715 static int nvme_tcp_alloc_tag_set(struct nvme_ctrl *nctrl) 1716 { 1717 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1718 struct blk_mq_tag_set *set = &ctrl->tag_set; 1719 int ret; 1720 1721 memset(set, 0, sizeof(*set)); 1722 set->ops = &nvme_tcp_mq_ops; 1723 set->queue_depth = nctrl->sqsize + 1; 1724 set->reserved_tags = NVMF_RESERVED_TAGS; 1725 set->numa_node = nctrl->numa_node; 1726 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING; 1727 set->cmd_size = sizeof(struct nvme_tcp_request); 1728 set->driver_data = ctrl; 1729 set->nr_hw_queues = nctrl->queue_count - 1; 1730 set->timeout = NVME_IO_TIMEOUT; 1731 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; 1732 ret = blk_mq_alloc_tag_set(set); 1733 if (!ret) 1734 nctrl->tagset = set; 1735 return ret; 1736 } 1737 1738 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 1739 { 1740 if (to_tcp_ctrl(ctrl)->async_req.pdu) { 1741 cancel_work_sync(&ctrl->async_event_work); 1742 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 1743 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 1744 } 1745 1746 nvme_tcp_free_queue(ctrl, 0); 1747 } 1748 1749 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 1750 { 1751 int i; 1752 1753 for (i = 1; i < ctrl->queue_count; i++) 1754 nvme_tcp_free_queue(ctrl, i); 1755 } 1756 1757 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 1758 { 1759 int i; 1760 1761 for (i = 1; i < ctrl->queue_count; i++) 1762 nvme_tcp_stop_queue(ctrl, i); 1763 } 1764 1765 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) 1766 { 1767 int i, ret; 1768 1769 for (i = 1; i < ctrl->queue_count; i++) { 1770 ret = nvme_tcp_start_queue(ctrl, i); 1771 if (ret) 1772 goto out_stop_queues; 1773 } 1774 1775 return 0; 1776 1777 out_stop_queues: 1778 for (i--; i >= 1; i--) 1779 nvme_tcp_stop_queue(ctrl, i); 1780 return ret; 1781 } 1782 1783 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 1784 { 1785 int ret; 1786 1787 ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); 1788 if (ret) 1789 return ret; 1790 1791 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 1792 if (ret) 1793 goto out_free_queue; 1794 1795 return 0; 1796 1797 out_free_queue: 1798 nvme_tcp_free_queue(ctrl, 0); 1799 return ret; 1800 } 1801 1802 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1803 { 1804 int i, ret; 1805 1806 for (i = 1; i < ctrl->queue_count; i++) { 1807 ret = nvme_tcp_alloc_queue(ctrl, i, ctrl->sqsize + 1); 1808 if (ret) 1809 goto out_free_queues; 1810 } 1811 1812 return 0; 1813 1814 out_free_queues: 1815 for (i--; i >= 1; i--) 1816 nvme_tcp_free_queue(ctrl, i); 1817 1818 return ret; 1819 } 1820 1821 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) 1822 { 1823 unsigned int nr_io_queues; 1824 1825 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); 1826 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); 1827 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); 1828 1829 return nr_io_queues; 1830 } 1831 1832 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl, 1833 unsigned int nr_io_queues) 1834 { 1835 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1836 struct nvmf_ctrl_options *opts = nctrl->opts; 1837 1838 if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { 1839 /* 1840 * separate read/write queues 1841 * hand out dedicated default queues only after we have 1842 * sufficient read queues. 1843 */ 1844 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; 1845 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; 1846 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1847 min(opts->nr_write_queues, nr_io_queues); 1848 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1849 } else { 1850 /* 1851 * shared read/write queues 1852 * either no write queues were requested, or we don't have 1853 * sufficient queue count to have dedicated default queues. 1854 */ 1855 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1856 min(opts->nr_io_queues, nr_io_queues); 1857 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1858 } 1859 1860 if (opts->nr_poll_queues && nr_io_queues) { 1861 /* map dedicated poll queues only if we have queues left */ 1862 ctrl->io_queues[HCTX_TYPE_POLL] = 1863 min(opts->nr_poll_queues, nr_io_queues); 1864 } 1865 } 1866 1867 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1868 { 1869 unsigned int nr_io_queues; 1870 int ret; 1871 1872 nr_io_queues = nvme_tcp_nr_io_queues(ctrl); 1873 ret = nvme_set_queue_count(ctrl, &nr_io_queues); 1874 if (ret) 1875 return ret; 1876 1877 if (nr_io_queues == 0) { 1878 dev_err(ctrl->device, 1879 "unable to set any I/O queues\n"); 1880 return -ENOMEM; 1881 } 1882 1883 ctrl->queue_count = nr_io_queues + 1; 1884 dev_info(ctrl->device, 1885 "creating %d I/O queues.\n", nr_io_queues); 1886 1887 nvme_tcp_set_io_queues(ctrl, nr_io_queues); 1888 1889 return __nvme_tcp_alloc_io_queues(ctrl); 1890 } 1891 1892 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 1893 { 1894 nvme_tcp_stop_io_queues(ctrl); 1895 if (remove) { 1896 blk_mq_destroy_queue(ctrl->connect_q); 1897 blk_mq_free_tag_set(ctrl->tagset); 1898 } 1899 nvme_tcp_free_io_queues(ctrl); 1900 } 1901 1902 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 1903 { 1904 int ret; 1905 1906 ret = nvme_tcp_alloc_io_queues(ctrl); 1907 if (ret) 1908 return ret; 1909 1910 if (new) { 1911 ret = nvme_tcp_alloc_tag_set(ctrl); 1912 if (ret) 1913 goto out_free_io_queues; 1914 1915 ret = nvme_ctrl_init_connect_q(ctrl); 1916 if (ret) 1917 goto out_free_tag_set; 1918 } 1919 1920 ret = nvme_tcp_start_io_queues(ctrl); 1921 if (ret) 1922 goto out_cleanup_connect_q; 1923 1924 if (!new) { 1925 nvme_start_queues(ctrl); 1926 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 1927 /* 1928 * If we timed out waiting for freeze we are likely to 1929 * be stuck. Fail the controller initialization just 1930 * to be safe. 1931 */ 1932 ret = -ENODEV; 1933 goto out_wait_freeze_timed_out; 1934 } 1935 blk_mq_update_nr_hw_queues(ctrl->tagset, 1936 ctrl->queue_count - 1); 1937 nvme_unfreeze(ctrl); 1938 } 1939 1940 return 0; 1941 1942 out_wait_freeze_timed_out: 1943 nvme_stop_queues(ctrl); 1944 nvme_sync_io_queues(ctrl); 1945 nvme_tcp_stop_io_queues(ctrl); 1946 out_cleanup_connect_q: 1947 nvme_cancel_tagset(ctrl); 1948 if (new) 1949 blk_mq_destroy_queue(ctrl->connect_q); 1950 out_free_tag_set: 1951 if (new) 1952 blk_mq_free_tag_set(ctrl->tagset); 1953 out_free_io_queues: 1954 nvme_tcp_free_io_queues(ctrl); 1955 return ret; 1956 } 1957 1958 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 1959 { 1960 nvme_tcp_stop_queue(ctrl, 0); 1961 if (remove) { 1962 blk_mq_destroy_queue(ctrl->admin_q); 1963 blk_mq_destroy_queue(ctrl->fabrics_q); 1964 blk_mq_free_tag_set(ctrl->admin_tagset); 1965 } 1966 nvme_tcp_free_admin_queue(ctrl); 1967 } 1968 1969 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 1970 { 1971 int error; 1972 1973 error = nvme_tcp_alloc_admin_queue(ctrl); 1974 if (error) 1975 return error; 1976 1977 if (new) { 1978 error = nvme_tcp_alloc_admin_tag_set(ctrl); 1979 if (error) 1980 goto out_free_queue; 1981 1982 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset); 1983 if (IS_ERR(ctrl->fabrics_q)) { 1984 error = PTR_ERR(ctrl->fabrics_q); 1985 goto out_free_tagset; 1986 } 1987 1988 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset); 1989 if (IS_ERR(ctrl->admin_q)) { 1990 error = PTR_ERR(ctrl->admin_q); 1991 goto out_cleanup_fabrics_q; 1992 } 1993 } 1994 1995 error = nvme_tcp_start_queue(ctrl, 0); 1996 if (error) 1997 goto out_cleanup_queue; 1998 1999 error = nvme_enable_ctrl(ctrl); 2000 if (error) 2001 goto out_stop_queue; 2002 2003 nvme_start_admin_queue(ctrl); 2004 2005 error = nvme_init_ctrl_finish(ctrl); 2006 if (error) 2007 goto out_quiesce_queue; 2008 2009 return 0; 2010 2011 out_quiesce_queue: 2012 nvme_stop_admin_queue(ctrl); 2013 blk_sync_queue(ctrl->admin_q); 2014 out_stop_queue: 2015 nvme_tcp_stop_queue(ctrl, 0); 2016 nvme_cancel_admin_tagset(ctrl); 2017 out_cleanup_queue: 2018 if (new) 2019 blk_mq_destroy_queue(ctrl->admin_q); 2020 out_cleanup_fabrics_q: 2021 if (new) 2022 blk_mq_destroy_queue(ctrl->fabrics_q); 2023 out_free_tagset: 2024 if (new) 2025 blk_mq_free_tag_set(ctrl->admin_tagset); 2026 out_free_queue: 2027 nvme_tcp_free_admin_queue(ctrl); 2028 return error; 2029 } 2030 2031 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 2032 bool remove) 2033 { 2034 nvme_stop_admin_queue(ctrl); 2035 blk_sync_queue(ctrl->admin_q); 2036 nvme_tcp_stop_queue(ctrl, 0); 2037 nvme_cancel_admin_tagset(ctrl); 2038 if (remove) 2039 nvme_start_admin_queue(ctrl); 2040 nvme_tcp_destroy_admin_queue(ctrl, remove); 2041 } 2042 2043 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 2044 bool remove) 2045 { 2046 if (ctrl->queue_count <= 1) 2047 return; 2048 nvme_stop_admin_queue(ctrl); 2049 nvme_start_freeze(ctrl); 2050 nvme_stop_queues(ctrl); 2051 nvme_sync_io_queues(ctrl); 2052 nvme_tcp_stop_io_queues(ctrl); 2053 nvme_cancel_tagset(ctrl); 2054 if (remove) 2055 nvme_start_queues(ctrl); 2056 nvme_tcp_destroy_io_queues(ctrl, remove); 2057 } 2058 2059 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 2060 { 2061 /* If we are resetting/deleting then do nothing */ 2062 if (ctrl->state != NVME_CTRL_CONNECTING) { 2063 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 2064 ctrl->state == NVME_CTRL_LIVE); 2065 return; 2066 } 2067 2068 if (nvmf_should_reconnect(ctrl)) { 2069 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 2070 ctrl->opts->reconnect_delay); 2071 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 2072 ctrl->opts->reconnect_delay * HZ); 2073 } else { 2074 dev_info(ctrl->device, "Removing controller...\n"); 2075 nvme_delete_ctrl(ctrl); 2076 } 2077 } 2078 2079 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 2080 { 2081 struct nvmf_ctrl_options *opts = ctrl->opts; 2082 int ret; 2083 2084 ret = nvme_tcp_configure_admin_queue(ctrl, new); 2085 if (ret) 2086 return ret; 2087 2088 if (ctrl->icdoff) { 2089 ret = -EOPNOTSUPP; 2090 dev_err(ctrl->device, "icdoff is not supported!\n"); 2091 goto destroy_admin; 2092 } 2093 2094 if (!nvme_ctrl_sgl_supported(ctrl)) { 2095 ret = -EOPNOTSUPP; 2096 dev_err(ctrl->device, "Mandatory sgls are not supported!\n"); 2097 goto destroy_admin; 2098 } 2099 2100 if (opts->queue_size > ctrl->sqsize + 1) 2101 dev_warn(ctrl->device, 2102 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2103 opts->queue_size, ctrl->sqsize + 1); 2104 2105 if (ctrl->sqsize + 1 > ctrl->maxcmd) { 2106 dev_warn(ctrl->device, 2107 "sqsize %u > ctrl maxcmd %u, clamping down\n", 2108 ctrl->sqsize + 1, ctrl->maxcmd); 2109 ctrl->sqsize = ctrl->maxcmd - 1; 2110 } 2111 2112 if (ctrl->queue_count > 1) { 2113 ret = nvme_tcp_configure_io_queues(ctrl, new); 2114 if (ret) 2115 goto destroy_admin; 2116 } 2117 2118 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2119 /* 2120 * state change failure is ok if we started ctrl delete, 2121 * unless we're during creation of a new controller to 2122 * avoid races with teardown flow. 2123 */ 2124 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2125 ctrl->state != NVME_CTRL_DELETING_NOIO); 2126 WARN_ON_ONCE(new); 2127 ret = -EINVAL; 2128 goto destroy_io; 2129 } 2130 2131 nvme_start_ctrl(ctrl); 2132 return 0; 2133 2134 destroy_io: 2135 if (ctrl->queue_count > 1) { 2136 nvme_stop_queues(ctrl); 2137 nvme_sync_io_queues(ctrl); 2138 nvme_tcp_stop_io_queues(ctrl); 2139 nvme_cancel_tagset(ctrl); 2140 nvme_tcp_destroy_io_queues(ctrl, new); 2141 } 2142 destroy_admin: 2143 nvme_stop_admin_queue(ctrl); 2144 blk_sync_queue(ctrl->admin_q); 2145 nvme_tcp_stop_queue(ctrl, 0); 2146 nvme_cancel_admin_tagset(ctrl); 2147 nvme_tcp_destroy_admin_queue(ctrl, new); 2148 return ret; 2149 } 2150 2151 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 2152 { 2153 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 2154 struct nvme_tcp_ctrl, connect_work); 2155 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2156 2157 ++ctrl->nr_reconnects; 2158 2159 if (nvme_tcp_setup_ctrl(ctrl, false)) 2160 goto requeue; 2161 2162 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 2163 ctrl->nr_reconnects); 2164 2165 ctrl->nr_reconnects = 0; 2166 2167 return; 2168 2169 requeue: 2170 dev_info(ctrl->device, "Failed reconnect attempt %d\n", 2171 ctrl->nr_reconnects); 2172 nvme_tcp_reconnect_or_remove(ctrl); 2173 } 2174 2175 static void nvme_tcp_error_recovery_work(struct work_struct *work) 2176 { 2177 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 2178 struct nvme_tcp_ctrl, err_work); 2179 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2180 2181 nvme_auth_stop(ctrl); 2182 nvme_stop_keep_alive(ctrl); 2183 flush_work(&ctrl->async_event_work); 2184 nvme_tcp_teardown_io_queues(ctrl, false); 2185 /* unquiesce to fail fast pending requests */ 2186 nvme_start_queues(ctrl); 2187 nvme_tcp_teardown_admin_queue(ctrl, false); 2188 nvme_start_admin_queue(ctrl); 2189 2190 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2191 /* state change failure is ok if we started ctrl delete */ 2192 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2193 ctrl->state != NVME_CTRL_DELETING_NOIO); 2194 return; 2195 } 2196 2197 nvme_tcp_reconnect_or_remove(ctrl); 2198 } 2199 2200 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 2201 { 2202 nvme_tcp_teardown_io_queues(ctrl, shutdown); 2203 nvme_stop_admin_queue(ctrl); 2204 if (shutdown) 2205 nvme_shutdown_ctrl(ctrl); 2206 else 2207 nvme_disable_ctrl(ctrl); 2208 nvme_tcp_teardown_admin_queue(ctrl, shutdown); 2209 } 2210 2211 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 2212 { 2213 nvme_tcp_teardown_ctrl(ctrl, true); 2214 } 2215 2216 static void nvme_reset_ctrl_work(struct work_struct *work) 2217 { 2218 struct nvme_ctrl *ctrl = 2219 container_of(work, struct nvme_ctrl, reset_work); 2220 2221 nvme_stop_ctrl(ctrl); 2222 nvme_tcp_teardown_ctrl(ctrl, false); 2223 2224 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2225 /* state change failure is ok if we started ctrl delete */ 2226 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2227 ctrl->state != NVME_CTRL_DELETING_NOIO); 2228 return; 2229 } 2230 2231 if (nvme_tcp_setup_ctrl(ctrl, false)) 2232 goto out_fail; 2233 2234 return; 2235 2236 out_fail: 2237 ++ctrl->nr_reconnects; 2238 nvme_tcp_reconnect_or_remove(ctrl); 2239 } 2240 2241 static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl) 2242 { 2243 cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work); 2244 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2245 } 2246 2247 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 2248 { 2249 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 2250 2251 if (list_empty(&ctrl->list)) 2252 goto free_ctrl; 2253 2254 mutex_lock(&nvme_tcp_ctrl_mutex); 2255 list_del(&ctrl->list); 2256 mutex_unlock(&nvme_tcp_ctrl_mutex); 2257 2258 nvmf_free_options(nctrl->opts); 2259 free_ctrl: 2260 kfree(ctrl->queues); 2261 kfree(ctrl); 2262 } 2263 2264 static void nvme_tcp_set_sg_null(struct nvme_command *c) 2265 { 2266 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2267 2268 sg->addr = 0; 2269 sg->length = 0; 2270 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2271 NVME_SGL_FMT_TRANSPORT_A; 2272 } 2273 2274 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 2275 struct nvme_command *c, u32 data_len) 2276 { 2277 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2278 2279 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 2280 sg->length = cpu_to_le32(data_len); 2281 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 2282 } 2283 2284 static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 2285 u32 data_len) 2286 { 2287 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2288 2289 sg->addr = 0; 2290 sg->length = cpu_to_le32(data_len); 2291 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2292 NVME_SGL_FMT_TRANSPORT_A; 2293 } 2294 2295 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 2296 { 2297 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 2298 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 2299 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 2300 struct nvme_command *cmd = &pdu->cmd; 2301 u8 hdgst = nvme_tcp_hdgst_len(queue); 2302 2303 memset(pdu, 0, sizeof(*pdu)); 2304 pdu->hdr.type = nvme_tcp_cmd; 2305 if (queue->hdr_digest) 2306 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2307 pdu->hdr.hlen = sizeof(*pdu); 2308 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 2309 2310 cmd->common.opcode = nvme_admin_async_event; 2311 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 2312 cmd->common.flags |= NVME_CMD_SGL_METABUF; 2313 nvme_tcp_set_sg_null(cmd); 2314 2315 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 2316 ctrl->async_req.offset = 0; 2317 ctrl->async_req.curr_bio = NULL; 2318 ctrl->async_req.data_len = 0; 2319 2320 nvme_tcp_queue_request(&ctrl->async_req, true, true); 2321 } 2322 2323 static void nvme_tcp_complete_timed_out(struct request *rq) 2324 { 2325 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2326 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2327 2328 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 2329 nvmf_complete_timed_out_request(rq); 2330 } 2331 2332 static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq) 2333 { 2334 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2335 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2336 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2337 2338 dev_warn(ctrl->device, 2339 "queue %d: timeout request %#x type %d\n", 2340 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type); 2341 2342 if (ctrl->state != NVME_CTRL_LIVE) { 2343 /* 2344 * If we are resetting, connecting or deleting we should 2345 * complete immediately because we may block controller 2346 * teardown or setup sequence 2347 * - ctrl disable/shutdown fabrics requests 2348 * - connect requests 2349 * - initialization admin requests 2350 * - I/O requests that entered after unquiescing and 2351 * the controller stopped responding 2352 * 2353 * All other requests should be cancelled by the error 2354 * recovery work, so it's fine that we fail it here. 2355 */ 2356 nvme_tcp_complete_timed_out(rq); 2357 return BLK_EH_DONE; 2358 } 2359 2360 /* 2361 * LIVE state should trigger the normal error recovery which will 2362 * handle completing this request. 2363 */ 2364 nvme_tcp_error_recovery(ctrl); 2365 return BLK_EH_RESET_TIMER; 2366 } 2367 2368 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 2369 struct request *rq) 2370 { 2371 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2372 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2373 struct nvme_command *c = &pdu->cmd; 2374 2375 c->common.flags |= NVME_CMD_SGL_METABUF; 2376 2377 if (!blk_rq_nr_phys_segments(rq)) 2378 nvme_tcp_set_sg_null(c); 2379 else if (rq_data_dir(rq) == WRITE && 2380 req->data_len <= nvme_tcp_inline_data_size(req)) 2381 nvme_tcp_set_sg_inline(queue, c, req->data_len); 2382 else 2383 nvme_tcp_set_sg_host_data(c, req->data_len); 2384 2385 return 0; 2386 } 2387 2388 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 2389 struct request *rq) 2390 { 2391 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2392 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2393 struct nvme_tcp_queue *queue = req->queue; 2394 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 2395 blk_status_t ret; 2396 2397 ret = nvme_setup_cmd(ns, rq); 2398 if (ret) 2399 return ret; 2400 2401 req->state = NVME_TCP_SEND_CMD_PDU; 2402 req->status = cpu_to_le16(NVME_SC_SUCCESS); 2403 req->offset = 0; 2404 req->data_sent = 0; 2405 req->pdu_len = 0; 2406 req->pdu_sent = 0; 2407 req->h2cdata_left = 0; 2408 req->data_len = blk_rq_nr_phys_segments(rq) ? 2409 blk_rq_payload_bytes(rq) : 0; 2410 req->curr_bio = rq->bio; 2411 if (req->curr_bio && req->data_len) 2412 nvme_tcp_init_iter(req, rq_data_dir(rq)); 2413 2414 if (rq_data_dir(rq) == WRITE && 2415 req->data_len <= nvme_tcp_inline_data_size(req)) 2416 req->pdu_len = req->data_len; 2417 2418 pdu->hdr.type = nvme_tcp_cmd; 2419 pdu->hdr.flags = 0; 2420 if (queue->hdr_digest) 2421 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2422 if (queue->data_digest && req->pdu_len) { 2423 pdu->hdr.flags |= NVME_TCP_F_DDGST; 2424 ddgst = nvme_tcp_ddgst_len(queue); 2425 } 2426 pdu->hdr.hlen = sizeof(*pdu); 2427 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 2428 pdu->hdr.plen = 2429 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 2430 2431 ret = nvme_tcp_map_data(queue, rq); 2432 if (unlikely(ret)) { 2433 nvme_cleanup_cmd(rq); 2434 dev_err(queue->ctrl->ctrl.device, 2435 "Failed to map data (%d)\n", ret); 2436 return ret; 2437 } 2438 2439 return 0; 2440 } 2441 2442 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 2443 { 2444 struct nvme_tcp_queue *queue = hctx->driver_data; 2445 2446 if (!llist_empty(&queue->req_list)) 2447 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2448 } 2449 2450 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 2451 const struct blk_mq_queue_data *bd) 2452 { 2453 struct nvme_ns *ns = hctx->queue->queuedata; 2454 struct nvme_tcp_queue *queue = hctx->driver_data; 2455 struct request *rq = bd->rq; 2456 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2457 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 2458 blk_status_t ret; 2459 2460 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2461 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 2462 2463 ret = nvme_tcp_setup_cmd_pdu(ns, rq); 2464 if (unlikely(ret)) 2465 return ret; 2466 2467 blk_mq_start_request(rq); 2468 2469 nvme_tcp_queue_request(req, true, bd->last); 2470 2471 return BLK_STS_OK; 2472 } 2473 2474 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2475 { 2476 struct nvme_tcp_ctrl *ctrl = set->driver_data; 2477 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2478 2479 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { 2480 /* separate read/write queues */ 2481 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2482 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2483 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2484 set->map[HCTX_TYPE_READ].nr_queues = 2485 ctrl->io_queues[HCTX_TYPE_READ]; 2486 set->map[HCTX_TYPE_READ].queue_offset = 2487 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2488 } else { 2489 /* shared read/write queues */ 2490 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2491 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2492 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2493 set->map[HCTX_TYPE_READ].nr_queues = 2494 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2495 set->map[HCTX_TYPE_READ].queue_offset = 0; 2496 } 2497 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 2498 blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 2499 2500 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { 2501 /* map dedicated poll queues only if we have queues left */ 2502 set->map[HCTX_TYPE_POLL].nr_queues = 2503 ctrl->io_queues[HCTX_TYPE_POLL]; 2504 set->map[HCTX_TYPE_POLL].queue_offset = 2505 ctrl->io_queues[HCTX_TYPE_DEFAULT] + 2506 ctrl->io_queues[HCTX_TYPE_READ]; 2507 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 2508 } 2509 2510 dev_info(ctrl->ctrl.device, 2511 "mapped %d/%d/%d default/read/poll queues.\n", 2512 ctrl->io_queues[HCTX_TYPE_DEFAULT], 2513 ctrl->io_queues[HCTX_TYPE_READ], 2514 ctrl->io_queues[HCTX_TYPE_POLL]); 2515 2516 return 0; 2517 } 2518 2519 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) 2520 { 2521 struct nvme_tcp_queue *queue = hctx->driver_data; 2522 struct sock *sk = queue->sock->sk; 2523 2524 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2525 return 0; 2526 2527 set_bit(NVME_TCP_Q_POLLING, &queue->flags); 2528 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 2529 sk_busy_loop(sk, true); 2530 nvme_tcp_try_recv(queue); 2531 clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 2532 return queue->nr_cqe; 2533 } 2534 2535 static const struct blk_mq_ops nvme_tcp_mq_ops = { 2536 .queue_rq = nvme_tcp_queue_rq, 2537 .commit_rqs = nvme_tcp_commit_rqs, 2538 .complete = nvme_complete_rq, 2539 .init_request = nvme_tcp_init_request, 2540 .exit_request = nvme_tcp_exit_request, 2541 .init_hctx = nvme_tcp_init_hctx, 2542 .timeout = nvme_tcp_timeout, 2543 .map_queues = nvme_tcp_map_queues, 2544 .poll = nvme_tcp_poll, 2545 }; 2546 2547 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 2548 .queue_rq = nvme_tcp_queue_rq, 2549 .complete = nvme_complete_rq, 2550 .init_request = nvme_tcp_init_request, 2551 .exit_request = nvme_tcp_exit_request, 2552 .init_hctx = nvme_tcp_init_admin_hctx, 2553 .timeout = nvme_tcp_timeout, 2554 }; 2555 2556 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 2557 .name = "tcp", 2558 .module = THIS_MODULE, 2559 .flags = NVME_F_FABRICS, 2560 .reg_read32 = nvmf_reg_read32, 2561 .reg_read64 = nvmf_reg_read64, 2562 .reg_write32 = nvmf_reg_write32, 2563 .free_ctrl = nvme_tcp_free_ctrl, 2564 .submit_async_event = nvme_tcp_submit_async_event, 2565 .delete_ctrl = nvme_tcp_delete_ctrl, 2566 .get_address = nvmf_get_address, 2567 .stop_ctrl = nvme_tcp_stop_ctrl, 2568 }; 2569 2570 static bool 2571 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 2572 { 2573 struct nvme_tcp_ctrl *ctrl; 2574 bool found = false; 2575 2576 mutex_lock(&nvme_tcp_ctrl_mutex); 2577 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 2578 found = nvmf_ip_options_match(&ctrl->ctrl, opts); 2579 if (found) 2580 break; 2581 } 2582 mutex_unlock(&nvme_tcp_ctrl_mutex); 2583 2584 return found; 2585 } 2586 2587 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 2588 struct nvmf_ctrl_options *opts) 2589 { 2590 struct nvme_tcp_ctrl *ctrl; 2591 int ret; 2592 2593 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 2594 if (!ctrl) 2595 return ERR_PTR(-ENOMEM); 2596 2597 INIT_LIST_HEAD(&ctrl->list); 2598 ctrl->ctrl.opts = opts; 2599 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 2600 opts->nr_poll_queues + 1; 2601 ctrl->ctrl.sqsize = opts->queue_size - 1; 2602 ctrl->ctrl.kato = opts->kato; 2603 2604 INIT_DELAYED_WORK(&ctrl->connect_work, 2605 nvme_tcp_reconnect_ctrl_work); 2606 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 2607 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 2608 2609 if (!(opts->mask & NVMF_OPT_TRSVCID)) { 2610 opts->trsvcid = 2611 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 2612 if (!opts->trsvcid) { 2613 ret = -ENOMEM; 2614 goto out_free_ctrl; 2615 } 2616 opts->mask |= NVMF_OPT_TRSVCID; 2617 } 2618 2619 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2620 opts->traddr, opts->trsvcid, &ctrl->addr); 2621 if (ret) { 2622 pr_err("malformed address passed: %s:%s\n", 2623 opts->traddr, opts->trsvcid); 2624 goto out_free_ctrl; 2625 } 2626 2627 if (opts->mask & NVMF_OPT_HOST_TRADDR) { 2628 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2629 opts->host_traddr, NULL, &ctrl->src_addr); 2630 if (ret) { 2631 pr_err("malformed src address passed: %s\n", 2632 opts->host_traddr); 2633 goto out_free_ctrl; 2634 } 2635 } 2636 2637 if (opts->mask & NVMF_OPT_HOST_IFACE) { 2638 if (!__dev_get_by_name(&init_net, opts->host_iface)) { 2639 pr_err("invalid interface passed: %s\n", 2640 opts->host_iface); 2641 ret = -ENODEV; 2642 goto out_free_ctrl; 2643 } 2644 } 2645 2646 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 2647 ret = -EALREADY; 2648 goto out_free_ctrl; 2649 } 2650 2651 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 2652 GFP_KERNEL); 2653 if (!ctrl->queues) { 2654 ret = -ENOMEM; 2655 goto out_free_ctrl; 2656 } 2657 2658 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 2659 if (ret) 2660 goto out_kfree_queues; 2661 2662 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2663 WARN_ON_ONCE(1); 2664 ret = -EINTR; 2665 goto out_uninit_ctrl; 2666 } 2667 2668 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 2669 if (ret) 2670 goto out_uninit_ctrl; 2671 2672 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 2673 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr); 2674 2675 mutex_lock(&nvme_tcp_ctrl_mutex); 2676 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 2677 mutex_unlock(&nvme_tcp_ctrl_mutex); 2678 2679 return &ctrl->ctrl; 2680 2681 out_uninit_ctrl: 2682 nvme_uninit_ctrl(&ctrl->ctrl); 2683 nvme_put_ctrl(&ctrl->ctrl); 2684 if (ret > 0) 2685 ret = -EIO; 2686 return ERR_PTR(ret); 2687 out_kfree_queues: 2688 kfree(ctrl->queues); 2689 out_free_ctrl: 2690 kfree(ctrl); 2691 return ERR_PTR(ret); 2692 } 2693 2694 static struct nvmf_transport_ops nvme_tcp_transport = { 2695 .name = "tcp", 2696 .module = THIS_MODULE, 2697 .required_opts = NVMF_OPT_TRADDR, 2698 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 2699 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2700 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2701 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 2702 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE, 2703 .create_ctrl = nvme_tcp_create_ctrl, 2704 }; 2705 2706 static int __init nvme_tcp_init_module(void) 2707 { 2708 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 2709 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 2710 if (!nvme_tcp_wq) 2711 return -ENOMEM; 2712 2713 nvmf_register_transport(&nvme_tcp_transport); 2714 return 0; 2715 } 2716 2717 static void __exit nvme_tcp_cleanup_module(void) 2718 { 2719 struct nvme_tcp_ctrl *ctrl; 2720 2721 nvmf_unregister_transport(&nvme_tcp_transport); 2722 2723 mutex_lock(&nvme_tcp_ctrl_mutex); 2724 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 2725 nvme_delete_ctrl(&ctrl->ctrl); 2726 mutex_unlock(&nvme_tcp_ctrl_mutex); 2727 flush_workqueue(nvme_delete_wq); 2728 2729 destroy_workqueue(nvme_tcp_wq); 2730 } 2731 2732 module_init(nvme_tcp_init_module); 2733 module_exit(nvme_tcp_cleanup_module); 2734 2735 MODULE_LICENSE("GPL v2"); 2736