1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * NVMe over Fabrics TCP host. 4 * Copyright (c) 2018 Lightbits Labs. All rights reserved. 5 */ 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 #include <linux/module.h> 8 #include <linux/init.h> 9 #include <linux/slab.h> 10 #include <linux/err.h> 11 #include <linux/nvme-tcp.h> 12 #include <net/sock.h> 13 #include <net/tcp.h> 14 #include <linux/blk-mq.h> 15 #include <crypto/hash.h> 16 #include <net/busy_poll.h> 17 18 #include "nvme.h" 19 #include "fabrics.h" 20 21 struct nvme_tcp_queue; 22 23 /* Define the socket priority to use for connections were it is desirable 24 * that the NIC consider performing optimized packet processing or filtering. 25 * A non-zero value being sufficient to indicate general consideration of any 26 * possible optimization. Making it a module param allows for alternative 27 * values that may be unique for some NIC implementations. 28 */ 29 static int so_priority; 30 module_param(so_priority, int, 0644); 31 MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority"); 32 33 enum nvme_tcp_send_state { 34 NVME_TCP_SEND_CMD_PDU = 0, 35 NVME_TCP_SEND_H2C_PDU, 36 NVME_TCP_SEND_DATA, 37 NVME_TCP_SEND_DDGST, 38 }; 39 40 struct nvme_tcp_request { 41 struct nvme_request req; 42 void *pdu; 43 struct nvme_tcp_queue *queue; 44 u32 data_len; 45 u32 pdu_len; 46 u32 pdu_sent; 47 u16 ttag; 48 __le16 status; 49 struct list_head entry; 50 struct llist_node lentry; 51 __le32 ddgst; 52 53 struct bio *curr_bio; 54 struct iov_iter iter; 55 56 /* send state */ 57 size_t offset; 58 size_t data_sent; 59 enum nvme_tcp_send_state state; 60 }; 61 62 enum nvme_tcp_queue_flags { 63 NVME_TCP_Q_ALLOCATED = 0, 64 NVME_TCP_Q_LIVE = 1, 65 NVME_TCP_Q_POLLING = 2, 66 }; 67 68 enum nvme_tcp_recv_state { 69 NVME_TCP_RECV_PDU = 0, 70 NVME_TCP_RECV_DATA, 71 NVME_TCP_RECV_DDGST, 72 }; 73 74 struct nvme_tcp_ctrl; 75 struct nvme_tcp_queue { 76 struct socket *sock; 77 struct work_struct io_work; 78 int io_cpu; 79 80 struct mutex queue_lock; 81 struct mutex send_mutex; 82 struct llist_head req_list; 83 struct list_head send_list; 84 bool more_requests; 85 86 /* recv state */ 87 void *pdu; 88 int pdu_remaining; 89 int pdu_offset; 90 size_t data_remaining; 91 size_t ddgst_remaining; 92 unsigned int nr_cqe; 93 94 /* send state */ 95 struct nvme_tcp_request *request; 96 97 int queue_size; 98 size_t cmnd_capsule_len; 99 struct nvme_tcp_ctrl *ctrl; 100 unsigned long flags; 101 bool rd_enabled; 102 103 bool hdr_digest; 104 bool data_digest; 105 struct ahash_request *rcv_hash; 106 struct ahash_request *snd_hash; 107 __le32 exp_ddgst; 108 __le32 recv_ddgst; 109 110 struct page_frag_cache pf_cache; 111 112 void (*state_change)(struct sock *); 113 void (*data_ready)(struct sock *); 114 void (*write_space)(struct sock *); 115 }; 116 117 struct nvme_tcp_ctrl { 118 /* read only in the hot path */ 119 struct nvme_tcp_queue *queues; 120 struct blk_mq_tag_set tag_set; 121 122 /* other member variables */ 123 struct list_head list; 124 struct blk_mq_tag_set admin_tag_set; 125 struct sockaddr_storage addr; 126 struct sockaddr_storage src_addr; 127 struct nvme_ctrl ctrl; 128 129 struct work_struct err_work; 130 struct delayed_work connect_work; 131 struct nvme_tcp_request async_req; 132 u32 io_queues[HCTX_MAX_TYPES]; 133 }; 134 135 static LIST_HEAD(nvme_tcp_ctrl_list); 136 static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); 137 static struct workqueue_struct *nvme_tcp_wq; 138 static const struct blk_mq_ops nvme_tcp_mq_ops; 139 static const struct blk_mq_ops nvme_tcp_admin_mq_ops; 140 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); 141 142 static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) 143 { 144 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl); 145 } 146 147 static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue) 148 { 149 return queue - queue->ctrl->queues; 150 } 151 152 static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue) 153 { 154 u32 queue_idx = nvme_tcp_queue_id(queue); 155 156 if (queue_idx == 0) 157 return queue->ctrl->admin_tag_set.tags[queue_idx]; 158 return queue->ctrl->tag_set.tags[queue_idx - 1]; 159 } 160 161 static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue) 162 { 163 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0; 164 } 165 166 static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue) 167 { 168 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0; 169 } 170 171 static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_queue *queue) 172 { 173 return queue->cmnd_capsule_len - sizeof(struct nvme_command); 174 } 175 176 static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req) 177 { 178 return req == &req->queue->ctrl->async_req; 179 } 180 181 static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req) 182 { 183 struct request *rq; 184 185 if (unlikely(nvme_tcp_async_req(req))) 186 return false; /* async events don't have a request */ 187 188 rq = blk_mq_rq_from_pdu(req); 189 190 return rq_data_dir(rq) == WRITE && req->data_len && 191 req->data_len <= nvme_tcp_inline_data_size(req->queue); 192 } 193 194 static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req) 195 { 196 return req->iter.bvec->bv_page; 197 } 198 199 static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req) 200 { 201 return req->iter.bvec->bv_offset + req->iter.iov_offset; 202 } 203 204 static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req) 205 { 206 return min_t(size_t, iov_iter_single_seg_count(&req->iter), 207 req->pdu_len - req->pdu_sent); 208 } 209 210 static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req) 211 { 212 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ? 213 req->pdu_len - req->pdu_sent : 0; 214 } 215 216 static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req, 217 int len) 218 { 219 return nvme_tcp_pdu_data_left(req) <= len; 220 } 221 222 static void nvme_tcp_init_iter(struct nvme_tcp_request *req, 223 unsigned int dir) 224 { 225 struct request *rq = blk_mq_rq_from_pdu(req); 226 struct bio_vec *vec; 227 unsigned int size; 228 int nr_bvec; 229 size_t offset; 230 231 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) { 232 vec = &rq->special_vec; 233 nr_bvec = 1; 234 size = blk_rq_payload_bytes(rq); 235 offset = 0; 236 } else { 237 struct bio *bio = req->curr_bio; 238 struct bvec_iter bi; 239 struct bio_vec bv; 240 241 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter); 242 nr_bvec = 0; 243 bio_for_each_bvec(bv, bio, bi) { 244 nr_bvec++; 245 } 246 size = bio->bi_iter.bi_size; 247 offset = bio->bi_iter.bi_bvec_done; 248 } 249 250 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size); 251 req->iter.iov_offset = offset; 252 } 253 254 static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req, 255 int len) 256 { 257 req->data_sent += len; 258 req->pdu_sent += len; 259 iov_iter_advance(&req->iter, len); 260 if (!iov_iter_count(&req->iter) && 261 req->data_sent < req->data_len) { 262 req->curr_bio = req->curr_bio->bi_next; 263 nvme_tcp_init_iter(req, WRITE); 264 } 265 } 266 267 static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue) 268 { 269 int ret; 270 271 /* drain the send queue as much as we can... */ 272 do { 273 ret = nvme_tcp_try_send(queue); 274 } while (ret > 0); 275 } 276 277 static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue) 278 { 279 return !list_empty(&queue->send_list) || 280 !llist_empty(&queue->req_list) || queue->more_requests; 281 } 282 283 static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req, 284 bool sync, bool last) 285 { 286 struct nvme_tcp_queue *queue = req->queue; 287 bool empty; 288 289 empty = llist_add(&req->lentry, &queue->req_list) && 290 list_empty(&queue->send_list) && !queue->request; 291 292 /* 293 * if we're the first on the send_list and we can try to send 294 * directly, otherwise queue io_work. Also, only do that if we 295 * are on the same cpu, so we don't introduce contention. 296 */ 297 if (queue->io_cpu == raw_smp_processor_id() && 298 sync && empty && mutex_trylock(&queue->send_mutex)) { 299 queue->more_requests = !last; 300 nvme_tcp_send_all(queue); 301 queue->more_requests = false; 302 mutex_unlock(&queue->send_mutex); 303 } 304 305 if (last && nvme_tcp_queue_more(queue)) 306 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 307 } 308 309 static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue) 310 { 311 struct nvme_tcp_request *req; 312 struct llist_node *node; 313 314 for (node = llist_del_all(&queue->req_list); node; node = node->next) { 315 req = llist_entry(node, struct nvme_tcp_request, lentry); 316 list_add(&req->entry, &queue->send_list); 317 } 318 } 319 320 static inline struct nvme_tcp_request * 321 nvme_tcp_fetch_request(struct nvme_tcp_queue *queue) 322 { 323 struct nvme_tcp_request *req; 324 325 req = list_first_entry_or_null(&queue->send_list, 326 struct nvme_tcp_request, entry); 327 if (!req) { 328 nvme_tcp_process_req_list(queue); 329 req = list_first_entry_or_null(&queue->send_list, 330 struct nvme_tcp_request, entry); 331 if (unlikely(!req)) 332 return NULL; 333 } 334 335 list_del(&req->entry); 336 return req; 337 } 338 339 static inline void nvme_tcp_ddgst_final(struct ahash_request *hash, 340 __le32 *dgst) 341 { 342 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0); 343 crypto_ahash_final(hash); 344 } 345 346 static inline void nvme_tcp_ddgst_update(struct ahash_request *hash, 347 struct page *page, off_t off, size_t len) 348 { 349 struct scatterlist sg; 350 351 sg_init_marker(&sg, 1); 352 sg_set_page(&sg, page, len, off); 353 ahash_request_set_crypt(hash, &sg, NULL, len); 354 crypto_ahash_update(hash); 355 } 356 357 static inline void nvme_tcp_hdgst(struct ahash_request *hash, 358 void *pdu, size_t len) 359 { 360 struct scatterlist sg; 361 362 sg_init_one(&sg, pdu, len); 363 ahash_request_set_crypt(hash, &sg, pdu + len, len); 364 crypto_ahash_digest(hash); 365 } 366 367 static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue, 368 void *pdu, size_t pdu_len) 369 { 370 struct nvme_tcp_hdr *hdr = pdu; 371 __le32 recv_digest; 372 __le32 exp_digest; 373 374 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) { 375 dev_err(queue->ctrl->ctrl.device, 376 "queue %d: header digest flag is cleared\n", 377 nvme_tcp_queue_id(queue)); 378 return -EPROTO; 379 } 380 381 recv_digest = *(__le32 *)(pdu + hdr->hlen); 382 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len); 383 exp_digest = *(__le32 *)(pdu + hdr->hlen); 384 if (recv_digest != exp_digest) { 385 dev_err(queue->ctrl->ctrl.device, 386 "header digest error: recv %#x expected %#x\n", 387 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest)); 388 return -EIO; 389 } 390 391 return 0; 392 } 393 394 static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu) 395 { 396 struct nvme_tcp_hdr *hdr = pdu; 397 u8 digest_len = nvme_tcp_hdgst_len(queue); 398 u32 len; 399 400 len = le32_to_cpu(hdr->plen) - hdr->hlen - 401 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0); 402 403 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) { 404 dev_err(queue->ctrl->ctrl.device, 405 "queue %d: data digest flag is cleared\n", 406 nvme_tcp_queue_id(queue)); 407 return -EPROTO; 408 } 409 crypto_ahash_init(queue->rcv_hash); 410 411 return 0; 412 } 413 414 static void nvme_tcp_exit_request(struct blk_mq_tag_set *set, 415 struct request *rq, unsigned int hctx_idx) 416 { 417 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 418 419 page_frag_free(req->pdu); 420 } 421 422 static int nvme_tcp_init_request(struct blk_mq_tag_set *set, 423 struct request *rq, unsigned int hctx_idx, 424 unsigned int numa_node) 425 { 426 struct nvme_tcp_ctrl *ctrl = set->driver_data; 427 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 428 struct nvme_tcp_cmd_pdu *pdu; 429 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0; 430 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx]; 431 u8 hdgst = nvme_tcp_hdgst_len(queue); 432 433 req->pdu = page_frag_alloc(&queue->pf_cache, 434 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 435 GFP_KERNEL | __GFP_ZERO); 436 if (!req->pdu) 437 return -ENOMEM; 438 439 pdu = req->pdu; 440 req->queue = queue; 441 nvme_req(rq)->ctrl = &ctrl->ctrl; 442 nvme_req(rq)->cmd = &pdu->cmd; 443 444 return 0; 445 } 446 447 static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, 448 unsigned int hctx_idx) 449 { 450 struct nvme_tcp_ctrl *ctrl = data; 451 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1]; 452 453 hctx->driver_data = queue; 454 return 0; 455 } 456 457 static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data, 458 unsigned int hctx_idx) 459 { 460 struct nvme_tcp_ctrl *ctrl = data; 461 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 462 463 hctx->driver_data = queue; 464 return 0; 465 } 466 467 static enum nvme_tcp_recv_state 468 nvme_tcp_recv_state(struct nvme_tcp_queue *queue) 469 { 470 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU : 471 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST : 472 NVME_TCP_RECV_DATA; 473 } 474 475 static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue) 476 { 477 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) + 478 nvme_tcp_hdgst_len(queue); 479 queue->pdu_offset = 0; 480 queue->data_remaining = -1; 481 queue->ddgst_remaining = 0; 482 } 483 484 static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl) 485 { 486 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING)) 487 return; 488 489 dev_warn(ctrl->device, "starting error recovery\n"); 490 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work); 491 } 492 493 static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, 494 struct nvme_completion *cqe) 495 { 496 struct nvme_tcp_request *req; 497 struct request *rq; 498 499 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id); 500 if (!rq) { 501 dev_err(queue->ctrl->ctrl.device, 502 "got bad cqe.command_id %#x on queue %d\n", 503 cqe->command_id, nvme_tcp_queue_id(queue)); 504 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 505 return -EINVAL; 506 } 507 508 req = blk_mq_rq_to_pdu(rq); 509 if (req->status == cpu_to_le16(NVME_SC_SUCCESS)) 510 req->status = cqe->status; 511 512 if (!nvme_try_complete_req(rq, req->status, cqe->result)) 513 nvme_complete_rq(rq); 514 queue->nr_cqe++; 515 516 return 0; 517 } 518 519 static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue, 520 struct nvme_tcp_data_pdu *pdu) 521 { 522 struct request *rq; 523 524 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 525 if (!rq) { 526 dev_err(queue->ctrl->ctrl.device, 527 "got bad c2hdata.command_id %#x on queue %d\n", 528 pdu->command_id, nvme_tcp_queue_id(queue)); 529 return -ENOENT; 530 } 531 532 if (!blk_rq_payload_bytes(rq)) { 533 dev_err(queue->ctrl->ctrl.device, 534 "queue %d tag %#x unexpected data\n", 535 nvme_tcp_queue_id(queue), rq->tag); 536 return -EIO; 537 } 538 539 queue->data_remaining = le32_to_cpu(pdu->data_length); 540 541 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS && 542 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) { 543 dev_err(queue->ctrl->ctrl.device, 544 "queue %d tag %#x SUCCESS set but not last PDU\n", 545 nvme_tcp_queue_id(queue), rq->tag); 546 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 547 return -EPROTO; 548 } 549 550 return 0; 551 } 552 553 static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue, 554 struct nvme_tcp_rsp_pdu *pdu) 555 { 556 struct nvme_completion *cqe = &pdu->cqe; 557 int ret = 0; 558 559 /* 560 * AEN requests are special as they don't time out and can 561 * survive any kind of queue freeze and often don't respond to 562 * aborts. We don't even bother to allocate a struct request 563 * for them but rather special case them here. 564 */ 565 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue), 566 cqe->command_id))) 567 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status, 568 &cqe->result); 569 else 570 ret = nvme_tcp_process_nvme_cqe(queue, cqe); 571 572 return ret; 573 } 574 575 static int nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req, 576 struct nvme_tcp_r2t_pdu *pdu) 577 { 578 struct nvme_tcp_data_pdu *data = req->pdu; 579 struct nvme_tcp_queue *queue = req->queue; 580 struct request *rq = blk_mq_rq_from_pdu(req); 581 u8 hdgst = nvme_tcp_hdgst_len(queue); 582 u8 ddgst = nvme_tcp_ddgst_len(queue); 583 584 req->pdu_len = le32_to_cpu(pdu->r2t_length); 585 req->pdu_sent = 0; 586 587 if (unlikely(!req->pdu_len)) { 588 dev_err(queue->ctrl->ctrl.device, 589 "req %d r2t len is %u, probably a bug...\n", 590 rq->tag, req->pdu_len); 591 return -EPROTO; 592 } 593 594 if (unlikely(req->data_sent + req->pdu_len > req->data_len)) { 595 dev_err(queue->ctrl->ctrl.device, 596 "req %d r2t len %u exceeded data len %u (%zu sent)\n", 597 rq->tag, req->pdu_len, req->data_len, 598 req->data_sent); 599 return -EPROTO; 600 } 601 602 if (unlikely(le32_to_cpu(pdu->r2t_offset) < req->data_sent)) { 603 dev_err(queue->ctrl->ctrl.device, 604 "req %d unexpected r2t offset %u (expected %zu)\n", 605 rq->tag, le32_to_cpu(pdu->r2t_offset), 606 req->data_sent); 607 return -EPROTO; 608 } 609 610 memset(data, 0, sizeof(*data)); 611 data->hdr.type = nvme_tcp_h2c_data; 612 data->hdr.flags = NVME_TCP_F_DATA_LAST; 613 if (queue->hdr_digest) 614 data->hdr.flags |= NVME_TCP_F_HDGST; 615 if (queue->data_digest) 616 data->hdr.flags |= NVME_TCP_F_DDGST; 617 data->hdr.hlen = sizeof(*data); 618 data->hdr.pdo = data->hdr.hlen + hdgst; 619 data->hdr.plen = 620 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst); 621 data->ttag = pdu->ttag; 622 data->command_id = nvme_cid(rq); 623 data->data_offset = pdu->r2t_offset; 624 data->data_length = cpu_to_le32(req->pdu_len); 625 return 0; 626 } 627 628 static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue, 629 struct nvme_tcp_r2t_pdu *pdu) 630 { 631 struct nvme_tcp_request *req; 632 struct request *rq; 633 int ret; 634 635 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id); 636 if (!rq) { 637 dev_err(queue->ctrl->ctrl.device, 638 "got bad r2t.command_id %#x on queue %d\n", 639 pdu->command_id, nvme_tcp_queue_id(queue)); 640 return -ENOENT; 641 } 642 req = blk_mq_rq_to_pdu(rq); 643 644 ret = nvme_tcp_setup_h2c_data_pdu(req, pdu); 645 if (unlikely(ret)) 646 return ret; 647 648 req->state = NVME_TCP_SEND_H2C_PDU; 649 req->offset = 0; 650 651 nvme_tcp_queue_request(req, false, true); 652 653 return 0; 654 } 655 656 static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb, 657 unsigned int *offset, size_t *len) 658 { 659 struct nvme_tcp_hdr *hdr; 660 char *pdu = queue->pdu; 661 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining); 662 int ret; 663 664 ret = skb_copy_bits(skb, *offset, 665 &pdu[queue->pdu_offset], rcv_len); 666 if (unlikely(ret)) 667 return ret; 668 669 queue->pdu_remaining -= rcv_len; 670 queue->pdu_offset += rcv_len; 671 *offset += rcv_len; 672 *len -= rcv_len; 673 if (queue->pdu_remaining) 674 return 0; 675 676 hdr = queue->pdu; 677 if (queue->hdr_digest) { 678 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen); 679 if (unlikely(ret)) 680 return ret; 681 } 682 683 684 if (queue->data_digest) { 685 ret = nvme_tcp_check_ddgst(queue, queue->pdu); 686 if (unlikely(ret)) 687 return ret; 688 } 689 690 switch (hdr->type) { 691 case nvme_tcp_c2h_data: 692 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu); 693 case nvme_tcp_rsp: 694 nvme_tcp_init_recv_ctx(queue); 695 return nvme_tcp_handle_comp(queue, (void *)queue->pdu); 696 case nvme_tcp_r2t: 697 nvme_tcp_init_recv_ctx(queue); 698 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu); 699 default: 700 dev_err(queue->ctrl->ctrl.device, 701 "unsupported pdu type (%d)\n", hdr->type); 702 return -EINVAL; 703 } 704 } 705 706 static inline void nvme_tcp_end_request(struct request *rq, u16 status) 707 { 708 union nvme_result res = {}; 709 710 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res)) 711 nvme_complete_rq(rq); 712 } 713 714 static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, 715 unsigned int *offset, size_t *len) 716 { 717 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 718 struct request *rq = 719 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id); 720 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 721 722 while (true) { 723 int recv_len, ret; 724 725 recv_len = min_t(size_t, *len, queue->data_remaining); 726 if (!recv_len) 727 break; 728 729 if (!iov_iter_count(&req->iter)) { 730 req->curr_bio = req->curr_bio->bi_next; 731 732 /* 733 * If we don`t have any bios it means that controller 734 * sent more data than we requested, hence error 735 */ 736 if (!req->curr_bio) { 737 dev_err(queue->ctrl->ctrl.device, 738 "queue %d no space in request %#x", 739 nvme_tcp_queue_id(queue), rq->tag); 740 nvme_tcp_init_recv_ctx(queue); 741 return -EIO; 742 } 743 nvme_tcp_init_iter(req, READ); 744 } 745 746 /* we can read only from what is left in this bio */ 747 recv_len = min_t(size_t, recv_len, 748 iov_iter_count(&req->iter)); 749 750 if (queue->data_digest) 751 ret = skb_copy_and_hash_datagram_iter(skb, *offset, 752 &req->iter, recv_len, queue->rcv_hash); 753 else 754 ret = skb_copy_datagram_iter(skb, *offset, 755 &req->iter, recv_len); 756 if (ret) { 757 dev_err(queue->ctrl->ctrl.device, 758 "queue %d failed to copy request %#x data", 759 nvme_tcp_queue_id(queue), rq->tag); 760 return ret; 761 } 762 763 *len -= recv_len; 764 *offset += recv_len; 765 queue->data_remaining -= recv_len; 766 } 767 768 if (!queue->data_remaining) { 769 if (queue->data_digest) { 770 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst); 771 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH; 772 } else { 773 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 774 nvme_tcp_end_request(rq, 775 le16_to_cpu(req->status)); 776 queue->nr_cqe++; 777 } 778 nvme_tcp_init_recv_ctx(queue); 779 } 780 } 781 782 return 0; 783 } 784 785 static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue, 786 struct sk_buff *skb, unsigned int *offset, size_t *len) 787 { 788 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu; 789 char *ddgst = (char *)&queue->recv_ddgst; 790 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining); 791 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining; 792 int ret; 793 794 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len); 795 if (unlikely(ret)) 796 return ret; 797 798 queue->ddgst_remaining -= recv_len; 799 *offset += recv_len; 800 *len -= recv_len; 801 if (queue->ddgst_remaining) 802 return 0; 803 804 if (queue->recv_ddgst != queue->exp_ddgst) { 805 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 806 pdu->command_id); 807 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 808 809 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR); 810 811 dev_err(queue->ctrl->ctrl.device, 812 "data digest error: recv %#x expected %#x\n", 813 le32_to_cpu(queue->recv_ddgst), 814 le32_to_cpu(queue->exp_ddgst)); 815 } 816 817 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) { 818 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue), 819 pdu->command_id); 820 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 821 822 nvme_tcp_end_request(rq, le16_to_cpu(req->status)); 823 queue->nr_cqe++; 824 } 825 826 nvme_tcp_init_recv_ctx(queue); 827 return 0; 828 } 829 830 static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb, 831 unsigned int offset, size_t len) 832 { 833 struct nvme_tcp_queue *queue = desc->arg.data; 834 size_t consumed = len; 835 int result; 836 837 while (len) { 838 switch (nvme_tcp_recv_state(queue)) { 839 case NVME_TCP_RECV_PDU: 840 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len); 841 break; 842 case NVME_TCP_RECV_DATA: 843 result = nvme_tcp_recv_data(queue, skb, &offset, &len); 844 break; 845 case NVME_TCP_RECV_DDGST: 846 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len); 847 break; 848 default: 849 result = -EFAULT; 850 } 851 if (result) { 852 dev_err(queue->ctrl->ctrl.device, 853 "receive failed: %d\n", result); 854 queue->rd_enabled = false; 855 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 856 return result; 857 } 858 } 859 860 return consumed; 861 } 862 863 static void nvme_tcp_data_ready(struct sock *sk) 864 { 865 struct nvme_tcp_queue *queue; 866 867 read_lock_bh(&sk->sk_callback_lock); 868 queue = sk->sk_user_data; 869 if (likely(queue && queue->rd_enabled) && 870 !test_bit(NVME_TCP_Q_POLLING, &queue->flags)) 871 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 872 read_unlock_bh(&sk->sk_callback_lock); 873 } 874 875 static void nvme_tcp_write_space(struct sock *sk) 876 { 877 struct nvme_tcp_queue *queue; 878 879 read_lock_bh(&sk->sk_callback_lock); 880 queue = sk->sk_user_data; 881 if (likely(queue && sk_stream_is_writeable(sk))) { 882 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 883 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 884 } 885 read_unlock_bh(&sk->sk_callback_lock); 886 } 887 888 static void nvme_tcp_state_change(struct sock *sk) 889 { 890 struct nvme_tcp_queue *queue; 891 892 read_lock_bh(&sk->sk_callback_lock); 893 queue = sk->sk_user_data; 894 if (!queue) 895 goto done; 896 897 switch (sk->sk_state) { 898 case TCP_CLOSE: 899 case TCP_CLOSE_WAIT: 900 case TCP_LAST_ACK: 901 case TCP_FIN_WAIT1: 902 case TCP_FIN_WAIT2: 903 nvme_tcp_error_recovery(&queue->ctrl->ctrl); 904 break; 905 default: 906 dev_info(queue->ctrl->ctrl.device, 907 "queue %d socket state %d\n", 908 nvme_tcp_queue_id(queue), sk->sk_state); 909 } 910 911 queue->state_change(sk); 912 done: 913 read_unlock_bh(&sk->sk_callback_lock); 914 } 915 916 static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue) 917 { 918 queue->request = NULL; 919 } 920 921 static void nvme_tcp_fail_request(struct nvme_tcp_request *req) 922 { 923 nvme_tcp_end_request(blk_mq_rq_from_pdu(req), NVME_SC_HOST_PATH_ERROR); 924 } 925 926 static int nvme_tcp_try_send_data(struct nvme_tcp_request *req) 927 { 928 struct nvme_tcp_queue *queue = req->queue; 929 930 while (true) { 931 struct page *page = nvme_tcp_req_cur_page(req); 932 size_t offset = nvme_tcp_req_cur_offset(req); 933 size_t len = nvme_tcp_req_cur_length(req); 934 bool last = nvme_tcp_pdu_last_send(req, len); 935 int ret, flags = MSG_DONTWAIT; 936 937 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue)) 938 flags |= MSG_EOR; 939 else 940 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 941 942 if (sendpage_ok(page)) { 943 ret = kernel_sendpage(queue->sock, page, offset, len, 944 flags); 945 } else { 946 ret = sock_no_sendpage(queue->sock, page, offset, len, 947 flags); 948 } 949 if (ret <= 0) 950 return ret; 951 952 if (queue->data_digest) 953 nvme_tcp_ddgst_update(queue->snd_hash, page, 954 offset, ret); 955 956 /* 957 * update the request iterator except for the last payload send 958 * in the request where we don't want to modify it as we may 959 * compete with the RX path completing the request. 960 */ 961 if (req->data_sent + ret < req->data_len) 962 nvme_tcp_advance_req(req, ret); 963 964 /* fully successful last send in current PDU */ 965 if (last && ret == len) { 966 if (queue->data_digest) { 967 nvme_tcp_ddgst_final(queue->snd_hash, 968 &req->ddgst); 969 req->state = NVME_TCP_SEND_DDGST; 970 req->offset = 0; 971 } else { 972 nvme_tcp_done_send_req(queue); 973 } 974 return 1; 975 } 976 } 977 return -EAGAIN; 978 } 979 980 static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req) 981 { 982 struct nvme_tcp_queue *queue = req->queue; 983 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 984 bool inline_data = nvme_tcp_has_inline_data(req); 985 u8 hdgst = nvme_tcp_hdgst_len(queue); 986 int len = sizeof(*pdu) + hdgst - req->offset; 987 int flags = MSG_DONTWAIT; 988 int ret; 989 990 if (inline_data || nvme_tcp_queue_more(queue)) 991 flags |= MSG_MORE | MSG_SENDPAGE_NOTLAST; 992 else 993 flags |= MSG_EOR; 994 995 if (queue->hdr_digest && !req->offset) 996 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 997 998 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 999 offset_in_page(pdu) + req->offset, len, flags); 1000 if (unlikely(ret <= 0)) 1001 return ret; 1002 1003 len -= ret; 1004 if (!len) { 1005 if (inline_data) { 1006 req->state = NVME_TCP_SEND_DATA; 1007 if (queue->data_digest) 1008 crypto_ahash_init(queue->snd_hash); 1009 } else { 1010 nvme_tcp_done_send_req(queue); 1011 } 1012 return 1; 1013 } 1014 req->offset += ret; 1015 1016 return -EAGAIN; 1017 } 1018 1019 static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req) 1020 { 1021 struct nvme_tcp_queue *queue = req->queue; 1022 struct nvme_tcp_data_pdu *pdu = req->pdu; 1023 u8 hdgst = nvme_tcp_hdgst_len(queue); 1024 int len = sizeof(*pdu) - req->offset + hdgst; 1025 int ret; 1026 1027 if (queue->hdr_digest && !req->offset) 1028 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu)); 1029 1030 ret = kernel_sendpage(queue->sock, virt_to_page(pdu), 1031 offset_in_page(pdu) + req->offset, len, 1032 MSG_DONTWAIT | MSG_MORE | MSG_SENDPAGE_NOTLAST); 1033 if (unlikely(ret <= 0)) 1034 return ret; 1035 1036 len -= ret; 1037 if (!len) { 1038 req->state = NVME_TCP_SEND_DATA; 1039 if (queue->data_digest) 1040 crypto_ahash_init(queue->snd_hash); 1041 return 1; 1042 } 1043 req->offset += ret; 1044 1045 return -EAGAIN; 1046 } 1047 1048 static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req) 1049 { 1050 struct nvme_tcp_queue *queue = req->queue; 1051 int ret; 1052 struct msghdr msg = { .msg_flags = MSG_DONTWAIT }; 1053 struct kvec iov = { 1054 .iov_base = &req->ddgst + req->offset, 1055 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset 1056 }; 1057 1058 if (nvme_tcp_queue_more(queue)) 1059 msg.msg_flags |= MSG_MORE; 1060 else 1061 msg.msg_flags |= MSG_EOR; 1062 1063 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1064 if (unlikely(ret <= 0)) 1065 return ret; 1066 1067 if (req->offset + ret == NVME_TCP_DIGEST_LENGTH) { 1068 nvme_tcp_done_send_req(queue); 1069 return 1; 1070 } 1071 1072 req->offset += ret; 1073 return -EAGAIN; 1074 } 1075 1076 static int nvme_tcp_try_send(struct nvme_tcp_queue *queue) 1077 { 1078 struct nvme_tcp_request *req; 1079 int ret = 1; 1080 1081 if (!queue->request) { 1082 queue->request = nvme_tcp_fetch_request(queue); 1083 if (!queue->request) 1084 return 0; 1085 } 1086 req = queue->request; 1087 1088 if (req->state == NVME_TCP_SEND_CMD_PDU) { 1089 ret = nvme_tcp_try_send_cmd_pdu(req); 1090 if (ret <= 0) 1091 goto done; 1092 if (!nvme_tcp_has_inline_data(req)) 1093 return ret; 1094 } 1095 1096 if (req->state == NVME_TCP_SEND_H2C_PDU) { 1097 ret = nvme_tcp_try_send_data_pdu(req); 1098 if (ret <= 0) 1099 goto done; 1100 } 1101 1102 if (req->state == NVME_TCP_SEND_DATA) { 1103 ret = nvme_tcp_try_send_data(req); 1104 if (ret <= 0) 1105 goto done; 1106 } 1107 1108 if (req->state == NVME_TCP_SEND_DDGST) 1109 ret = nvme_tcp_try_send_ddgst(req); 1110 done: 1111 if (ret == -EAGAIN) { 1112 ret = 0; 1113 } else if (ret < 0) { 1114 dev_err(queue->ctrl->ctrl.device, 1115 "failed to send request %d\n", ret); 1116 if (ret != -EPIPE && ret != -ECONNRESET) 1117 nvme_tcp_fail_request(queue->request); 1118 nvme_tcp_done_send_req(queue); 1119 } 1120 return ret; 1121 } 1122 1123 static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue) 1124 { 1125 struct socket *sock = queue->sock; 1126 struct sock *sk = sock->sk; 1127 read_descriptor_t rd_desc; 1128 int consumed; 1129 1130 rd_desc.arg.data = queue; 1131 rd_desc.count = 1; 1132 lock_sock(sk); 1133 queue->nr_cqe = 0; 1134 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb); 1135 release_sock(sk); 1136 return consumed; 1137 } 1138 1139 static void nvme_tcp_io_work(struct work_struct *w) 1140 { 1141 struct nvme_tcp_queue *queue = 1142 container_of(w, struct nvme_tcp_queue, io_work); 1143 unsigned long deadline = jiffies + msecs_to_jiffies(1); 1144 1145 do { 1146 bool pending = false; 1147 int result; 1148 1149 if (mutex_trylock(&queue->send_mutex)) { 1150 result = nvme_tcp_try_send(queue); 1151 mutex_unlock(&queue->send_mutex); 1152 if (result > 0) 1153 pending = true; 1154 else if (unlikely(result < 0)) 1155 break; 1156 } 1157 1158 result = nvme_tcp_try_recv(queue); 1159 if (result > 0) 1160 pending = true; 1161 else if (unlikely(result < 0)) 1162 return; 1163 1164 if (!pending) 1165 return; 1166 1167 } while (!time_after(jiffies, deadline)); /* quota is exhausted */ 1168 1169 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 1170 } 1171 1172 static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue) 1173 { 1174 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash); 1175 1176 ahash_request_free(queue->rcv_hash); 1177 ahash_request_free(queue->snd_hash); 1178 crypto_free_ahash(tfm); 1179 } 1180 1181 static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue) 1182 { 1183 struct crypto_ahash *tfm; 1184 1185 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC); 1186 if (IS_ERR(tfm)) 1187 return PTR_ERR(tfm); 1188 1189 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1190 if (!queue->snd_hash) 1191 goto free_tfm; 1192 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL); 1193 1194 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL); 1195 if (!queue->rcv_hash) 1196 goto free_snd_hash; 1197 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL); 1198 1199 return 0; 1200 free_snd_hash: 1201 ahash_request_free(queue->snd_hash); 1202 free_tfm: 1203 crypto_free_ahash(tfm); 1204 return -ENOMEM; 1205 } 1206 1207 static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl) 1208 { 1209 struct nvme_tcp_request *async = &ctrl->async_req; 1210 1211 page_frag_free(async->pdu); 1212 } 1213 1214 static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl) 1215 { 1216 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 1217 struct nvme_tcp_request *async = &ctrl->async_req; 1218 u8 hdgst = nvme_tcp_hdgst_len(queue); 1219 1220 async->pdu = page_frag_alloc(&queue->pf_cache, 1221 sizeof(struct nvme_tcp_cmd_pdu) + hdgst, 1222 GFP_KERNEL | __GFP_ZERO); 1223 if (!async->pdu) 1224 return -ENOMEM; 1225 1226 async->queue = &ctrl->queues[0]; 1227 return 0; 1228 } 1229 1230 static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid) 1231 { 1232 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1233 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1234 1235 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags)) 1236 return; 1237 1238 if (queue->hdr_digest || queue->data_digest) 1239 nvme_tcp_free_crypto(queue); 1240 1241 sock_release(queue->sock); 1242 kfree(queue->pdu); 1243 mutex_destroy(&queue->send_mutex); 1244 mutex_destroy(&queue->queue_lock); 1245 } 1246 1247 static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue) 1248 { 1249 struct nvme_tcp_icreq_pdu *icreq; 1250 struct nvme_tcp_icresp_pdu *icresp; 1251 struct msghdr msg = {}; 1252 struct kvec iov; 1253 bool ctrl_hdgst, ctrl_ddgst; 1254 int ret; 1255 1256 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL); 1257 if (!icreq) 1258 return -ENOMEM; 1259 1260 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL); 1261 if (!icresp) { 1262 ret = -ENOMEM; 1263 goto free_icreq; 1264 } 1265 1266 icreq->hdr.type = nvme_tcp_icreq; 1267 icreq->hdr.hlen = sizeof(*icreq); 1268 icreq->hdr.pdo = 0; 1269 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen); 1270 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0); 1271 icreq->maxr2t = 0; /* single inflight r2t supported */ 1272 icreq->hpda = 0; /* no alignment constraint */ 1273 if (queue->hdr_digest) 1274 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE; 1275 if (queue->data_digest) 1276 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE; 1277 1278 iov.iov_base = icreq; 1279 iov.iov_len = sizeof(*icreq); 1280 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len); 1281 if (ret < 0) 1282 goto free_icresp; 1283 1284 memset(&msg, 0, sizeof(msg)); 1285 iov.iov_base = icresp; 1286 iov.iov_len = sizeof(*icresp); 1287 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1, 1288 iov.iov_len, msg.msg_flags); 1289 if (ret < 0) 1290 goto free_icresp; 1291 1292 ret = -EINVAL; 1293 if (icresp->hdr.type != nvme_tcp_icresp) { 1294 pr_err("queue %d: bad type returned %d\n", 1295 nvme_tcp_queue_id(queue), icresp->hdr.type); 1296 goto free_icresp; 1297 } 1298 1299 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) { 1300 pr_err("queue %d: bad pdu length returned %d\n", 1301 nvme_tcp_queue_id(queue), icresp->hdr.plen); 1302 goto free_icresp; 1303 } 1304 1305 if (icresp->pfv != NVME_TCP_PFV_1_0) { 1306 pr_err("queue %d: bad pfv returned %d\n", 1307 nvme_tcp_queue_id(queue), icresp->pfv); 1308 goto free_icresp; 1309 } 1310 1311 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE); 1312 if ((queue->data_digest && !ctrl_ddgst) || 1313 (!queue->data_digest && ctrl_ddgst)) { 1314 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n", 1315 nvme_tcp_queue_id(queue), 1316 queue->data_digest ? "enabled" : "disabled", 1317 ctrl_ddgst ? "enabled" : "disabled"); 1318 goto free_icresp; 1319 } 1320 1321 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE); 1322 if ((queue->hdr_digest && !ctrl_hdgst) || 1323 (!queue->hdr_digest && ctrl_hdgst)) { 1324 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n", 1325 nvme_tcp_queue_id(queue), 1326 queue->hdr_digest ? "enabled" : "disabled", 1327 ctrl_hdgst ? "enabled" : "disabled"); 1328 goto free_icresp; 1329 } 1330 1331 if (icresp->cpda != 0) { 1332 pr_err("queue %d: unsupported cpda returned %d\n", 1333 nvme_tcp_queue_id(queue), icresp->cpda); 1334 goto free_icresp; 1335 } 1336 1337 ret = 0; 1338 free_icresp: 1339 kfree(icresp); 1340 free_icreq: 1341 kfree(icreq); 1342 return ret; 1343 } 1344 1345 static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue) 1346 { 1347 return nvme_tcp_queue_id(queue) == 0; 1348 } 1349 1350 static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue) 1351 { 1352 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1353 int qid = nvme_tcp_queue_id(queue); 1354 1355 return !nvme_tcp_admin_queue(queue) && 1356 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1357 } 1358 1359 static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue) 1360 { 1361 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1362 int qid = nvme_tcp_queue_id(queue); 1363 1364 return !nvme_tcp_admin_queue(queue) && 1365 !nvme_tcp_default_queue(queue) && 1366 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1367 ctrl->io_queues[HCTX_TYPE_READ]; 1368 } 1369 1370 static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue) 1371 { 1372 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1373 int qid = nvme_tcp_queue_id(queue); 1374 1375 return !nvme_tcp_admin_queue(queue) && 1376 !nvme_tcp_default_queue(queue) && 1377 !nvme_tcp_read_queue(queue) && 1378 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] + 1379 ctrl->io_queues[HCTX_TYPE_READ] + 1380 ctrl->io_queues[HCTX_TYPE_POLL]; 1381 } 1382 1383 static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue) 1384 { 1385 struct nvme_tcp_ctrl *ctrl = queue->ctrl; 1386 int qid = nvme_tcp_queue_id(queue); 1387 int n = 0; 1388 1389 if (nvme_tcp_default_queue(queue)) 1390 n = qid - 1; 1391 else if (nvme_tcp_read_queue(queue)) 1392 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1; 1393 else if (nvme_tcp_poll_queue(queue)) 1394 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1395 ctrl->io_queues[HCTX_TYPE_READ] - 1; 1396 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false); 1397 } 1398 1399 static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, 1400 int qid, size_t queue_size) 1401 { 1402 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1403 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1404 int ret, rcv_pdu_size; 1405 1406 mutex_init(&queue->queue_lock); 1407 queue->ctrl = ctrl; 1408 init_llist_head(&queue->req_list); 1409 INIT_LIST_HEAD(&queue->send_list); 1410 mutex_init(&queue->send_mutex); 1411 INIT_WORK(&queue->io_work, nvme_tcp_io_work); 1412 queue->queue_size = queue_size; 1413 1414 if (qid > 0) 1415 queue->cmnd_capsule_len = nctrl->ioccsz * 16; 1416 else 1417 queue->cmnd_capsule_len = sizeof(struct nvme_command) + 1418 NVME_TCP_ADMIN_CCSZ; 1419 1420 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM, 1421 IPPROTO_TCP, &queue->sock); 1422 if (ret) { 1423 dev_err(nctrl->device, 1424 "failed to create socket: %d\n", ret); 1425 goto err_destroy_mutex; 1426 } 1427 1428 /* Single syn retry */ 1429 tcp_sock_set_syncnt(queue->sock->sk, 1); 1430 1431 /* Set TCP no delay */ 1432 tcp_sock_set_nodelay(queue->sock->sk); 1433 1434 /* 1435 * Cleanup whatever is sitting in the TCP transmit queue on socket 1436 * close. This is done to prevent stale data from being sent should 1437 * the network connection be restored before TCP times out. 1438 */ 1439 sock_no_linger(queue->sock->sk); 1440 1441 if (so_priority > 0) 1442 sock_set_priority(queue->sock->sk, so_priority); 1443 1444 /* Set socket type of service */ 1445 if (nctrl->opts->tos >= 0) 1446 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos); 1447 1448 /* Set 10 seconds timeout for icresp recvmsg */ 1449 queue->sock->sk->sk_rcvtimeo = 10 * HZ; 1450 1451 queue->sock->sk->sk_allocation = GFP_ATOMIC; 1452 nvme_tcp_set_queue_io_cpu(queue); 1453 queue->request = NULL; 1454 queue->data_remaining = 0; 1455 queue->ddgst_remaining = 0; 1456 queue->pdu_remaining = 0; 1457 queue->pdu_offset = 0; 1458 sk_set_memalloc(queue->sock->sk); 1459 1460 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) { 1461 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr, 1462 sizeof(ctrl->src_addr)); 1463 if (ret) { 1464 dev_err(nctrl->device, 1465 "failed to bind queue %d socket %d\n", 1466 qid, ret); 1467 goto err_sock; 1468 } 1469 } 1470 1471 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) { 1472 char *iface = nctrl->opts->host_iface; 1473 sockptr_t optval = KERNEL_SOCKPTR(iface); 1474 1475 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE, 1476 optval, strlen(iface)); 1477 if (ret) { 1478 dev_err(nctrl->device, 1479 "failed to bind to interface %s queue %d err %d\n", 1480 iface, qid, ret); 1481 goto err_sock; 1482 } 1483 } 1484 1485 queue->hdr_digest = nctrl->opts->hdr_digest; 1486 queue->data_digest = nctrl->opts->data_digest; 1487 if (queue->hdr_digest || queue->data_digest) { 1488 ret = nvme_tcp_alloc_crypto(queue); 1489 if (ret) { 1490 dev_err(nctrl->device, 1491 "failed to allocate queue %d crypto\n", qid); 1492 goto err_sock; 1493 } 1494 } 1495 1496 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) + 1497 nvme_tcp_hdgst_len(queue); 1498 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL); 1499 if (!queue->pdu) { 1500 ret = -ENOMEM; 1501 goto err_crypto; 1502 } 1503 1504 dev_dbg(nctrl->device, "connecting queue %d\n", 1505 nvme_tcp_queue_id(queue)); 1506 1507 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr, 1508 sizeof(ctrl->addr), 0); 1509 if (ret) { 1510 dev_err(nctrl->device, 1511 "failed to connect socket: %d\n", ret); 1512 goto err_rcv_pdu; 1513 } 1514 1515 ret = nvme_tcp_init_connection(queue); 1516 if (ret) 1517 goto err_init_connect; 1518 1519 queue->rd_enabled = true; 1520 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags); 1521 nvme_tcp_init_recv_ctx(queue); 1522 1523 write_lock_bh(&queue->sock->sk->sk_callback_lock); 1524 queue->sock->sk->sk_user_data = queue; 1525 queue->state_change = queue->sock->sk->sk_state_change; 1526 queue->data_ready = queue->sock->sk->sk_data_ready; 1527 queue->write_space = queue->sock->sk->sk_write_space; 1528 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready; 1529 queue->sock->sk->sk_state_change = nvme_tcp_state_change; 1530 queue->sock->sk->sk_write_space = nvme_tcp_write_space; 1531 #ifdef CONFIG_NET_RX_BUSY_POLL 1532 queue->sock->sk->sk_ll_usec = 1; 1533 #endif 1534 write_unlock_bh(&queue->sock->sk->sk_callback_lock); 1535 1536 return 0; 1537 1538 err_init_connect: 1539 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1540 err_rcv_pdu: 1541 kfree(queue->pdu); 1542 err_crypto: 1543 if (queue->hdr_digest || queue->data_digest) 1544 nvme_tcp_free_crypto(queue); 1545 err_sock: 1546 sock_release(queue->sock); 1547 queue->sock = NULL; 1548 err_destroy_mutex: 1549 mutex_destroy(&queue->send_mutex); 1550 mutex_destroy(&queue->queue_lock); 1551 return ret; 1552 } 1553 1554 static void nvme_tcp_restore_sock_calls(struct nvme_tcp_queue *queue) 1555 { 1556 struct socket *sock = queue->sock; 1557 1558 write_lock_bh(&sock->sk->sk_callback_lock); 1559 sock->sk->sk_user_data = NULL; 1560 sock->sk->sk_data_ready = queue->data_ready; 1561 sock->sk->sk_state_change = queue->state_change; 1562 sock->sk->sk_write_space = queue->write_space; 1563 write_unlock_bh(&sock->sk->sk_callback_lock); 1564 } 1565 1566 static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue) 1567 { 1568 kernel_sock_shutdown(queue->sock, SHUT_RDWR); 1569 nvme_tcp_restore_sock_calls(queue); 1570 cancel_work_sync(&queue->io_work); 1571 } 1572 1573 static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid) 1574 { 1575 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1576 struct nvme_tcp_queue *queue = &ctrl->queues[qid]; 1577 1578 mutex_lock(&queue->queue_lock); 1579 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags)) 1580 __nvme_tcp_stop_queue(queue); 1581 mutex_unlock(&queue->queue_lock); 1582 } 1583 1584 static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx) 1585 { 1586 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1587 int ret; 1588 1589 if (idx) 1590 ret = nvmf_connect_io_queue(nctrl, idx); 1591 else 1592 ret = nvmf_connect_admin_queue(nctrl); 1593 1594 if (!ret) { 1595 set_bit(NVME_TCP_Q_LIVE, &ctrl->queues[idx].flags); 1596 } else { 1597 if (test_bit(NVME_TCP_Q_ALLOCATED, &ctrl->queues[idx].flags)) 1598 __nvme_tcp_stop_queue(&ctrl->queues[idx]); 1599 dev_err(nctrl->device, 1600 "failed to connect queue: %d ret=%d\n", idx, ret); 1601 } 1602 return ret; 1603 } 1604 1605 static struct blk_mq_tag_set *nvme_tcp_alloc_tagset(struct nvme_ctrl *nctrl, 1606 bool admin) 1607 { 1608 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1609 struct blk_mq_tag_set *set; 1610 int ret; 1611 1612 if (admin) { 1613 set = &ctrl->admin_tag_set; 1614 memset(set, 0, sizeof(*set)); 1615 set->ops = &nvme_tcp_admin_mq_ops; 1616 set->queue_depth = NVME_AQ_MQ_TAG_DEPTH; 1617 set->reserved_tags = NVMF_RESERVED_TAGS; 1618 set->numa_node = nctrl->numa_node; 1619 set->flags = BLK_MQ_F_BLOCKING; 1620 set->cmd_size = sizeof(struct nvme_tcp_request); 1621 set->driver_data = ctrl; 1622 set->nr_hw_queues = 1; 1623 set->timeout = NVME_ADMIN_TIMEOUT; 1624 } else { 1625 set = &ctrl->tag_set; 1626 memset(set, 0, sizeof(*set)); 1627 set->ops = &nvme_tcp_mq_ops; 1628 set->queue_depth = nctrl->sqsize + 1; 1629 set->reserved_tags = NVMF_RESERVED_TAGS; 1630 set->numa_node = nctrl->numa_node; 1631 set->flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING; 1632 set->cmd_size = sizeof(struct nvme_tcp_request); 1633 set->driver_data = ctrl; 1634 set->nr_hw_queues = nctrl->queue_count - 1; 1635 set->timeout = NVME_IO_TIMEOUT; 1636 set->nr_maps = nctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2; 1637 } 1638 1639 ret = blk_mq_alloc_tag_set(set); 1640 if (ret) 1641 return ERR_PTR(ret); 1642 1643 return set; 1644 } 1645 1646 static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl) 1647 { 1648 if (to_tcp_ctrl(ctrl)->async_req.pdu) { 1649 cancel_work_sync(&ctrl->async_event_work); 1650 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl)); 1651 to_tcp_ctrl(ctrl)->async_req.pdu = NULL; 1652 } 1653 1654 nvme_tcp_free_queue(ctrl, 0); 1655 } 1656 1657 static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl) 1658 { 1659 int i; 1660 1661 for (i = 1; i < ctrl->queue_count; i++) 1662 nvme_tcp_free_queue(ctrl, i); 1663 } 1664 1665 static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl) 1666 { 1667 int i; 1668 1669 for (i = 1; i < ctrl->queue_count; i++) 1670 nvme_tcp_stop_queue(ctrl, i); 1671 } 1672 1673 static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl) 1674 { 1675 int i, ret = 0; 1676 1677 for (i = 1; i < ctrl->queue_count; i++) { 1678 ret = nvme_tcp_start_queue(ctrl, i); 1679 if (ret) 1680 goto out_stop_queues; 1681 } 1682 1683 return 0; 1684 1685 out_stop_queues: 1686 for (i--; i >= 1; i--) 1687 nvme_tcp_stop_queue(ctrl, i); 1688 return ret; 1689 } 1690 1691 static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl) 1692 { 1693 int ret; 1694 1695 ret = nvme_tcp_alloc_queue(ctrl, 0, NVME_AQ_DEPTH); 1696 if (ret) 1697 return ret; 1698 1699 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl)); 1700 if (ret) 1701 goto out_free_queue; 1702 1703 return 0; 1704 1705 out_free_queue: 1706 nvme_tcp_free_queue(ctrl, 0); 1707 return ret; 1708 } 1709 1710 static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1711 { 1712 int i, ret; 1713 1714 for (i = 1; i < ctrl->queue_count; i++) { 1715 ret = nvme_tcp_alloc_queue(ctrl, i, 1716 ctrl->sqsize + 1); 1717 if (ret) 1718 goto out_free_queues; 1719 } 1720 1721 return 0; 1722 1723 out_free_queues: 1724 for (i--; i >= 1; i--) 1725 nvme_tcp_free_queue(ctrl, i); 1726 1727 return ret; 1728 } 1729 1730 static unsigned int nvme_tcp_nr_io_queues(struct nvme_ctrl *ctrl) 1731 { 1732 unsigned int nr_io_queues; 1733 1734 nr_io_queues = min(ctrl->opts->nr_io_queues, num_online_cpus()); 1735 nr_io_queues += min(ctrl->opts->nr_write_queues, num_online_cpus()); 1736 nr_io_queues += min(ctrl->opts->nr_poll_queues, num_online_cpus()); 1737 1738 return nr_io_queues; 1739 } 1740 1741 static void nvme_tcp_set_io_queues(struct nvme_ctrl *nctrl, 1742 unsigned int nr_io_queues) 1743 { 1744 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 1745 struct nvmf_ctrl_options *opts = nctrl->opts; 1746 1747 if (opts->nr_write_queues && opts->nr_io_queues < nr_io_queues) { 1748 /* 1749 * separate read/write queues 1750 * hand out dedicated default queues only after we have 1751 * sufficient read queues. 1752 */ 1753 ctrl->io_queues[HCTX_TYPE_READ] = opts->nr_io_queues; 1754 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_READ]; 1755 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1756 min(opts->nr_write_queues, nr_io_queues); 1757 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1758 } else { 1759 /* 1760 * shared read/write queues 1761 * either no write queues were requested, or we don't have 1762 * sufficient queue count to have dedicated default queues. 1763 */ 1764 ctrl->io_queues[HCTX_TYPE_DEFAULT] = 1765 min(opts->nr_io_queues, nr_io_queues); 1766 nr_io_queues -= ctrl->io_queues[HCTX_TYPE_DEFAULT]; 1767 } 1768 1769 if (opts->nr_poll_queues && nr_io_queues) { 1770 /* map dedicated poll queues only if we have queues left */ 1771 ctrl->io_queues[HCTX_TYPE_POLL] = 1772 min(opts->nr_poll_queues, nr_io_queues); 1773 } 1774 } 1775 1776 static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl) 1777 { 1778 unsigned int nr_io_queues; 1779 int ret; 1780 1781 nr_io_queues = nvme_tcp_nr_io_queues(ctrl); 1782 ret = nvme_set_queue_count(ctrl, &nr_io_queues); 1783 if (ret) 1784 return ret; 1785 1786 if (nr_io_queues == 0) { 1787 dev_err(ctrl->device, 1788 "unable to set any I/O queues\n"); 1789 return -ENOMEM; 1790 } 1791 1792 ctrl->queue_count = nr_io_queues + 1; 1793 dev_info(ctrl->device, 1794 "creating %d I/O queues.\n", nr_io_queues); 1795 1796 nvme_tcp_set_io_queues(ctrl, nr_io_queues); 1797 1798 return __nvme_tcp_alloc_io_queues(ctrl); 1799 } 1800 1801 static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove) 1802 { 1803 nvme_tcp_stop_io_queues(ctrl); 1804 if (remove) { 1805 blk_cleanup_queue(ctrl->connect_q); 1806 blk_mq_free_tag_set(ctrl->tagset); 1807 } 1808 nvme_tcp_free_io_queues(ctrl); 1809 } 1810 1811 static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) 1812 { 1813 int ret; 1814 1815 ret = nvme_tcp_alloc_io_queues(ctrl); 1816 if (ret) 1817 return ret; 1818 1819 if (new) { 1820 ctrl->tagset = nvme_tcp_alloc_tagset(ctrl, false); 1821 if (IS_ERR(ctrl->tagset)) { 1822 ret = PTR_ERR(ctrl->tagset); 1823 goto out_free_io_queues; 1824 } 1825 1826 ctrl->connect_q = blk_mq_init_queue(ctrl->tagset); 1827 if (IS_ERR(ctrl->connect_q)) { 1828 ret = PTR_ERR(ctrl->connect_q); 1829 goto out_free_tag_set; 1830 } 1831 } 1832 1833 ret = nvme_tcp_start_io_queues(ctrl); 1834 if (ret) 1835 goto out_cleanup_connect_q; 1836 1837 if (!new) { 1838 nvme_start_queues(ctrl); 1839 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) { 1840 /* 1841 * If we timed out waiting for freeze we are likely to 1842 * be stuck. Fail the controller initialization just 1843 * to be safe. 1844 */ 1845 ret = -ENODEV; 1846 goto out_wait_freeze_timed_out; 1847 } 1848 blk_mq_update_nr_hw_queues(ctrl->tagset, 1849 ctrl->queue_count - 1); 1850 nvme_unfreeze(ctrl); 1851 } 1852 1853 return 0; 1854 1855 out_wait_freeze_timed_out: 1856 nvme_stop_queues(ctrl); 1857 nvme_sync_io_queues(ctrl); 1858 nvme_tcp_stop_io_queues(ctrl); 1859 out_cleanup_connect_q: 1860 nvme_cancel_tagset(ctrl); 1861 if (new) 1862 blk_cleanup_queue(ctrl->connect_q); 1863 out_free_tag_set: 1864 if (new) 1865 blk_mq_free_tag_set(ctrl->tagset); 1866 out_free_io_queues: 1867 nvme_tcp_free_io_queues(ctrl); 1868 return ret; 1869 } 1870 1871 static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove) 1872 { 1873 nvme_tcp_stop_queue(ctrl, 0); 1874 if (remove) { 1875 blk_cleanup_queue(ctrl->admin_q); 1876 blk_cleanup_queue(ctrl->fabrics_q); 1877 blk_mq_free_tag_set(ctrl->admin_tagset); 1878 } 1879 nvme_tcp_free_admin_queue(ctrl); 1880 } 1881 1882 static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new) 1883 { 1884 int error; 1885 1886 error = nvme_tcp_alloc_admin_queue(ctrl); 1887 if (error) 1888 return error; 1889 1890 if (new) { 1891 ctrl->admin_tagset = nvme_tcp_alloc_tagset(ctrl, true); 1892 if (IS_ERR(ctrl->admin_tagset)) { 1893 error = PTR_ERR(ctrl->admin_tagset); 1894 goto out_free_queue; 1895 } 1896 1897 ctrl->fabrics_q = blk_mq_init_queue(ctrl->admin_tagset); 1898 if (IS_ERR(ctrl->fabrics_q)) { 1899 error = PTR_ERR(ctrl->fabrics_q); 1900 goto out_free_tagset; 1901 } 1902 1903 ctrl->admin_q = blk_mq_init_queue(ctrl->admin_tagset); 1904 if (IS_ERR(ctrl->admin_q)) { 1905 error = PTR_ERR(ctrl->admin_q); 1906 goto out_cleanup_fabrics_q; 1907 } 1908 } 1909 1910 error = nvme_tcp_start_queue(ctrl, 0); 1911 if (error) 1912 goto out_cleanup_queue; 1913 1914 error = nvme_enable_ctrl(ctrl); 1915 if (error) 1916 goto out_stop_queue; 1917 1918 blk_mq_unquiesce_queue(ctrl->admin_q); 1919 1920 error = nvme_init_ctrl_finish(ctrl); 1921 if (error) 1922 goto out_quiesce_queue; 1923 1924 return 0; 1925 1926 out_quiesce_queue: 1927 blk_mq_quiesce_queue(ctrl->admin_q); 1928 blk_sync_queue(ctrl->admin_q); 1929 out_stop_queue: 1930 nvme_tcp_stop_queue(ctrl, 0); 1931 nvme_cancel_admin_tagset(ctrl); 1932 out_cleanup_queue: 1933 if (new) 1934 blk_cleanup_queue(ctrl->admin_q); 1935 out_cleanup_fabrics_q: 1936 if (new) 1937 blk_cleanup_queue(ctrl->fabrics_q); 1938 out_free_tagset: 1939 if (new) 1940 blk_mq_free_tag_set(ctrl->admin_tagset); 1941 out_free_queue: 1942 nvme_tcp_free_admin_queue(ctrl); 1943 return error; 1944 } 1945 1946 static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl, 1947 bool remove) 1948 { 1949 blk_mq_quiesce_queue(ctrl->admin_q); 1950 blk_sync_queue(ctrl->admin_q); 1951 nvme_tcp_stop_queue(ctrl, 0); 1952 nvme_cancel_admin_tagset(ctrl); 1953 if (remove) 1954 blk_mq_unquiesce_queue(ctrl->admin_q); 1955 nvme_tcp_destroy_admin_queue(ctrl, remove); 1956 } 1957 1958 static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl, 1959 bool remove) 1960 { 1961 if (ctrl->queue_count <= 1) 1962 return; 1963 blk_mq_quiesce_queue(ctrl->admin_q); 1964 nvme_start_freeze(ctrl); 1965 nvme_stop_queues(ctrl); 1966 nvme_sync_io_queues(ctrl); 1967 nvme_tcp_stop_io_queues(ctrl); 1968 nvme_cancel_tagset(ctrl); 1969 if (remove) 1970 nvme_start_queues(ctrl); 1971 nvme_tcp_destroy_io_queues(ctrl, remove); 1972 } 1973 1974 static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl) 1975 { 1976 /* If we are resetting/deleting then do nothing */ 1977 if (ctrl->state != NVME_CTRL_CONNECTING) { 1978 WARN_ON_ONCE(ctrl->state == NVME_CTRL_NEW || 1979 ctrl->state == NVME_CTRL_LIVE); 1980 return; 1981 } 1982 1983 if (nvmf_should_reconnect(ctrl)) { 1984 dev_info(ctrl->device, "Reconnecting in %d seconds...\n", 1985 ctrl->opts->reconnect_delay); 1986 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work, 1987 ctrl->opts->reconnect_delay * HZ); 1988 } else { 1989 dev_info(ctrl->device, "Removing controller...\n"); 1990 nvme_delete_ctrl(ctrl); 1991 } 1992 } 1993 1994 static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new) 1995 { 1996 struct nvmf_ctrl_options *opts = ctrl->opts; 1997 int ret; 1998 1999 ret = nvme_tcp_configure_admin_queue(ctrl, new); 2000 if (ret) 2001 return ret; 2002 2003 if (ctrl->icdoff) { 2004 ret = -EOPNOTSUPP; 2005 dev_err(ctrl->device, "icdoff is not supported!\n"); 2006 goto destroy_admin; 2007 } 2008 2009 if (!nvme_ctrl_sgl_supported(ctrl)) { 2010 ret = -EOPNOTSUPP; 2011 dev_err(ctrl->device, "Mandatory sgls are not supported!\n"); 2012 goto destroy_admin; 2013 } 2014 2015 if (opts->queue_size > ctrl->sqsize + 1) 2016 dev_warn(ctrl->device, 2017 "queue_size %zu > ctrl sqsize %u, clamping down\n", 2018 opts->queue_size, ctrl->sqsize + 1); 2019 2020 if (ctrl->sqsize + 1 > ctrl->maxcmd) { 2021 dev_warn(ctrl->device, 2022 "sqsize %u > ctrl maxcmd %u, clamping down\n", 2023 ctrl->sqsize + 1, ctrl->maxcmd); 2024 ctrl->sqsize = ctrl->maxcmd - 1; 2025 } 2026 2027 if (ctrl->queue_count > 1) { 2028 ret = nvme_tcp_configure_io_queues(ctrl, new); 2029 if (ret) 2030 goto destroy_admin; 2031 } 2032 2033 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) { 2034 /* 2035 * state change failure is ok if we started ctrl delete, 2036 * unless we're during creation of a new controller to 2037 * avoid races with teardown flow. 2038 */ 2039 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2040 ctrl->state != NVME_CTRL_DELETING_NOIO); 2041 WARN_ON_ONCE(new); 2042 ret = -EINVAL; 2043 goto destroy_io; 2044 } 2045 2046 nvme_start_ctrl(ctrl); 2047 return 0; 2048 2049 destroy_io: 2050 if (ctrl->queue_count > 1) { 2051 nvme_stop_queues(ctrl); 2052 nvme_sync_io_queues(ctrl); 2053 nvme_tcp_stop_io_queues(ctrl); 2054 nvme_cancel_tagset(ctrl); 2055 nvme_tcp_destroy_io_queues(ctrl, new); 2056 } 2057 destroy_admin: 2058 blk_mq_quiesce_queue(ctrl->admin_q); 2059 blk_sync_queue(ctrl->admin_q); 2060 nvme_tcp_stop_queue(ctrl, 0); 2061 nvme_cancel_admin_tagset(ctrl); 2062 nvme_tcp_destroy_admin_queue(ctrl, new); 2063 return ret; 2064 } 2065 2066 static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work) 2067 { 2068 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work), 2069 struct nvme_tcp_ctrl, connect_work); 2070 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2071 2072 ++ctrl->nr_reconnects; 2073 2074 if (nvme_tcp_setup_ctrl(ctrl, false)) 2075 goto requeue; 2076 2077 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n", 2078 ctrl->nr_reconnects); 2079 2080 ctrl->nr_reconnects = 0; 2081 2082 return; 2083 2084 requeue: 2085 dev_info(ctrl->device, "Failed reconnect attempt %d\n", 2086 ctrl->nr_reconnects); 2087 nvme_tcp_reconnect_or_remove(ctrl); 2088 } 2089 2090 static void nvme_tcp_error_recovery_work(struct work_struct *work) 2091 { 2092 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work, 2093 struct nvme_tcp_ctrl, err_work); 2094 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl; 2095 2096 nvme_stop_keep_alive(ctrl); 2097 nvme_tcp_teardown_io_queues(ctrl, false); 2098 /* unquiesce to fail fast pending requests */ 2099 nvme_start_queues(ctrl); 2100 nvme_tcp_teardown_admin_queue(ctrl, false); 2101 blk_mq_unquiesce_queue(ctrl->admin_q); 2102 2103 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2104 /* state change failure is ok if we started ctrl delete */ 2105 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2106 ctrl->state != NVME_CTRL_DELETING_NOIO); 2107 return; 2108 } 2109 2110 nvme_tcp_reconnect_or_remove(ctrl); 2111 } 2112 2113 static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown) 2114 { 2115 cancel_work_sync(&to_tcp_ctrl(ctrl)->err_work); 2116 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work); 2117 2118 nvme_tcp_teardown_io_queues(ctrl, shutdown); 2119 blk_mq_quiesce_queue(ctrl->admin_q); 2120 if (shutdown) 2121 nvme_shutdown_ctrl(ctrl); 2122 else 2123 nvme_disable_ctrl(ctrl); 2124 nvme_tcp_teardown_admin_queue(ctrl, shutdown); 2125 } 2126 2127 static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl) 2128 { 2129 nvme_tcp_teardown_ctrl(ctrl, true); 2130 } 2131 2132 static void nvme_reset_ctrl_work(struct work_struct *work) 2133 { 2134 struct nvme_ctrl *ctrl = 2135 container_of(work, struct nvme_ctrl, reset_work); 2136 2137 nvme_stop_ctrl(ctrl); 2138 nvme_tcp_teardown_ctrl(ctrl, false); 2139 2140 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) { 2141 /* state change failure is ok if we started ctrl delete */ 2142 WARN_ON_ONCE(ctrl->state != NVME_CTRL_DELETING && 2143 ctrl->state != NVME_CTRL_DELETING_NOIO); 2144 return; 2145 } 2146 2147 if (nvme_tcp_setup_ctrl(ctrl, false)) 2148 goto out_fail; 2149 2150 return; 2151 2152 out_fail: 2153 ++ctrl->nr_reconnects; 2154 nvme_tcp_reconnect_or_remove(ctrl); 2155 } 2156 2157 static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl) 2158 { 2159 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl); 2160 2161 if (list_empty(&ctrl->list)) 2162 goto free_ctrl; 2163 2164 mutex_lock(&nvme_tcp_ctrl_mutex); 2165 list_del(&ctrl->list); 2166 mutex_unlock(&nvme_tcp_ctrl_mutex); 2167 2168 nvmf_free_options(nctrl->opts); 2169 free_ctrl: 2170 kfree(ctrl->queues); 2171 kfree(ctrl); 2172 } 2173 2174 static void nvme_tcp_set_sg_null(struct nvme_command *c) 2175 { 2176 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2177 2178 sg->addr = 0; 2179 sg->length = 0; 2180 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2181 NVME_SGL_FMT_TRANSPORT_A; 2182 } 2183 2184 static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue, 2185 struct nvme_command *c, u32 data_len) 2186 { 2187 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2188 2189 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff); 2190 sg->length = cpu_to_le32(data_len); 2191 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET; 2192 } 2193 2194 static void nvme_tcp_set_sg_host_data(struct nvme_command *c, 2195 u32 data_len) 2196 { 2197 struct nvme_sgl_desc *sg = &c->common.dptr.sgl; 2198 2199 sg->addr = 0; 2200 sg->length = cpu_to_le32(data_len); 2201 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) | 2202 NVME_SGL_FMT_TRANSPORT_A; 2203 } 2204 2205 static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg) 2206 { 2207 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg); 2208 struct nvme_tcp_queue *queue = &ctrl->queues[0]; 2209 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu; 2210 struct nvme_command *cmd = &pdu->cmd; 2211 u8 hdgst = nvme_tcp_hdgst_len(queue); 2212 2213 memset(pdu, 0, sizeof(*pdu)); 2214 pdu->hdr.type = nvme_tcp_cmd; 2215 if (queue->hdr_digest) 2216 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2217 pdu->hdr.hlen = sizeof(*pdu); 2218 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst); 2219 2220 cmd->common.opcode = nvme_admin_async_event; 2221 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH; 2222 cmd->common.flags |= NVME_CMD_SGL_METABUF; 2223 nvme_tcp_set_sg_null(cmd); 2224 2225 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU; 2226 ctrl->async_req.offset = 0; 2227 ctrl->async_req.curr_bio = NULL; 2228 ctrl->async_req.data_len = 0; 2229 2230 nvme_tcp_queue_request(&ctrl->async_req, true, true); 2231 } 2232 2233 static void nvme_tcp_complete_timed_out(struct request *rq) 2234 { 2235 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2236 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2237 2238 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue)); 2239 if (blk_mq_request_started(rq) && !blk_mq_request_completed(rq)) { 2240 nvme_req(rq)->status = NVME_SC_HOST_ABORTED_CMD; 2241 blk_mq_complete_request(rq); 2242 } 2243 } 2244 2245 static enum blk_eh_timer_return 2246 nvme_tcp_timeout(struct request *rq, bool reserved) 2247 { 2248 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2249 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl; 2250 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2251 2252 dev_warn(ctrl->device, 2253 "queue %d: timeout request %#x type %d\n", 2254 nvme_tcp_queue_id(req->queue), rq->tag, pdu->hdr.type); 2255 2256 if (ctrl->state != NVME_CTRL_LIVE) { 2257 /* 2258 * If we are resetting, connecting or deleting we should 2259 * complete immediately because we may block controller 2260 * teardown or setup sequence 2261 * - ctrl disable/shutdown fabrics requests 2262 * - connect requests 2263 * - initialization admin requests 2264 * - I/O requests that entered after unquiescing and 2265 * the controller stopped responding 2266 * 2267 * All other requests should be cancelled by the error 2268 * recovery work, so it's fine that we fail it here. 2269 */ 2270 nvme_tcp_complete_timed_out(rq); 2271 return BLK_EH_DONE; 2272 } 2273 2274 /* 2275 * LIVE state should trigger the normal error recovery which will 2276 * handle completing this request. 2277 */ 2278 nvme_tcp_error_recovery(ctrl); 2279 return BLK_EH_RESET_TIMER; 2280 } 2281 2282 static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue, 2283 struct request *rq) 2284 { 2285 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2286 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2287 struct nvme_command *c = &pdu->cmd; 2288 2289 c->common.flags |= NVME_CMD_SGL_METABUF; 2290 2291 if (!blk_rq_nr_phys_segments(rq)) 2292 nvme_tcp_set_sg_null(c); 2293 else if (rq_data_dir(rq) == WRITE && 2294 req->data_len <= nvme_tcp_inline_data_size(queue)) 2295 nvme_tcp_set_sg_inline(queue, c, req->data_len); 2296 else 2297 nvme_tcp_set_sg_host_data(c, req->data_len); 2298 2299 return 0; 2300 } 2301 2302 static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns, 2303 struct request *rq) 2304 { 2305 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2306 struct nvme_tcp_cmd_pdu *pdu = req->pdu; 2307 struct nvme_tcp_queue *queue = req->queue; 2308 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0; 2309 blk_status_t ret; 2310 2311 ret = nvme_setup_cmd(ns, rq); 2312 if (ret) 2313 return ret; 2314 2315 req->state = NVME_TCP_SEND_CMD_PDU; 2316 req->status = cpu_to_le16(NVME_SC_SUCCESS); 2317 req->offset = 0; 2318 req->data_sent = 0; 2319 req->pdu_len = 0; 2320 req->pdu_sent = 0; 2321 req->data_len = blk_rq_nr_phys_segments(rq) ? 2322 blk_rq_payload_bytes(rq) : 0; 2323 req->curr_bio = rq->bio; 2324 if (req->curr_bio && req->data_len) 2325 nvme_tcp_init_iter(req, rq_data_dir(rq)); 2326 2327 if (rq_data_dir(rq) == WRITE && 2328 req->data_len <= nvme_tcp_inline_data_size(queue)) 2329 req->pdu_len = req->data_len; 2330 2331 pdu->hdr.type = nvme_tcp_cmd; 2332 pdu->hdr.flags = 0; 2333 if (queue->hdr_digest) 2334 pdu->hdr.flags |= NVME_TCP_F_HDGST; 2335 if (queue->data_digest && req->pdu_len) { 2336 pdu->hdr.flags |= NVME_TCP_F_DDGST; 2337 ddgst = nvme_tcp_ddgst_len(queue); 2338 } 2339 pdu->hdr.hlen = sizeof(*pdu); 2340 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0; 2341 pdu->hdr.plen = 2342 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst); 2343 2344 ret = nvme_tcp_map_data(queue, rq); 2345 if (unlikely(ret)) { 2346 nvme_cleanup_cmd(rq); 2347 dev_err(queue->ctrl->ctrl.device, 2348 "Failed to map data (%d)\n", ret); 2349 return ret; 2350 } 2351 2352 return 0; 2353 } 2354 2355 static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx) 2356 { 2357 struct nvme_tcp_queue *queue = hctx->driver_data; 2358 2359 if (!llist_empty(&queue->req_list)) 2360 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work); 2361 } 2362 2363 static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx, 2364 const struct blk_mq_queue_data *bd) 2365 { 2366 struct nvme_ns *ns = hctx->queue->queuedata; 2367 struct nvme_tcp_queue *queue = hctx->driver_data; 2368 struct request *rq = bd->rq; 2369 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq); 2370 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags); 2371 blk_status_t ret; 2372 2373 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready)) 2374 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq); 2375 2376 ret = nvme_tcp_setup_cmd_pdu(ns, rq); 2377 if (unlikely(ret)) 2378 return ret; 2379 2380 blk_mq_start_request(rq); 2381 2382 nvme_tcp_queue_request(req, true, bd->last); 2383 2384 return BLK_STS_OK; 2385 } 2386 2387 static int nvme_tcp_map_queues(struct blk_mq_tag_set *set) 2388 { 2389 struct nvme_tcp_ctrl *ctrl = set->driver_data; 2390 struct nvmf_ctrl_options *opts = ctrl->ctrl.opts; 2391 2392 if (opts->nr_write_queues && ctrl->io_queues[HCTX_TYPE_READ]) { 2393 /* separate read/write queues */ 2394 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2395 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2396 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2397 set->map[HCTX_TYPE_READ].nr_queues = 2398 ctrl->io_queues[HCTX_TYPE_READ]; 2399 set->map[HCTX_TYPE_READ].queue_offset = 2400 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2401 } else { 2402 /* shared read/write queues */ 2403 set->map[HCTX_TYPE_DEFAULT].nr_queues = 2404 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2405 set->map[HCTX_TYPE_DEFAULT].queue_offset = 0; 2406 set->map[HCTX_TYPE_READ].nr_queues = 2407 ctrl->io_queues[HCTX_TYPE_DEFAULT]; 2408 set->map[HCTX_TYPE_READ].queue_offset = 0; 2409 } 2410 blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]); 2411 blk_mq_map_queues(&set->map[HCTX_TYPE_READ]); 2412 2413 if (opts->nr_poll_queues && ctrl->io_queues[HCTX_TYPE_POLL]) { 2414 /* map dedicated poll queues only if we have queues left */ 2415 set->map[HCTX_TYPE_POLL].nr_queues = 2416 ctrl->io_queues[HCTX_TYPE_POLL]; 2417 set->map[HCTX_TYPE_POLL].queue_offset = 2418 ctrl->io_queues[HCTX_TYPE_DEFAULT] + 2419 ctrl->io_queues[HCTX_TYPE_READ]; 2420 blk_mq_map_queues(&set->map[HCTX_TYPE_POLL]); 2421 } 2422 2423 dev_info(ctrl->ctrl.device, 2424 "mapped %d/%d/%d default/read/poll queues.\n", 2425 ctrl->io_queues[HCTX_TYPE_DEFAULT], 2426 ctrl->io_queues[HCTX_TYPE_READ], 2427 ctrl->io_queues[HCTX_TYPE_POLL]); 2428 2429 return 0; 2430 } 2431 2432 static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) 2433 { 2434 struct nvme_tcp_queue *queue = hctx->driver_data; 2435 struct sock *sk = queue->sock->sk; 2436 2437 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags)) 2438 return 0; 2439 2440 set_bit(NVME_TCP_Q_POLLING, &queue->flags); 2441 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue)) 2442 sk_busy_loop(sk, true); 2443 nvme_tcp_try_recv(queue); 2444 clear_bit(NVME_TCP_Q_POLLING, &queue->flags); 2445 return queue->nr_cqe; 2446 } 2447 2448 static const struct blk_mq_ops nvme_tcp_mq_ops = { 2449 .queue_rq = nvme_tcp_queue_rq, 2450 .commit_rqs = nvme_tcp_commit_rqs, 2451 .complete = nvme_complete_rq, 2452 .init_request = nvme_tcp_init_request, 2453 .exit_request = nvme_tcp_exit_request, 2454 .init_hctx = nvme_tcp_init_hctx, 2455 .timeout = nvme_tcp_timeout, 2456 .map_queues = nvme_tcp_map_queues, 2457 .poll = nvme_tcp_poll, 2458 }; 2459 2460 static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { 2461 .queue_rq = nvme_tcp_queue_rq, 2462 .complete = nvme_complete_rq, 2463 .init_request = nvme_tcp_init_request, 2464 .exit_request = nvme_tcp_exit_request, 2465 .init_hctx = nvme_tcp_init_admin_hctx, 2466 .timeout = nvme_tcp_timeout, 2467 }; 2468 2469 static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { 2470 .name = "tcp", 2471 .module = THIS_MODULE, 2472 .flags = NVME_F_FABRICS, 2473 .reg_read32 = nvmf_reg_read32, 2474 .reg_read64 = nvmf_reg_read64, 2475 .reg_write32 = nvmf_reg_write32, 2476 .free_ctrl = nvme_tcp_free_ctrl, 2477 .submit_async_event = nvme_tcp_submit_async_event, 2478 .delete_ctrl = nvme_tcp_delete_ctrl, 2479 .get_address = nvmf_get_address, 2480 }; 2481 2482 static bool 2483 nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts) 2484 { 2485 struct nvme_tcp_ctrl *ctrl; 2486 bool found = false; 2487 2488 mutex_lock(&nvme_tcp_ctrl_mutex); 2489 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) { 2490 found = nvmf_ip_options_match(&ctrl->ctrl, opts); 2491 if (found) 2492 break; 2493 } 2494 mutex_unlock(&nvme_tcp_ctrl_mutex); 2495 2496 return found; 2497 } 2498 2499 static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev, 2500 struct nvmf_ctrl_options *opts) 2501 { 2502 struct nvme_tcp_ctrl *ctrl; 2503 int ret; 2504 2505 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL); 2506 if (!ctrl) 2507 return ERR_PTR(-ENOMEM); 2508 2509 INIT_LIST_HEAD(&ctrl->list); 2510 ctrl->ctrl.opts = opts; 2511 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues + 2512 opts->nr_poll_queues + 1; 2513 ctrl->ctrl.sqsize = opts->queue_size - 1; 2514 ctrl->ctrl.kato = opts->kato; 2515 2516 INIT_DELAYED_WORK(&ctrl->connect_work, 2517 nvme_tcp_reconnect_ctrl_work); 2518 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work); 2519 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work); 2520 2521 if (!(opts->mask & NVMF_OPT_TRSVCID)) { 2522 opts->trsvcid = 2523 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL); 2524 if (!opts->trsvcid) { 2525 ret = -ENOMEM; 2526 goto out_free_ctrl; 2527 } 2528 opts->mask |= NVMF_OPT_TRSVCID; 2529 } 2530 2531 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2532 opts->traddr, opts->trsvcid, &ctrl->addr); 2533 if (ret) { 2534 pr_err("malformed address passed: %s:%s\n", 2535 opts->traddr, opts->trsvcid); 2536 goto out_free_ctrl; 2537 } 2538 2539 if (opts->mask & NVMF_OPT_HOST_TRADDR) { 2540 ret = inet_pton_with_scope(&init_net, AF_UNSPEC, 2541 opts->host_traddr, NULL, &ctrl->src_addr); 2542 if (ret) { 2543 pr_err("malformed src address passed: %s\n", 2544 opts->host_traddr); 2545 goto out_free_ctrl; 2546 } 2547 } 2548 2549 if (opts->mask & NVMF_OPT_HOST_IFACE) { 2550 if (!__dev_get_by_name(&init_net, opts->host_iface)) { 2551 pr_err("invalid interface passed: %s\n", 2552 opts->host_iface); 2553 ret = -ENODEV; 2554 goto out_free_ctrl; 2555 } 2556 } 2557 2558 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) { 2559 ret = -EALREADY; 2560 goto out_free_ctrl; 2561 } 2562 2563 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues), 2564 GFP_KERNEL); 2565 if (!ctrl->queues) { 2566 ret = -ENOMEM; 2567 goto out_free_ctrl; 2568 } 2569 2570 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0); 2571 if (ret) 2572 goto out_kfree_queues; 2573 2574 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) { 2575 WARN_ON_ONCE(1); 2576 ret = -EINTR; 2577 goto out_uninit_ctrl; 2578 } 2579 2580 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true); 2581 if (ret) 2582 goto out_uninit_ctrl; 2583 2584 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp\n", 2585 ctrl->ctrl.opts->subsysnqn, &ctrl->addr); 2586 2587 mutex_lock(&nvme_tcp_ctrl_mutex); 2588 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list); 2589 mutex_unlock(&nvme_tcp_ctrl_mutex); 2590 2591 return &ctrl->ctrl; 2592 2593 out_uninit_ctrl: 2594 nvme_uninit_ctrl(&ctrl->ctrl); 2595 nvme_put_ctrl(&ctrl->ctrl); 2596 if (ret > 0) 2597 ret = -EIO; 2598 return ERR_PTR(ret); 2599 out_kfree_queues: 2600 kfree(ctrl->queues); 2601 out_free_ctrl: 2602 kfree(ctrl); 2603 return ERR_PTR(ret); 2604 } 2605 2606 static struct nvmf_transport_ops nvme_tcp_transport = { 2607 .name = "tcp", 2608 .module = THIS_MODULE, 2609 .required_opts = NVMF_OPT_TRADDR, 2610 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY | 2611 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO | 2612 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST | 2613 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES | 2614 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE, 2615 .create_ctrl = nvme_tcp_create_ctrl, 2616 }; 2617 2618 static int __init nvme_tcp_init_module(void) 2619 { 2620 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", 2621 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0); 2622 if (!nvme_tcp_wq) 2623 return -ENOMEM; 2624 2625 nvmf_register_transport(&nvme_tcp_transport); 2626 return 0; 2627 } 2628 2629 static void __exit nvme_tcp_cleanup_module(void) 2630 { 2631 struct nvme_tcp_ctrl *ctrl; 2632 2633 nvmf_unregister_transport(&nvme_tcp_transport); 2634 2635 mutex_lock(&nvme_tcp_ctrl_mutex); 2636 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) 2637 nvme_delete_ctrl(&ctrl->ctrl); 2638 mutex_unlock(&nvme_tcp_ctrl_mutex); 2639 flush_workqueue(nvme_delete_wq); 2640 2641 destroy_workqueue(nvme_tcp_wq); 2642 } 2643 2644 module_init(nvme_tcp_init_module); 2645 module_exit(nvme_tcp_cleanup_module); 2646 2647 MODULE_LICENSE("GPL v2"); 2648