1 /* 2 * NVMe over Fabrics RDMA target. 3 * Copyright (c) 2015-2016 HGST, a Western Digital Company. 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 */ 14 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 15 #include <linux/atomic.h> 16 #include <linux/ctype.h> 17 #include <linux/delay.h> 18 #include <linux/err.h> 19 #include <linux/init.h> 20 #include <linux/module.h> 21 #include <linux/nvme.h> 22 #include <linux/slab.h> 23 #include <linux/string.h> 24 #include <linux/wait.h> 25 #include <linux/inet.h> 26 #include <asm/unaligned.h> 27 28 #include <rdma/ib_verbs.h> 29 #include <rdma/rdma_cm.h> 30 #include <rdma/rw.h> 31 32 #include <linux/nvme-rdma.h> 33 #include "nvmet.h" 34 35 /* 36 * We allow at least 1 page, up to 4 SGEs, and up to 16KB of inline data 37 */ 38 #define NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE PAGE_SIZE 39 #define NVMET_RDMA_MAX_INLINE_SGE 4 40 #define NVMET_RDMA_MAX_INLINE_DATA_SIZE max_t(int, SZ_16K, PAGE_SIZE) 41 42 struct nvmet_rdma_cmd { 43 struct ib_sge sge[NVMET_RDMA_MAX_INLINE_SGE + 1]; 44 struct ib_cqe cqe; 45 struct ib_recv_wr wr; 46 struct scatterlist inline_sg[NVMET_RDMA_MAX_INLINE_SGE]; 47 struct nvme_command *nvme_cmd; 48 struct nvmet_rdma_queue *queue; 49 }; 50 51 enum { 52 NVMET_RDMA_REQ_INLINE_DATA = (1 << 0), 53 NVMET_RDMA_REQ_INVALIDATE_RKEY = (1 << 1), 54 }; 55 56 struct nvmet_rdma_rsp { 57 struct ib_sge send_sge; 58 struct ib_cqe send_cqe; 59 struct ib_send_wr send_wr; 60 61 struct nvmet_rdma_cmd *cmd; 62 struct nvmet_rdma_queue *queue; 63 64 struct ib_cqe read_cqe; 65 struct rdma_rw_ctx rw; 66 67 struct nvmet_req req; 68 69 bool allocated; 70 u8 n_rdma; 71 u32 flags; 72 u32 invalidate_rkey; 73 74 struct list_head wait_list; 75 struct list_head free_list; 76 }; 77 78 enum nvmet_rdma_queue_state { 79 NVMET_RDMA_Q_CONNECTING, 80 NVMET_RDMA_Q_LIVE, 81 NVMET_RDMA_Q_DISCONNECTING, 82 }; 83 84 struct nvmet_rdma_queue { 85 struct rdma_cm_id *cm_id; 86 struct nvmet_port *port; 87 struct ib_cq *cq; 88 atomic_t sq_wr_avail; 89 struct nvmet_rdma_device *dev; 90 spinlock_t state_lock; 91 enum nvmet_rdma_queue_state state; 92 struct nvmet_cq nvme_cq; 93 struct nvmet_sq nvme_sq; 94 95 struct nvmet_rdma_rsp *rsps; 96 struct list_head free_rsps; 97 spinlock_t rsps_lock; 98 struct nvmet_rdma_cmd *cmds; 99 100 struct work_struct release_work; 101 struct list_head rsp_wait_list; 102 struct list_head rsp_wr_wait_list; 103 spinlock_t rsp_wr_wait_lock; 104 105 int idx; 106 int host_qid; 107 int recv_queue_size; 108 int send_queue_size; 109 110 struct list_head queue_list; 111 }; 112 113 struct nvmet_rdma_device { 114 struct ib_device *device; 115 struct ib_pd *pd; 116 struct ib_srq *srq; 117 struct nvmet_rdma_cmd *srq_cmds; 118 size_t srq_size; 119 struct kref ref; 120 struct list_head entry; 121 int inline_data_size; 122 int inline_page_count; 123 }; 124 125 static bool nvmet_rdma_use_srq; 126 module_param_named(use_srq, nvmet_rdma_use_srq, bool, 0444); 127 MODULE_PARM_DESC(use_srq, "Use shared receive queue."); 128 129 static DEFINE_IDA(nvmet_rdma_queue_ida); 130 static LIST_HEAD(nvmet_rdma_queue_list); 131 static DEFINE_MUTEX(nvmet_rdma_queue_mutex); 132 133 static LIST_HEAD(device_list); 134 static DEFINE_MUTEX(device_list_mutex); 135 136 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp); 137 static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc); 138 static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc); 139 static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc); 140 static void nvmet_rdma_qp_event(struct ib_event *event, void *priv); 141 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue); 142 143 static const struct nvmet_fabrics_ops nvmet_rdma_ops; 144 145 static int num_pages(int len) 146 { 147 return 1 + (((len - 1) & PAGE_MASK) >> PAGE_SHIFT); 148 } 149 150 /* XXX: really should move to a generic header sooner or later.. */ 151 static inline u32 get_unaligned_le24(const u8 *p) 152 { 153 return (u32)p[0] | (u32)p[1] << 8 | (u32)p[2] << 16; 154 } 155 156 static inline bool nvmet_rdma_need_data_in(struct nvmet_rdma_rsp *rsp) 157 { 158 return nvme_is_write(rsp->req.cmd) && 159 rsp->req.transfer_len && 160 !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA); 161 } 162 163 static inline bool nvmet_rdma_need_data_out(struct nvmet_rdma_rsp *rsp) 164 { 165 return !nvme_is_write(rsp->req.cmd) && 166 rsp->req.transfer_len && 167 !rsp->req.rsp->status && 168 !(rsp->flags & NVMET_RDMA_REQ_INLINE_DATA); 169 } 170 171 static inline struct nvmet_rdma_rsp * 172 nvmet_rdma_get_rsp(struct nvmet_rdma_queue *queue) 173 { 174 struct nvmet_rdma_rsp *rsp; 175 unsigned long flags; 176 177 spin_lock_irqsave(&queue->rsps_lock, flags); 178 rsp = list_first_entry_or_null(&queue->free_rsps, 179 struct nvmet_rdma_rsp, free_list); 180 if (likely(rsp)) 181 list_del(&rsp->free_list); 182 spin_unlock_irqrestore(&queue->rsps_lock, flags); 183 184 if (unlikely(!rsp)) { 185 rsp = kmalloc(sizeof(*rsp), GFP_KERNEL); 186 if (unlikely(!rsp)) 187 return NULL; 188 rsp->allocated = true; 189 } 190 191 return rsp; 192 } 193 194 static inline void 195 nvmet_rdma_put_rsp(struct nvmet_rdma_rsp *rsp) 196 { 197 unsigned long flags; 198 199 if (rsp->allocated) { 200 kfree(rsp); 201 return; 202 } 203 204 spin_lock_irqsave(&rsp->queue->rsps_lock, flags); 205 list_add_tail(&rsp->free_list, &rsp->queue->free_rsps); 206 spin_unlock_irqrestore(&rsp->queue->rsps_lock, flags); 207 } 208 209 static void nvmet_rdma_free_inline_pages(struct nvmet_rdma_device *ndev, 210 struct nvmet_rdma_cmd *c) 211 { 212 struct scatterlist *sg; 213 struct ib_sge *sge; 214 int i; 215 216 if (!ndev->inline_data_size) 217 return; 218 219 sg = c->inline_sg; 220 sge = &c->sge[1]; 221 222 for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) { 223 if (sge->length) 224 ib_dma_unmap_page(ndev->device, sge->addr, 225 sge->length, DMA_FROM_DEVICE); 226 if (sg_page(sg)) 227 __free_page(sg_page(sg)); 228 } 229 } 230 231 static int nvmet_rdma_alloc_inline_pages(struct nvmet_rdma_device *ndev, 232 struct nvmet_rdma_cmd *c) 233 { 234 struct scatterlist *sg; 235 struct ib_sge *sge; 236 struct page *pg; 237 int len; 238 int i; 239 240 if (!ndev->inline_data_size) 241 return 0; 242 243 sg = c->inline_sg; 244 sg_init_table(sg, ndev->inline_page_count); 245 sge = &c->sge[1]; 246 len = ndev->inline_data_size; 247 248 for (i = 0; i < ndev->inline_page_count; i++, sg++, sge++) { 249 pg = alloc_page(GFP_KERNEL); 250 if (!pg) 251 goto out_err; 252 sg_assign_page(sg, pg); 253 sge->addr = ib_dma_map_page(ndev->device, 254 pg, 0, PAGE_SIZE, DMA_FROM_DEVICE); 255 if (ib_dma_mapping_error(ndev->device, sge->addr)) 256 goto out_err; 257 sge->length = min_t(int, len, PAGE_SIZE); 258 sge->lkey = ndev->pd->local_dma_lkey; 259 len -= sge->length; 260 } 261 262 return 0; 263 out_err: 264 for (; i >= 0; i--, sg--, sge--) { 265 if (sge->length) 266 ib_dma_unmap_page(ndev->device, sge->addr, 267 sge->length, DMA_FROM_DEVICE); 268 if (sg_page(sg)) 269 __free_page(sg_page(sg)); 270 } 271 return -ENOMEM; 272 } 273 274 static int nvmet_rdma_alloc_cmd(struct nvmet_rdma_device *ndev, 275 struct nvmet_rdma_cmd *c, bool admin) 276 { 277 /* NVMe command / RDMA RECV */ 278 c->nvme_cmd = kmalloc(sizeof(*c->nvme_cmd), GFP_KERNEL); 279 if (!c->nvme_cmd) 280 goto out; 281 282 c->sge[0].addr = ib_dma_map_single(ndev->device, c->nvme_cmd, 283 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE); 284 if (ib_dma_mapping_error(ndev->device, c->sge[0].addr)) 285 goto out_free_cmd; 286 287 c->sge[0].length = sizeof(*c->nvme_cmd); 288 c->sge[0].lkey = ndev->pd->local_dma_lkey; 289 290 if (!admin && nvmet_rdma_alloc_inline_pages(ndev, c)) 291 goto out_unmap_cmd; 292 293 c->cqe.done = nvmet_rdma_recv_done; 294 295 c->wr.wr_cqe = &c->cqe; 296 c->wr.sg_list = c->sge; 297 c->wr.num_sge = admin ? 1 : ndev->inline_page_count + 1; 298 299 return 0; 300 301 out_unmap_cmd: 302 ib_dma_unmap_single(ndev->device, c->sge[0].addr, 303 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE); 304 out_free_cmd: 305 kfree(c->nvme_cmd); 306 307 out: 308 return -ENOMEM; 309 } 310 311 static void nvmet_rdma_free_cmd(struct nvmet_rdma_device *ndev, 312 struct nvmet_rdma_cmd *c, bool admin) 313 { 314 if (!admin) 315 nvmet_rdma_free_inline_pages(ndev, c); 316 ib_dma_unmap_single(ndev->device, c->sge[0].addr, 317 sizeof(*c->nvme_cmd), DMA_FROM_DEVICE); 318 kfree(c->nvme_cmd); 319 } 320 321 static struct nvmet_rdma_cmd * 322 nvmet_rdma_alloc_cmds(struct nvmet_rdma_device *ndev, 323 int nr_cmds, bool admin) 324 { 325 struct nvmet_rdma_cmd *cmds; 326 int ret = -EINVAL, i; 327 328 cmds = kcalloc(nr_cmds, sizeof(struct nvmet_rdma_cmd), GFP_KERNEL); 329 if (!cmds) 330 goto out; 331 332 for (i = 0; i < nr_cmds; i++) { 333 ret = nvmet_rdma_alloc_cmd(ndev, cmds + i, admin); 334 if (ret) 335 goto out_free; 336 } 337 338 return cmds; 339 340 out_free: 341 while (--i >= 0) 342 nvmet_rdma_free_cmd(ndev, cmds + i, admin); 343 kfree(cmds); 344 out: 345 return ERR_PTR(ret); 346 } 347 348 static void nvmet_rdma_free_cmds(struct nvmet_rdma_device *ndev, 349 struct nvmet_rdma_cmd *cmds, int nr_cmds, bool admin) 350 { 351 int i; 352 353 for (i = 0; i < nr_cmds; i++) 354 nvmet_rdma_free_cmd(ndev, cmds + i, admin); 355 kfree(cmds); 356 } 357 358 static int nvmet_rdma_alloc_rsp(struct nvmet_rdma_device *ndev, 359 struct nvmet_rdma_rsp *r) 360 { 361 /* NVMe CQE / RDMA SEND */ 362 r->req.rsp = kmalloc(sizeof(*r->req.rsp), GFP_KERNEL); 363 if (!r->req.rsp) 364 goto out; 365 366 r->send_sge.addr = ib_dma_map_single(ndev->device, r->req.rsp, 367 sizeof(*r->req.rsp), DMA_TO_DEVICE); 368 if (ib_dma_mapping_error(ndev->device, r->send_sge.addr)) 369 goto out_free_rsp; 370 371 r->send_sge.length = sizeof(*r->req.rsp); 372 r->send_sge.lkey = ndev->pd->local_dma_lkey; 373 374 r->send_cqe.done = nvmet_rdma_send_done; 375 376 r->send_wr.wr_cqe = &r->send_cqe; 377 r->send_wr.sg_list = &r->send_sge; 378 r->send_wr.num_sge = 1; 379 r->send_wr.send_flags = IB_SEND_SIGNALED; 380 381 /* Data In / RDMA READ */ 382 r->read_cqe.done = nvmet_rdma_read_data_done; 383 return 0; 384 385 out_free_rsp: 386 kfree(r->req.rsp); 387 out: 388 return -ENOMEM; 389 } 390 391 static void nvmet_rdma_free_rsp(struct nvmet_rdma_device *ndev, 392 struct nvmet_rdma_rsp *r) 393 { 394 ib_dma_unmap_single(ndev->device, r->send_sge.addr, 395 sizeof(*r->req.rsp), DMA_TO_DEVICE); 396 kfree(r->req.rsp); 397 } 398 399 static int 400 nvmet_rdma_alloc_rsps(struct nvmet_rdma_queue *queue) 401 { 402 struct nvmet_rdma_device *ndev = queue->dev; 403 int nr_rsps = queue->recv_queue_size * 2; 404 int ret = -EINVAL, i; 405 406 queue->rsps = kcalloc(nr_rsps, sizeof(struct nvmet_rdma_rsp), 407 GFP_KERNEL); 408 if (!queue->rsps) 409 goto out; 410 411 for (i = 0; i < nr_rsps; i++) { 412 struct nvmet_rdma_rsp *rsp = &queue->rsps[i]; 413 414 ret = nvmet_rdma_alloc_rsp(ndev, rsp); 415 if (ret) 416 goto out_free; 417 418 list_add_tail(&rsp->free_list, &queue->free_rsps); 419 } 420 421 return 0; 422 423 out_free: 424 while (--i >= 0) { 425 struct nvmet_rdma_rsp *rsp = &queue->rsps[i]; 426 427 list_del(&rsp->free_list); 428 nvmet_rdma_free_rsp(ndev, rsp); 429 } 430 kfree(queue->rsps); 431 out: 432 return ret; 433 } 434 435 static void nvmet_rdma_free_rsps(struct nvmet_rdma_queue *queue) 436 { 437 struct nvmet_rdma_device *ndev = queue->dev; 438 int i, nr_rsps = queue->recv_queue_size * 2; 439 440 for (i = 0; i < nr_rsps; i++) { 441 struct nvmet_rdma_rsp *rsp = &queue->rsps[i]; 442 443 list_del(&rsp->free_list); 444 nvmet_rdma_free_rsp(ndev, rsp); 445 } 446 kfree(queue->rsps); 447 } 448 449 static int nvmet_rdma_post_recv(struct nvmet_rdma_device *ndev, 450 struct nvmet_rdma_cmd *cmd) 451 { 452 int ret; 453 454 ib_dma_sync_single_for_device(ndev->device, 455 cmd->sge[0].addr, cmd->sge[0].length, 456 DMA_FROM_DEVICE); 457 458 if (ndev->srq) 459 ret = ib_post_srq_recv(ndev->srq, &cmd->wr, NULL); 460 else 461 ret = ib_post_recv(cmd->queue->cm_id->qp, &cmd->wr, NULL); 462 463 if (unlikely(ret)) 464 pr_err("post_recv cmd failed\n"); 465 466 return ret; 467 } 468 469 static void nvmet_rdma_process_wr_wait_list(struct nvmet_rdma_queue *queue) 470 { 471 spin_lock(&queue->rsp_wr_wait_lock); 472 while (!list_empty(&queue->rsp_wr_wait_list)) { 473 struct nvmet_rdma_rsp *rsp; 474 bool ret; 475 476 rsp = list_entry(queue->rsp_wr_wait_list.next, 477 struct nvmet_rdma_rsp, wait_list); 478 list_del(&rsp->wait_list); 479 480 spin_unlock(&queue->rsp_wr_wait_lock); 481 ret = nvmet_rdma_execute_command(rsp); 482 spin_lock(&queue->rsp_wr_wait_lock); 483 484 if (!ret) { 485 list_add(&rsp->wait_list, &queue->rsp_wr_wait_list); 486 break; 487 } 488 } 489 spin_unlock(&queue->rsp_wr_wait_lock); 490 } 491 492 493 static void nvmet_rdma_release_rsp(struct nvmet_rdma_rsp *rsp) 494 { 495 struct nvmet_rdma_queue *queue = rsp->queue; 496 497 atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); 498 499 if (rsp->n_rdma) { 500 rdma_rw_ctx_destroy(&rsp->rw, queue->cm_id->qp, 501 queue->cm_id->port_num, rsp->req.sg, 502 rsp->req.sg_cnt, nvmet_data_dir(&rsp->req)); 503 } 504 505 if (rsp->req.sg != rsp->cmd->inline_sg) 506 nvmet_req_free_sgl(&rsp->req); 507 508 if (unlikely(!list_empty_careful(&queue->rsp_wr_wait_list))) 509 nvmet_rdma_process_wr_wait_list(queue); 510 511 nvmet_rdma_put_rsp(rsp); 512 } 513 514 static void nvmet_rdma_error_comp(struct nvmet_rdma_queue *queue) 515 { 516 if (queue->nvme_sq.ctrl) { 517 nvmet_ctrl_fatal_error(queue->nvme_sq.ctrl); 518 } else { 519 /* 520 * we didn't setup the controller yet in case 521 * of admin connect error, just disconnect and 522 * cleanup the queue 523 */ 524 nvmet_rdma_queue_disconnect(queue); 525 } 526 } 527 528 static void nvmet_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc) 529 { 530 struct nvmet_rdma_rsp *rsp = 531 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, send_cqe); 532 533 nvmet_rdma_release_rsp(rsp); 534 535 if (unlikely(wc->status != IB_WC_SUCCESS && 536 wc->status != IB_WC_WR_FLUSH_ERR)) { 537 pr_err("SEND for CQE 0x%p failed with status %s (%d).\n", 538 wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status); 539 nvmet_rdma_error_comp(rsp->queue); 540 } 541 } 542 543 static void nvmet_rdma_queue_response(struct nvmet_req *req) 544 { 545 struct nvmet_rdma_rsp *rsp = 546 container_of(req, struct nvmet_rdma_rsp, req); 547 struct rdma_cm_id *cm_id = rsp->queue->cm_id; 548 struct ib_send_wr *first_wr; 549 550 if (rsp->flags & NVMET_RDMA_REQ_INVALIDATE_RKEY) { 551 rsp->send_wr.opcode = IB_WR_SEND_WITH_INV; 552 rsp->send_wr.ex.invalidate_rkey = rsp->invalidate_rkey; 553 } else { 554 rsp->send_wr.opcode = IB_WR_SEND; 555 } 556 557 if (nvmet_rdma_need_data_out(rsp)) 558 first_wr = rdma_rw_ctx_wrs(&rsp->rw, cm_id->qp, 559 cm_id->port_num, NULL, &rsp->send_wr); 560 else 561 first_wr = &rsp->send_wr; 562 563 nvmet_rdma_post_recv(rsp->queue->dev, rsp->cmd); 564 565 ib_dma_sync_single_for_device(rsp->queue->dev->device, 566 rsp->send_sge.addr, rsp->send_sge.length, 567 DMA_TO_DEVICE); 568 569 if (unlikely(ib_post_send(cm_id->qp, first_wr, NULL))) { 570 pr_err("sending cmd response failed\n"); 571 nvmet_rdma_release_rsp(rsp); 572 } 573 } 574 575 static void nvmet_rdma_read_data_done(struct ib_cq *cq, struct ib_wc *wc) 576 { 577 struct nvmet_rdma_rsp *rsp = 578 container_of(wc->wr_cqe, struct nvmet_rdma_rsp, read_cqe); 579 struct nvmet_rdma_queue *queue = cq->cq_context; 580 581 WARN_ON(rsp->n_rdma <= 0); 582 atomic_add(rsp->n_rdma, &queue->sq_wr_avail); 583 rdma_rw_ctx_destroy(&rsp->rw, queue->cm_id->qp, 584 queue->cm_id->port_num, rsp->req.sg, 585 rsp->req.sg_cnt, nvmet_data_dir(&rsp->req)); 586 rsp->n_rdma = 0; 587 588 if (unlikely(wc->status != IB_WC_SUCCESS)) { 589 nvmet_req_uninit(&rsp->req); 590 nvmet_rdma_release_rsp(rsp); 591 if (wc->status != IB_WC_WR_FLUSH_ERR) { 592 pr_info("RDMA READ for CQE 0x%p failed with status %s (%d).\n", 593 wc->wr_cqe, ib_wc_status_msg(wc->status), wc->status); 594 nvmet_rdma_error_comp(queue); 595 } 596 return; 597 } 598 599 nvmet_req_execute(&rsp->req); 600 } 601 602 static void nvmet_rdma_use_inline_sg(struct nvmet_rdma_rsp *rsp, u32 len, 603 u64 off) 604 { 605 int sg_count = num_pages(len); 606 struct scatterlist *sg; 607 int i; 608 609 sg = rsp->cmd->inline_sg; 610 for (i = 0; i < sg_count; i++, sg++) { 611 if (i < sg_count - 1) 612 sg_unmark_end(sg); 613 else 614 sg_mark_end(sg); 615 sg->offset = off; 616 sg->length = min_t(int, len, PAGE_SIZE - off); 617 len -= sg->length; 618 if (!i) 619 off = 0; 620 } 621 622 rsp->req.sg = rsp->cmd->inline_sg; 623 rsp->req.sg_cnt = sg_count; 624 } 625 626 static u16 nvmet_rdma_map_sgl_inline(struct nvmet_rdma_rsp *rsp) 627 { 628 struct nvme_sgl_desc *sgl = &rsp->req.cmd->common.dptr.sgl; 629 u64 off = le64_to_cpu(sgl->addr); 630 u32 len = le32_to_cpu(sgl->length); 631 632 if (!nvme_is_write(rsp->req.cmd)) 633 return NVME_SC_INVALID_FIELD | NVME_SC_DNR; 634 635 if (off + len > rsp->queue->dev->inline_data_size) { 636 pr_err("invalid inline data offset!\n"); 637 return NVME_SC_SGL_INVALID_OFFSET | NVME_SC_DNR; 638 } 639 640 /* no data command? */ 641 if (!len) 642 return 0; 643 644 nvmet_rdma_use_inline_sg(rsp, len, off); 645 rsp->flags |= NVMET_RDMA_REQ_INLINE_DATA; 646 rsp->req.transfer_len += len; 647 return 0; 648 } 649 650 static u16 nvmet_rdma_map_sgl_keyed(struct nvmet_rdma_rsp *rsp, 651 struct nvme_keyed_sgl_desc *sgl, bool invalidate) 652 { 653 struct rdma_cm_id *cm_id = rsp->queue->cm_id; 654 u64 addr = le64_to_cpu(sgl->addr); 655 u32 key = get_unaligned_le32(sgl->key); 656 int ret; 657 658 rsp->req.transfer_len = get_unaligned_le24(sgl->length); 659 660 /* no data command? */ 661 if (!rsp->req.transfer_len) 662 return 0; 663 664 ret = nvmet_req_alloc_sgl(&rsp->req); 665 if (ret < 0) 666 goto error_out; 667 668 ret = rdma_rw_ctx_init(&rsp->rw, cm_id->qp, cm_id->port_num, 669 rsp->req.sg, rsp->req.sg_cnt, 0, addr, key, 670 nvmet_data_dir(&rsp->req)); 671 if (ret < 0) 672 goto error_out; 673 rsp->n_rdma += ret; 674 675 if (invalidate) { 676 rsp->invalidate_rkey = key; 677 rsp->flags |= NVMET_RDMA_REQ_INVALIDATE_RKEY; 678 } 679 680 return 0; 681 682 error_out: 683 rsp->req.transfer_len = 0; 684 return NVME_SC_INTERNAL; 685 } 686 687 static u16 nvmet_rdma_map_sgl(struct nvmet_rdma_rsp *rsp) 688 { 689 struct nvme_keyed_sgl_desc *sgl = &rsp->req.cmd->common.dptr.ksgl; 690 691 switch (sgl->type >> 4) { 692 case NVME_SGL_FMT_DATA_DESC: 693 switch (sgl->type & 0xf) { 694 case NVME_SGL_FMT_OFFSET: 695 return nvmet_rdma_map_sgl_inline(rsp); 696 default: 697 pr_err("invalid SGL subtype: %#x\n", sgl->type); 698 return NVME_SC_INVALID_FIELD | NVME_SC_DNR; 699 } 700 case NVME_KEY_SGL_FMT_DATA_DESC: 701 switch (sgl->type & 0xf) { 702 case NVME_SGL_FMT_ADDRESS | NVME_SGL_FMT_INVALIDATE: 703 return nvmet_rdma_map_sgl_keyed(rsp, sgl, true); 704 case NVME_SGL_FMT_ADDRESS: 705 return nvmet_rdma_map_sgl_keyed(rsp, sgl, false); 706 default: 707 pr_err("invalid SGL subtype: %#x\n", sgl->type); 708 return NVME_SC_INVALID_FIELD | NVME_SC_DNR; 709 } 710 default: 711 pr_err("invalid SGL type: %#x\n", sgl->type); 712 return NVME_SC_SGL_INVALID_TYPE | NVME_SC_DNR; 713 } 714 } 715 716 static bool nvmet_rdma_execute_command(struct nvmet_rdma_rsp *rsp) 717 { 718 struct nvmet_rdma_queue *queue = rsp->queue; 719 720 if (unlikely(atomic_sub_return(1 + rsp->n_rdma, 721 &queue->sq_wr_avail) < 0)) { 722 pr_debug("IB send queue full (needed %d): queue %u cntlid %u\n", 723 1 + rsp->n_rdma, queue->idx, 724 queue->nvme_sq.ctrl->cntlid); 725 atomic_add(1 + rsp->n_rdma, &queue->sq_wr_avail); 726 return false; 727 } 728 729 if (nvmet_rdma_need_data_in(rsp)) { 730 if (rdma_rw_ctx_post(&rsp->rw, queue->cm_id->qp, 731 queue->cm_id->port_num, &rsp->read_cqe, NULL)) 732 nvmet_req_complete(&rsp->req, NVME_SC_DATA_XFER_ERROR); 733 } else { 734 nvmet_req_execute(&rsp->req); 735 } 736 737 return true; 738 } 739 740 static void nvmet_rdma_handle_command(struct nvmet_rdma_queue *queue, 741 struct nvmet_rdma_rsp *cmd) 742 { 743 u16 status; 744 745 ib_dma_sync_single_for_cpu(queue->dev->device, 746 cmd->cmd->sge[0].addr, cmd->cmd->sge[0].length, 747 DMA_FROM_DEVICE); 748 ib_dma_sync_single_for_cpu(queue->dev->device, 749 cmd->send_sge.addr, cmd->send_sge.length, 750 DMA_TO_DEVICE); 751 752 cmd->req.p2p_client = &queue->dev->device->dev; 753 754 if (!nvmet_req_init(&cmd->req, &queue->nvme_cq, 755 &queue->nvme_sq, &nvmet_rdma_ops)) 756 return; 757 758 status = nvmet_rdma_map_sgl(cmd); 759 if (status) 760 goto out_err; 761 762 if (unlikely(!nvmet_rdma_execute_command(cmd))) { 763 spin_lock(&queue->rsp_wr_wait_lock); 764 list_add_tail(&cmd->wait_list, &queue->rsp_wr_wait_list); 765 spin_unlock(&queue->rsp_wr_wait_lock); 766 } 767 768 return; 769 770 out_err: 771 nvmet_req_complete(&cmd->req, status); 772 } 773 774 static void nvmet_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc) 775 { 776 struct nvmet_rdma_cmd *cmd = 777 container_of(wc->wr_cqe, struct nvmet_rdma_cmd, cqe); 778 struct nvmet_rdma_queue *queue = cq->cq_context; 779 struct nvmet_rdma_rsp *rsp; 780 781 if (unlikely(wc->status != IB_WC_SUCCESS)) { 782 if (wc->status != IB_WC_WR_FLUSH_ERR) { 783 pr_err("RECV for CQE 0x%p failed with status %s (%d)\n", 784 wc->wr_cqe, ib_wc_status_msg(wc->status), 785 wc->status); 786 nvmet_rdma_error_comp(queue); 787 } 788 return; 789 } 790 791 if (unlikely(wc->byte_len < sizeof(struct nvme_command))) { 792 pr_err("Ctrl Fatal Error: capsule size less than 64 bytes\n"); 793 nvmet_rdma_error_comp(queue); 794 return; 795 } 796 797 cmd->queue = queue; 798 rsp = nvmet_rdma_get_rsp(queue); 799 if (unlikely(!rsp)) { 800 /* 801 * we get here only under memory pressure, 802 * silently drop and have the host retry 803 * as we can't even fail it. 804 */ 805 nvmet_rdma_post_recv(queue->dev, cmd); 806 return; 807 } 808 rsp->queue = queue; 809 rsp->cmd = cmd; 810 rsp->flags = 0; 811 rsp->req.cmd = cmd->nvme_cmd; 812 rsp->req.port = queue->port; 813 rsp->n_rdma = 0; 814 815 if (unlikely(queue->state != NVMET_RDMA_Q_LIVE)) { 816 unsigned long flags; 817 818 spin_lock_irqsave(&queue->state_lock, flags); 819 if (queue->state == NVMET_RDMA_Q_CONNECTING) 820 list_add_tail(&rsp->wait_list, &queue->rsp_wait_list); 821 else 822 nvmet_rdma_put_rsp(rsp); 823 spin_unlock_irqrestore(&queue->state_lock, flags); 824 return; 825 } 826 827 nvmet_rdma_handle_command(queue, rsp); 828 } 829 830 static void nvmet_rdma_destroy_srq(struct nvmet_rdma_device *ndev) 831 { 832 if (!ndev->srq) 833 return; 834 835 nvmet_rdma_free_cmds(ndev, ndev->srq_cmds, ndev->srq_size, false); 836 ib_destroy_srq(ndev->srq); 837 } 838 839 static int nvmet_rdma_init_srq(struct nvmet_rdma_device *ndev) 840 { 841 struct ib_srq_init_attr srq_attr = { NULL, }; 842 struct ib_srq *srq; 843 size_t srq_size; 844 int ret, i; 845 846 srq_size = 4095; /* XXX: tune */ 847 848 srq_attr.attr.max_wr = srq_size; 849 srq_attr.attr.max_sge = 1 + ndev->inline_page_count; 850 srq_attr.attr.srq_limit = 0; 851 srq_attr.srq_type = IB_SRQT_BASIC; 852 srq = ib_create_srq(ndev->pd, &srq_attr); 853 if (IS_ERR(srq)) { 854 /* 855 * If SRQs aren't supported we just go ahead and use normal 856 * non-shared receive queues. 857 */ 858 pr_info("SRQ requested but not supported.\n"); 859 return 0; 860 } 861 862 ndev->srq_cmds = nvmet_rdma_alloc_cmds(ndev, srq_size, false); 863 if (IS_ERR(ndev->srq_cmds)) { 864 ret = PTR_ERR(ndev->srq_cmds); 865 goto out_destroy_srq; 866 } 867 868 ndev->srq = srq; 869 ndev->srq_size = srq_size; 870 871 for (i = 0; i < srq_size; i++) { 872 ret = nvmet_rdma_post_recv(ndev, &ndev->srq_cmds[i]); 873 if (ret) 874 goto out_free_cmds; 875 } 876 877 return 0; 878 879 out_free_cmds: 880 nvmet_rdma_free_cmds(ndev, ndev->srq_cmds, ndev->srq_size, false); 881 out_destroy_srq: 882 ib_destroy_srq(srq); 883 return ret; 884 } 885 886 static void nvmet_rdma_free_dev(struct kref *ref) 887 { 888 struct nvmet_rdma_device *ndev = 889 container_of(ref, struct nvmet_rdma_device, ref); 890 891 mutex_lock(&device_list_mutex); 892 list_del(&ndev->entry); 893 mutex_unlock(&device_list_mutex); 894 895 nvmet_rdma_destroy_srq(ndev); 896 ib_dealloc_pd(ndev->pd); 897 898 kfree(ndev); 899 } 900 901 static struct nvmet_rdma_device * 902 nvmet_rdma_find_get_device(struct rdma_cm_id *cm_id) 903 { 904 struct nvmet_port *port = cm_id->context; 905 struct nvmet_rdma_device *ndev; 906 int inline_page_count; 907 int inline_sge_count; 908 int ret; 909 910 mutex_lock(&device_list_mutex); 911 list_for_each_entry(ndev, &device_list, entry) { 912 if (ndev->device->node_guid == cm_id->device->node_guid && 913 kref_get_unless_zero(&ndev->ref)) 914 goto out_unlock; 915 } 916 917 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); 918 if (!ndev) 919 goto out_err; 920 921 inline_page_count = num_pages(port->inline_data_size); 922 inline_sge_count = max(cm_id->device->attrs.max_sge_rd, 923 cm_id->device->attrs.max_recv_sge) - 1; 924 if (inline_page_count > inline_sge_count) { 925 pr_warn("inline_data_size %d cannot be supported by device %s. Reducing to %lu.\n", 926 port->inline_data_size, cm_id->device->name, 927 inline_sge_count * PAGE_SIZE); 928 port->inline_data_size = inline_sge_count * PAGE_SIZE; 929 inline_page_count = inline_sge_count; 930 } 931 ndev->inline_data_size = port->inline_data_size; 932 ndev->inline_page_count = inline_page_count; 933 ndev->device = cm_id->device; 934 kref_init(&ndev->ref); 935 936 ndev->pd = ib_alloc_pd(ndev->device, 0); 937 if (IS_ERR(ndev->pd)) 938 goto out_free_dev; 939 940 if (nvmet_rdma_use_srq) { 941 ret = nvmet_rdma_init_srq(ndev); 942 if (ret) 943 goto out_free_pd; 944 } 945 946 list_add(&ndev->entry, &device_list); 947 out_unlock: 948 mutex_unlock(&device_list_mutex); 949 pr_debug("added %s.\n", ndev->device->name); 950 return ndev; 951 952 out_free_pd: 953 ib_dealloc_pd(ndev->pd); 954 out_free_dev: 955 kfree(ndev); 956 out_err: 957 mutex_unlock(&device_list_mutex); 958 return NULL; 959 } 960 961 static int nvmet_rdma_create_queue_ib(struct nvmet_rdma_queue *queue) 962 { 963 struct ib_qp_init_attr qp_attr; 964 struct nvmet_rdma_device *ndev = queue->dev; 965 int comp_vector, nr_cqe, ret, i; 966 967 /* 968 * Spread the io queues across completion vectors, 969 * but still keep all admin queues on vector 0. 970 */ 971 comp_vector = !queue->host_qid ? 0 : 972 queue->idx % ndev->device->num_comp_vectors; 973 974 /* 975 * Reserve CQ slots for RECV + RDMA_READ/RDMA_WRITE + RDMA_SEND. 976 */ 977 nr_cqe = queue->recv_queue_size + 2 * queue->send_queue_size; 978 979 queue->cq = ib_alloc_cq(ndev->device, queue, 980 nr_cqe + 1, comp_vector, 981 IB_POLL_WORKQUEUE); 982 if (IS_ERR(queue->cq)) { 983 ret = PTR_ERR(queue->cq); 984 pr_err("failed to create CQ cqe= %d ret= %d\n", 985 nr_cqe + 1, ret); 986 goto out; 987 } 988 989 memset(&qp_attr, 0, sizeof(qp_attr)); 990 qp_attr.qp_context = queue; 991 qp_attr.event_handler = nvmet_rdma_qp_event; 992 qp_attr.send_cq = queue->cq; 993 qp_attr.recv_cq = queue->cq; 994 qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; 995 qp_attr.qp_type = IB_QPT_RC; 996 /* +1 for drain */ 997 qp_attr.cap.max_send_wr = queue->send_queue_size + 1; 998 qp_attr.cap.max_rdma_ctxs = queue->send_queue_size; 999 qp_attr.cap.max_send_sge = max(ndev->device->attrs.max_sge_rd, 1000 ndev->device->attrs.max_send_sge); 1001 1002 if (ndev->srq) { 1003 qp_attr.srq = ndev->srq; 1004 } else { 1005 /* +1 for drain */ 1006 qp_attr.cap.max_recv_wr = 1 + queue->recv_queue_size; 1007 qp_attr.cap.max_recv_sge = 1 + ndev->inline_page_count; 1008 } 1009 1010 ret = rdma_create_qp(queue->cm_id, ndev->pd, &qp_attr); 1011 if (ret) { 1012 pr_err("failed to create_qp ret= %d\n", ret); 1013 goto err_destroy_cq; 1014 } 1015 1016 atomic_set(&queue->sq_wr_avail, qp_attr.cap.max_send_wr); 1017 1018 pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n", 1019 __func__, queue->cq->cqe, qp_attr.cap.max_send_sge, 1020 qp_attr.cap.max_send_wr, queue->cm_id); 1021 1022 if (!ndev->srq) { 1023 for (i = 0; i < queue->recv_queue_size; i++) { 1024 queue->cmds[i].queue = queue; 1025 ret = nvmet_rdma_post_recv(ndev, &queue->cmds[i]); 1026 if (ret) 1027 goto err_destroy_qp; 1028 } 1029 } 1030 1031 out: 1032 return ret; 1033 1034 err_destroy_qp: 1035 rdma_destroy_qp(queue->cm_id); 1036 err_destroy_cq: 1037 ib_free_cq(queue->cq); 1038 goto out; 1039 } 1040 1041 static void nvmet_rdma_destroy_queue_ib(struct nvmet_rdma_queue *queue) 1042 { 1043 struct ib_qp *qp = queue->cm_id->qp; 1044 1045 ib_drain_qp(qp); 1046 rdma_destroy_id(queue->cm_id); 1047 ib_destroy_qp(qp); 1048 ib_free_cq(queue->cq); 1049 } 1050 1051 static void nvmet_rdma_free_queue(struct nvmet_rdma_queue *queue) 1052 { 1053 pr_debug("freeing queue %d\n", queue->idx); 1054 1055 nvmet_sq_destroy(&queue->nvme_sq); 1056 1057 nvmet_rdma_destroy_queue_ib(queue); 1058 if (!queue->dev->srq) { 1059 nvmet_rdma_free_cmds(queue->dev, queue->cmds, 1060 queue->recv_queue_size, 1061 !queue->host_qid); 1062 } 1063 nvmet_rdma_free_rsps(queue); 1064 ida_simple_remove(&nvmet_rdma_queue_ida, queue->idx); 1065 kfree(queue); 1066 } 1067 1068 static void nvmet_rdma_release_queue_work(struct work_struct *w) 1069 { 1070 struct nvmet_rdma_queue *queue = 1071 container_of(w, struct nvmet_rdma_queue, release_work); 1072 struct nvmet_rdma_device *dev = queue->dev; 1073 1074 nvmet_rdma_free_queue(queue); 1075 1076 kref_put(&dev->ref, nvmet_rdma_free_dev); 1077 } 1078 1079 static int 1080 nvmet_rdma_parse_cm_connect_req(struct rdma_conn_param *conn, 1081 struct nvmet_rdma_queue *queue) 1082 { 1083 struct nvme_rdma_cm_req *req; 1084 1085 req = (struct nvme_rdma_cm_req *)conn->private_data; 1086 if (!req || conn->private_data_len == 0) 1087 return NVME_RDMA_CM_INVALID_LEN; 1088 1089 if (le16_to_cpu(req->recfmt) != NVME_RDMA_CM_FMT_1_0) 1090 return NVME_RDMA_CM_INVALID_RECFMT; 1091 1092 queue->host_qid = le16_to_cpu(req->qid); 1093 1094 /* 1095 * req->hsqsize corresponds to our recv queue size plus 1 1096 * req->hrqsize corresponds to our send queue size 1097 */ 1098 queue->recv_queue_size = le16_to_cpu(req->hsqsize) + 1; 1099 queue->send_queue_size = le16_to_cpu(req->hrqsize); 1100 1101 if (!queue->host_qid && queue->recv_queue_size > NVME_AQ_DEPTH) 1102 return NVME_RDMA_CM_INVALID_HSQSIZE; 1103 1104 /* XXX: Should we enforce some kind of max for IO queues? */ 1105 1106 return 0; 1107 } 1108 1109 static int nvmet_rdma_cm_reject(struct rdma_cm_id *cm_id, 1110 enum nvme_rdma_cm_status status) 1111 { 1112 struct nvme_rdma_cm_rej rej; 1113 1114 pr_debug("rejecting connect request: status %d (%s)\n", 1115 status, nvme_rdma_cm_msg(status)); 1116 1117 rej.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0); 1118 rej.sts = cpu_to_le16(status); 1119 1120 return rdma_reject(cm_id, (void *)&rej, sizeof(rej)); 1121 } 1122 1123 static struct nvmet_rdma_queue * 1124 nvmet_rdma_alloc_queue(struct nvmet_rdma_device *ndev, 1125 struct rdma_cm_id *cm_id, 1126 struct rdma_cm_event *event) 1127 { 1128 struct nvmet_rdma_queue *queue; 1129 int ret; 1130 1131 queue = kzalloc(sizeof(*queue), GFP_KERNEL); 1132 if (!queue) { 1133 ret = NVME_RDMA_CM_NO_RSC; 1134 goto out_reject; 1135 } 1136 1137 ret = nvmet_sq_init(&queue->nvme_sq); 1138 if (ret) { 1139 ret = NVME_RDMA_CM_NO_RSC; 1140 goto out_free_queue; 1141 } 1142 1143 ret = nvmet_rdma_parse_cm_connect_req(&event->param.conn, queue); 1144 if (ret) 1145 goto out_destroy_sq; 1146 1147 /* 1148 * Schedules the actual release because calling rdma_destroy_id from 1149 * inside a CM callback would trigger a deadlock. (great API design..) 1150 */ 1151 INIT_WORK(&queue->release_work, nvmet_rdma_release_queue_work); 1152 queue->dev = ndev; 1153 queue->cm_id = cm_id; 1154 1155 spin_lock_init(&queue->state_lock); 1156 queue->state = NVMET_RDMA_Q_CONNECTING; 1157 INIT_LIST_HEAD(&queue->rsp_wait_list); 1158 INIT_LIST_HEAD(&queue->rsp_wr_wait_list); 1159 spin_lock_init(&queue->rsp_wr_wait_lock); 1160 INIT_LIST_HEAD(&queue->free_rsps); 1161 spin_lock_init(&queue->rsps_lock); 1162 INIT_LIST_HEAD(&queue->queue_list); 1163 1164 queue->idx = ida_simple_get(&nvmet_rdma_queue_ida, 0, 0, GFP_KERNEL); 1165 if (queue->idx < 0) { 1166 ret = NVME_RDMA_CM_NO_RSC; 1167 goto out_destroy_sq; 1168 } 1169 1170 ret = nvmet_rdma_alloc_rsps(queue); 1171 if (ret) { 1172 ret = NVME_RDMA_CM_NO_RSC; 1173 goto out_ida_remove; 1174 } 1175 1176 if (!ndev->srq) { 1177 queue->cmds = nvmet_rdma_alloc_cmds(ndev, 1178 queue->recv_queue_size, 1179 !queue->host_qid); 1180 if (IS_ERR(queue->cmds)) { 1181 ret = NVME_RDMA_CM_NO_RSC; 1182 goto out_free_responses; 1183 } 1184 } 1185 1186 ret = nvmet_rdma_create_queue_ib(queue); 1187 if (ret) { 1188 pr_err("%s: creating RDMA queue failed (%d).\n", 1189 __func__, ret); 1190 ret = NVME_RDMA_CM_NO_RSC; 1191 goto out_free_cmds; 1192 } 1193 1194 return queue; 1195 1196 out_free_cmds: 1197 if (!ndev->srq) { 1198 nvmet_rdma_free_cmds(queue->dev, queue->cmds, 1199 queue->recv_queue_size, 1200 !queue->host_qid); 1201 } 1202 out_free_responses: 1203 nvmet_rdma_free_rsps(queue); 1204 out_ida_remove: 1205 ida_simple_remove(&nvmet_rdma_queue_ida, queue->idx); 1206 out_destroy_sq: 1207 nvmet_sq_destroy(&queue->nvme_sq); 1208 out_free_queue: 1209 kfree(queue); 1210 out_reject: 1211 nvmet_rdma_cm_reject(cm_id, ret); 1212 return NULL; 1213 } 1214 1215 static void nvmet_rdma_qp_event(struct ib_event *event, void *priv) 1216 { 1217 struct nvmet_rdma_queue *queue = priv; 1218 1219 switch (event->event) { 1220 case IB_EVENT_COMM_EST: 1221 rdma_notify(queue->cm_id, event->event); 1222 break; 1223 default: 1224 pr_err("received IB QP event: %s (%d)\n", 1225 ib_event_msg(event->event), event->event); 1226 break; 1227 } 1228 } 1229 1230 static int nvmet_rdma_cm_accept(struct rdma_cm_id *cm_id, 1231 struct nvmet_rdma_queue *queue, 1232 struct rdma_conn_param *p) 1233 { 1234 struct rdma_conn_param param = { }; 1235 struct nvme_rdma_cm_rep priv = { }; 1236 int ret = -ENOMEM; 1237 1238 param.rnr_retry_count = 7; 1239 param.flow_control = 1; 1240 param.initiator_depth = min_t(u8, p->initiator_depth, 1241 queue->dev->device->attrs.max_qp_init_rd_atom); 1242 param.private_data = &priv; 1243 param.private_data_len = sizeof(priv); 1244 priv.recfmt = cpu_to_le16(NVME_RDMA_CM_FMT_1_0); 1245 priv.crqsize = cpu_to_le16(queue->recv_queue_size); 1246 1247 ret = rdma_accept(cm_id, ¶m); 1248 if (ret) 1249 pr_err("rdma_accept failed (error code = %d)\n", ret); 1250 1251 return ret; 1252 } 1253 1254 static int nvmet_rdma_queue_connect(struct rdma_cm_id *cm_id, 1255 struct rdma_cm_event *event) 1256 { 1257 struct nvmet_rdma_device *ndev; 1258 struct nvmet_rdma_queue *queue; 1259 int ret = -EINVAL; 1260 1261 ndev = nvmet_rdma_find_get_device(cm_id); 1262 if (!ndev) { 1263 nvmet_rdma_cm_reject(cm_id, NVME_RDMA_CM_NO_RSC); 1264 return -ECONNREFUSED; 1265 } 1266 1267 queue = nvmet_rdma_alloc_queue(ndev, cm_id, event); 1268 if (!queue) { 1269 ret = -ENOMEM; 1270 goto put_device; 1271 } 1272 queue->port = cm_id->context; 1273 1274 if (queue->host_qid == 0) { 1275 /* Let inflight controller teardown complete */ 1276 flush_scheduled_work(); 1277 } 1278 1279 ret = nvmet_rdma_cm_accept(cm_id, queue, &event->param.conn); 1280 if (ret) { 1281 schedule_work(&queue->release_work); 1282 /* Destroying rdma_cm id is not needed here */ 1283 return 0; 1284 } 1285 1286 mutex_lock(&nvmet_rdma_queue_mutex); 1287 list_add_tail(&queue->queue_list, &nvmet_rdma_queue_list); 1288 mutex_unlock(&nvmet_rdma_queue_mutex); 1289 1290 return 0; 1291 1292 put_device: 1293 kref_put(&ndev->ref, nvmet_rdma_free_dev); 1294 1295 return ret; 1296 } 1297 1298 static void nvmet_rdma_queue_established(struct nvmet_rdma_queue *queue) 1299 { 1300 unsigned long flags; 1301 1302 spin_lock_irqsave(&queue->state_lock, flags); 1303 if (queue->state != NVMET_RDMA_Q_CONNECTING) { 1304 pr_warn("trying to establish a connected queue\n"); 1305 goto out_unlock; 1306 } 1307 queue->state = NVMET_RDMA_Q_LIVE; 1308 1309 while (!list_empty(&queue->rsp_wait_list)) { 1310 struct nvmet_rdma_rsp *cmd; 1311 1312 cmd = list_first_entry(&queue->rsp_wait_list, 1313 struct nvmet_rdma_rsp, wait_list); 1314 list_del(&cmd->wait_list); 1315 1316 spin_unlock_irqrestore(&queue->state_lock, flags); 1317 nvmet_rdma_handle_command(queue, cmd); 1318 spin_lock_irqsave(&queue->state_lock, flags); 1319 } 1320 1321 out_unlock: 1322 spin_unlock_irqrestore(&queue->state_lock, flags); 1323 } 1324 1325 static void __nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) 1326 { 1327 bool disconnect = false; 1328 unsigned long flags; 1329 1330 pr_debug("cm_id= %p queue->state= %d\n", queue->cm_id, queue->state); 1331 1332 spin_lock_irqsave(&queue->state_lock, flags); 1333 switch (queue->state) { 1334 case NVMET_RDMA_Q_CONNECTING: 1335 case NVMET_RDMA_Q_LIVE: 1336 queue->state = NVMET_RDMA_Q_DISCONNECTING; 1337 disconnect = true; 1338 break; 1339 case NVMET_RDMA_Q_DISCONNECTING: 1340 break; 1341 } 1342 spin_unlock_irqrestore(&queue->state_lock, flags); 1343 1344 if (disconnect) { 1345 rdma_disconnect(queue->cm_id); 1346 schedule_work(&queue->release_work); 1347 } 1348 } 1349 1350 static void nvmet_rdma_queue_disconnect(struct nvmet_rdma_queue *queue) 1351 { 1352 bool disconnect = false; 1353 1354 mutex_lock(&nvmet_rdma_queue_mutex); 1355 if (!list_empty(&queue->queue_list)) { 1356 list_del_init(&queue->queue_list); 1357 disconnect = true; 1358 } 1359 mutex_unlock(&nvmet_rdma_queue_mutex); 1360 1361 if (disconnect) 1362 __nvmet_rdma_queue_disconnect(queue); 1363 } 1364 1365 static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, 1366 struct nvmet_rdma_queue *queue) 1367 { 1368 WARN_ON_ONCE(queue->state != NVMET_RDMA_Q_CONNECTING); 1369 1370 mutex_lock(&nvmet_rdma_queue_mutex); 1371 if (!list_empty(&queue->queue_list)) 1372 list_del_init(&queue->queue_list); 1373 mutex_unlock(&nvmet_rdma_queue_mutex); 1374 1375 pr_err("failed to connect queue %d\n", queue->idx); 1376 schedule_work(&queue->release_work); 1377 } 1378 1379 /** 1380 * nvme_rdma_device_removal() - Handle RDMA device removal 1381 * @cm_id: rdma_cm id, used for nvmet port 1382 * @queue: nvmet rdma queue (cm id qp_context) 1383 * 1384 * DEVICE_REMOVAL event notifies us that the RDMA device is about 1385 * to unplug. Note that this event can be generated on a normal 1386 * queue cm_id and/or a device bound listener cm_id (where in this 1387 * case queue will be null). 1388 * 1389 * We registered an ib_client to handle device removal for queues, 1390 * so we only need to handle the listening port cm_ids. In this case 1391 * we nullify the priv to prevent double cm_id destruction and destroying 1392 * the cm_id implicitely by returning a non-zero rc to the callout. 1393 */ 1394 static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, 1395 struct nvmet_rdma_queue *queue) 1396 { 1397 struct nvmet_port *port; 1398 1399 if (queue) { 1400 /* 1401 * This is a queue cm_id. we have registered 1402 * an ib_client to handle queues removal 1403 * so don't interfear and just return. 1404 */ 1405 return 0; 1406 } 1407 1408 port = cm_id->context; 1409 1410 /* 1411 * This is a listener cm_id. Make sure that 1412 * future remove_port won't invoke a double 1413 * cm_id destroy. use atomic xchg to make sure 1414 * we don't compete with remove_port. 1415 */ 1416 if (xchg(&port->priv, NULL) != cm_id) 1417 return 0; 1418 1419 /* 1420 * We need to return 1 so that the core will destroy 1421 * it's own ID. What a great API design.. 1422 */ 1423 return 1; 1424 } 1425 1426 static int nvmet_rdma_cm_handler(struct rdma_cm_id *cm_id, 1427 struct rdma_cm_event *event) 1428 { 1429 struct nvmet_rdma_queue *queue = NULL; 1430 int ret = 0; 1431 1432 if (cm_id->qp) 1433 queue = cm_id->qp->qp_context; 1434 1435 pr_debug("%s (%d): status %d id %p\n", 1436 rdma_event_msg(event->event), event->event, 1437 event->status, cm_id); 1438 1439 switch (event->event) { 1440 case RDMA_CM_EVENT_CONNECT_REQUEST: 1441 ret = nvmet_rdma_queue_connect(cm_id, event); 1442 break; 1443 case RDMA_CM_EVENT_ESTABLISHED: 1444 nvmet_rdma_queue_established(queue); 1445 break; 1446 case RDMA_CM_EVENT_ADDR_CHANGE: 1447 case RDMA_CM_EVENT_DISCONNECTED: 1448 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 1449 nvmet_rdma_queue_disconnect(queue); 1450 break; 1451 case RDMA_CM_EVENT_DEVICE_REMOVAL: 1452 ret = nvmet_rdma_device_removal(cm_id, queue); 1453 break; 1454 case RDMA_CM_EVENT_REJECTED: 1455 pr_debug("Connection rejected: %s\n", 1456 rdma_reject_msg(cm_id, event->status)); 1457 /* FALLTHROUGH */ 1458 case RDMA_CM_EVENT_UNREACHABLE: 1459 case RDMA_CM_EVENT_CONNECT_ERROR: 1460 nvmet_rdma_queue_connect_fail(cm_id, queue); 1461 break; 1462 default: 1463 pr_err("received unrecognized RDMA CM event %d\n", 1464 event->event); 1465 break; 1466 } 1467 1468 return ret; 1469 } 1470 1471 static void nvmet_rdma_delete_ctrl(struct nvmet_ctrl *ctrl) 1472 { 1473 struct nvmet_rdma_queue *queue; 1474 1475 restart: 1476 mutex_lock(&nvmet_rdma_queue_mutex); 1477 list_for_each_entry(queue, &nvmet_rdma_queue_list, queue_list) { 1478 if (queue->nvme_sq.ctrl == ctrl) { 1479 list_del_init(&queue->queue_list); 1480 mutex_unlock(&nvmet_rdma_queue_mutex); 1481 1482 __nvmet_rdma_queue_disconnect(queue); 1483 goto restart; 1484 } 1485 } 1486 mutex_unlock(&nvmet_rdma_queue_mutex); 1487 } 1488 1489 static int nvmet_rdma_add_port(struct nvmet_port *port) 1490 { 1491 struct rdma_cm_id *cm_id; 1492 struct sockaddr_storage addr = { }; 1493 __kernel_sa_family_t af; 1494 int ret; 1495 1496 switch (port->disc_addr.adrfam) { 1497 case NVMF_ADDR_FAMILY_IP4: 1498 af = AF_INET; 1499 break; 1500 case NVMF_ADDR_FAMILY_IP6: 1501 af = AF_INET6; 1502 break; 1503 default: 1504 pr_err("address family %d not supported\n", 1505 port->disc_addr.adrfam); 1506 return -EINVAL; 1507 } 1508 1509 if (port->inline_data_size < 0) { 1510 port->inline_data_size = NVMET_RDMA_DEFAULT_INLINE_DATA_SIZE; 1511 } else if (port->inline_data_size > NVMET_RDMA_MAX_INLINE_DATA_SIZE) { 1512 pr_warn("inline_data_size %u is too large, reducing to %u\n", 1513 port->inline_data_size, 1514 NVMET_RDMA_MAX_INLINE_DATA_SIZE); 1515 port->inline_data_size = NVMET_RDMA_MAX_INLINE_DATA_SIZE; 1516 } 1517 1518 ret = inet_pton_with_scope(&init_net, af, port->disc_addr.traddr, 1519 port->disc_addr.trsvcid, &addr); 1520 if (ret) { 1521 pr_err("malformed ip/port passed: %s:%s\n", 1522 port->disc_addr.traddr, port->disc_addr.trsvcid); 1523 return ret; 1524 } 1525 1526 cm_id = rdma_create_id(&init_net, nvmet_rdma_cm_handler, port, 1527 RDMA_PS_TCP, IB_QPT_RC); 1528 if (IS_ERR(cm_id)) { 1529 pr_err("CM ID creation failed\n"); 1530 return PTR_ERR(cm_id); 1531 } 1532 1533 /* 1534 * Allow both IPv4 and IPv6 sockets to bind a single port 1535 * at the same time. 1536 */ 1537 ret = rdma_set_afonly(cm_id, 1); 1538 if (ret) { 1539 pr_err("rdma_set_afonly failed (%d)\n", ret); 1540 goto out_destroy_id; 1541 } 1542 1543 ret = rdma_bind_addr(cm_id, (struct sockaddr *)&addr); 1544 if (ret) { 1545 pr_err("binding CM ID to %pISpcs failed (%d)\n", 1546 (struct sockaddr *)&addr, ret); 1547 goto out_destroy_id; 1548 } 1549 1550 ret = rdma_listen(cm_id, 128); 1551 if (ret) { 1552 pr_err("listening to %pISpcs failed (%d)\n", 1553 (struct sockaddr *)&addr, ret); 1554 goto out_destroy_id; 1555 } 1556 1557 pr_info("enabling port %d (%pISpcs)\n", 1558 le16_to_cpu(port->disc_addr.portid), (struct sockaddr *)&addr); 1559 port->priv = cm_id; 1560 return 0; 1561 1562 out_destroy_id: 1563 rdma_destroy_id(cm_id); 1564 return ret; 1565 } 1566 1567 static void nvmet_rdma_remove_port(struct nvmet_port *port) 1568 { 1569 struct rdma_cm_id *cm_id = xchg(&port->priv, NULL); 1570 1571 if (cm_id) 1572 rdma_destroy_id(cm_id); 1573 } 1574 1575 static void nvmet_rdma_disc_port_addr(struct nvmet_req *req, 1576 struct nvmet_port *port, char *traddr) 1577 { 1578 struct rdma_cm_id *cm_id = port->priv; 1579 1580 if (inet_addr_is_any((struct sockaddr *)&cm_id->route.addr.src_addr)) { 1581 struct nvmet_rdma_rsp *rsp = 1582 container_of(req, struct nvmet_rdma_rsp, req); 1583 struct rdma_cm_id *req_cm_id = rsp->queue->cm_id; 1584 struct sockaddr *addr = (void *)&req_cm_id->route.addr.src_addr; 1585 1586 sprintf(traddr, "%pISc", addr); 1587 } else { 1588 memcpy(traddr, port->disc_addr.traddr, NVMF_TRADDR_SIZE); 1589 } 1590 } 1591 1592 static const struct nvmet_fabrics_ops nvmet_rdma_ops = { 1593 .owner = THIS_MODULE, 1594 .type = NVMF_TRTYPE_RDMA, 1595 .msdbd = 1, 1596 .has_keyed_sgls = 1, 1597 .add_port = nvmet_rdma_add_port, 1598 .remove_port = nvmet_rdma_remove_port, 1599 .queue_response = nvmet_rdma_queue_response, 1600 .delete_ctrl = nvmet_rdma_delete_ctrl, 1601 .disc_traddr = nvmet_rdma_disc_port_addr, 1602 }; 1603 1604 static void nvmet_rdma_remove_one(struct ib_device *ib_device, void *client_data) 1605 { 1606 struct nvmet_rdma_queue *queue, *tmp; 1607 struct nvmet_rdma_device *ndev; 1608 bool found = false; 1609 1610 mutex_lock(&device_list_mutex); 1611 list_for_each_entry(ndev, &device_list, entry) { 1612 if (ndev->device == ib_device) { 1613 found = true; 1614 break; 1615 } 1616 } 1617 mutex_unlock(&device_list_mutex); 1618 1619 if (!found) 1620 return; 1621 1622 /* 1623 * IB Device that is used by nvmet controllers is being removed, 1624 * delete all queues using this device. 1625 */ 1626 mutex_lock(&nvmet_rdma_queue_mutex); 1627 list_for_each_entry_safe(queue, tmp, &nvmet_rdma_queue_list, 1628 queue_list) { 1629 if (queue->dev->device != ib_device) 1630 continue; 1631 1632 pr_info("Removing queue %d\n", queue->idx); 1633 list_del_init(&queue->queue_list); 1634 __nvmet_rdma_queue_disconnect(queue); 1635 } 1636 mutex_unlock(&nvmet_rdma_queue_mutex); 1637 1638 flush_scheduled_work(); 1639 } 1640 1641 static struct ib_client nvmet_rdma_ib_client = { 1642 .name = "nvmet_rdma", 1643 .remove = nvmet_rdma_remove_one 1644 }; 1645 1646 static int __init nvmet_rdma_init(void) 1647 { 1648 int ret; 1649 1650 ret = ib_register_client(&nvmet_rdma_ib_client); 1651 if (ret) 1652 return ret; 1653 1654 ret = nvmet_register_transport(&nvmet_rdma_ops); 1655 if (ret) 1656 goto err_ib_client; 1657 1658 return 0; 1659 1660 err_ib_client: 1661 ib_unregister_client(&nvmet_rdma_ib_client); 1662 return ret; 1663 } 1664 1665 static void __exit nvmet_rdma_exit(void) 1666 { 1667 nvmet_unregister_transport(&nvmet_rdma_ops); 1668 ib_unregister_client(&nvmet_rdma_ib_client); 1669 WARN_ON_ONCE(!list_empty(&nvmet_rdma_queue_list)); 1670 ida_destroy(&nvmet_rdma_queue_ida); 1671 } 1672 1673 module_init(nvmet_rdma_init); 1674 module_exit(nvmet_rdma_exit); 1675 1676 MODULE_LICENSE("GPL v2"); 1677 MODULE_ALIAS("nvmet-transport-1"); /* 1 == NVMF_TRTYPE_RDMA */ 1678