1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 static int isert_debug_level; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static void 51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static void 56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 57 static int 58 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 59 struct isert_rdma_wr *wr); 60 static int 61 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 62 static int 63 isert_rdma_post_recvl(struct isert_conn *isert_conn); 64 static int 65 isert_rdma_accept(struct isert_conn *isert_conn); 66 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 67 68 static inline bool 69 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 70 { 71 return (conn->pi_support && 72 cmd->prot_op != TARGET_PROT_NORMAL); 73 } 74 75 76 static void 77 isert_qp_event_callback(struct ib_event *e, void *context) 78 { 79 struct isert_conn *isert_conn = context; 80 81 isert_err("conn %p event: %d\n", isert_conn, e->event); 82 switch (e->event) { 83 case IB_EVENT_COMM_EST: 84 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 85 break; 86 case IB_EVENT_QP_LAST_WQE_REACHED: 87 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 88 break; 89 default: 90 break; 91 } 92 } 93 94 static int 95 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 96 { 97 int ret; 98 99 ret = ib_query_device(ib_dev, devattr); 100 if (ret) { 101 isert_err("ib_query_device() failed: %d\n", ret); 102 return ret; 103 } 104 isert_dbg("devattr->max_sge: %d\n", devattr->max_sge); 105 isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 106 107 return 0; 108 } 109 110 static struct isert_comp * 111 isert_comp_get(struct isert_conn *isert_conn) 112 { 113 struct isert_device *device = isert_conn->device; 114 struct isert_comp *comp; 115 int i, min = 0; 116 117 mutex_lock(&device_list_mutex); 118 for (i = 0; i < device->comps_used; i++) 119 if (device->comps[i].active_qps < 120 device->comps[min].active_qps) 121 min = i; 122 comp = &device->comps[min]; 123 comp->active_qps++; 124 mutex_unlock(&device_list_mutex); 125 126 isert_info("conn %p, using comp %p min_index: %d\n", 127 isert_conn, comp, min); 128 129 return comp; 130 } 131 132 static void 133 isert_comp_put(struct isert_comp *comp) 134 { 135 mutex_lock(&device_list_mutex); 136 comp->active_qps--; 137 mutex_unlock(&device_list_mutex); 138 } 139 140 static struct ib_qp * 141 isert_create_qp(struct isert_conn *isert_conn, 142 struct isert_comp *comp, 143 struct rdma_cm_id *cma_id) 144 { 145 struct isert_device *device = isert_conn->device; 146 struct ib_qp_init_attr attr; 147 int ret; 148 149 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 150 attr.event_handler = isert_qp_event_callback; 151 attr.qp_context = isert_conn; 152 attr.send_cq = comp->cq; 153 attr.recv_cq = comp->cq; 154 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 155 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 156 /* 157 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 158 * work-around for RDMA_READs with ConnectX-2. 159 * 160 * Also, still make sure to have at least two SGEs for 161 * outgoing control PDU responses. 162 */ 163 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 164 isert_conn->max_sge = attr.cap.max_send_sge; 165 166 attr.cap.max_recv_sge = 1; 167 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 168 attr.qp_type = IB_QPT_RC; 169 if (device->pi_capable) 170 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 171 172 ret = rdma_create_qp(cma_id, device->pd, &attr); 173 if (ret) { 174 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 175 return ERR_PTR(ret); 176 } 177 178 return cma_id->qp; 179 } 180 181 static int 182 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 183 { 184 struct isert_comp *comp; 185 int ret; 186 187 comp = isert_comp_get(isert_conn); 188 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 189 if (IS_ERR(isert_conn->qp)) { 190 ret = PTR_ERR(isert_conn->qp); 191 goto err; 192 } 193 194 return 0; 195 err: 196 isert_comp_put(comp); 197 return ret; 198 } 199 200 static void 201 isert_cq_event_callback(struct ib_event *e, void *context) 202 { 203 isert_dbg("event: %d\n", e->event); 204 } 205 206 static int 207 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 208 { 209 struct isert_device *device = isert_conn->device; 210 struct ib_device *ib_dev = device->ib_device; 211 struct iser_rx_desc *rx_desc; 212 struct ib_sge *rx_sg; 213 u64 dma_addr; 214 int i, j; 215 216 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 217 sizeof(struct iser_rx_desc), GFP_KERNEL); 218 if (!isert_conn->rx_descs) 219 goto fail; 220 221 rx_desc = isert_conn->rx_descs; 222 223 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 224 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 225 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 226 if (ib_dma_mapping_error(ib_dev, dma_addr)) 227 goto dma_map_fail; 228 229 rx_desc->dma_addr = dma_addr; 230 231 rx_sg = &rx_desc->rx_sg; 232 rx_sg->addr = rx_desc->dma_addr; 233 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 234 rx_sg->lkey = device->mr->lkey; 235 } 236 237 isert_conn->rx_desc_head = 0; 238 239 return 0; 240 241 dma_map_fail: 242 rx_desc = isert_conn->rx_descs; 243 for (j = 0; j < i; j++, rx_desc++) { 244 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 245 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 246 } 247 kfree(isert_conn->rx_descs); 248 isert_conn->rx_descs = NULL; 249 fail: 250 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 251 252 return -ENOMEM; 253 } 254 255 static void 256 isert_free_rx_descriptors(struct isert_conn *isert_conn) 257 { 258 struct ib_device *ib_dev = isert_conn->device->ib_device; 259 struct iser_rx_desc *rx_desc; 260 int i; 261 262 if (!isert_conn->rx_descs) 263 return; 264 265 rx_desc = isert_conn->rx_descs; 266 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 267 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 268 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 269 } 270 271 kfree(isert_conn->rx_descs); 272 isert_conn->rx_descs = NULL; 273 } 274 275 static void isert_cq_work(struct work_struct *); 276 static void isert_cq_callback(struct ib_cq *, void *); 277 278 static void 279 isert_free_comps(struct isert_device *device) 280 { 281 int i; 282 283 for (i = 0; i < device->comps_used; i++) { 284 struct isert_comp *comp = &device->comps[i]; 285 286 if (comp->cq) { 287 cancel_work_sync(&comp->work); 288 ib_destroy_cq(comp->cq); 289 } 290 } 291 kfree(device->comps); 292 } 293 294 static int 295 isert_alloc_comps(struct isert_device *device, 296 struct ib_device_attr *attr) 297 { 298 int i, max_cqe, ret = 0; 299 300 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 301 device->ib_device->num_comp_vectors)); 302 303 isert_info("Using %d CQs, %s supports %d vectors support " 304 "Fast registration %d pi_capable %d\n", 305 device->comps_used, device->ib_device->name, 306 device->ib_device->num_comp_vectors, device->use_fastreg, 307 device->pi_capable); 308 309 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 310 GFP_KERNEL); 311 if (!device->comps) { 312 isert_err("Unable to allocate completion contexts\n"); 313 return -ENOMEM; 314 } 315 316 max_cqe = min(ISER_MAX_CQ_LEN, attr->max_cqe); 317 318 for (i = 0; i < device->comps_used; i++) { 319 struct isert_comp *comp = &device->comps[i]; 320 321 comp->device = device; 322 INIT_WORK(&comp->work, isert_cq_work); 323 comp->cq = ib_create_cq(device->ib_device, 324 isert_cq_callback, 325 isert_cq_event_callback, 326 (void *)comp, 327 max_cqe, i); 328 if (IS_ERR(comp->cq)) { 329 isert_err("Unable to allocate cq\n"); 330 ret = PTR_ERR(comp->cq); 331 comp->cq = NULL; 332 goto out_cq; 333 } 334 335 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 336 if (ret) 337 goto out_cq; 338 } 339 340 return 0; 341 out_cq: 342 isert_free_comps(device); 343 return ret; 344 } 345 346 static int 347 isert_create_device_ib_res(struct isert_device *device) 348 { 349 struct ib_device_attr *dev_attr; 350 int ret; 351 352 dev_attr = &device->dev_attr; 353 ret = isert_query_device(device->ib_device, dev_attr); 354 if (ret) 355 return ret; 356 357 /* asign function handlers */ 358 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 359 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 360 device->use_fastreg = 1; 361 device->reg_rdma_mem = isert_reg_rdma; 362 device->unreg_rdma_mem = isert_unreg_rdma; 363 } else { 364 device->use_fastreg = 0; 365 device->reg_rdma_mem = isert_map_rdma; 366 device->unreg_rdma_mem = isert_unmap_cmd; 367 } 368 369 ret = isert_alloc_comps(device, dev_attr); 370 if (ret) 371 return ret; 372 373 device->pd = ib_alloc_pd(device->ib_device); 374 if (IS_ERR(device->pd)) { 375 ret = PTR_ERR(device->pd); 376 isert_err("failed to allocate pd, device %p, ret=%d\n", 377 device, ret); 378 goto out_cq; 379 } 380 381 device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE); 382 if (IS_ERR(device->mr)) { 383 ret = PTR_ERR(device->mr); 384 isert_err("failed to create dma mr, device %p, ret=%d\n", 385 device, ret); 386 goto out_mr; 387 } 388 389 /* Check signature cap */ 390 device->pi_capable = dev_attr->device_cap_flags & 391 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 392 393 return 0; 394 395 out_mr: 396 ib_dealloc_pd(device->pd); 397 out_cq: 398 isert_free_comps(device); 399 return ret; 400 } 401 402 static void 403 isert_free_device_ib_res(struct isert_device *device) 404 { 405 isert_info("device %p\n", device); 406 407 ib_dereg_mr(device->mr); 408 ib_dealloc_pd(device->pd); 409 isert_free_comps(device); 410 } 411 412 static void 413 isert_device_put(struct isert_device *device) 414 { 415 mutex_lock(&device_list_mutex); 416 device->refcount--; 417 isert_info("device %p refcount %d\n", device, device->refcount); 418 if (!device->refcount) { 419 isert_free_device_ib_res(device); 420 list_del(&device->dev_node); 421 kfree(device); 422 } 423 mutex_unlock(&device_list_mutex); 424 } 425 426 static struct isert_device * 427 isert_device_get(struct rdma_cm_id *cma_id) 428 { 429 struct isert_device *device; 430 int ret; 431 432 mutex_lock(&device_list_mutex); 433 list_for_each_entry(device, &device_list, dev_node) { 434 if (device->ib_device->node_guid == cma_id->device->node_guid) { 435 device->refcount++; 436 isert_info("Found iser device %p refcount %d\n", 437 device, device->refcount); 438 mutex_unlock(&device_list_mutex); 439 return device; 440 } 441 } 442 443 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 444 if (!device) { 445 mutex_unlock(&device_list_mutex); 446 return ERR_PTR(-ENOMEM); 447 } 448 449 INIT_LIST_HEAD(&device->dev_node); 450 451 device->ib_device = cma_id->device; 452 ret = isert_create_device_ib_res(device); 453 if (ret) { 454 kfree(device); 455 mutex_unlock(&device_list_mutex); 456 return ERR_PTR(ret); 457 } 458 459 device->refcount++; 460 list_add_tail(&device->dev_node, &device_list); 461 isert_info("Created a new iser device %p refcount %d\n", 462 device, device->refcount); 463 mutex_unlock(&device_list_mutex); 464 465 return device; 466 } 467 468 static void 469 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 470 { 471 struct fast_reg_descriptor *fr_desc, *tmp; 472 int i = 0; 473 474 if (list_empty(&isert_conn->fr_pool)) 475 return; 476 477 isert_info("Freeing conn %p fastreg pool", isert_conn); 478 479 list_for_each_entry_safe(fr_desc, tmp, 480 &isert_conn->fr_pool, list) { 481 list_del(&fr_desc->list); 482 ib_free_fast_reg_page_list(fr_desc->data_frpl); 483 ib_dereg_mr(fr_desc->data_mr); 484 if (fr_desc->pi_ctx) { 485 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 486 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 487 ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 488 kfree(fr_desc->pi_ctx); 489 } 490 kfree(fr_desc); 491 ++i; 492 } 493 494 if (i < isert_conn->fr_pool_size) 495 isert_warn("Pool still has %d regions registered\n", 496 isert_conn->fr_pool_size - i); 497 } 498 499 static int 500 isert_create_pi_ctx(struct fast_reg_descriptor *desc, 501 struct ib_device *device, 502 struct ib_pd *pd) 503 { 504 struct ib_mr_init_attr mr_init_attr; 505 struct pi_context *pi_ctx; 506 int ret; 507 508 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 509 if (!pi_ctx) { 510 isert_err("Failed to allocate pi context\n"); 511 return -ENOMEM; 512 } 513 514 pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(device, 515 ISCSI_ISER_SG_TABLESIZE); 516 if (IS_ERR(pi_ctx->prot_frpl)) { 517 isert_err("Failed to allocate prot frpl err=%ld\n", 518 PTR_ERR(pi_ctx->prot_frpl)); 519 ret = PTR_ERR(pi_ctx->prot_frpl); 520 goto err_pi_ctx; 521 } 522 523 pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 524 if (IS_ERR(pi_ctx->prot_mr)) { 525 isert_err("Failed to allocate prot frmr err=%ld\n", 526 PTR_ERR(pi_ctx->prot_mr)); 527 ret = PTR_ERR(pi_ctx->prot_mr); 528 goto err_prot_frpl; 529 } 530 desc->ind |= ISERT_PROT_KEY_VALID; 531 532 memset(&mr_init_attr, 0, sizeof(mr_init_attr)); 533 mr_init_attr.max_reg_descriptors = 2; 534 mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 535 pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 536 if (IS_ERR(pi_ctx->sig_mr)) { 537 isert_err("Failed to allocate signature enabled mr err=%ld\n", 538 PTR_ERR(pi_ctx->sig_mr)); 539 ret = PTR_ERR(pi_ctx->sig_mr); 540 goto err_prot_mr; 541 } 542 543 desc->pi_ctx = pi_ctx; 544 desc->ind |= ISERT_SIG_KEY_VALID; 545 desc->ind &= ~ISERT_PROTECTED; 546 547 return 0; 548 549 err_prot_mr: 550 ib_dereg_mr(desc->pi_ctx->prot_mr); 551 err_prot_frpl: 552 ib_free_fast_reg_page_list(desc->pi_ctx->prot_frpl); 553 err_pi_ctx: 554 kfree(desc->pi_ctx); 555 556 return ret; 557 } 558 559 static int 560 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 561 struct fast_reg_descriptor *fr_desc) 562 { 563 int ret; 564 565 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 566 ISCSI_ISER_SG_TABLESIZE); 567 if (IS_ERR(fr_desc->data_frpl)) { 568 isert_err("Failed to allocate data frpl err=%ld\n", 569 PTR_ERR(fr_desc->data_frpl)); 570 return PTR_ERR(fr_desc->data_frpl); 571 } 572 573 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 574 if (IS_ERR(fr_desc->data_mr)) { 575 isert_err("Failed to allocate data frmr err=%ld\n", 576 PTR_ERR(fr_desc->data_mr)); 577 ret = PTR_ERR(fr_desc->data_mr); 578 goto err_data_frpl; 579 } 580 fr_desc->ind |= ISERT_DATA_KEY_VALID; 581 582 isert_dbg("Created fr_desc %p\n", fr_desc); 583 584 return 0; 585 586 err_data_frpl: 587 ib_free_fast_reg_page_list(fr_desc->data_frpl); 588 589 return ret; 590 } 591 592 static int 593 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 594 { 595 struct fast_reg_descriptor *fr_desc; 596 struct isert_device *device = isert_conn->device; 597 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 598 struct se_node_acl *se_nacl = se_sess->se_node_acl; 599 int i, ret, tag_num; 600 /* 601 * Setup the number of FRMRs based upon the number of tags 602 * available to session in iscsi_target_locate_portal(). 603 */ 604 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 605 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 606 607 isert_conn->fr_pool_size = 0; 608 for (i = 0; i < tag_num; i++) { 609 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 610 if (!fr_desc) { 611 isert_err("Failed to allocate fast_reg descriptor\n"); 612 ret = -ENOMEM; 613 goto err; 614 } 615 616 ret = isert_create_fr_desc(device->ib_device, 617 device->pd, fr_desc); 618 if (ret) { 619 isert_err("Failed to create fastreg descriptor err=%d\n", 620 ret); 621 kfree(fr_desc); 622 goto err; 623 } 624 625 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 626 isert_conn->fr_pool_size++; 627 } 628 629 isert_dbg("Creating conn %p fastreg pool size=%d", 630 isert_conn, isert_conn->fr_pool_size); 631 632 return 0; 633 634 err: 635 isert_conn_free_fastreg_pool(isert_conn); 636 return ret; 637 } 638 639 static void 640 isert_init_conn(struct isert_conn *isert_conn) 641 { 642 isert_conn->state = ISER_CONN_INIT; 643 INIT_LIST_HEAD(&isert_conn->accept_node); 644 init_completion(&isert_conn->login_comp); 645 init_completion(&isert_conn->login_req_comp); 646 init_completion(&isert_conn->wait); 647 kref_init(&isert_conn->kref); 648 mutex_init(&isert_conn->mutex); 649 spin_lock_init(&isert_conn->pool_lock); 650 INIT_LIST_HEAD(&isert_conn->fr_pool); 651 } 652 653 static void 654 isert_free_login_buf(struct isert_conn *isert_conn) 655 { 656 struct ib_device *ib_dev = isert_conn->device->ib_device; 657 658 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 659 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 660 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 661 ISCSI_DEF_MAX_RECV_SEG_LEN, 662 DMA_FROM_DEVICE); 663 kfree(isert_conn->login_buf); 664 } 665 666 static int 667 isert_alloc_login_buf(struct isert_conn *isert_conn, 668 struct ib_device *ib_dev) 669 { 670 int ret; 671 672 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 673 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 674 if (!isert_conn->login_buf) { 675 isert_err("Unable to allocate isert_conn->login_buf\n"); 676 return -ENOMEM; 677 } 678 679 isert_conn->login_req_buf = isert_conn->login_buf; 680 isert_conn->login_rsp_buf = isert_conn->login_buf + 681 ISCSI_DEF_MAX_RECV_SEG_LEN; 682 683 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 684 isert_conn->login_buf, isert_conn->login_req_buf, 685 isert_conn->login_rsp_buf); 686 687 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 688 (void *)isert_conn->login_req_buf, 689 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 690 691 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 692 if (ret) { 693 isert_err("login_req_dma mapping error: %d\n", ret); 694 isert_conn->login_req_dma = 0; 695 goto out_login_buf; 696 } 697 698 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 699 (void *)isert_conn->login_rsp_buf, 700 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 701 702 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 703 if (ret) { 704 isert_err("login_rsp_dma mapping error: %d\n", ret); 705 isert_conn->login_rsp_dma = 0; 706 goto out_req_dma_map; 707 } 708 709 return 0; 710 711 out_req_dma_map: 712 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 713 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 714 out_login_buf: 715 kfree(isert_conn->login_buf); 716 return ret; 717 } 718 719 static int 720 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 721 { 722 struct isert_np *isert_np = cma_id->context; 723 struct iscsi_np *np = isert_np->np; 724 struct isert_conn *isert_conn; 725 struct isert_device *device; 726 int ret = 0; 727 728 spin_lock_bh(&np->np_thread_lock); 729 if (!np->enabled) { 730 spin_unlock_bh(&np->np_thread_lock); 731 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 732 return rdma_reject(cma_id, NULL, 0); 733 } 734 spin_unlock_bh(&np->np_thread_lock); 735 736 isert_dbg("cma_id: %p, portal: %p\n", 737 cma_id, cma_id->context); 738 739 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 740 if (!isert_conn) 741 return -ENOMEM; 742 743 isert_init_conn(isert_conn); 744 isert_conn->cm_id = cma_id; 745 746 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 747 if (ret) 748 goto out; 749 750 device = isert_device_get(cma_id); 751 if (IS_ERR(device)) { 752 ret = PTR_ERR(device); 753 goto out_rsp_dma_map; 754 } 755 isert_conn->device = device; 756 757 /* Set max inflight RDMA READ requests */ 758 isert_conn->initiator_depth = min_t(u8, 759 event->param.conn.initiator_depth, 760 device->dev_attr.max_qp_init_rd_atom); 761 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 762 763 ret = isert_conn_setup_qp(isert_conn, cma_id); 764 if (ret) 765 goto out_conn_dev; 766 767 ret = isert_rdma_post_recvl(isert_conn); 768 if (ret) 769 goto out_conn_dev; 770 771 ret = isert_rdma_accept(isert_conn); 772 if (ret) 773 goto out_conn_dev; 774 775 mutex_lock(&isert_np->np_accept_mutex); 776 list_add_tail(&isert_conn->accept_node, &isert_np->np_accept_list); 777 mutex_unlock(&isert_np->np_accept_mutex); 778 779 isert_info("np %p: Allow accept_np to continue\n", np); 780 up(&isert_np->np_sem); 781 return 0; 782 783 out_conn_dev: 784 isert_device_put(device); 785 out_rsp_dma_map: 786 isert_free_login_buf(isert_conn); 787 out: 788 kfree(isert_conn); 789 rdma_reject(cma_id, NULL, 0); 790 return ret; 791 } 792 793 static void 794 isert_connect_release(struct isert_conn *isert_conn) 795 { 796 struct isert_device *device = isert_conn->device; 797 798 isert_dbg("conn %p\n", isert_conn); 799 800 BUG_ON(!device); 801 802 if (device->use_fastreg) 803 isert_conn_free_fastreg_pool(isert_conn); 804 805 isert_free_rx_descriptors(isert_conn); 806 if (isert_conn->cm_id) 807 rdma_destroy_id(isert_conn->cm_id); 808 809 if (isert_conn->qp) { 810 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 811 812 isert_comp_put(comp); 813 ib_destroy_qp(isert_conn->qp); 814 } 815 816 if (isert_conn->login_buf) 817 isert_free_login_buf(isert_conn); 818 819 isert_device_put(device); 820 821 kfree(isert_conn); 822 } 823 824 static void 825 isert_connected_handler(struct rdma_cm_id *cma_id) 826 { 827 struct isert_conn *isert_conn = cma_id->qp->qp_context; 828 829 isert_info("conn %p\n", isert_conn); 830 831 if (!kref_get_unless_zero(&isert_conn->kref)) { 832 isert_warn("conn %p connect_release is running\n", isert_conn); 833 return; 834 } 835 836 mutex_lock(&isert_conn->mutex); 837 if (isert_conn->state != ISER_CONN_FULL_FEATURE) 838 isert_conn->state = ISER_CONN_UP; 839 mutex_unlock(&isert_conn->mutex); 840 } 841 842 static void 843 isert_release_kref(struct kref *kref) 844 { 845 struct isert_conn *isert_conn = container_of(kref, 846 struct isert_conn, kref); 847 848 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 849 current->pid); 850 851 isert_connect_release(isert_conn); 852 } 853 854 static void 855 isert_put_conn(struct isert_conn *isert_conn) 856 { 857 kref_put(&isert_conn->kref, isert_release_kref); 858 } 859 860 /** 861 * isert_conn_terminate() - Initiate connection termination 862 * @isert_conn: isert connection struct 863 * 864 * Notes: 865 * In case the connection state is FULL_FEATURE, move state 866 * to TEMINATING and start teardown sequence (rdma_disconnect). 867 * In case the connection state is UP, complete flush as well. 868 * 869 * This routine must be called with mutex held. Thus it is 870 * safe to call multiple times. 871 */ 872 static void 873 isert_conn_terminate(struct isert_conn *isert_conn) 874 { 875 int err; 876 877 switch (isert_conn->state) { 878 case ISER_CONN_TERMINATING: 879 break; 880 case ISER_CONN_UP: 881 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 882 isert_info("Terminating conn %p state %d\n", 883 isert_conn, isert_conn->state); 884 isert_conn->state = ISER_CONN_TERMINATING; 885 err = rdma_disconnect(isert_conn->cm_id); 886 if (err) 887 isert_warn("Failed rdma_disconnect isert_conn %p\n", 888 isert_conn); 889 break; 890 default: 891 isert_warn("conn %p teminating in state %d\n", 892 isert_conn, isert_conn->state); 893 } 894 } 895 896 static int 897 isert_np_cma_handler(struct isert_np *isert_np, 898 enum rdma_cm_event_type event) 899 { 900 isert_dbg("isert np %p, handling event %d\n", isert_np, event); 901 902 switch (event) { 903 case RDMA_CM_EVENT_DEVICE_REMOVAL: 904 isert_np->np_cm_id = NULL; 905 break; 906 case RDMA_CM_EVENT_ADDR_CHANGE: 907 isert_np->np_cm_id = isert_setup_id(isert_np); 908 if (IS_ERR(isert_np->np_cm_id)) { 909 isert_err("isert np %p setup id failed: %ld\n", 910 isert_np, PTR_ERR(isert_np->np_cm_id)); 911 isert_np->np_cm_id = NULL; 912 } 913 break; 914 default: 915 isert_err("isert np %p Unexpected event %d\n", 916 isert_np, event); 917 } 918 919 return -1; 920 } 921 922 static int 923 isert_disconnected_handler(struct rdma_cm_id *cma_id, 924 enum rdma_cm_event_type event) 925 { 926 struct isert_np *isert_np = cma_id->context; 927 struct isert_conn *isert_conn; 928 929 if (isert_np->np_cm_id == cma_id) 930 return isert_np_cma_handler(cma_id->context, event); 931 932 isert_conn = cma_id->qp->qp_context; 933 934 mutex_lock(&isert_conn->mutex); 935 isert_conn_terminate(isert_conn); 936 mutex_unlock(&isert_conn->mutex); 937 938 isert_info("conn %p completing wait\n", isert_conn); 939 complete(&isert_conn->wait); 940 941 return 0; 942 } 943 944 static int 945 isert_connect_error(struct rdma_cm_id *cma_id) 946 { 947 struct isert_conn *isert_conn = cma_id->qp->qp_context; 948 949 isert_conn->cm_id = NULL; 950 isert_put_conn(isert_conn); 951 952 return -1; 953 } 954 955 static int 956 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 957 { 958 int ret = 0; 959 960 isert_info("event %d status %d id %p np %p\n", event->event, 961 event->status, cma_id, cma_id->context); 962 963 switch (event->event) { 964 case RDMA_CM_EVENT_CONNECT_REQUEST: 965 ret = isert_connect_request(cma_id, event); 966 if (ret) 967 isert_err("failed handle connect request %d\n", ret); 968 break; 969 case RDMA_CM_EVENT_ESTABLISHED: 970 isert_connected_handler(cma_id); 971 break; 972 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 973 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 974 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 975 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 976 ret = isert_disconnected_handler(cma_id, event->event); 977 break; 978 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 979 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 980 case RDMA_CM_EVENT_CONNECT_ERROR: 981 ret = isert_connect_error(cma_id); 982 break; 983 default: 984 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 985 break; 986 } 987 988 return ret; 989 } 990 991 static int 992 isert_post_recv(struct isert_conn *isert_conn, u32 count) 993 { 994 struct ib_recv_wr *rx_wr, *rx_wr_failed; 995 int i, ret; 996 unsigned int rx_head = isert_conn->rx_desc_head; 997 struct iser_rx_desc *rx_desc; 998 999 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 1000 rx_desc = &isert_conn->rx_descs[rx_head]; 1001 rx_wr->wr_id = (uintptr_t)rx_desc; 1002 rx_wr->sg_list = &rx_desc->rx_sg; 1003 rx_wr->num_sge = 1; 1004 rx_wr->next = rx_wr + 1; 1005 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 1006 } 1007 1008 rx_wr--; 1009 rx_wr->next = NULL; /* mark end of work requests list */ 1010 1011 isert_conn->post_recv_buf_count += count; 1012 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 1013 &rx_wr_failed); 1014 if (ret) { 1015 isert_err("ib_post_recv() failed with ret: %d\n", ret); 1016 isert_conn->post_recv_buf_count -= count; 1017 } else { 1018 isert_dbg("Posted %d RX buffers\n", count); 1019 isert_conn->rx_desc_head = rx_head; 1020 } 1021 return ret; 1022 } 1023 1024 static int 1025 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1026 { 1027 struct ib_device *ib_dev = isert_conn->cm_id->device; 1028 struct ib_send_wr send_wr, *send_wr_failed; 1029 int ret; 1030 1031 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1032 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1033 1034 send_wr.next = NULL; 1035 send_wr.wr_id = (uintptr_t)tx_desc; 1036 send_wr.sg_list = tx_desc->tx_sg; 1037 send_wr.num_sge = tx_desc->num_sge; 1038 send_wr.opcode = IB_WR_SEND; 1039 send_wr.send_flags = IB_SEND_SIGNALED; 1040 1041 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1042 if (ret) 1043 isert_err("ib_post_send() failed, ret: %d\n", ret); 1044 1045 return ret; 1046 } 1047 1048 static void 1049 isert_create_send_desc(struct isert_conn *isert_conn, 1050 struct isert_cmd *isert_cmd, 1051 struct iser_tx_desc *tx_desc) 1052 { 1053 struct isert_device *device = isert_conn->device; 1054 struct ib_device *ib_dev = device->ib_device; 1055 1056 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1057 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1058 1059 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 1060 tx_desc->iser_header.flags = ISER_VER; 1061 1062 tx_desc->num_sge = 1; 1063 tx_desc->isert_cmd = isert_cmd; 1064 1065 if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { 1066 tx_desc->tx_sg[0].lkey = device->mr->lkey; 1067 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1068 } 1069 } 1070 1071 static int 1072 isert_init_tx_hdrs(struct isert_conn *isert_conn, 1073 struct iser_tx_desc *tx_desc) 1074 { 1075 struct isert_device *device = isert_conn->device; 1076 struct ib_device *ib_dev = device->ib_device; 1077 u64 dma_addr; 1078 1079 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1080 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1081 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1082 isert_err("ib_dma_mapping_error() failed\n"); 1083 return -ENOMEM; 1084 } 1085 1086 tx_desc->dma_addr = dma_addr; 1087 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1088 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1089 tx_desc->tx_sg[0].lkey = device->mr->lkey; 1090 1091 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 1092 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 1093 tx_desc->tx_sg[0].lkey); 1094 1095 return 0; 1096 } 1097 1098 static void 1099 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1100 struct ib_send_wr *send_wr) 1101 { 1102 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 1103 1104 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1105 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1106 send_wr->opcode = IB_WR_SEND; 1107 send_wr->sg_list = &tx_desc->tx_sg[0]; 1108 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 1109 send_wr->send_flags = IB_SEND_SIGNALED; 1110 } 1111 1112 static int 1113 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1114 { 1115 struct ib_recv_wr rx_wr, *rx_wr_fail; 1116 struct ib_sge sge; 1117 int ret; 1118 1119 memset(&sge, 0, sizeof(struct ib_sge)); 1120 sge.addr = isert_conn->login_req_dma; 1121 sge.length = ISER_RX_LOGIN_SIZE; 1122 sge.lkey = isert_conn->device->mr->lkey; 1123 1124 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1125 sge.addr, sge.length, sge.lkey); 1126 1127 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1128 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1129 rx_wr.sg_list = &sge; 1130 rx_wr.num_sge = 1; 1131 1132 isert_conn->post_recv_buf_count++; 1133 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 1134 if (ret) { 1135 isert_err("ib_post_recv() failed: %d\n", ret); 1136 isert_conn->post_recv_buf_count--; 1137 } 1138 1139 return ret; 1140 } 1141 1142 static int 1143 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1144 u32 length) 1145 { 1146 struct isert_conn *isert_conn = conn->context; 1147 struct isert_device *device = isert_conn->device; 1148 struct ib_device *ib_dev = device->ib_device; 1149 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1150 int ret; 1151 1152 isert_create_send_desc(isert_conn, NULL, tx_desc); 1153 1154 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1155 sizeof(struct iscsi_hdr)); 1156 1157 isert_init_tx_hdrs(isert_conn, tx_desc); 1158 1159 if (length > 0) { 1160 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1161 1162 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1163 length, DMA_TO_DEVICE); 1164 1165 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1166 1167 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1168 length, DMA_TO_DEVICE); 1169 1170 tx_dsg->addr = isert_conn->login_rsp_dma; 1171 tx_dsg->length = length; 1172 tx_dsg->lkey = isert_conn->device->mr->lkey; 1173 tx_desc->num_sge = 2; 1174 } 1175 if (!login->login_failed) { 1176 if (login->login_complete) { 1177 if (!conn->sess->sess_ops->SessionType && 1178 isert_conn->device->use_fastreg) { 1179 ret = isert_conn_create_fastreg_pool(isert_conn); 1180 if (ret) { 1181 isert_err("Conn: %p failed to create" 1182 " fastreg pool\n", isert_conn); 1183 return ret; 1184 } 1185 } 1186 1187 ret = isert_alloc_rx_descriptors(isert_conn); 1188 if (ret) 1189 return ret; 1190 1191 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1192 if (ret) 1193 return ret; 1194 1195 /* Now we are in FULL_FEATURE phase */ 1196 mutex_lock(&isert_conn->mutex); 1197 isert_conn->state = ISER_CONN_FULL_FEATURE; 1198 mutex_unlock(&isert_conn->mutex); 1199 goto post_send; 1200 } 1201 1202 ret = isert_rdma_post_recvl(isert_conn); 1203 if (ret) 1204 return ret; 1205 } 1206 post_send: 1207 ret = isert_post_send(isert_conn, tx_desc); 1208 if (ret) 1209 return ret; 1210 1211 return 0; 1212 } 1213 1214 static void 1215 isert_rx_login_req(struct isert_conn *isert_conn) 1216 { 1217 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 1218 int rx_buflen = isert_conn->login_req_len; 1219 struct iscsi_conn *conn = isert_conn->conn; 1220 struct iscsi_login *login = conn->conn_login; 1221 int size; 1222 1223 isert_info("conn %p\n", isert_conn); 1224 1225 WARN_ON_ONCE(!login); 1226 1227 if (login->first_request) { 1228 struct iscsi_login_req *login_req = 1229 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1230 /* 1231 * Setup the initial iscsi_login values from the leading 1232 * login request PDU. 1233 */ 1234 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1235 login->current_stage = 1236 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1237 >> 2; 1238 login->version_min = login_req->min_version; 1239 login->version_max = login_req->max_version; 1240 memcpy(login->isid, login_req->isid, 6); 1241 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1242 login->init_task_tag = login_req->itt; 1243 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1244 login->cid = be16_to_cpu(login_req->cid); 1245 login->tsih = be16_to_cpu(login_req->tsih); 1246 } 1247 1248 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1249 1250 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1251 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1252 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1253 MAX_KEY_VALUE_PAIRS); 1254 memcpy(login->req_buf, &rx_desc->data[0], size); 1255 1256 if (login->first_request) { 1257 complete(&isert_conn->login_comp); 1258 return; 1259 } 1260 schedule_delayed_work(&conn->login_work, 0); 1261 } 1262 1263 static struct iscsi_cmd 1264 *isert_allocate_cmd(struct iscsi_conn *conn) 1265 { 1266 struct isert_conn *isert_conn = conn->context; 1267 struct isert_cmd *isert_cmd; 1268 struct iscsi_cmd *cmd; 1269 1270 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1271 if (!cmd) { 1272 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1273 return NULL; 1274 } 1275 isert_cmd = iscsit_priv_cmd(cmd); 1276 isert_cmd->conn = isert_conn; 1277 isert_cmd->iscsi_cmd = cmd; 1278 1279 return cmd; 1280 } 1281 1282 static int 1283 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1284 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1285 struct iser_rx_desc *rx_desc, unsigned char *buf) 1286 { 1287 struct iscsi_conn *conn = isert_conn->conn; 1288 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1289 struct scatterlist *sg; 1290 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1291 bool dump_payload = false; 1292 1293 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1294 if (rc < 0) 1295 return rc; 1296 1297 imm_data = cmd->immediate_data; 1298 imm_data_len = cmd->first_burst_len; 1299 unsol_data = cmd->unsolicited_data; 1300 1301 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1302 if (rc < 0) { 1303 return 0; 1304 } else if (rc > 0) { 1305 dump_payload = true; 1306 goto sequence_cmd; 1307 } 1308 1309 if (!imm_data) 1310 return 0; 1311 1312 sg = &cmd->se_cmd.t_data_sg[0]; 1313 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1314 1315 isert_dbg("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1316 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1317 1318 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1319 1320 cmd->write_data_done += imm_data_len; 1321 1322 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1323 spin_lock_bh(&cmd->istate_lock); 1324 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1325 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1326 spin_unlock_bh(&cmd->istate_lock); 1327 } 1328 1329 sequence_cmd: 1330 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1331 1332 if (!rc && dump_payload == false && unsol_data) 1333 iscsit_set_unsoliticed_dataout(cmd); 1334 else if (dump_payload && imm_data) 1335 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); 1336 1337 return 0; 1338 } 1339 1340 static int 1341 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1342 struct iser_rx_desc *rx_desc, unsigned char *buf) 1343 { 1344 struct scatterlist *sg_start; 1345 struct iscsi_conn *conn = isert_conn->conn; 1346 struct iscsi_cmd *cmd = NULL; 1347 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1348 u32 unsol_data_len = ntoh24(hdr->dlength); 1349 int rc, sg_nents, sg_off, page_off; 1350 1351 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1352 if (rc < 0) 1353 return rc; 1354 else if (!cmd) 1355 return 0; 1356 /* 1357 * FIXME: Unexpected unsolicited_data out 1358 */ 1359 if (!cmd->unsolicited_data) { 1360 isert_err("Received unexpected solicited data payload\n"); 1361 dump_stack(); 1362 return -1; 1363 } 1364 1365 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1366 "write_data_done: %u, data_length: %u\n", 1367 unsol_data_len, cmd->write_data_done, 1368 cmd->se_cmd.data_length); 1369 1370 sg_off = cmd->write_data_done / PAGE_SIZE; 1371 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1372 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1373 page_off = cmd->write_data_done % PAGE_SIZE; 1374 /* 1375 * FIXME: Non page-aligned unsolicited_data out 1376 */ 1377 if (page_off) { 1378 isert_err("unexpected non-page aligned data payload\n"); 1379 dump_stack(); 1380 return -1; 1381 } 1382 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1383 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1384 sg_nents, &rx_desc->data[0], unsol_data_len); 1385 1386 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1387 unsol_data_len); 1388 1389 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1390 if (rc < 0) 1391 return rc; 1392 1393 return 0; 1394 } 1395 1396 static int 1397 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1398 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1399 unsigned char *buf) 1400 { 1401 struct iscsi_conn *conn = isert_conn->conn; 1402 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1403 int rc; 1404 1405 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1406 if (rc < 0) 1407 return rc; 1408 /* 1409 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1410 */ 1411 1412 return iscsit_process_nop_out(conn, cmd, hdr); 1413 } 1414 1415 static int 1416 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1417 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1418 struct iscsi_text *hdr) 1419 { 1420 struct iscsi_conn *conn = isert_conn->conn; 1421 u32 payload_length = ntoh24(hdr->dlength); 1422 int rc; 1423 unsigned char *text_in = NULL; 1424 1425 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1426 if (rc < 0) 1427 return rc; 1428 1429 if (payload_length) { 1430 text_in = kzalloc(payload_length, GFP_KERNEL); 1431 if (!text_in) { 1432 isert_err("Unable to allocate text_in of payload_length: %u\n", 1433 payload_length); 1434 return -ENOMEM; 1435 } 1436 } 1437 cmd->text_in_ptr = text_in; 1438 1439 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1440 1441 return iscsit_process_text_cmd(conn, cmd, hdr); 1442 } 1443 1444 static int 1445 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1446 uint32_t read_stag, uint64_t read_va, 1447 uint32_t write_stag, uint64_t write_va) 1448 { 1449 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1450 struct iscsi_conn *conn = isert_conn->conn; 1451 struct iscsi_cmd *cmd; 1452 struct isert_cmd *isert_cmd; 1453 int ret = -EINVAL; 1454 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1455 1456 if (conn->sess->sess_ops->SessionType && 1457 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1458 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1459 " ignoring\n", opcode); 1460 return 0; 1461 } 1462 1463 switch (opcode) { 1464 case ISCSI_OP_SCSI_CMD: 1465 cmd = isert_allocate_cmd(conn); 1466 if (!cmd) 1467 break; 1468 1469 isert_cmd = iscsit_priv_cmd(cmd); 1470 isert_cmd->read_stag = read_stag; 1471 isert_cmd->read_va = read_va; 1472 isert_cmd->write_stag = write_stag; 1473 isert_cmd->write_va = write_va; 1474 1475 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1476 rx_desc, (unsigned char *)hdr); 1477 break; 1478 case ISCSI_OP_NOOP_OUT: 1479 cmd = isert_allocate_cmd(conn); 1480 if (!cmd) 1481 break; 1482 1483 isert_cmd = iscsit_priv_cmd(cmd); 1484 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1485 rx_desc, (unsigned char *)hdr); 1486 break; 1487 case ISCSI_OP_SCSI_DATA_OUT: 1488 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1489 (unsigned char *)hdr); 1490 break; 1491 case ISCSI_OP_SCSI_TMFUNC: 1492 cmd = isert_allocate_cmd(conn); 1493 if (!cmd) 1494 break; 1495 1496 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1497 (unsigned char *)hdr); 1498 break; 1499 case ISCSI_OP_LOGOUT: 1500 cmd = isert_allocate_cmd(conn); 1501 if (!cmd) 1502 break; 1503 1504 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1505 break; 1506 case ISCSI_OP_TEXT: 1507 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) { 1508 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1509 if (!cmd) 1510 break; 1511 } else { 1512 cmd = isert_allocate_cmd(conn); 1513 if (!cmd) 1514 break; 1515 } 1516 1517 isert_cmd = iscsit_priv_cmd(cmd); 1518 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1519 rx_desc, (struct iscsi_text *)hdr); 1520 break; 1521 default: 1522 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1523 dump_stack(); 1524 break; 1525 } 1526 1527 return ret; 1528 } 1529 1530 static void 1531 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1532 { 1533 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1534 uint64_t read_va = 0, write_va = 0; 1535 uint32_t read_stag = 0, write_stag = 0; 1536 int rc; 1537 1538 switch (iser_hdr->flags & 0xF0) { 1539 case ISCSI_CTRL: 1540 if (iser_hdr->flags & ISER_RSV) { 1541 read_stag = be32_to_cpu(iser_hdr->read_stag); 1542 read_va = be64_to_cpu(iser_hdr->read_va); 1543 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1544 read_stag, (unsigned long long)read_va); 1545 } 1546 if (iser_hdr->flags & ISER_WSV) { 1547 write_stag = be32_to_cpu(iser_hdr->write_stag); 1548 write_va = be64_to_cpu(iser_hdr->write_va); 1549 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1550 write_stag, (unsigned long long)write_va); 1551 } 1552 1553 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1554 break; 1555 case ISER_HELLO: 1556 isert_err("iSER Hello message\n"); 1557 break; 1558 default: 1559 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1560 break; 1561 } 1562 1563 rc = isert_rx_opcode(isert_conn, rx_desc, 1564 read_stag, read_va, write_stag, write_va); 1565 } 1566 1567 static void 1568 isert_rcv_completion(struct iser_rx_desc *desc, 1569 struct isert_conn *isert_conn, 1570 u32 xfer_len) 1571 { 1572 struct ib_device *ib_dev = isert_conn->cm_id->device; 1573 struct iscsi_hdr *hdr; 1574 u64 rx_dma; 1575 int rx_buflen, outstanding; 1576 1577 if ((char *)desc == isert_conn->login_req_buf) { 1578 rx_dma = isert_conn->login_req_dma; 1579 rx_buflen = ISER_RX_LOGIN_SIZE; 1580 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1581 rx_dma, rx_buflen); 1582 } else { 1583 rx_dma = desc->dma_addr; 1584 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1585 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1586 rx_dma, rx_buflen); 1587 } 1588 1589 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1590 1591 hdr = &desc->iscsi_header; 1592 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1593 hdr->opcode, hdr->itt, hdr->flags, 1594 (int)(xfer_len - ISER_HEADERS_LEN)); 1595 1596 if ((char *)desc == isert_conn->login_req_buf) { 1597 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 1598 if (isert_conn->conn) { 1599 struct iscsi_login *login = isert_conn->conn->conn_login; 1600 1601 if (login && !login->first_request) 1602 isert_rx_login_req(isert_conn); 1603 } 1604 mutex_lock(&isert_conn->mutex); 1605 complete(&isert_conn->login_req_comp); 1606 mutex_unlock(&isert_conn->mutex); 1607 } else { 1608 isert_rx_do_work(desc, isert_conn); 1609 } 1610 1611 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1612 DMA_FROM_DEVICE); 1613 1614 isert_conn->post_recv_buf_count--; 1615 isert_dbg("Decremented post_recv_buf_count: %d\n", 1616 isert_conn->post_recv_buf_count); 1617 1618 if ((char *)desc == isert_conn->login_req_buf) 1619 return; 1620 1621 outstanding = isert_conn->post_recv_buf_count; 1622 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1623 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1624 ISERT_MIN_POSTED_RX); 1625 err = isert_post_recv(isert_conn, count); 1626 if (err) { 1627 isert_err("isert_post_recv() count: %d failed, %d\n", 1628 count, err); 1629 } 1630 } 1631 } 1632 1633 static int 1634 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1635 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1636 enum iser_ib_op_code op, struct isert_data_buf *data) 1637 { 1638 struct ib_device *ib_dev = isert_conn->cm_id->device; 1639 1640 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1641 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1642 1643 data->len = length - offset; 1644 data->offset = offset; 1645 data->sg_off = data->offset / PAGE_SIZE; 1646 1647 data->sg = &sg[data->sg_off]; 1648 data->nents = min_t(unsigned int, nents - data->sg_off, 1649 ISCSI_ISER_SG_TABLESIZE); 1650 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1651 PAGE_SIZE); 1652 1653 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1654 data->dma_dir); 1655 if (unlikely(!data->dma_nents)) { 1656 isert_err("Cmd: unable to dma map SGs %p\n", sg); 1657 return -EINVAL; 1658 } 1659 1660 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1661 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1662 1663 return 0; 1664 } 1665 1666 static void 1667 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1668 { 1669 struct ib_device *ib_dev = isert_conn->cm_id->device; 1670 1671 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1672 memset(data, 0, sizeof(*data)); 1673 } 1674 1675 1676 1677 static void 1678 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1679 { 1680 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1681 1682 isert_dbg("Cmd %p\n", isert_cmd); 1683 1684 if (wr->data.sg) { 1685 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1686 isert_unmap_data_buf(isert_conn, &wr->data); 1687 } 1688 1689 if (wr->send_wr) { 1690 isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1691 kfree(wr->send_wr); 1692 wr->send_wr = NULL; 1693 } 1694 1695 if (wr->ib_sge) { 1696 isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1697 kfree(wr->ib_sge); 1698 wr->ib_sge = NULL; 1699 } 1700 } 1701 1702 static void 1703 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1704 { 1705 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1706 1707 isert_dbg("Cmd %p\n", isert_cmd); 1708 1709 if (wr->fr_desc) { 1710 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 1711 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1712 isert_unmap_data_buf(isert_conn, &wr->prot); 1713 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1714 } 1715 spin_lock_bh(&isert_conn->pool_lock); 1716 list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); 1717 spin_unlock_bh(&isert_conn->pool_lock); 1718 wr->fr_desc = NULL; 1719 } 1720 1721 if (wr->data.sg) { 1722 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1723 isert_unmap_data_buf(isert_conn, &wr->data); 1724 } 1725 1726 wr->ib_sge = NULL; 1727 wr->send_wr = NULL; 1728 } 1729 1730 static void 1731 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1732 { 1733 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1734 struct isert_conn *isert_conn = isert_cmd->conn; 1735 struct iscsi_conn *conn = isert_conn->conn; 1736 struct isert_device *device = isert_conn->device; 1737 struct iscsi_text_rsp *hdr; 1738 1739 isert_dbg("Cmd %p\n", isert_cmd); 1740 1741 switch (cmd->iscsi_opcode) { 1742 case ISCSI_OP_SCSI_CMD: 1743 spin_lock_bh(&conn->cmd_lock); 1744 if (!list_empty(&cmd->i_conn_node)) 1745 list_del_init(&cmd->i_conn_node); 1746 spin_unlock_bh(&conn->cmd_lock); 1747 1748 if (cmd->data_direction == DMA_TO_DEVICE) { 1749 iscsit_stop_dataout_timer(cmd); 1750 /* 1751 * Check for special case during comp_err where 1752 * WRITE_PENDING has been handed off from core, 1753 * but requires an extra target_put_sess_cmd() 1754 * before transport_generic_free_cmd() below. 1755 */ 1756 if (comp_err && 1757 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1758 struct se_cmd *se_cmd = &cmd->se_cmd; 1759 1760 target_put_sess_cmd(se_cmd->se_sess, se_cmd); 1761 } 1762 } 1763 1764 device->unreg_rdma_mem(isert_cmd, isert_conn); 1765 transport_generic_free_cmd(&cmd->se_cmd, 0); 1766 break; 1767 case ISCSI_OP_SCSI_TMFUNC: 1768 spin_lock_bh(&conn->cmd_lock); 1769 if (!list_empty(&cmd->i_conn_node)) 1770 list_del_init(&cmd->i_conn_node); 1771 spin_unlock_bh(&conn->cmd_lock); 1772 1773 transport_generic_free_cmd(&cmd->se_cmd, 0); 1774 break; 1775 case ISCSI_OP_REJECT: 1776 case ISCSI_OP_NOOP_OUT: 1777 case ISCSI_OP_TEXT: 1778 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1779 /* If the continue bit is on, keep the command alive */ 1780 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1781 break; 1782 1783 spin_lock_bh(&conn->cmd_lock); 1784 if (!list_empty(&cmd->i_conn_node)) 1785 list_del_init(&cmd->i_conn_node); 1786 spin_unlock_bh(&conn->cmd_lock); 1787 1788 /* 1789 * Handle special case for REJECT when iscsi_add_reject*() has 1790 * overwritten the original iscsi_opcode assignment, and the 1791 * associated cmd->se_cmd needs to be released. 1792 */ 1793 if (cmd->se_cmd.se_tfo != NULL) { 1794 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1795 cmd->iscsi_opcode); 1796 transport_generic_free_cmd(&cmd->se_cmd, 0); 1797 break; 1798 } 1799 /* 1800 * Fall-through 1801 */ 1802 default: 1803 iscsit_release_cmd(cmd); 1804 break; 1805 } 1806 } 1807 1808 static void 1809 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1810 { 1811 if (tx_desc->dma_addr != 0) { 1812 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1813 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1814 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1815 tx_desc->dma_addr = 0; 1816 } 1817 } 1818 1819 static void 1820 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1821 struct ib_device *ib_dev, bool comp_err) 1822 { 1823 if (isert_cmd->pdu_buf_dma != 0) { 1824 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1825 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1826 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1827 isert_cmd->pdu_buf_dma = 0; 1828 } 1829 1830 isert_unmap_tx_desc(tx_desc, ib_dev); 1831 isert_put_cmd(isert_cmd, comp_err); 1832 } 1833 1834 static int 1835 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1836 { 1837 struct ib_mr_status mr_status; 1838 int ret; 1839 1840 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1841 if (ret) { 1842 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1843 goto fail_mr_status; 1844 } 1845 1846 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1847 u64 sec_offset_err; 1848 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1849 1850 switch (mr_status.sig_err.err_type) { 1851 case IB_SIG_BAD_GUARD: 1852 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1853 break; 1854 case IB_SIG_BAD_REFTAG: 1855 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1856 break; 1857 case IB_SIG_BAD_APPTAG: 1858 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1859 break; 1860 } 1861 sec_offset_err = mr_status.sig_err.sig_err_offset; 1862 do_div(sec_offset_err, block_size); 1863 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1864 1865 isert_err("PI error found type %d at sector 0x%llx " 1866 "expected 0x%x vs actual 0x%x\n", 1867 mr_status.sig_err.err_type, 1868 (unsigned long long)se_cmd->bad_sector, 1869 mr_status.sig_err.expected, 1870 mr_status.sig_err.actual); 1871 ret = 1; 1872 } 1873 1874 fail_mr_status: 1875 return ret; 1876 } 1877 1878 static void 1879 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1880 struct isert_cmd *isert_cmd) 1881 { 1882 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1883 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1884 struct se_cmd *se_cmd = &cmd->se_cmd; 1885 struct isert_conn *isert_conn = isert_cmd->conn; 1886 struct isert_device *device = isert_conn->device; 1887 int ret = 0; 1888 1889 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1890 ret = isert_check_pi_status(se_cmd, 1891 wr->fr_desc->pi_ctx->sig_mr); 1892 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1893 } 1894 1895 device->unreg_rdma_mem(isert_cmd, isert_conn); 1896 wr->send_wr_num = 0; 1897 if (ret) 1898 transport_send_check_condition_and_sense(se_cmd, 1899 se_cmd->pi_err, 0); 1900 else 1901 isert_put_response(isert_conn->conn, cmd); 1902 } 1903 1904 static void 1905 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1906 struct isert_cmd *isert_cmd) 1907 { 1908 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1909 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1910 struct se_cmd *se_cmd = &cmd->se_cmd; 1911 struct isert_conn *isert_conn = isert_cmd->conn; 1912 struct isert_device *device = isert_conn->device; 1913 int ret = 0; 1914 1915 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1916 ret = isert_check_pi_status(se_cmd, 1917 wr->fr_desc->pi_ctx->sig_mr); 1918 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1919 } 1920 1921 iscsit_stop_dataout_timer(cmd); 1922 device->unreg_rdma_mem(isert_cmd, isert_conn); 1923 cmd->write_data_done = wr->data.len; 1924 wr->send_wr_num = 0; 1925 1926 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1927 spin_lock_bh(&cmd->istate_lock); 1928 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1929 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1930 spin_unlock_bh(&cmd->istate_lock); 1931 1932 if (ret) { 1933 target_put_sess_cmd(se_cmd->se_sess, se_cmd); 1934 transport_send_check_condition_and_sense(se_cmd, 1935 se_cmd->pi_err, 0); 1936 } else { 1937 target_execute_cmd(se_cmd); 1938 } 1939 } 1940 1941 static void 1942 isert_do_control_comp(struct work_struct *work) 1943 { 1944 struct isert_cmd *isert_cmd = container_of(work, 1945 struct isert_cmd, comp_work); 1946 struct isert_conn *isert_conn = isert_cmd->conn; 1947 struct ib_device *ib_dev = isert_conn->cm_id->device; 1948 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1949 1950 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1951 1952 switch (cmd->i_state) { 1953 case ISTATE_SEND_TASKMGTRSP: 1954 iscsit_tmr_post_handler(cmd, cmd->conn); 1955 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1956 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1957 cmd->i_state = ISTATE_SENT_STATUS; 1958 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1959 ib_dev, false); 1960 break; 1961 case ISTATE_SEND_LOGOUTRSP: 1962 iscsit_logout_post_handler(cmd, cmd->conn); 1963 break; 1964 default: 1965 isert_err("Unknown i_state %d\n", cmd->i_state); 1966 dump_stack(); 1967 break; 1968 } 1969 } 1970 1971 static void 1972 isert_response_completion(struct iser_tx_desc *tx_desc, 1973 struct isert_cmd *isert_cmd, 1974 struct isert_conn *isert_conn, 1975 struct ib_device *ib_dev) 1976 { 1977 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1978 1979 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1980 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1981 cmd->i_state == ISTATE_SEND_REJECT || 1982 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1983 isert_unmap_tx_desc(tx_desc, ib_dev); 1984 1985 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1986 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1987 return; 1988 } 1989 1990 cmd->i_state = ISTATE_SENT_STATUS; 1991 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1992 } 1993 1994 static void 1995 isert_snd_completion(struct iser_tx_desc *tx_desc, 1996 struct isert_conn *isert_conn) 1997 { 1998 struct ib_device *ib_dev = isert_conn->cm_id->device; 1999 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2000 struct isert_rdma_wr *wr; 2001 2002 if (!isert_cmd) { 2003 isert_unmap_tx_desc(tx_desc, ib_dev); 2004 return; 2005 } 2006 wr = &isert_cmd->rdma_wr; 2007 2008 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 2009 2010 switch (wr->iser_ib_op) { 2011 case ISER_IB_SEND: 2012 isert_response_completion(tx_desc, isert_cmd, 2013 isert_conn, ib_dev); 2014 break; 2015 case ISER_IB_RDMA_WRITE: 2016 isert_completion_rdma_write(tx_desc, isert_cmd); 2017 break; 2018 case ISER_IB_RDMA_READ: 2019 isert_completion_rdma_read(tx_desc, isert_cmd); 2020 break; 2021 default: 2022 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 2023 dump_stack(); 2024 break; 2025 } 2026 } 2027 2028 /** 2029 * is_isert_tx_desc() - Indicate if the completion wr_id 2030 * is a TX descriptor or not. 2031 * @isert_conn: iser connection 2032 * @wr_id: completion WR identifier 2033 * 2034 * Since we cannot rely on wc opcode in FLUSH errors 2035 * we must work around it by checking if the wr_id address 2036 * falls in the iser connection rx_descs buffer. If so 2037 * it is an RX descriptor, otherwize it is a TX. 2038 */ 2039 static inline bool 2040 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 2041 { 2042 void *start = isert_conn->rx_descs; 2043 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); 2044 2045 if (wr_id >= start && wr_id < start + len) 2046 return false; 2047 2048 return true; 2049 } 2050 2051 static void 2052 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 2053 { 2054 if (wc->wr_id == ISER_BEACON_WRID) { 2055 isert_info("conn %p completing wait_comp_err\n", 2056 isert_conn); 2057 complete(&isert_conn->wait_comp_err); 2058 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 2059 struct ib_device *ib_dev = isert_conn->cm_id->device; 2060 struct isert_cmd *isert_cmd; 2061 struct iser_tx_desc *desc; 2062 2063 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2064 isert_cmd = desc->isert_cmd; 2065 if (!isert_cmd) 2066 isert_unmap_tx_desc(desc, ib_dev); 2067 else 2068 isert_completion_put(desc, isert_cmd, ib_dev, true); 2069 } else { 2070 isert_conn->post_recv_buf_count--; 2071 if (!isert_conn->post_recv_buf_count) 2072 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2073 } 2074 } 2075 2076 static void 2077 isert_handle_wc(struct ib_wc *wc) 2078 { 2079 struct isert_conn *isert_conn; 2080 struct iser_tx_desc *tx_desc; 2081 struct iser_rx_desc *rx_desc; 2082 2083 isert_conn = wc->qp->qp_context; 2084 if (likely(wc->status == IB_WC_SUCCESS)) { 2085 if (wc->opcode == IB_WC_RECV) { 2086 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 2087 isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); 2088 } else { 2089 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2090 isert_snd_completion(tx_desc, isert_conn); 2091 } 2092 } else { 2093 if (wc->status != IB_WC_WR_FLUSH_ERR) 2094 isert_err("wr id %llx status %d vend_err %x\n", 2095 wc->wr_id, wc->status, wc->vendor_err); 2096 else 2097 isert_dbg("flush error: wr id %llx\n", wc->wr_id); 2098 2099 if (wc->wr_id != ISER_FASTREG_LI_WRID) 2100 isert_cq_comp_err(isert_conn, wc); 2101 } 2102 } 2103 2104 static void 2105 isert_cq_work(struct work_struct *work) 2106 { 2107 enum { isert_poll_budget = 65536 }; 2108 struct isert_comp *comp = container_of(work, struct isert_comp, 2109 work); 2110 struct ib_wc *const wcs = comp->wcs; 2111 int i, n, completed = 0; 2112 2113 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 2114 for (i = 0; i < n; i++) 2115 isert_handle_wc(&wcs[i]); 2116 2117 completed += n; 2118 if (completed >= isert_poll_budget) 2119 break; 2120 } 2121 2122 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2123 } 2124 2125 static void 2126 isert_cq_callback(struct ib_cq *cq, void *context) 2127 { 2128 struct isert_comp *comp = context; 2129 2130 queue_work(isert_comp_wq, &comp->work); 2131 } 2132 2133 static int 2134 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2135 { 2136 struct ib_send_wr *wr_failed; 2137 int ret; 2138 2139 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2140 &wr_failed); 2141 if (ret) { 2142 isert_err("ib_post_send failed with %d\n", ret); 2143 return ret; 2144 } 2145 return ret; 2146 } 2147 2148 static int 2149 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2150 { 2151 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2152 struct isert_conn *isert_conn = conn->context; 2153 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2154 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2155 &isert_cmd->tx_desc.iscsi_header; 2156 2157 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2158 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2159 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2160 /* 2161 * Attach SENSE DATA payload to iSCSI Response PDU 2162 */ 2163 if (cmd->se_cmd.sense_buffer && 2164 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2165 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2166 struct isert_device *device = isert_conn->device; 2167 struct ib_device *ib_dev = device->ib_device; 2168 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2169 u32 padding, pdu_len; 2170 2171 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2172 cmd->sense_buffer); 2173 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2174 2175 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2176 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2177 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2178 2179 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2180 (void *)cmd->sense_buffer, pdu_len, 2181 DMA_TO_DEVICE); 2182 2183 isert_cmd->pdu_buf_len = pdu_len; 2184 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2185 tx_dsg->length = pdu_len; 2186 tx_dsg->lkey = device->mr->lkey; 2187 isert_cmd->tx_desc.num_sge = 2; 2188 } 2189 2190 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2191 2192 isert_dbg("Posting SCSI Response\n"); 2193 2194 return isert_post_response(isert_conn, isert_cmd); 2195 } 2196 2197 static void 2198 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2199 { 2200 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2201 struct isert_conn *isert_conn = conn->context; 2202 struct isert_device *device = isert_conn->device; 2203 2204 spin_lock_bh(&conn->cmd_lock); 2205 if (!list_empty(&cmd->i_conn_node)) 2206 list_del_init(&cmd->i_conn_node); 2207 spin_unlock_bh(&conn->cmd_lock); 2208 2209 if (cmd->data_direction == DMA_TO_DEVICE) 2210 iscsit_stop_dataout_timer(cmd); 2211 2212 device->unreg_rdma_mem(isert_cmd, isert_conn); 2213 } 2214 2215 static enum target_prot_op 2216 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2217 { 2218 struct isert_conn *isert_conn = conn->context; 2219 struct isert_device *device = isert_conn->device; 2220 2221 if (conn->tpg->tpg_attrib.t10_pi) { 2222 if (device->pi_capable) { 2223 isert_info("conn %p PI offload enabled\n", isert_conn); 2224 isert_conn->pi_support = true; 2225 return TARGET_PROT_ALL; 2226 } 2227 } 2228 2229 isert_info("conn %p PI offload disabled\n", isert_conn); 2230 isert_conn->pi_support = false; 2231 2232 return TARGET_PROT_NORMAL; 2233 } 2234 2235 static int 2236 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2237 bool nopout_response) 2238 { 2239 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2240 struct isert_conn *isert_conn = conn->context; 2241 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2242 2243 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2244 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2245 &isert_cmd->tx_desc.iscsi_header, 2246 nopout_response); 2247 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2248 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2249 2250 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2251 2252 return isert_post_response(isert_conn, isert_cmd); 2253 } 2254 2255 static int 2256 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2257 { 2258 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2259 struct isert_conn *isert_conn = conn->context; 2260 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2261 2262 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2263 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2264 &isert_cmd->tx_desc.iscsi_header); 2265 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2266 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2267 2268 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2269 2270 return isert_post_response(isert_conn, isert_cmd); 2271 } 2272 2273 static int 2274 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2275 { 2276 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2277 struct isert_conn *isert_conn = conn->context; 2278 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2279 2280 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2281 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2282 &isert_cmd->tx_desc.iscsi_header); 2283 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2284 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2285 2286 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2287 2288 return isert_post_response(isert_conn, isert_cmd); 2289 } 2290 2291 static int 2292 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2293 { 2294 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2295 struct isert_conn *isert_conn = conn->context; 2296 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2297 struct isert_device *device = isert_conn->device; 2298 struct ib_device *ib_dev = device->ib_device; 2299 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2300 struct iscsi_reject *hdr = 2301 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2302 2303 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2304 iscsit_build_reject(cmd, conn, hdr); 2305 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2306 2307 hton24(hdr->dlength, ISCSI_HDR_LEN); 2308 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2309 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2310 DMA_TO_DEVICE); 2311 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2312 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2313 tx_dsg->length = ISCSI_HDR_LEN; 2314 tx_dsg->lkey = device->mr->lkey; 2315 isert_cmd->tx_desc.num_sge = 2; 2316 2317 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2318 2319 isert_dbg("conn %p Posting Reject\n", isert_conn); 2320 2321 return isert_post_response(isert_conn, isert_cmd); 2322 } 2323 2324 static int 2325 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2326 { 2327 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2328 struct isert_conn *isert_conn = conn->context; 2329 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2330 struct iscsi_text_rsp *hdr = 2331 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2332 u32 txt_rsp_len; 2333 int rc; 2334 2335 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2336 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2337 if (rc < 0) 2338 return rc; 2339 2340 txt_rsp_len = rc; 2341 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2342 2343 if (txt_rsp_len) { 2344 struct isert_device *device = isert_conn->device; 2345 struct ib_device *ib_dev = device->ib_device; 2346 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2347 void *txt_rsp_buf = cmd->buf_ptr; 2348 2349 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2350 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2351 2352 isert_cmd->pdu_buf_len = txt_rsp_len; 2353 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2354 tx_dsg->length = txt_rsp_len; 2355 tx_dsg->lkey = device->mr->lkey; 2356 isert_cmd->tx_desc.num_sge = 2; 2357 } 2358 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2359 2360 isert_dbg("conn %p Text Response\n", isert_conn); 2361 2362 return isert_post_response(isert_conn, isert_cmd); 2363 } 2364 2365 static int 2366 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2367 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2368 u32 data_left, u32 offset) 2369 { 2370 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2371 struct scatterlist *sg_start, *tmp_sg; 2372 struct isert_device *device = isert_conn->device; 2373 struct ib_device *ib_dev = device->ib_device; 2374 u32 sg_off, page_off; 2375 int i = 0, sg_nents; 2376 2377 sg_off = offset / PAGE_SIZE; 2378 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2379 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2380 page_off = offset % PAGE_SIZE; 2381 2382 send_wr->sg_list = ib_sge; 2383 send_wr->num_sge = sg_nents; 2384 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2385 /* 2386 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2387 */ 2388 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2389 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 2390 "page_off: %u\n", 2391 (unsigned long long)tmp_sg->dma_address, 2392 tmp_sg->length, page_off); 2393 2394 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2395 ib_sge->length = min_t(u32, data_left, 2396 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2397 ib_sge->lkey = device->mr->lkey; 2398 2399 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 2400 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2401 page_off = 0; 2402 data_left -= ib_sge->length; 2403 ib_sge++; 2404 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2405 } 2406 2407 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2408 send_wr->sg_list, send_wr->num_sge); 2409 2410 return sg_nents; 2411 } 2412 2413 static int 2414 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2415 struct isert_rdma_wr *wr) 2416 { 2417 struct se_cmd *se_cmd = &cmd->se_cmd; 2418 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2419 struct isert_conn *isert_conn = conn->context; 2420 struct isert_data_buf *data = &wr->data; 2421 struct ib_send_wr *send_wr; 2422 struct ib_sge *ib_sge; 2423 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2424 int ret = 0, i, ib_sge_cnt; 2425 2426 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2427 2428 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2429 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2430 se_cmd->t_data_nents, se_cmd->data_length, 2431 offset, wr->iser_ib_op, &wr->data); 2432 if (ret) 2433 return ret; 2434 2435 data_left = data->len; 2436 offset = data->offset; 2437 2438 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2439 if (!ib_sge) { 2440 isert_warn("Unable to allocate ib_sge\n"); 2441 ret = -ENOMEM; 2442 goto unmap_cmd; 2443 } 2444 wr->ib_sge = ib_sge; 2445 2446 wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2447 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2448 GFP_KERNEL); 2449 if (!wr->send_wr) { 2450 isert_dbg("Unable to allocate wr->send_wr\n"); 2451 ret = -ENOMEM; 2452 goto unmap_cmd; 2453 } 2454 2455 wr->isert_cmd = isert_cmd; 2456 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2457 2458 for (i = 0; i < wr->send_wr_num; i++) { 2459 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2460 data_len = min(data_left, rdma_write_max); 2461 2462 send_wr->send_flags = 0; 2463 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2464 send_wr->opcode = IB_WR_RDMA_WRITE; 2465 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2466 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2467 if (i + 1 == wr->send_wr_num) 2468 send_wr->next = &isert_cmd->tx_desc.send_wr; 2469 else 2470 send_wr->next = &wr->send_wr[i + 1]; 2471 } else { 2472 send_wr->opcode = IB_WR_RDMA_READ; 2473 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2474 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2475 if (i + 1 == wr->send_wr_num) 2476 send_wr->send_flags = IB_SEND_SIGNALED; 2477 else 2478 send_wr->next = &wr->send_wr[i + 1]; 2479 } 2480 2481 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2482 send_wr, data_len, offset); 2483 ib_sge += ib_sge_cnt; 2484 2485 offset += data_len; 2486 va_offset += data_len; 2487 data_left -= data_len; 2488 } 2489 2490 return 0; 2491 unmap_cmd: 2492 isert_unmap_data_buf(isert_conn, data); 2493 2494 return ret; 2495 } 2496 2497 static int 2498 isert_map_fr_pagelist(struct ib_device *ib_dev, 2499 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2500 { 2501 u64 start_addr, end_addr, page, chunk_start = 0; 2502 struct scatterlist *tmp_sg; 2503 int i = 0, new_chunk, last_ent, n_pages; 2504 2505 n_pages = 0; 2506 new_chunk = 1; 2507 last_ent = sg_nents - 1; 2508 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2509 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2510 if (new_chunk) 2511 chunk_start = start_addr; 2512 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2513 2514 isert_dbg("SGL[%d] dma_addr: 0x%llx len: %u\n", 2515 i, (unsigned long long)tmp_sg->dma_address, 2516 tmp_sg->length); 2517 2518 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2519 new_chunk = 0; 2520 continue; 2521 } 2522 new_chunk = 1; 2523 2524 page = chunk_start & PAGE_MASK; 2525 do { 2526 fr_pl[n_pages++] = page; 2527 isert_dbg("Mapped page_list[%d] page_addr: 0x%llx\n", 2528 n_pages - 1, page); 2529 page += PAGE_SIZE; 2530 } while (page < end_addr); 2531 } 2532 2533 return n_pages; 2534 } 2535 2536 static inline void 2537 isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 2538 { 2539 u32 rkey; 2540 2541 memset(inv_wr, 0, sizeof(*inv_wr)); 2542 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 2543 inv_wr->opcode = IB_WR_LOCAL_INV; 2544 inv_wr->ex.invalidate_rkey = mr->rkey; 2545 2546 /* Bump the key */ 2547 rkey = ib_inc_rkey(mr->rkey); 2548 ib_update_fast_reg_key(mr, rkey); 2549 } 2550 2551 static int 2552 isert_fast_reg_mr(struct isert_conn *isert_conn, 2553 struct fast_reg_descriptor *fr_desc, 2554 struct isert_data_buf *mem, 2555 enum isert_indicator ind, 2556 struct ib_sge *sge) 2557 { 2558 struct isert_device *device = isert_conn->device; 2559 struct ib_device *ib_dev = device->ib_device; 2560 struct ib_mr *mr; 2561 struct ib_fast_reg_page_list *frpl; 2562 struct ib_send_wr fr_wr, inv_wr; 2563 struct ib_send_wr *bad_wr, *wr = NULL; 2564 int ret, pagelist_len; 2565 u32 page_off; 2566 2567 if (mem->dma_nents == 1) { 2568 sge->lkey = device->mr->lkey; 2569 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2570 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2571 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2572 sge->addr, sge->length, sge->lkey); 2573 return 0; 2574 } 2575 2576 if (ind == ISERT_DATA_KEY_VALID) { 2577 /* Registering data buffer */ 2578 mr = fr_desc->data_mr; 2579 frpl = fr_desc->data_frpl; 2580 } else { 2581 /* Registering protection buffer */ 2582 mr = fr_desc->pi_ctx->prot_mr; 2583 frpl = fr_desc->pi_ctx->prot_frpl; 2584 } 2585 2586 page_off = mem->offset % PAGE_SIZE; 2587 2588 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2589 fr_desc, mem->nents, mem->offset); 2590 2591 pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 2592 &frpl->page_list[0]); 2593 2594 if (!(fr_desc->ind & ind)) { 2595 isert_inv_rkey(&inv_wr, mr); 2596 wr = &inv_wr; 2597 } 2598 2599 /* Prepare FASTREG WR */ 2600 memset(&fr_wr, 0, sizeof(fr_wr)); 2601 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2602 fr_wr.opcode = IB_WR_FAST_REG_MR; 2603 fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 2604 fr_wr.wr.fast_reg.page_list = frpl; 2605 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2606 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2607 fr_wr.wr.fast_reg.length = mem->len; 2608 fr_wr.wr.fast_reg.rkey = mr->rkey; 2609 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2610 2611 if (!wr) 2612 wr = &fr_wr; 2613 else 2614 wr->next = &fr_wr; 2615 2616 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2617 if (ret) { 2618 isert_err("fast registration failed, ret:%d\n", ret); 2619 return ret; 2620 } 2621 fr_desc->ind &= ~ind; 2622 2623 sge->lkey = mr->lkey; 2624 sge->addr = frpl->page_list[0] + page_off; 2625 sge->length = mem->len; 2626 2627 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2628 sge->addr, sge->length, sge->lkey); 2629 2630 return ret; 2631 } 2632 2633 static inline void 2634 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2635 struct ib_sig_domain *domain) 2636 { 2637 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2638 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2639 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2640 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2641 /* 2642 * At the moment we hard code those, but if in the future 2643 * the target core would like to use it, we will take it 2644 * from se_cmd. 2645 */ 2646 domain->sig.dif.apptag_check_mask = 0xffff; 2647 domain->sig.dif.app_escape = true; 2648 domain->sig.dif.ref_escape = true; 2649 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2650 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2651 domain->sig.dif.ref_remap = true; 2652 }; 2653 2654 static int 2655 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2656 { 2657 switch (se_cmd->prot_op) { 2658 case TARGET_PROT_DIN_INSERT: 2659 case TARGET_PROT_DOUT_STRIP: 2660 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2661 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2662 break; 2663 case TARGET_PROT_DOUT_INSERT: 2664 case TARGET_PROT_DIN_STRIP: 2665 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2666 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2667 break; 2668 case TARGET_PROT_DIN_PASS: 2669 case TARGET_PROT_DOUT_PASS: 2670 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2671 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2672 break; 2673 default: 2674 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2675 return -EINVAL; 2676 } 2677 2678 return 0; 2679 } 2680 2681 static inline u8 2682 isert_set_prot_checks(u8 prot_checks) 2683 { 2684 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2685 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2686 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2687 } 2688 2689 static int 2690 isert_reg_sig_mr(struct isert_conn *isert_conn, 2691 struct se_cmd *se_cmd, 2692 struct isert_rdma_wr *rdma_wr, 2693 struct fast_reg_descriptor *fr_desc) 2694 { 2695 struct ib_send_wr sig_wr, inv_wr; 2696 struct ib_send_wr *bad_wr, *wr = NULL; 2697 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2698 struct ib_sig_attrs sig_attrs; 2699 int ret; 2700 2701 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2702 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2703 if (ret) 2704 goto err; 2705 2706 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2707 2708 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2709 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 2710 wr = &inv_wr; 2711 } 2712 2713 memset(&sig_wr, 0, sizeof(sig_wr)); 2714 sig_wr.opcode = IB_WR_REG_SIG_MR; 2715 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2716 sig_wr.sg_list = &rdma_wr->ib_sg[DATA]; 2717 sig_wr.num_sge = 1; 2718 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 2719 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 2720 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 2721 if (se_cmd->t_prot_sg) 2722 sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT]; 2723 2724 if (!wr) 2725 wr = &sig_wr; 2726 else 2727 wr->next = &sig_wr; 2728 2729 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2730 if (ret) { 2731 isert_err("fast registration failed, ret:%d\n", ret); 2732 goto err; 2733 } 2734 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2735 2736 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2737 rdma_wr->ib_sg[SIG].addr = 0; 2738 rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 2739 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2740 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2741 /* 2742 * We have protection guards on the wire 2743 * so we need to set a larget transfer 2744 */ 2745 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 2746 2747 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2748 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2749 rdma_wr->ib_sg[SIG].lkey); 2750 err: 2751 return ret; 2752 } 2753 2754 static int 2755 isert_handle_prot_cmd(struct isert_conn *isert_conn, 2756 struct isert_cmd *isert_cmd, 2757 struct isert_rdma_wr *wr) 2758 { 2759 struct isert_device *device = isert_conn->device; 2760 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2761 int ret; 2762 2763 if (!wr->fr_desc->pi_ctx) { 2764 ret = isert_create_pi_ctx(wr->fr_desc, 2765 device->ib_device, 2766 device->pd); 2767 if (ret) { 2768 isert_err("conn %p failed to allocate pi_ctx\n", 2769 isert_conn); 2770 return ret; 2771 } 2772 } 2773 2774 if (se_cmd->t_prot_sg) { 2775 ret = isert_map_data_buf(isert_conn, isert_cmd, 2776 se_cmd->t_prot_sg, 2777 se_cmd->t_prot_nents, 2778 se_cmd->prot_length, 2779 0, wr->iser_ib_op, &wr->prot); 2780 if (ret) { 2781 isert_err("conn %p failed to map protection buffer\n", 2782 isert_conn); 2783 return ret; 2784 } 2785 2786 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2787 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2788 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2789 if (ret) { 2790 isert_err("conn %p failed to fast reg mr\n", 2791 isert_conn); 2792 goto unmap_prot_cmd; 2793 } 2794 } 2795 2796 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2797 if (ret) { 2798 isert_err("conn %p failed to fast reg mr\n", 2799 isert_conn); 2800 goto unmap_prot_cmd; 2801 } 2802 wr->fr_desc->ind |= ISERT_PROTECTED; 2803 2804 return 0; 2805 2806 unmap_prot_cmd: 2807 if (se_cmd->t_prot_sg) 2808 isert_unmap_data_buf(isert_conn, &wr->prot); 2809 2810 return ret; 2811 } 2812 2813 static int 2814 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2815 struct isert_rdma_wr *wr) 2816 { 2817 struct se_cmd *se_cmd = &cmd->se_cmd; 2818 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2819 struct isert_conn *isert_conn = conn->context; 2820 struct fast_reg_descriptor *fr_desc = NULL; 2821 struct ib_send_wr *send_wr; 2822 struct ib_sge *ib_sg; 2823 u32 offset; 2824 int ret = 0; 2825 unsigned long flags; 2826 2827 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2828 2829 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2830 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2831 se_cmd->t_data_nents, se_cmd->data_length, 2832 offset, wr->iser_ib_op, &wr->data); 2833 if (ret) 2834 return ret; 2835 2836 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2837 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2838 fr_desc = list_first_entry(&isert_conn->fr_pool, 2839 struct fast_reg_descriptor, list); 2840 list_del(&fr_desc->list); 2841 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2842 wr->fr_desc = fr_desc; 2843 } 2844 2845 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2846 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2847 if (ret) 2848 goto unmap_cmd; 2849 2850 if (isert_prot_cmd(isert_conn, se_cmd)) { 2851 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 2852 if (ret) 2853 goto unmap_cmd; 2854 2855 ib_sg = &wr->ib_sg[SIG]; 2856 } else { 2857 ib_sg = &wr->ib_sg[DATA]; 2858 } 2859 2860 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2861 wr->ib_sge = &wr->s_ib_sge; 2862 wr->send_wr_num = 1; 2863 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2864 wr->send_wr = &wr->s_send_wr; 2865 wr->isert_cmd = isert_cmd; 2866 2867 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2868 send_wr->sg_list = &wr->s_ib_sge; 2869 send_wr->num_sge = 1; 2870 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2871 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2872 send_wr->opcode = IB_WR_RDMA_WRITE; 2873 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2874 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2875 send_wr->send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 2876 0 : IB_SEND_SIGNALED; 2877 } else { 2878 send_wr->opcode = IB_WR_RDMA_READ; 2879 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2880 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2881 send_wr->send_flags = IB_SEND_SIGNALED; 2882 } 2883 2884 return 0; 2885 2886 unmap_cmd: 2887 if (fr_desc) { 2888 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2889 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2890 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2891 } 2892 isert_unmap_data_buf(isert_conn, &wr->data); 2893 2894 return ret; 2895 } 2896 2897 static int 2898 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2899 { 2900 struct se_cmd *se_cmd = &cmd->se_cmd; 2901 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2902 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2903 struct isert_conn *isert_conn = conn->context; 2904 struct isert_device *device = isert_conn->device; 2905 struct ib_send_wr *wr_failed; 2906 int rc; 2907 2908 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2909 isert_cmd, se_cmd->data_length); 2910 2911 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2912 rc = device->reg_rdma_mem(conn, cmd, wr); 2913 if (rc) { 2914 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2915 return rc; 2916 } 2917 2918 if (!isert_prot_cmd(isert_conn, se_cmd)) { 2919 /* 2920 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2921 */ 2922 isert_create_send_desc(isert_conn, isert_cmd, 2923 &isert_cmd->tx_desc); 2924 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2925 &isert_cmd->tx_desc.iscsi_header); 2926 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2927 isert_init_send_wr(isert_conn, isert_cmd, 2928 &isert_cmd->tx_desc.send_wr); 2929 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2930 wr->send_wr_num += 1; 2931 } 2932 2933 rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed); 2934 if (rc) 2935 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2936 2937 if (!isert_prot_cmd(isert_conn, se_cmd)) 2938 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2939 "READ\n", isert_cmd); 2940 else 2941 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2942 isert_cmd); 2943 2944 return 1; 2945 } 2946 2947 static int 2948 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2949 { 2950 struct se_cmd *se_cmd = &cmd->se_cmd; 2951 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2952 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2953 struct isert_conn *isert_conn = conn->context; 2954 struct isert_device *device = isert_conn->device; 2955 struct ib_send_wr *wr_failed; 2956 int rc; 2957 2958 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2959 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2960 wr->iser_ib_op = ISER_IB_RDMA_READ; 2961 rc = device->reg_rdma_mem(conn, cmd, wr); 2962 if (rc) { 2963 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2964 return rc; 2965 } 2966 2967 rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed); 2968 if (rc) 2969 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2970 2971 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2972 isert_cmd); 2973 2974 return 0; 2975 } 2976 2977 static int 2978 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2979 { 2980 int ret; 2981 2982 switch (state) { 2983 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2984 ret = isert_put_nopin(cmd, conn, false); 2985 break; 2986 default: 2987 isert_err("Unknown immediate state: 0x%02x\n", state); 2988 ret = -EINVAL; 2989 break; 2990 } 2991 2992 return ret; 2993 } 2994 2995 static int 2996 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2997 { 2998 struct isert_conn *isert_conn = conn->context; 2999 int ret; 3000 3001 switch (state) { 3002 case ISTATE_SEND_LOGOUTRSP: 3003 ret = isert_put_logout_rsp(cmd, conn); 3004 if (!ret) 3005 isert_conn->logout_posted = true; 3006 break; 3007 case ISTATE_SEND_NOPIN: 3008 ret = isert_put_nopin(cmd, conn, true); 3009 break; 3010 case ISTATE_SEND_TASKMGTRSP: 3011 ret = isert_put_tm_rsp(cmd, conn); 3012 break; 3013 case ISTATE_SEND_REJECT: 3014 ret = isert_put_reject(cmd, conn); 3015 break; 3016 case ISTATE_SEND_TEXTRSP: 3017 ret = isert_put_text_rsp(cmd, conn); 3018 break; 3019 case ISTATE_SEND_STATUS: 3020 /* 3021 * Special case for sending non GOOD SCSI status from TX thread 3022 * context during pre se_cmd excecution failure. 3023 */ 3024 ret = isert_put_response(conn, cmd); 3025 break; 3026 default: 3027 isert_err("Unknown response state: 0x%02x\n", state); 3028 ret = -EINVAL; 3029 break; 3030 } 3031 3032 return ret; 3033 } 3034 3035 struct rdma_cm_id * 3036 isert_setup_id(struct isert_np *isert_np) 3037 { 3038 struct iscsi_np *np = isert_np->np; 3039 struct rdma_cm_id *id; 3040 struct sockaddr *sa; 3041 int ret; 3042 3043 sa = (struct sockaddr *)&np->np_sockaddr; 3044 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3045 3046 id = rdma_create_id(isert_cma_handler, isert_np, 3047 RDMA_PS_TCP, IB_QPT_RC); 3048 if (IS_ERR(id)) { 3049 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3050 ret = PTR_ERR(id); 3051 goto out; 3052 } 3053 isert_dbg("id %p context %p\n", id, id->context); 3054 3055 ret = rdma_bind_addr(id, sa); 3056 if (ret) { 3057 isert_err("rdma_bind_addr() failed: %d\n", ret); 3058 goto out_id; 3059 } 3060 3061 ret = rdma_listen(id, 0); 3062 if (ret) { 3063 isert_err("rdma_listen() failed: %d\n", ret); 3064 goto out_id; 3065 } 3066 3067 return id; 3068 out_id: 3069 rdma_destroy_id(id); 3070 out: 3071 return ERR_PTR(ret); 3072 } 3073 3074 static int 3075 isert_setup_np(struct iscsi_np *np, 3076 struct __kernel_sockaddr_storage *ksockaddr) 3077 { 3078 struct isert_np *isert_np; 3079 struct rdma_cm_id *isert_lid; 3080 int ret; 3081 3082 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3083 if (!isert_np) { 3084 isert_err("Unable to allocate struct isert_np\n"); 3085 return -ENOMEM; 3086 } 3087 sema_init(&isert_np->np_sem, 0); 3088 mutex_init(&isert_np->np_accept_mutex); 3089 INIT_LIST_HEAD(&isert_np->np_accept_list); 3090 init_completion(&isert_np->np_login_comp); 3091 isert_np->np = np; 3092 3093 /* 3094 * Setup the np->np_sockaddr from the passed sockaddr setup 3095 * in iscsi_target_configfs.c code.. 3096 */ 3097 memcpy(&np->np_sockaddr, ksockaddr, 3098 sizeof(struct __kernel_sockaddr_storage)); 3099 3100 isert_lid = isert_setup_id(isert_np); 3101 if (IS_ERR(isert_lid)) { 3102 ret = PTR_ERR(isert_lid); 3103 goto out; 3104 } 3105 3106 isert_np->np_cm_id = isert_lid; 3107 np->np_context = isert_np; 3108 3109 return 0; 3110 3111 out: 3112 kfree(isert_np); 3113 3114 return ret; 3115 } 3116 3117 static int 3118 isert_rdma_accept(struct isert_conn *isert_conn) 3119 { 3120 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3121 struct rdma_conn_param cp; 3122 int ret; 3123 3124 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3125 cp.initiator_depth = isert_conn->initiator_depth; 3126 cp.retry_count = 7; 3127 cp.rnr_retry_count = 7; 3128 3129 ret = rdma_accept(cm_id, &cp); 3130 if (ret) { 3131 isert_err("rdma_accept() failed with: %d\n", ret); 3132 return ret; 3133 } 3134 3135 return 0; 3136 } 3137 3138 static int 3139 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3140 { 3141 struct isert_conn *isert_conn = conn->context; 3142 int ret; 3143 3144 isert_info("before login_req comp conn: %p\n", isert_conn); 3145 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 3146 if (ret) { 3147 isert_err("isert_conn %p interrupted before got login req\n", 3148 isert_conn); 3149 return ret; 3150 } 3151 reinit_completion(&isert_conn->login_req_comp); 3152 3153 /* 3154 * For login requests after the first PDU, isert_rx_login_req() will 3155 * kick schedule_delayed_work(&conn->login_work) as the packet is 3156 * received, which turns this callback from iscsi_target_do_login_rx() 3157 * into a NOP. 3158 */ 3159 if (!login->first_request) 3160 return 0; 3161 3162 isert_rx_login_req(isert_conn); 3163 3164 isert_info("before login_comp conn: %p\n", conn); 3165 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3166 if (ret) 3167 return ret; 3168 3169 isert_info("processing login->req: %p\n", login->req); 3170 3171 return 0; 3172 } 3173 3174 static void 3175 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3176 struct isert_conn *isert_conn) 3177 { 3178 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3179 struct rdma_route *cm_route = &cm_id->route; 3180 struct sockaddr_in *sock_in; 3181 struct sockaddr_in6 *sock_in6; 3182 3183 conn->login_family = np->np_sockaddr.ss_family; 3184 3185 if (np->np_sockaddr.ss_family == AF_INET6) { 3186 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3187 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3188 &sock_in6->sin6_addr.in6_u); 3189 conn->login_port = ntohs(sock_in6->sin6_port); 3190 3191 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3192 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3193 &sock_in6->sin6_addr.in6_u); 3194 conn->local_port = ntohs(sock_in6->sin6_port); 3195 } else { 3196 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3197 sprintf(conn->login_ip, "%pI4", 3198 &sock_in->sin_addr.s_addr); 3199 conn->login_port = ntohs(sock_in->sin_port); 3200 3201 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3202 sprintf(conn->local_ip, "%pI4", 3203 &sock_in->sin_addr.s_addr); 3204 conn->local_port = ntohs(sock_in->sin_port); 3205 } 3206 } 3207 3208 static int 3209 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3210 { 3211 struct isert_np *isert_np = np->np_context; 3212 struct isert_conn *isert_conn; 3213 int ret; 3214 3215 accept_wait: 3216 ret = down_interruptible(&isert_np->np_sem); 3217 if (ret) 3218 return -ENODEV; 3219 3220 spin_lock_bh(&np->np_thread_lock); 3221 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3222 spin_unlock_bh(&np->np_thread_lock); 3223 isert_dbg("np_thread_state %d\n", 3224 np->np_thread_state); 3225 /** 3226 * No point in stalling here when np_thread 3227 * is in state RESET/SHUTDOWN/EXIT - bail 3228 **/ 3229 return -ENODEV; 3230 } 3231 spin_unlock_bh(&np->np_thread_lock); 3232 3233 mutex_lock(&isert_np->np_accept_mutex); 3234 if (list_empty(&isert_np->np_accept_list)) { 3235 mutex_unlock(&isert_np->np_accept_mutex); 3236 goto accept_wait; 3237 } 3238 isert_conn = list_first_entry(&isert_np->np_accept_list, 3239 struct isert_conn, accept_node); 3240 list_del_init(&isert_conn->accept_node); 3241 mutex_unlock(&isert_np->np_accept_mutex); 3242 3243 conn->context = isert_conn; 3244 isert_conn->conn = conn; 3245 3246 isert_set_conn_info(np, conn, isert_conn); 3247 3248 isert_dbg("Processing isert_conn: %p\n", isert_conn); 3249 3250 return 0; 3251 } 3252 3253 static void 3254 isert_free_np(struct iscsi_np *np) 3255 { 3256 struct isert_np *isert_np = np->np_context; 3257 struct isert_conn *isert_conn, *n; 3258 3259 if (isert_np->np_cm_id) 3260 rdma_destroy_id(isert_np->np_cm_id); 3261 3262 /* 3263 * FIXME: At this point we don't have a good way to insure 3264 * that at this point we don't have hanging connections that 3265 * completed RDMA establishment but didn't start iscsi login 3266 * process. So work-around this by cleaning up what ever piled 3267 * up in np_accept_list. 3268 */ 3269 mutex_lock(&isert_np->np_accept_mutex); 3270 if (!list_empty(&isert_np->np_accept_list)) { 3271 isert_info("Still have isert connections, cleaning up...\n"); 3272 list_for_each_entry_safe(isert_conn, n, 3273 &isert_np->np_accept_list, 3274 accept_node) { 3275 isert_info("cleaning isert_conn %p state (%d)\n", 3276 isert_conn, isert_conn->state); 3277 isert_connect_release(isert_conn); 3278 } 3279 } 3280 mutex_unlock(&isert_np->np_accept_mutex); 3281 3282 np->np_context = NULL; 3283 kfree(isert_np); 3284 } 3285 3286 static void isert_release_work(struct work_struct *work) 3287 { 3288 struct isert_conn *isert_conn = container_of(work, 3289 struct isert_conn, 3290 release_work); 3291 3292 isert_info("Starting release conn %p\n", isert_conn); 3293 3294 wait_for_completion(&isert_conn->wait); 3295 3296 mutex_lock(&isert_conn->mutex); 3297 isert_conn->state = ISER_CONN_DOWN; 3298 mutex_unlock(&isert_conn->mutex); 3299 3300 isert_info("Destroying conn %p\n", isert_conn); 3301 isert_put_conn(isert_conn); 3302 } 3303 3304 static void 3305 isert_wait4logout(struct isert_conn *isert_conn) 3306 { 3307 struct iscsi_conn *conn = isert_conn->conn; 3308 3309 isert_info("conn %p\n", isert_conn); 3310 3311 if (isert_conn->logout_posted) { 3312 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3313 wait_for_completion_timeout(&conn->conn_logout_comp, 3314 SECONDS_FOR_LOGOUT_COMP * HZ); 3315 } 3316 } 3317 3318 static void 3319 isert_wait4cmds(struct iscsi_conn *conn) 3320 { 3321 isert_info("iscsi_conn %p\n", conn); 3322 3323 if (conn->sess) { 3324 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3325 target_wait_for_sess_cmds(conn->sess->se_sess); 3326 } 3327 } 3328 3329 static void 3330 isert_wait4flush(struct isert_conn *isert_conn) 3331 { 3332 struct ib_recv_wr *bad_wr; 3333 3334 isert_info("conn %p\n", isert_conn); 3335 3336 init_completion(&isert_conn->wait_comp_err); 3337 isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3338 /* post an indication that all flush errors were consumed */ 3339 if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { 3340 isert_err("conn %p failed to post beacon", isert_conn); 3341 return; 3342 } 3343 3344 wait_for_completion(&isert_conn->wait_comp_err); 3345 } 3346 3347 static void isert_wait_conn(struct iscsi_conn *conn) 3348 { 3349 struct isert_conn *isert_conn = conn->context; 3350 3351 isert_info("Starting conn %p\n", isert_conn); 3352 3353 mutex_lock(&isert_conn->mutex); 3354 /* 3355 * Only wait for wait_comp_err if the isert_conn made it 3356 * into full feature phase.. 3357 */ 3358 if (isert_conn->state == ISER_CONN_INIT) { 3359 mutex_unlock(&isert_conn->mutex); 3360 return; 3361 } 3362 isert_conn_terminate(isert_conn); 3363 mutex_unlock(&isert_conn->mutex); 3364 3365 isert_wait4cmds(conn); 3366 isert_wait4flush(isert_conn); 3367 isert_wait4logout(isert_conn); 3368 3369 INIT_WORK(&isert_conn->release_work, isert_release_work); 3370 queue_work(isert_release_wq, &isert_conn->release_work); 3371 } 3372 3373 static void isert_free_conn(struct iscsi_conn *conn) 3374 { 3375 struct isert_conn *isert_conn = conn->context; 3376 3377 isert_put_conn(isert_conn); 3378 } 3379 3380 static struct iscsit_transport iser_target_transport = { 3381 .name = "IB/iSER", 3382 .transport_type = ISCSI_INFINIBAND, 3383 .priv_size = sizeof(struct isert_cmd), 3384 .owner = THIS_MODULE, 3385 .iscsit_setup_np = isert_setup_np, 3386 .iscsit_accept_np = isert_accept_np, 3387 .iscsit_free_np = isert_free_np, 3388 .iscsit_wait_conn = isert_wait_conn, 3389 .iscsit_free_conn = isert_free_conn, 3390 .iscsit_get_login_rx = isert_get_login_rx, 3391 .iscsit_put_login_tx = isert_put_login_tx, 3392 .iscsit_immediate_queue = isert_immediate_queue, 3393 .iscsit_response_queue = isert_response_queue, 3394 .iscsit_get_dataout = isert_get_dataout, 3395 .iscsit_queue_data_in = isert_put_datain, 3396 .iscsit_queue_status = isert_put_response, 3397 .iscsit_aborted_task = isert_aborted_task, 3398 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3399 }; 3400 3401 static int __init isert_init(void) 3402 { 3403 int ret; 3404 3405 isert_comp_wq = alloc_workqueue("isert_comp_wq", 3406 WQ_UNBOUND | WQ_HIGHPRI, 0); 3407 if (!isert_comp_wq) { 3408 isert_err("Unable to allocate isert_comp_wq\n"); 3409 ret = -ENOMEM; 3410 return -ENOMEM; 3411 } 3412 3413 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3414 WQ_UNBOUND_MAX_ACTIVE); 3415 if (!isert_release_wq) { 3416 isert_err("Unable to allocate isert_release_wq\n"); 3417 ret = -ENOMEM; 3418 goto destroy_comp_wq; 3419 } 3420 3421 iscsit_register_transport(&iser_target_transport); 3422 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3423 3424 return 0; 3425 3426 destroy_comp_wq: 3427 destroy_workqueue(isert_comp_wq); 3428 3429 return ret; 3430 } 3431 3432 static void __exit isert_exit(void) 3433 { 3434 flush_scheduled_work(); 3435 destroy_workqueue(isert_release_wq); 3436 destroy_workqueue(isert_comp_wq); 3437 iscsit_unregister_transport(&iser_target_transport); 3438 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3439 } 3440 3441 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3442 MODULE_VERSION("1.0"); 3443 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3444 MODULE_LICENSE("GPL"); 3445 3446 module_init(isert_init); 3447 module_exit(isert_exit); 3448