1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 static int isert_debug_level; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static void 51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static void 56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 57 static int 58 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 59 struct isert_rdma_wr *wr); 60 static int 61 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 62 static int 63 isert_rdma_post_recvl(struct isert_conn *isert_conn); 64 static int 65 isert_rdma_accept(struct isert_conn *isert_conn); 66 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 67 68 static void isert_release_work(struct work_struct *work); 69 70 static inline bool 71 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 72 { 73 return (conn->pi_support && 74 cmd->prot_op != TARGET_PROT_NORMAL); 75 } 76 77 78 static void 79 isert_qp_event_callback(struct ib_event *e, void *context) 80 { 81 struct isert_conn *isert_conn = context; 82 83 isert_err("%s (%d): conn %p\n", 84 ib_event_msg(e->event), e->event, isert_conn); 85 86 switch (e->event) { 87 case IB_EVENT_COMM_EST: 88 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 89 break; 90 case IB_EVENT_QP_LAST_WQE_REACHED: 91 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 92 break; 93 default: 94 break; 95 } 96 } 97 98 static int 99 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 100 { 101 int ret; 102 103 ret = ib_query_device(ib_dev, devattr); 104 if (ret) { 105 isert_err("ib_query_device() failed: %d\n", ret); 106 return ret; 107 } 108 isert_dbg("devattr->max_sge: %d\n", devattr->max_sge); 109 isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 110 111 return 0; 112 } 113 114 static struct isert_comp * 115 isert_comp_get(struct isert_conn *isert_conn) 116 { 117 struct isert_device *device = isert_conn->device; 118 struct isert_comp *comp; 119 int i, min = 0; 120 121 mutex_lock(&device_list_mutex); 122 for (i = 0; i < device->comps_used; i++) 123 if (device->comps[i].active_qps < 124 device->comps[min].active_qps) 125 min = i; 126 comp = &device->comps[min]; 127 comp->active_qps++; 128 mutex_unlock(&device_list_mutex); 129 130 isert_info("conn %p, using comp %p min_index: %d\n", 131 isert_conn, comp, min); 132 133 return comp; 134 } 135 136 static void 137 isert_comp_put(struct isert_comp *comp) 138 { 139 mutex_lock(&device_list_mutex); 140 comp->active_qps--; 141 mutex_unlock(&device_list_mutex); 142 } 143 144 static struct ib_qp * 145 isert_create_qp(struct isert_conn *isert_conn, 146 struct isert_comp *comp, 147 struct rdma_cm_id *cma_id) 148 { 149 struct isert_device *device = isert_conn->device; 150 struct ib_qp_init_attr attr; 151 int ret; 152 153 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 154 attr.event_handler = isert_qp_event_callback; 155 attr.qp_context = isert_conn; 156 attr.send_cq = comp->cq; 157 attr.recv_cq = comp->cq; 158 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 159 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 160 /* 161 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 162 * work-around for RDMA_READs with ConnectX-2. 163 * 164 * Also, still make sure to have at least two SGEs for 165 * outgoing control PDU responses. 166 */ 167 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 168 isert_conn->max_sge = attr.cap.max_send_sge; 169 170 attr.cap.max_recv_sge = 1; 171 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 172 attr.qp_type = IB_QPT_RC; 173 if (device->pi_capable) 174 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 175 176 ret = rdma_create_qp(cma_id, device->pd, &attr); 177 if (ret) { 178 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 179 return ERR_PTR(ret); 180 } 181 182 return cma_id->qp; 183 } 184 185 static int 186 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 187 { 188 struct isert_comp *comp; 189 int ret; 190 191 comp = isert_comp_get(isert_conn); 192 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 193 if (IS_ERR(isert_conn->qp)) { 194 ret = PTR_ERR(isert_conn->qp); 195 goto err; 196 } 197 198 return 0; 199 err: 200 isert_comp_put(comp); 201 return ret; 202 } 203 204 static void 205 isert_cq_event_callback(struct ib_event *e, void *context) 206 { 207 isert_dbg("event: %d\n", e->event); 208 } 209 210 static int 211 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 212 { 213 struct isert_device *device = isert_conn->device; 214 struct ib_device *ib_dev = device->ib_device; 215 struct iser_rx_desc *rx_desc; 216 struct ib_sge *rx_sg; 217 u64 dma_addr; 218 int i, j; 219 220 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 221 sizeof(struct iser_rx_desc), GFP_KERNEL); 222 if (!isert_conn->rx_descs) 223 goto fail; 224 225 rx_desc = isert_conn->rx_descs; 226 227 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 228 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 229 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 230 if (ib_dma_mapping_error(ib_dev, dma_addr)) 231 goto dma_map_fail; 232 233 rx_desc->dma_addr = dma_addr; 234 235 rx_sg = &rx_desc->rx_sg; 236 rx_sg->addr = rx_desc->dma_addr; 237 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 238 rx_sg->lkey = device->mr->lkey; 239 } 240 241 isert_conn->rx_desc_head = 0; 242 243 return 0; 244 245 dma_map_fail: 246 rx_desc = isert_conn->rx_descs; 247 for (j = 0; j < i; j++, rx_desc++) { 248 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 249 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 250 } 251 kfree(isert_conn->rx_descs); 252 isert_conn->rx_descs = NULL; 253 fail: 254 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 255 256 return -ENOMEM; 257 } 258 259 static void 260 isert_free_rx_descriptors(struct isert_conn *isert_conn) 261 { 262 struct ib_device *ib_dev = isert_conn->device->ib_device; 263 struct iser_rx_desc *rx_desc; 264 int i; 265 266 if (!isert_conn->rx_descs) 267 return; 268 269 rx_desc = isert_conn->rx_descs; 270 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 271 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 272 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 273 } 274 275 kfree(isert_conn->rx_descs); 276 isert_conn->rx_descs = NULL; 277 } 278 279 static void isert_cq_work(struct work_struct *); 280 static void isert_cq_callback(struct ib_cq *, void *); 281 282 static void 283 isert_free_comps(struct isert_device *device) 284 { 285 int i; 286 287 for (i = 0; i < device->comps_used; i++) { 288 struct isert_comp *comp = &device->comps[i]; 289 290 if (comp->cq) { 291 cancel_work_sync(&comp->work); 292 ib_destroy_cq(comp->cq); 293 } 294 } 295 kfree(device->comps); 296 } 297 298 static int 299 isert_alloc_comps(struct isert_device *device, 300 struct ib_device_attr *attr) 301 { 302 int i, max_cqe, ret = 0; 303 304 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 305 device->ib_device->num_comp_vectors)); 306 307 isert_info("Using %d CQs, %s supports %d vectors support " 308 "Fast registration %d pi_capable %d\n", 309 device->comps_used, device->ib_device->name, 310 device->ib_device->num_comp_vectors, device->use_fastreg, 311 device->pi_capable); 312 313 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 314 GFP_KERNEL); 315 if (!device->comps) { 316 isert_err("Unable to allocate completion contexts\n"); 317 return -ENOMEM; 318 } 319 320 max_cqe = min(ISER_MAX_CQ_LEN, attr->max_cqe); 321 322 for (i = 0; i < device->comps_used; i++) { 323 struct ib_cq_init_attr cq_attr = {}; 324 struct isert_comp *comp = &device->comps[i]; 325 326 comp->device = device; 327 INIT_WORK(&comp->work, isert_cq_work); 328 cq_attr.cqe = max_cqe; 329 cq_attr.comp_vector = i; 330 comp->cq = ib_create_cq(device->ib_device, 331 isert_cq_callback, 332 isert_cq_event_callback, 333 (void *)comp, 334 &cq_attr); 335 if (IS_ERR(comp->cq)) { 336 isert_err("Unable to allocate cq\n"); 337 ret = PTR_ERR(comp->cq); 338 comp->cq = NULL; 339 goto out_cq; 340 } 341 342 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 343 if (ret) 344 goto out_cq; 345 } 346 347 return 0; 348 out_cq: 349 isert_free_comps(device); 350 return ret; 351 } 352 353 static int 354 isert_create_device_ib_res(struct isert_device *device) 355 { 356 struct ib_device_attr *dev_attr; 357 int ret; 358 359 dev_attr = &device->dev_attr; 360 ret = isert_query_device(device->ib_device, dev_attr); 361 if (ret) 362 return ret; 363 364 /* asign function handlers */ 365 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 366 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 367 device->use_fastreg = 1; 368 device->reg_rdma_mem = isert_reg_rdma; 369 device->unreg_rdma_mem = isert_unreg_rdma; 370 } else { 371 device->use_fastreg = 0; 372 device->reg_rdma_mem = isert_map_rdma; 373 device->unreg_rdma_mem = isert_unmap_cmd; 374 } 375 376 ret = isert_alloc_comps(device, dev_attr); 377 if (ret) 378 return ret; 379 380 device->pd = ib_alloc_pd(device->ib_device); 381 if (IS_ERR(device->pd)) { 382 ret = PTR_ERR(device->pd); 383 isert_err("failed to allocate pd, device %p, ret=%d\n", 384 device, ret); 385 goto out_cq; 386 } 387 388 device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE); 389 if (IS_ERR(device->mr)) { 390 ret = PTR_ERR(device->mr); 391 isert_err("failed to create dma mr, device %p, ret=%d\n", 392 device, ret); 393 goto out_mr; 394 } 395 396 /* Check signature cap */ 397 device->pi_capable = dev_attr->device_cap_flags & 398 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 399 400 return 0; 401 402 out_mr: 403 ib_dealloc_pd(device->pd); 404 out_cq: 405 isert_free_comps(device); 406 return ret; 407 } 408 409 static void 410 isert_free_device_ib_res(struct isert_device *device) 411 { 412 isert_info("device %p\n", device); 413 414 ib_dereg_mr(device->mr); 415 ib_dealloc_pd(device->pd); 416 isert_free_comps(device); 417 } 418 419 static void 420 isert_device_put(struct isert_device *device) 421 { 422 mutex_lock(&device_list_mutex); 423 device->refcount--; 424 isert_info("device %p refcount %d\n", device, device->refcount); 425 if (!device->refcount) { 426 isert_free_device_ib_res(device); 427 list_del(&device->dev_node); 428 kfree(device); 429 } 430 mutex_unlock(&device_list_mutex); 431 } 432 433 static struct isert_device * 434 isert_device_get(struct rdma_cm_id *cma_id) 435 { 436 struct isert_device *device; 437 int ret; 438 439 mutex_lock(&device_list_mutex); 440 list_for_each_entry(device, &device_list, dev_node) { 441 if (device->ib_device->node_guid == cma_id->device->node_guid) { 442 device->refcount++; 443 isert_info("Found iser device %p refcount %d\n", 444 device, device->refcount); 445 mutex_unlock(&device_list_mutex); 446 return device; 447 } 448 } 449 450 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 451 if (!device) { 452 mutex_unlock(&device_list_mutex); 453 return ERR_PTR(-ENOMEM); 454 } 455 456 INIT_LIST_HEAD(&device->dev_node); 457 458 device->ib_device = cma_id->device; 459 ret = isert_create_device_ib_res(device); 460 if (ret) { 461 kfree(device); 462 mutex_unlock(&device_list_mutex); 463 return ERR_PTR(ret); 464 } 465 466 device->refcount++; 467 list_add_tail(&device->dev_node, &device_list); 468 isert_info("Created a new iser device %p refcount %d\n", 469 device, device->refcount); 470 mutex_unlock(&device_list_mutex); 471 472 return device; 473 } 474 475 static void 476 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 477 { 478 struct fast_reg_descriptor *fr_desc, *tmp; 479 int i = 0; 480 481 if (list_empty(&isert_conn->fr_pool)) 482 return; 483 484 isert_info("Freeing conn %p fastreg pool", isert_conn); 485 486 list_for_each_entry_safe(fr_desc, tmp, 487 &isert_conn->fr_pool, list) { 488 list_del(&fr_desc->list); 489 ib_free_fast_reg_page_list(fr_desc->data_frpl); 490 ib_dereg_mr(fr_desc->data_mr); 491 if (fr_desc->pi_ctx) { 492 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 493 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 494 ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 495 kfree(fr_desc->pi_ctx); 496 } 497 kfree(fr_desc); 498 ++i; 499 } 500 501 if (i < isert_conn->fr_pool_size) 502 isert_warn("Pool still has %d regions registered\n", 503 isert_conn->fr_pool_size - i); 504 } 505 506 static int 507 isert_create_pi_ctx(struct fast_reg_descriptor *desc, 508 struct ib_device *device, 509 struct ib_pd *pd) 510 { 511 struct ib_mr_init_attr mr_init_attr; 512 struct pi_context *pi_ctx; 513 int ret; 514 515 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 516 if (!pi_ctx) { 517 isert_err("Failed to allocate pi context\n"); 518 return -ENOMEM; 519 } 520 521 pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(device, 522 ISCSI_ISER_SG_TABLESIZE); 523 if (IS_ERR(pi_ctx->prot_frpl)) { 524 isert_err("Failed to allocate prot frpl err=%ld\n", 525 PTR_ERR(pi_ctx->prot_frpl)); 526 ret = PTR_ERR(pi_ctx->prot_frpl); 527 goto err_pi_ctx; 528 } 529 530 pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 531 if (IS_ERR(pi_ctx->prot_mr)) { 532 isert_err("Failed to allocate prot frmr err=%ld\n", 533 PTR_ERR(pi_ctx->prot_mr)); 534 ret = PTR_ERR(pi_ctx->prot_mr); 535 goto err_prot_frpl; 536 } 537 desc->ind |= ISERT_PROT_KEY_VALID; 538 539 memset(&mr_init_attr, 0, sizeof(mr_init_attr)); 540 mr_init_attr.max_reg_descriptors = 2; 541 mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 542 pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 543 if (IS_ERR(pi_ctx->sig_mr)) { 544 isert_err("Failed to allocate signature enabled mr err=%ld\n", 545 PTR_ERR(pi_ctx->sig_mr)); 546 ret = PTR_ERR(pi_ctx->sig_mr); 547 goto err_prot_mr; 548 } 549 550 desc->pi_ctx = pi_ctx; 551 desc->ind |= ISERT_SIG_KEY_VALID; 552 desc->ind &= ~ISERT_PROTECTED; 553 554 return 0; 555 556 err_prot_mr: 557 ib_dereg_mr(pi_ctx->prot_mr); 558 err_prot_frpl: 559 ib_free_fast_reg_page_list(pi_ctx->prot_frpl); 560 err_pi_ctx: 561 kfree(pi_ctx); 562 563 return ret; 564 } 565 566 static int 567 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 568 struct fast_reg_descriptor *fr_desc) 569 { 570 int ret; 571 572 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 573 ISCSI_ISER_SG_TABLESIZE); 574 if (IS_ERR(fr_desc->data_frpl)) { 575 isert_err("Failed to allocate data frpl err=%ld\n", 576 PTR_ERR(fr_desc->data_frpl)); 577 return PTR_ERR(fr_desc->data_frpl); 578 } 579 580 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 581 if (IS_ERR(fr_desc->data_mr)) { 582 isert_err("Failed to allocate data frmr err=%ld\n", 583 PTR_ERR(fr_desc->data_mr)); 584 ret = PTR_ERR(fr_desc->data_mr); 585 goto err_data_frpl; 586 } 587 fr_desc->ind |= ISERT_DATA_KEY_VALID; 588 589 isert_dbg("Created fr_desc %p\n", fr_desc); 590 591 return 0; 592 593 err_data_frpl: 594 ib_free_fast_reg_page_list(fr_desc->data_frpl); 595 596 return ret; 597 } 598 599 static int 600 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 601 { 602 struct fast_reg_descriptor *fr_desc; 603 struct isert_device *device = isert_conn->device; 604 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 605 struct se_node_acl *se_nacl = se_sess->se_node_acl; 606 int i, ret, tag_num; 607 /* 608 * Setup the number of FRMRs based upon the number of tags 609 * available to session in iscsi_target_locate_portal(). 610 */ 611 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 612 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 613 614 isert_conn->fr_pool_size = 0; 615 for (i = 0; i < tag_num; i++) { 616 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 617 if (!fr_desc) { 618 isert_err("Failed to allocate fast_reg descriptor\n"); 619 ret = -ENOMEM; 620 goto err; 621 } 622 623 ret = isert_create_fr_desc(device->ib_device, 624 device->pd, fr_desc); 625 if (ret) { 626 isert_err("Failed to create fastreg descriptor err=%d\n", 627 ret); 628 kfree(fr_desc); 629 goto err; 630 } 631 632 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 633 isert_conn->fr_pool_size++; 634 } 635 636 isert_dbg("Creating conn %p fastreg pool size=%d", 637 isert_conn, isert_conn->fr_pool_size); 638 639 return 0; 640 641 err: 642 isert_conn_free_fastreg_pool(isert_conn); 643 return ret; 644 } 645 646 static void 647 isert_init_conn(struct isert_conn *isert_conn) 648 { 649 isert_conn->state = ISER_CONN_INIT; 650 INIT_LIST_HEAD(&isert_conn->accept_node); 651 init_completion(&isert_conn->login_comp); 652 init_completion(&isert_conn->login_req_comp); 653 init_completion(&isert_conn->wait); 654 kref_init(&isert_conn->kref); 655 mutex_init(&isert_conn->mutex); 656 spin_lock_init(&isert_conn->pool_lock); 657 INIT_LIST_HEAD(&isert_conn->fr_pool); 658 INIT_WORK(&isert_conn->release_work, isert_release_work); 659 } 660 661 static void 662 isert_free_login_buf(struct isert_conn *isert_conn) 663 { 664 struct ib_device *ib_dev = isert_conn->device->ib_device; 665 666 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 667 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 668 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 669 ISCSI_DEF_MAX_RECV_SEG_LEN, 670 DMA_FROM_DEVICE); 671 kfree(isert_conn->login_buf); 672 } 673 674 static int 675 isert_alloc_login_buf(struct isert_conn *isert_conn, 676 struct ib_device *ib_dev) 677 { 678 int ret; 679 680 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 681 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 682 if (!isert_conn->login_buf) { 683 isert_err("Unable to allocate isert_conn->login_buf\n"); 684 return -ENOMEM; 685 } 686 687 isert_conn->login_req_buf = isert_conn->login_buf; 688 isert_conn->login_rsp_buf = isert_conn->login_buf + 689 ISCSI_DEF_MAX_RECV_SEG_LEN; 690 691 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 692 isert_conn->login_buf, isert_conn->login_req_buf, 693 isert_conn->login_rsp_buf); 694 695 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 696 (void *)isert_conn->login_req_buf, 697 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 698 699 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 700 if (ret) { 701 isert_err("login_req_dma mapping error: %d\n", ret); 702 isert_conn->login_req_dma = 0; 703 goto out_login_buf; 704 } 705 706 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 707 (void *)isert_conn->login_rsp_buf, 708 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 709 710 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 711 if (ret) { 712 isert_err("login_rsp_dma mapping error: %d\n", ret); 713 isert_conn->login_rsp_dma = 0; 714 goto out_req_dma_map; 715 } 716 717 return 0; 718 719 out_req_dma_map: 720 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 721 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 722 out_login_buf: 723 kfree(isert_conn->login_buf); 724 return ret; 725 } 726 727 static int 728 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 729 { 730 struct isert_np *isert_np = cma_id->context; 731 struct iscsi_np *np = isert_np->np; 732 struct isert_conn *isert_conn; 733 struct isert_device *device; 734 int ret = 0; 735 736 spin_lock_bh(&np->np_thread_lock); 737 if (!np->enabled) { 738 spin_unlock_bh(&np->np_thread_lock); 739 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 740 return rdma_reject(cma_id, NULL, 0); 741 } 742 spin_unlock_bh(&np->np_thread_lock); 743 744 isert_dbg("cma_id: %p, portal: %p\n", 745 cma_id, cma_id->context); 746 747 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 748 if (!isert_conn) 749 return -ENOMEM; 750 751 isert_init_conn(isert_conn); 752 isert_conn->cm_id = cma_id; 753 754 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 755 if (ret) 756 goto out; 757 758 device = isert_device_get(cma_id); 759 if (IS_ERR(device)) { 760 ret = PTR_ERR(device); 761 goto out_rsp_dma_map; 762 } 763 isert_conn->device = device; 764 765 /* Set max inflight RDMA READ requests */ 766 isert_conn->initiator_depth = min_t(u8, 767 event->param.conn.initiator_depth, 768 device->dev_attr.max_qp_init_rd_atom); 769 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 770 771 ret = isert_conn_setup_qp(isert_conn, cma_id); 772 if (ret) 773 goto out_conn_dev; 774 775 ret = isert_rdma_post_recvl(isert_conn); 776 if (ret) 777 goto out_conn_dev; 778 /* 779 * Obtain the second reference now before isert_rdma_accept() to 780 * ensure that any initiator generated REJECT CM event that occurs 781 * asynchronously won't drop the last reference until the error path 782 * in iscsi_target_login_sess_out() does it's ->iscsit_free_conn() -> 783 * isert_free_conn() -> isert_put_conn() -> kref_put(). 784 */ 785 if (!kref_get_unless_zero(&isert_conn->kref)) { 786 isert_warn("conn %p connect_release is running\n", isert_conn); 787 goto out_conn_dev; 788 } 789 790 ret = isert_rdma_accept(isert_conn); 791 if (ret) 792 goto out_conn_dev; 793 794 mutex_lock(&isert_np->np_accept_mutex); 795 list_add_tail(&isert_conn->accept_node, &isert_np->np_accept_list); 796 mutex_unlock(&isert_np->np_accept_mutex); 797 798 isert_info("np %p: Allow accept_np to continue\n", np); 799 up(&isert_np->np_sem); 800 return 0; 801 802 out_conn_dev: 803 isert_device_put(device); 804 out_rsp_dma_map: 805 isert_free_login_buf(isert_conn); 806 out: 807 kfree(isert_conn); 808 rdma_reject(cma_id, NULL, 0); 809 return ret; 810 } 811 812 static void 813 isert_connect_release(struct isert_conn *isert_conn) 814 { 815 struct isert_device *device = isert_conn->device; 816 817 isert_dbg("conn %p\n", isert_conn); 818 819 BUG_ON(!device); 820 821 if (device->use_fastreg) 822 isert_conn_free_fastreg_pool(isert_conn); 823 824 isert_free_rx_descriptors(isert_conn); 825 if (isert_conn->cm_id) 826 rdma_destroy_id(isert_conn->cm_id); 827 828 if (isert_conn->qp) { 829 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 830 831 isert_comp_put(comp); 832 ib_destroy_qp(isert_conn->qp); 833 } 834 835 if (isert_conn->login_buf) 836 isert_free_login_buf(isert_conn); 837 838 isert_device_put(device); 839 840 kfree(isert_conn); 841 } 842 843 static void 844 isert_connected_handler(struct rdma_cm_id *cma_id) 845 { 846 struct isert_conn *isert_conn = cma_id->qp->qp_context; 847 848 isert_info("conn %p\n", isert_conn); 849 850 mutex_lock(&isert_conn->mutex); 851 if (isert_conn->state != ISER_CONN_FULL_FEATURE) 852 isert_conn->state = ISER_CONN_UP; 853 mutex_unlock(&isert_conn->mutex); 854 } 855 856 static void 857 isert_release_kref(struct kref *kref) 858 { 859 struct isert_conn *isert_conn = container_of(kref, 860 struct isert_conn, kref); 861 862 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 863 current->pid); 864 865 isert_connect_release(isert_conn); 866 } 867 868 static void 869 isert_put_conn(struct isert_conn *isert_conn) 870 { 871 kref_put(&isert_conn->kref, isert_release_kref); 872 } 873 874 /** 875 * isert_conn_terminate() - Initiate connection termination 876 * @isert_conn: isert connection struct 877 * 878 * Notes: 879 * In case the connection state is FULL_FEATURE, move state 880 * to TEMINATING and start teardown sequence (rdma_disconnect). 881 * In case the connection state is UP, complete flush as well. 882 * 883 * This routine must be called with mutex held. Thus it is 884 * safe to call multiple times. 885 */ 886 static void 887 isert_conn_terminate(struct isert_conn *isert_conn) 888 { 889 int err; 890 891 switch (isert_conn->state) { 892 case ISER_CONN_TERMINATING: 893 break; 894 case ISER_CONN_UP: 895 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 896 isert_info("Terminating conn %p state %d\n", 897 isert_conn, isert_conn->state); 898 isert_conn->state = ISER_CONN_TERMINATING; 899 err = rdma_disconnect(isert_conn->cm_id); 900 if (err) 901 isert_warn("Failed rdma_disconnect isert_conn %p\n", 902 isert_conn); 903 break; 904 default: 905 isert_warn("conn %p teminating in state %d\n", 906 isert_conn, isert_conn->state); 907 } 908 } 909 910 static int 911 isert_np_cma_handler(struct isert_np *isert_np, 912 enum rdma_cm_event_type event) 913 { 914 isert_dbg("%s (%d): isert np %p\n", 915 rdma_event_msg(event), event, isert_np); 916 917 switch (event) { 918 case RDMA_CM_EVENT_DEVICE_REMOVAL: 919 isert_np->np_cm_id = NULL; 920 break; 921 case RDMA_CM_EVENT_ADDR_CHANGE: 922 isert_np->np_cm_id = isert_setup_id(isert_np); 923 if (IS_ERR(isert_np->np_cm_id)) { 924 isert_err("isert np %p setup id failed: %ld\n", 925 isert_np, PTR_ERR(isert_np->np_cm_id)); 926 isert_np->np_cm_id = NULL; 927 } 928 break; 929 default: 930 isert_err("isert np %p Unexpected event %d\n", 931 isert_np, event); 932 } 933 934 return -1; 935 } 936 937 static int 938 isert_disconnected_handler(struct rdma_cm_id *cma_id, 939 enum rdma_cm_event_type event) 940 { 941 struct isert_np *isert_np = cma_id->context; 942 struct isert_conn *isert_conn; 943 bool terminating = false; 944 945 if (isert_np->np_cm_id == cma_id) 946 return isert_np_cma_handler(cma_id->context, event); 947 948 isert_conn = cma_id->qp->qp_context; 949 950 mutex_lock(&isert_conn->mutex); 951 terminating = (isert_conn->state == ISER_CONN_TERMINATING); 952 isert_conn_terminate(isert_conn); 953 mutex_unlock(&isert_conn->mutex); 954 955 isert_info("conn %p completing wait\n", isert_conn); 956 complete(&isert_conn->wait); 957 958 if (terminating) 959 goto out; 960 961 mutex_lock(&isert_np->np_accept_mutex); 962 if (!list_empty(&isert_conn->accept_node)) { 963 list_del_init(&isert_conn->accept_node); 964 isert_put_conn(isert_conn); 965 queue_work(isert_release_wq, &isert_conn->release_work); 966 } 967 mutex_unlock(&isert_np->np_accept_mutex); 968 969 out: 970 return 0; 971 } 972 973 static int 974 isert_connect_error(struct rdma_cm_id *cma_id) 975 { 976 struct isert_conn *isert_conn = cma_id->qp->qp_context; 977 978 isert_conn->cm_id = NULL; 979 isert_put_conn(isert_conn); 980 981 return -1; 982 } 983 984 static int 985 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 986 { 987 int ret = 0; 988 989 isert_info("%s (%d): status %d id %p np %p\n", 990 rdma_event_msg(event->event), event->event, 991 event->status, cma_id, cma_id->context); 992 993 switch (event->event) { 994 case RDMA_CM_EVENT_CONNECT_REQUEST: 995 ret = isert_connect_request(cma_id, event); 996 if (ret) 997 isert_err("failed handle connect request %d\n", ret); 998 break; 999 case RDMA_CM_EVENT_ESTABLISHED: 1000 isert_connected_handler(cma_id); 1001 break; 1002 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 1003 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 1004 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 1005 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 1006 ret = isert_disconnected_handler(cma_id, event->event); 1007 break; 1008 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 1009 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 1010 case RDMA_CM_EVENT_CONNECT_ERROR: 1011 ret = isert_connect_error(cma_id); 1012 break; 1013 default: 1014 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 1015 break; 1016 } 1017 1018 return ret; 1019 } 1020 1021 static int 1022 isert_post_recv(struct isert_conn *isert_conn, u32 count) 1023 { 1024 struct ib_recv_wr *rx_wr, *rx_wr_failed; 1025 int i, ret; 1026 unsigned int rx_head = isert_conn->rx_desc_head; 1027 struct iser_rx_desc *rx_desc; 1028 1029 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 1030 rx_desc = &isert_conn->rx_descs[rx_head]; 1031 rx_wr->wr_id = (uintptr_t)rx_desc; 1032 rx_wr->sg_list = &rx_desc->rx_sg; 1033 rx_wr->num_sge = 1; 1034 rx_wr->next = rx_wr + 1; 1035 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 1036 } 1037 1038 rx_wr--; 1039 rx_wr->next = NULL; /* mark end of work requests list */ 1040 1041 isert_conn->post_recv_buf_count += count; 1042 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 1043 &rx_wr_failed); 1044 if (ret) { 1045 isert_err("ib_post_recv() failed with ret: %d\n", ret); 1046 isert_conn->post_recv_buf_count -= count; 1047 } else { 1048 isert_dbg("Posted %d RX buffers\n", count); 1049 isert_conn->rx_desc_head = rx_head; 1050 } 1051 return ret; 1052 } 1053 1054 static int 1055 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1056 { 1057 struct ib_device *ib_dev = isert_conn->cm_id->device; 1058 struct ib_send_wr send_wr, *send_wr_failed; 1059 int ret; 1060 1061 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1062 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1063 1064 send_wr.next = NULL; 1065 send_wr.wr_id = (uintptr_t)tx_desc; 1066 send_wr.sg_list = tx_desc->tx_sg; 1067 send_wr.num_sge = tx_desc->num_sge; 1068 send_wr.opcode = IB_WR_SEND; 1069 send_wr.send_flags = IB_SEND_SIGNALED; 1070 1071 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1072 if (ret) 1073 isert_err("ib_post_send() failed, ret: %d\n", ret); 1074 1075 return ret; 1076 } 1077 1078 static void 1079 isert_create_send_desc(struct isert_conn *isert_conn, 1080 struct isert_cmd *isert_cmd, 1081 struct iser_tx_desc *tx_desc) 1082 { 1083 struct isert_device *device = isert_conn->device; 1084 struct ib_device *ib_dev = device->ib_device; 1085 1086 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1087 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1088 1089 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 1090 tx_desc->iser_header.flags = ISER_VER; 1091 1092 tx_desc->num_sge = 1; 1093 tx_desc->isert_cmd = isert_cmd; 1094 1095 if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { 1096 tx_desc->tx_sg[0].lkey = device->mr->lkey; 1097 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1098 } 1099 } 1100 1101 static int 1102 isert_init_tx_hdrs(struct isert_conn *isert_conn, 1103 struct iser_tx_desc *tx_desc) 1104 { 1105 struct isert_device *device = isert_conn->device; 1106 struct ib_device *ib_dev = device->ib_device; 1107 u64 dma_addr; 1108 1109 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1110 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1111 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1112 isert_err("ib_dma_mapping_error() failed\n"); 1113 return -ENOMEM; 1114 } 1115 1116 tx_desc->dma_addr = dma_addr; 1117 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1118 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1119 tx_desc->tx_sg[0].lkey = device->mr->lkey; 1120 1121 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 1122 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 1123 tx_desc->tx_sg[0].lkey); 1124 1125 return 0; 1126 } 1127 1128 static void 1129 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1130 struct ib_send_wr *send_wr) 1131 { 1132 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 1133 1134 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1135 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1136 send_wr->opcode = IB_WR_SEND; 1137 send_wr->sg_list = &tx_desc->tx_sg[0]; 1138 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 1139 send_wr->send_flags = IB_SEND_SIGNALED; 1140 } 1141 1142 static int 1143 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1144 { 1145 struct ib_recv_wr rx_wr, *rx_wr_fail; 1146 struct ib_sge sge; 1147 int ret; 1148 1149 memset(&sge, 0, sizeof(struct ib_sge)); 1150 sge.addr = isert_conn->login_req_dma; 1151 sge.length = ISER_RX_LOGIN_SIZE; 1152 sge.lkey = isert_conn->device->mr->lkey; 1153 1154 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1155 sge.addr, sge.length, sge.lkey); 1156 1157 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1158 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1159 rx_wr.sg_list = &sge; 1160 rx_wr.num_sge = 1; 1161 1162 isert_conn->post_recv_buf_count++; 1163 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 1164 if (ret) { 1165 isert_err("ib_post_recv() failed: %d\n", ret); 1166 isert_conn->post_recv_buf_count--; 1167 } 1168 1169 return ret; 1170 } 1171 1172 static int 1173 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1174 u32 length) 1175 { 1176 struct isert_conn *isert_conn = conn->context; 1177 struct isert_device *device = isert_conn->device; 1178 struct ib_device *ib_dev = device->ib_device; 1179 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1180 int ret; 1181 1182 isert_create_send_desc(isert_conn, NULL, tx_desc); 1183 1184 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1185 sizeof(struct iscsi_hdr)); 1186 1187 isert_init_tx_hdrs(isert_conn, tx_desc); 1188 1189 if (length > 0) { 1190 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1191 1192 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1193 length, DMA_TO_DEVICE); 1194 1195 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1196 1197 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1198 length, DMA_TO_DEVICE); 1199 1200 tx_dsg->addr = isert_conn->login_rsp_dma; 1201 tx_dsg->length = length; 1202 tx_dsg->lkey = isert_conn->device->mr->lkey; 1203 tx_desc->num_sge = 2; 1204 } 1205 if (!login->login_failed) { 1206 if (login->login_complete) { 1207 if (!conn->sess->sess_ops->SessionType && 1208 isert_conn->device->use_fastreg) { 1209 ret = isert_conn_create_fastreg_pool(isert_conn); 1210 if (ret) { 1211 isert_err("Conn: %p failed to create" 1212 " fastreg pool\n", isert_conn); 1213 return ret; 1214 } 1215 } 1216 1217 ret = isert_alloc_rx_descriptors(isert_conn); 1218 if (ret) 1219 return ret; 1220 1221 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1222 if (ret) 1223 return ret; 1224 1225 /* Now we are in FULL_FEATURE phase */ 1226 mutex_lock(&isert_conn->mutex); 1227 isert_conn->state = ISER_CONN_FULL_FEATURE; 1228 mutex_unlock(&isert_conn->mutex); 1229 goto post_send; 1230 } 1231 1232 ret = isert_rdma_post_recvl(isert_conn); 1233 if (ret) 1234 return ret; 1235 } 1236 post_send: 1237 ret = isert_post_send(isert_conn, tx_desc); 1238 if (ret) 1239 return ret; 1240 1241 return 0; 1242 } 1243 1244 static void 1245 isert_rx_login_req(struct isert_conn *isert_conn) 1246 { 1247 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 1248 int rx_buflen = isert_conn->login_req_len; 1249 struct iscsi_conn *conn = isert_conn->conn; 1250 struct iscsi_login *login = conn->conn_login; 1251 int size; 1252 1253 isert_info("conn %p\n", isert_conn); 1254 1255 WARN_ON_ONCE(!login); 1256 1257 if (login->first_request) { 1258 struct iscsi_login_req *login_req = 1259 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1260 /* 1261 * Setup the initial iscsi_login values from the leading 1262 * login request PDU. 1263 */ 1264 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1265 login->current_stage = 1266 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1267 >> 2; 1268 login->version_min = login_req->min_version; 1269 login->version_max = login_req->max_version; 1270 memcpy(login->isid, login_req->isid, 6); 1271 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1272 login->init_task_tag = login_req->itt; 1273 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1274 login->cid = be16_to_cpu(login_req->cid); 1275 login->tsih = be16_to_cpu(login_req->tsih); 1276 } 1277 1278 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1279 1280 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1281 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1282 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1283 MAX_KEY_VALUE_PAIRS); 1284 memcpy(login->req_buf, &rx_desc->data[0], size); 1285 1286 if (login->first_request) { 1287 complete(&isert_conn->login_comp); 1288 return; 1289 } 1290 schedule_delayed_work(&conn->login_work, 0); 1291 } 1292 1293 static struct iscsi_cmd 1294 *isert_allocate_cmd(struct iscsi_conn *conn) 1295 { 1296 struct isert_conn *isert_conn = conn->context; 1297 struct isert_cmd *isert_cmd; 1298 struct iscsi_cmd *cmd; 1299 1300 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1301 if (!cmd) { 1302 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1303 return NULL; 1304 } 1305 isert_cmd = iscsit_priv_cmd(cmd); 1306 isert_cmd->conn = isert_conn; 1307 isert_cmd->iscsi_cmd = cmd; 1308 1309 return cmd; 1310 } 1311 1312 static int 1313 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1314 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1315 struct iser_rx_desc *rx_desc, unsigned char *buf) 1316 { 1317 struct iscsi_conn *conn = isert_conn->conn; 1318 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1319 struct scatterlist *sg; 1320 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1321 bool dump_payload = false; 1322 1323 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1324 if (rc < 0) 1325 return rc; 1326 1327 imm_data = cmd->immediate_data; 1328 imm_data_len = cmd->first_burst_len; 1329 unsol_data = cmd->unsolicited_data; 1330 1331 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1332 if (rc < 0) { 1333 return 0; 1334 } else if (rc > 0) { 1335 dump_payload = true; 1336 goto sequence_cmd; 1337 } 1338 1339 if (!imm_data) 1340 return 0; 1341 1342 sg = &cmd->se_cmd.t_data_sg[0]; 1343 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1344 1345 isert_dbg("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1346 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1347 1348 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1349 1350 cmd->write_data_done += imm_data_len; 1351 1352 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1353 spin_lock_bh(&cmd->istate_lock); 1354 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1355 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1356 spin_unlock_bh(&cmd->istate_lock); 1357 } 1358 1359 sequence_cmd: 1360 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1361 1362 if (!rc && dump_payload == false && unsol_data) 1363 iscsit_set_unsoliticed_dataout(cmd); 1364 else if (dump_payload && imm_data) 1365 target_put_sess_cmd(&cmd->se_cmd); 1366 1367 return 0; 1368 } 1369 1370 static int 1371 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1372 struct iser_rx_desc *rx_desc, unsigned char *buf) 1373 { 1374 struct scatterlist *sg_start; 1375 struct iscsi_conn *conn = isert_conn->conn; 1376 struct iscsi_cmd *cmd = NULL; 1377 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1378 u32 unsol_data_len = ntoh24(hdr->dlength); 1379 int rc, sg_nents, sg_off, page_off; 1380 1381 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1382 if (rc < 0) 1383 return rc; 1384 else if (!cmd) 1385 return 0; 1386 /* 1387 * FIXME: Unexpected unsolicited_data out 1388 */ 1389 if (!cmd->unsolicited_data) { 1390 isert_err("Received unexpected solicited data payload\n"); 1391 dump_stack(); 1392 return -1; 1393 } 1394 1395 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1396 "write_data_done: %u, data_length: %u\n", 1397 unsol_data_len, cmd->write_data_done, 1398 cmd->se_cmd.data_length); 1399 1400 sg_off = cmd->write_data_done / PAGE_SIZE; 1401 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1402 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1403 page_off = cmd->write_data_done % PAGE_SIZE; 1404 /* 1405 * FIXME: Non page-aligned unsolicited_data out 1406 */ 1407 if (page_off) { 1408 isert_err("unexpected non-page aligned data payload\n"); 1409 dump_stack(); 1410 return -1; 1411 } 1412 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1413 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1414 sg_nents, &rx_desc->data[0], unsol_data_len); 1415 1416 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1417 unsol_data_len); 1418 1419 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1420 if (rc < 0) 1421 return rc; 1422 1423 return 0; 1424 } 1425 1426 static int 1427 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1428 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1429 unsigned char *buf) 1430 { 1431 struct iscsi_conn *conn = isert_conn->conn; 1432 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1433 int rc; 1434 1435 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1436 if (rc < 0) 1437 return rc; 1438 /* 1439 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1440 */ 1441 1442 return iscsit_process_nop_out(conn, cmd, hdr); 1443 } 1444 1445 static int 1446 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1447 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1448 struct iscsi_text *hdr) 1449 { 1450 struct iscsi_conn *conn = isert_conn->conn; 1451 u32 payload_length = ntoh24(hdr->dlength); 1452 int rc; 1453 unsigned char *text_in = NULL; 1454 1455 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1456 if (rc < 0) 1457 return rc; 1458 1459 if (payload_length) { 1460 text_in = kzalloc(payload_length, GFP_KERNEL); 1461 if (!text_in) { 1462 isert_err("Unable to allocate text_in of payload_length: %u\n", 1463 payload_length); 1464 return -ENOMEM; 1465 } 1466 } 1467 cmd->text_in_ptr = text_in; 1468 1469 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1470 1471 return iscsit_process_text_cmd(conn, cmd, hdr); 1472 } 1473 1474 static int 1475 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1476 uint32_t read_stag, uint64_t read_va, 1477 uint32_t write_stag, uint64_t write_va) 1478 { 1479 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1480 struct iscsi_conn *conn = isert_conn->conn; 1481 struct iscsi_cmd *cmd; 1482 struct isert_cmd *isert_cmd; 1483 int ret = -EINVAL; 1484 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1485 1486 if (conn->sess->sess_ops->SessionType && 1487 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1488 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1489 " ignoring\n", opcode); 1490 return 0; 1491 } 1492 1493 switch (opcode) { 1494 case ISCSI_OP_SCSI_CMD: 1495 cmd = isert_allocate_cmd(conn); 1496 if (!cmd) 1497 break; 1498 1499 isert_cmd = iscsit_priv_cmd(cmd); 1500 isert_cmd->read_stag = read_stag; 1501 isert_cmd->read_va = read_va; 1502 isert_cmd->write_stag = write_stag; 1503 isert_cmd->write_va = write_va; 1504 1505 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1506 rx_desc, (unsigned char *)hdr); 1507 break; 1508 case ISCSI_OP_NOOP_OUT: 1509 cmd = isert_allocate_cmd(conn); 1510 if (!cmd) 1511 break; 1512 1513 isert_cmd = iscsit_priv_cmd(cmd); 1514 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1515 rx_desc, (unsigned char *)hdr); 1516 break; 1517 case ISCSI_OP_SCSI_DATA_OUT: 1518 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1519 (unsigned char *)hdr); 1520 break; 1521 case ISCSI_OP_SCSI_TMFUNC: 1522 cmd = isert_allocate_cmd(conn); 1523 if (!cmd) 1524 break; 1525 1526 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1527 (unsigned char *)hdr); 1528 break; 1529 case ISCSI_OP_LOGOUT: 1530 cmd = isert_allocate_cmd(conn); 1531 if (!cmd) 1532 break; 1533 1534 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1535 break; 1536 case ISCSI_OP_TEXT: 1537 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) { 1538 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1539 if (!cmd) 1540 break; 1541 } else { 1542 cmd = isert_allocate_cmd(conn); 1543 if (!cmd) 1544 break; 1545 } 1546 1547 isert_cmd = iscsit_priv_cmd(cmd); 1548 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1549 rx_desc, (struct iscsi_text *)hdr); 1550 break; 1551 default: 1552 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1553 dump_stack(); 1554 break; 1555 } 1556 1557 return ret; 1558 } 1559 1560 static void 1561 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1562 { 1563 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1564 uint64_t read_va = 0, write_va = 0; 1565 uint32_t read_stag = 0, write_stag = 0; 1566 int rc; 1567 1568 switch (iser_hdr->flags & 0xF0) { 1569 case ISCSI_CTRL: 1570 if (iser_hdr->flags & ISER_RSV) { 1571 read_stag = be32_to_cpu(iser_hdr->read_stag); 1572 read_va = be64_to_cpu(iser_hdr->read_va); 1573 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1574 read_stag, (unsigned long long)read_va); 1575 } 1576 if (iser_hdr->flags & ISER_WSV) { 1577 write_stag = be32_to_cpu(iser_hdr->write_stag); 1578 write_va = be64_to_cpu(iser_hdr->write_va); 1579 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1580 write_stag, (unsigned long long)write_va); 1581 } 1582 1583 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1584 break; 1585 case ISER_HELLO: 1586 isert_err("iSER Hello message\n"); 1587 break; 1588 default: 1589 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1590 break; 1591 } 1592 1593 rc = isert_rx_opcode(isert_conn, rx_desc, 1594 read_stag, read_va, write_stag, write_va); 1595 } 1596 1597 static void 1598 isert_rcv_completion(struct iser_rx_desc *desc, 1599 struct isert_conn *isert_conn, 1600 u32 xfer_len) 1601 { 1602 struct ib_device *ib_dev = isert_conn->cm_id->device; 1603 struct iscsi_hdr *hdr; 1604 u64 rx_dma; 1605 int rx_buflen, outstanding; 1606 1607 if ((char *)desc == isert_conn->login_req_buf) { 1608 rx_dma = isert_conn->login_req_dma; 1609 rx_buflen = ISER_RX_LOGIN_SIZE; 1610 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1611 rx_dma, rx_buflen); 1612 } else { 1613 rx_dma = desc->dma_addr; 1614 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1615 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1616 rx_dma, rx_buflen); 1617 } 1618 1619 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1620 1621 hdr = &desc->iscsi_header; 1622 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1623 hdr->opcode, hdr->itt, hdr->flags, 1624 (int)(xfer_len - ISER_HEADERS_LEN)); 1625 1626 if ((char *)desc == isert_conn->login_req_buf) { 1627 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 1628 if (isert_conn->conn) { 1629 struct iscsi_login *login = isert_conn->conn->conn_login; 1630 1631 if (login && !login->first_request) 1632 isert_rx_login_req(isert_conn); 1633 } 1634 mutex_lock(&isert_conn->mutex); 1635 complete(&isert_conn->login_req_comp); 1636 mutex_unlock(&isert_conn->mutex); 1637 } else { 1638 isert_rx_do_work(desc, isert_conn); 1639 } 1640 1641 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1642 DMA_FROM_DEVICE); 1643 1644 isert_conn->post_recv_buf_count--; 1645 isert_dbg("Decremented post_recv_buf_count: %d\n", 1646 isert_conn->post_recv_buf_count); 1647 1648 if ((char *)desc == isert_conn->login_req_buf) 1649 return; 1650 1651 outstanding = isert_conn->post_recv_buf_count; 1652 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1653 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1654 ISERT_MIN_POSTED_RX); 1655 err = isert_post_recv(isert_conn, count); 1656 if (err) { 1657 isert_err("isert_post_recv() count: %d failed, %d\n", 1658 count, err); 1659 } 1660 } 1661 } 1662 1663 static int 1664 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1665 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1666 enum iser_ib_op_code op, struct isert_data_buf *data) 1667 { 1668 struct ib_device *ib_dev = isert_conn->cm_id->device; 1669 1670 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1671 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1672 1673 data->len = length - offset; 1674 data->offset = offset; 1675 data->sg_off = data->offset / PAGE_SIZE; 1676 1677 data->sg = &sg[data->sg_off]; 1678 data->nents = min_t(unsigned int, nents - data->sg_off, 1679 ISCSI_ISER_SG_TABLESIZE); 1680 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1681 PAGE_SIZE); 1682 1683 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1684 data->dma_dir); 1685 if (unlikely(!data->dma_nents)) { 1686 isert_err("Cmd: unable to dma map SGs %p\n", sg); 1687 return -EINVAL; 1688 } 1689 1690 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1691 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1692 1693 return 0; 1694 } 1695 1696 static void 1697 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1698 { 1699 struct ib_device *ib_dev = isert_conn->cm_id->device; 1700 1701 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1702 memset(data, 0, sizeof(*data)); 1703 } 1704 1705 1706 1707 static void 1708 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1709 { 1710 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1711 1712 isert_dbg("Cmd %p\n", isert_cmd); 1713 1714 if (wr->data.sg) { 1715 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1716 isert_unmap_data_buf(isert_conn, &wr->data); 1717 } 1718 1719 if (wr->send_wr) { 1720 isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1721 kfree(wr->send_wr); 1722 wr->send_wr = NULL; 1723 } 1724 1725 if (wr->ib_sge) { 1726 isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1727 kfree(wr->ib_sge); 1728 wr->ib_sge = NULL; 1729 } 1730 } 1731 1732 static void 1733 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1734 { 1735 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1736 1737 isert_dbg("Cmd %p\n", isert_cmd); 1738 1739 if (wr->fr_desc) { 1740 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 1741 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1742 isert_unmap_data_buf(isert_conn, &wr->prot); 1743 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1744 } 1745 spin_lock_bh(&isert_conn->pool_lock); 1746 list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); 1747 spin_unlock_bh(&isert_conn->pool_lock); 1748 wr->fr_desc = NULL; 1749 } 1750 1751 if (wr->data.sg) { 1752 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1753 isert_unmap_data_buf(isert_conn, &wr->data); 1754 } 1755 1756 wr->ib_sge = NULL; 1757 wr->send_wr = NULL; 1758 } 1759 1760 static void 1761 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1762 { 1763 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1764 struct isert_conn *isert_conn = isert_cmd->conn; 1765 struct iscsi_conn *conn = isert_conn->conn; 1766 struct isert_device *device = isert_conn->device; 1767 struct iscsi_text_rsp *hdr; 1768 1769 isert_dbg("Cmd %p\n", isert_cmd); 1770 1771 switch (cmd->iscsi_opcode) { 1772 case ISCSI_OP_SCSI_CMD: 1773 spin_lock_bh(&conn->cmd_lock); 1774 if (!list_empty(&cmd->i_conn_node)) 1775 list_del_init(&cmd->i_conn_node); 1776 spin_unlock_bh(&conn->cmd_lock); 1777 1778 if (cmd->data_direction == DMA_TO_DEVICE) { 1779 iscsit_stop_dataout_timer(cmd); 1780 /* 1781 * Check for special case during comp_err where 1782 * WRITE_PENDING has been handed off from core, 1783 * but requires an extra target_put_sess_cmd() 1784 * before transport_generic_free_cmd() below. 1785 */ 1786 if (comp_err && 1787 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1788 struct se_cmd *se_cmd = &cmd->se_cmd; 1789 1790 target_put_sess_cmd(se_cmd); 1791 } 1792 } 1793 1794 device->unreg_rdma_mem(isert_cmd, isert_conn); 1795 transport_generic_free_cmd(&cmd->se_cmd, 0); 1796 break; 1797 case ISCSI_OP_SCSI_TMFUNC: 1798 spin_lock_bh(&conn->cmd_lock); 1799 if (!list_empty(&cmd->i_conn_node)) 1800 list_del_init(&cmd->i_conn_node); 1801 spin_unlock_bh(&conn->cmd_lock); 1802 1803 transport_generic_free_cmd(&cmd->se_cmd, 0); 1804 break; 1805 case ISCSI_OP_REJECT: 1806 case ISCSI_OP_NOOP_OUT: 1807 case ISCSI_OP_TEXT: 1808 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1809 /* If the continue bit is on, keep the command alive */ 1810 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1811 break; 1812 1813 spin_lock_bh(&conn->cmd_lock); 1814 if (!list_empty(&cmd->i_conn_node)) 1815 list_del_init(&cmd->i_conn_node); 1816 spin_unlock_bh(&conn->cmd_lock); 1817 1818 /* 1819 * Handle special case for REJECT when iscsi_add_reject*() has 1820 * overwritten the original iscsi_opcode assignment, and the 1821 * associated cmd->se_cmd needs to be released. 1822 */ 1823 if (cmd->se_cmd.se_tfo != NULL) { 1824 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1825 cmd->iscsi_opcode); 1826 transport_generic_free_cmd(&cmd->se_cmd, 0); 1827 break; 1828 } 1829 /* 1830 * Fall-through 1831 */ 1832 default: 1833 iscsit_release_cmd(cmd); 1834 break; 1835 } 1836 } 1837 1838 static void 1839 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1840 { 1841 if (tx_desc->dma_addr != 0) { 1842 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1843 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1844 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1845 tx_desc->dma_addr = 0; 1846 } 1847 } 1848 1849 static void 1850 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1851 struct ib_device *ib_dev, bool comp_err) 1852 { 1853 if (isert_cmd->pdu_buf_dma != 0) { 1854 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1855 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1856 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1857 isert_cmd->pdu_buf_dma = 0; 1858 } 1859 1860 isert_unmap_tx_desc(tx_desc, ib_dev); 1861 isert_put_cmd(isert_cmd, comp_err); 1862 } 1863 1864 static int 1865 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1866 { 1867 struct ib_mr_status mr_status; 1868 int ret; 1869 1870 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1871 if (ret) { 1872 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1873 goto fail_mr_status; 1874 } 1875 1876 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1877 u64 sec_offset_err; 1878 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1879 1880 switch (mr_status.sig_err.err_type) { 1881 case IB_SIG_BAD_GUARD: 1882 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1883 break; 1884 case IB_SIG_BAD_REFTAG: 1885 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1886 break; 1887 case IB_SIG_BAD_APPTAG: 1888 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1889 break; 1890 } 1891 sec_offset_err = mr_status.sig_err.sig_err_offset; 1892 do_div(sec_offset_err, block_size); 1893 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1894 1895 isert_err("PI error found type %d at sector 0x%llx " 1896 "expected 0x%x vs actual 0x%x\n", 1897 mr_status.sig_err.err_type, 1898 (unsigned long long)se_cmd->bad_sector, 1899 mr_status.sig_err.expected, 1900 mr_status.sig_err.actual); 1901 ret = 1; 1902 } 1903 1904 fail_mr_status: 1905 return ret; 1906 } 1907 1908 static void 1909 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1910 struct isert_cmd *isert_cmd) 1911 { 1912 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1913 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1914 struct se_cmd *se_cmd = &cmd->se_cmd; 1915 struct isert_conn *isert_conn = isert_cmd->conn; 1916 struct isert_device *device = isert_conn->device; 1917 int ret = 0; 1918 1919 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1920 ret = isert_check_pi_status(se_cmd, 1921 wr->fr_desc->pi_ctx->sig_mr); 1922 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1923 } 1924 1925 device->unreg_rdma_mem(isert_cmd, isert_conn); 1926 wr->send_wr_num = 0; 1927 if (ret) 1928 transport_send_check_condition_and_sense(se_cmd, 1929 se_cmd->pi_err, 0); 1930 else 1931 isert_put_response(isert_conn->conn, cmd); 1932 } 1933 1934 static void 1935 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1936 struct isert_cmd *isert_cmd) 1937 { 1938 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1939 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1940 struct se_cmd *se_cmd = &cmd->se_cmd; 1941 struct isert_conn *isert_conn = isert_cmd->conn; 1942 struct isert_device *device = isert_conn->device; 1943 int ret = 0; 1944 1945 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1946 ret = isert_check_pi_status(se_cmd, 1947 wr->fr_desc->pi_ctx->sig_mr); 1948 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1949 } 1950 1951 iscsit_stop_dataout_timer(cmd); 1952 device->unreg_rdma_mem(isert_cmd, isert_conn); 1953 cmd->write_data_done = wr->data.len; 1954 wr->send_wr_num = 0; 1955 1956 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1957 spin_lock_bh(&cmd->istate_lock); 1958 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1959 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1960 spin_unlock_bh(&cmd->istate_lock); 1961 1962 if (ret) { 1963 target_put_sess_cmd(se_cmd); 1964 transport_send_check_condition_and_sense(se_cmd, 1965 se_cmd->pi_err, 0); 1966 } else { 1967 target_execute_cmd(se_cmd); 1968 } 1969 } 1970 1971 static void 1972 isert_do_control_comp(struct work_struct *work) 1973 { 1974 struct isert_cmd *isert_cmd = container_of(work, 1975 struct isert_cmd, comp_work); 1976 struct isert_conn *isert_conn = isert_cmd->conn; 1977 struct ib_device *ib_dev = isert_conn->cm_id->device; 1978 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1979 1980 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1981 1982 switch (cmd->i_state) { 1983 case ISTATE_SEND_TASKMGTRSP: 1984 iscsit_tmr_post_handler(cmd, cmd->conn); 1985 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1986 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1987 cmd->i_state = ISTATE_SENT_STATUS; 1988 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1989 ib_dev, false); 1990 break; 1991 case ISTATE_SEND_LOGOUTRSP: 1992 iscsit_logout_post_handler(cmd, cmd->conn); 1993 break; 1994 default: 1995 isert_err("Unknown i_state %d\n", cmd->i_state); 1996 dump_stack(); 1997 break; 1998 } 1999 } 2000 2001 static void 2002 isert_response_completion(struct iser_tx_desc *tx_desc, 2003 struct isert_cmd *isert_cmd, 2004 struct isert_conn *isert_conn, 2005 struct ib_device *ib_dev) 2006 { 2007 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2008 2009 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 2010 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 2011 cmd->i_state == ISTATE_SEND_REJECT || 2012 cmd->i_state == ISTATE_SEND_TEXTRSP) { 2013 isert_unmap_tx_desc(tx_desc, ib_dev); 2014 2015 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 2016 queue_work(isert_comp_wq, &isert_cmd->comp_work); 2017 return; 2018 } 2019 2020 cmd->i_state = ISTATE_SENT_STATUS; 2021 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 2022 } 2023 2024 static void 2025 isert_snd_completion(struct iser_tx_desc *tx_desc, 2026 struct isert_conn *isert_conn) 2027 { 2028 struct ib_device *ib_dev = isert_conn->cm_id->device; 2029 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2030 struct isert_rdma_wr *wr; 2031 2032 if (!isert_cmd) { 2033 isert_unmap_tx_desc(tx_desc, ib_dev); 2034 return; 2035 } 2036 wr = &isert_cmd->rdma_wr; 2037 2038 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 2039 2040 switch (wr->iser_ib_op) { 2041 case ISER_IB_SEND: 2042 isert_response_completion(tx_desc, isert_cmd, 2043 isert_conn, ib_dev); 2044 break; 2045 case ISER_IB_RDMA_WRITE: 2046 isert_completion_rdma_write(tx_desc, isert_cmd); 2047 break; 2048 case ISER_IB_RDMA_READ: 2049 isert_completion_rdma_read(tx_desc, isert_cmd); 2050 break; 2051 default: 2052 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 2053 dump_stack(); 2054 break; 2055 } 2056 } 2057 2058 /** 2059 * is_isert_tx_desc() - Indicate if the completion wr_id 2060 * is a TX descriptor or not. 2061 * @isert_conn: iser connection 2062 * @wr_id: completion WR identifier 2063 * 2064 * Since we cannot rely on wc opcode in FLUSH errors 2065 * we must work around it by checking if the wr_id address 2066 * falls in the iser connection rx_descs buffer. If so 2067 * it is an RX descriptor, otherwize it is a TX. 2068 */ 2069 static inline bool 2070 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 2071 { 2072 void *start = isert_conn->rx_descs; 2073 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); 2074 2075 if (wr_id >= start && wr_id < start + len) 2076 return false; 2077 2078 return true; 2079 } 2080 2081 static void 2082 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 2083 { 2084 if (wc->wr_id == ISER_BEACON_WRID) { 2085 isert_info("conn %p completing wait_comp_err\n", 2086 isert_conn); 2087 complete(&isert_conn->wait_comp_err); 2088 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 2089 struct ib_device *ib_dev = isert_conn->cm_id->device; 2090 struct isert_cmd *isert_cmd; 2091 struct iser_tx_desc *desc; 2092 2093 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2094 isert_cmd = desc->isert_cmd; 2095 if (!isert_cmd) 2096 isert_unmap_tx_desc(desc, ib_dev); 2097 else 2098 isert_completion_put(desc, isert_cmd, ib_dev, true); 2099 } else { 2100 isert_conn->post_recv_buf_count--; 2101 if (!isert_conn->post_recv_buf_count) 2102 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2103 } 2104 } 2105 2106 static void 2107 isert_handle_wc(struct ib_wc *wc) 2108 { 2109 struct isert_conn *isert_conn; 2110 struct iser_tx_desc *tx_desc; 2111 struct iser_rx_desc *rx_desc; 2112 2113 isert_conn = wc->qp->qp_context; 2114 if (likely(wc->status == IB_WC_SUCCESS)) { 2115 if (wc->opcode == IB_WC_RECV) { 2116 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 2117 isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); 2118 } else { 2119 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2120 isert_snd_completion(tx_desc, isert_conn); 2121 } 2122 } else { 2123 if (wc->status != IB_WC_WR_FLUSH_ERR) 2124 isert_err("%s (%d): wr id %llx vend_err %x\n", 2125 ib_wc_status_msg(wc->status), wc->status, 2126 wc->wr_id, wc->vendor_err); 2127 else 2128 isert_dbg("%s (%d): wr id %llx\n", 2129 ib_wc_status_msg(wc->status), wc->status, 2130 wc->wr_id); 2131 2132 if (wc->wr_id != ISER_FASTREG_LI_WRID) 2133 isert_cq_comp_err(isert_conn, wc); 2134 } 2135 } 2136 2137 static void 2138 isert_cq_work(struct work_struct *work) 2139 { 2140 enum { isert_poll_budget = 65536 }; 2141 struct isert_comp *comp = container_of(work, struct isert_comp, 2142 work); 2143 struct ib_wc *const wcs = comp->wcs; 2144 int i, n, completed = 0; 2145 2146 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 2147 for (i = 0; i < n; i++) 2148 isert_handle_wc(&wcs[i]); 2149 2150 completed += n; 2151 if (completed >= isert_poll_budget) 2152 break; 2153 } 2154 2155 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2156 } 2157 2158 static void 2159 isert_cq_callback(struct ib_cq *cq, void *context) 2160 { 2161 struct isert_comp *comp = context; 2162 2163 queue_work(isert_comp_wq, &comp->work); 2164 } 2165 2166 static int 2167 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2168 { 2169 struct ib_send_wr *wr_failed; 2170 int ret; 2171 2172 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2173 &wr_failed); 2174 if (ret) { 2175 isert_err("ib_post_send failed with %d\n", ret); 2176 return ret; 2177 } 2178 return ret; 2179 } 2180 2181 static int 2182 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2183 { 2184 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2185 struct isert_conn *isert_conn = conn->context; 2186 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2187 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2188 &isert_cmd->tx_desc.iscsi_header; 2189 2190 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2191 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2192 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2193 /* 2194 * Attach SENSE DATA payload to iSCSI Response PDU 2195 */ 2196 if (cmd->se_cmd.sense_buffer && 2197 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2198 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2199 struct isert_device *device = isert_conn->device; 2200 struct ib_device *ib_dev = device->ib_device; 2201 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2202 u32 padding, pdu_len; 2203 2204 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2205 cmd->sense_buffer); 2206 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2207 2208 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2209 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2210 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2211 2212 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2213 (void *)cmd->sense_buffer, pdu_len, 2214 DMA_TO_DEVICE); 2215 2216 isert_cmd->pdu_buf_len = pdu_len; 2217 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2218 tx_dsg->length = pdu_len; 2219 tx_dsg->lkey = device->mr->lkey; 2220 isert_cmd->tx_desc.num_sge = 2; 2221 } 2222 2223 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2224 2225 isert_dbg("Posting SCSI Response\n"); 2226 2227 return isert_post_response(isert_conn, isert_cmd); 2228 } 2229 2230 static void 2231 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2232 { 2233 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2234 struct isert_conn *isert_conn = conn->context; 2235 struct isert_device *device = isert_conn->device; 2236 2237 spin_lock_bh(&conn->cmd_lock); 2238 if (!list_empty(&cmd->i_conn_node)) 2239 list_del_init(&cmd->i_conn_node); 2240 spin_unlock_bh(&conn->cmd_lock); 2241 2242 if (cmd->data_direction == DMA_TO_DEVICE) 2243 iscsit_stop_dataout_timer(cmd); 2244 2245 device->unreg_rdma_mem(isert_cmd, isert_conn); 2246 } 2247 2248 static enum target_prot_op 2249 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2250 { 2251 struct isert_conn *isert_conn = conn->context; 2252 struct isert_device *device = isert_conn->device; 2253 2254 if (conn->tpg->tpg_attrib.t10_pi) { 2255 if (device->pi_capable) { 2256 isert_info("conn %p PI offload enabled\n", isert_conn); 2257 isert_conn->pi_support = true; 2258 return TARGET_PROT_ALL; 2259 } 2260 } 2261 2262 isert_info("conn %p PI offload disabled\n", isert_conn); 2263 isert_conn->pi_support = false; 2264 2265 return TARGET_PROT_NORMAL; 2266 } 2267 2268 static int 2269 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2270 bool nopout_response) 2271 { 2272 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2273 struct isert_conn *isert_conn = conn->context; 2274 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2275 2276 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2277 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2278 &isert_cmd->tx_desc.iscsi_header, 2279 nopout_response); 2280 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2281 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2282 2283 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2284 2285 return isert_post_response(isert_conn, isert_cmd); 2286 } 2287 2288 static int 2289 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2290 { 2291 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2292 struct isert_conn *isert_conn = conn->context; 2293 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2294 2295 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2296 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2297 &isert_cmd->tx_desc.iscsi_header); 2298 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2299 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2300 2301 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2302 2303 return isert_post_response(isert_conn, isert_cmd); 2304 } 2305 2306 static int 2307 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2308 { 2309 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2310 struct isert_conn *isert_conn = conn->context; 2311 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2312 2313 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2314 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2315 &isert_cmd->tx_desc.iscsi_header); 2316 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2317 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2318 2319 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2320 2321 return isert_post_response(isert_conn, isert_cmd); 2322 } 2323 2324 static int 2325 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2326 { 2327 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2328 struct isert_conn *isert_conn = conn->context; 2329 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2330 struct isert_device *device = isert_conn->device; 2331 struct ib_device *ib_dev = device->ib_device; 2332 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2333 struct iscsi_reject *hdr = 2334 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2335 2336 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2337 iscsit_build_reject(cmd, conn, hdr); 2338 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2339 2340 hton24(hdr->dlength, ISCSI_HDR_LEN); 2341 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2342 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2343 DMA_TO_DEVICE); 2344 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2345 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2346 tx_dsg->length = ISCSI_HDR_LEN; 2347 tx_dsg->lkey = device->mr->lkey; 2348 isert_cmd->tx_desc.num_sge = 2; 2349 2350 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2351 2352 isert_dbg("conn %p Posting Reject\n", isert_conn); 2353 2354 return isert_post_response(isert_conn, isert_cmd); 2355 } 2356 2357 static int 2358 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2359 { 2360 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2361 struct isert_conn *isert_conn = conn->context; 2362 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2363 struct iscsi_text_rsp *hdr = 2364 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2365 u32 txt_rsp_len; 2366 int rc; 2367 2368 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2369 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2370 if (rc < 0) 2371 return rc; 2372 2373 txt_rsp_len = rc; 2374 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2375 2376 if (txt_rsp_len) { 2377 struct isert_device *device = isert_conn->device; 2378 struct ib_device *ib_dev = device->ib_device; 2379 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2380 void *txt_rsp_buf = cmd->buf_ptr; 2381 2382 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2383 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2384 2385 isert_cmd->pdu_buf_len = txt_rsp_len; 2386 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2387 tx_dsg->length = txt_rsp_len; 2388 tx_dsg->lkey = device->mr->lkey; 2389 isert_cmd->tx_desc.num_sge = 2; 2390 } 2391 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2392 2393 isert_dbg("conn %p Text Response\n", isert_conn); 2394 2395 return isert_post_response(isert_conn, isert_cmd); 2396 } 2397 2398 static int 2399 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2400 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2401 u32 data_left, u32 offset) 2402 { 2403 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2404 struct scatterlist *sg_start, *tmp_sg; 2405 struct isert_device *device = isert_conn->device; 2406 struct ib_device *ib_dev = device->ib_device; 2407 u32 sg_off, page_off; 2408 int i = 0, sg_nents; 2409 2410 sg_off = offset / PAGE_SIZE; 2411 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2412 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2413 page_off = offset % PAGE_SIZE; 2414 2415 send_wr->sg_list = ib_sge; 2416 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2417 /* 2418 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2419 */ 2420 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2421 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 2422 "page_off: %u\n", 2423 (unsigned long long)tmp_sg->dma_address, 2424 tmp_sg->length, page_off); 2425 2426 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2427 ib_sge->length = min_t(u32, data_left, 2428 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2429 ib_sge->lkey = device->mr->lkey; 2430 2431 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 2432 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2433 page_off = 0; 2434 data_left -= ib_sge->length; 2435 if (!data_left) 2436 break; 2437 ib_sge++; 2438 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2439 } 2440 2441 send_wr->num_sge = ++i; 2442 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2443 send_wr->sg_list, send_wr->num_sge); 2444 2445 return send_wr->num_sge; 2446 } 2447 2448 static int 2449 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2450 struct isert_rdma_wr *wr) 2451 { 2452 struct se_cmd *se_cmd = &cmd->se_cmd; 2453 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2454 struct isert_conn *isert_conn = conn->context; 2455 struct isert_data_buf *data = &wr->data; 2456 struct ib_send_wr *send_wr; 2457 struct ib_sge *ib_sge; 2458 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2459 int ret = 0, i, ib_sge_cnt; 2460 2461 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2462 2463 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2464 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2465 se_cmd->t_data_nents, se_cmd->data_length, 2466 offset, wr->iser_ib_op, &wr->data); 2467 if (ret) 2468 return ret; 2469 2470 data_left = data->len; 2471 offset = data->offset; 2472 2473 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2474 if (!ib_sge) { 2475 isert_warn("Unable to allocate ib_sge\n"); 2476 ret = -ENOMEM; 2477 goto unmap_cmd; 2478 } 2479 wr->ib_sge = ib_sge; 2480 2481 wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2482 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2483 GFP_KERNEL); 2484 if (!wr->send_wr) { 2485 isert_dbg("Unable to allocate wr->send_wr\n"); 2486 ret = -ENOMEM; 2487 goto unmap_cmd; 2488 } 2489 2490 wr->isert_cmd = isert_cmd; 2491 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2492 2493 for (i = 0; i < wr->send_wr_num; i++) { 2494 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2495 data_len = min(data_left, rdma_write_max); 2496 2497 send_wr->send_flags = 0; 2498 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2499 send_wr->opcode = IB_WR_RDMA_WRITE; 2500 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2501 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2502 if (i + 1 == wr->send_wr_num) 2503 send_wr->next = &isert_cmd->tx_desc.send_wr; 2504 else 2505 send_wr->next = &wr->send_wr[i + 1]; 2506 } else { 2507 send_wr->opcode = IB_WR_RDMA_READ; 2508 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2509 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2510 if (i + 1 == wr->send_wr_num) 2511 send_wr->send_flags = IB_SEND_SIGNALED; 2512 else 2513 send_wr->next = &wr->send_wr[i + 1]; 2514 } 2515 2516 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2517 send_wr, data_len, offset); 2518 ib_sge += ib_sge_cnt; 2519 2520 offset += data_len; 2521 va_offset += data_len; 2522 data_left -= data_len; 2523 } 2524 2525 return 0; 2526 unmap_cmd: 2527 isert_unmap_data_buf(isert_conn, data); 2528 2529 return ret; 2530 } 2531 2532 static int 2533 isert_map_fr_pagelist(struct ib_device *ib_dev, 2534 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2535 { 2536 u64 start_addr, end_addr, page, chunk_start = 0; 2537 struct scatterlist *tmp_sg; 2538 int i = 0, new_chunk, last_ent, n_pages; 2539 2540 n_pages = 0; 2541 new_chunk = 1; 2542 last_ent = sg_nents - 1; 2543 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2544 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2545 if (new_chunk) 2546 chunk_start = start_addr; 2547 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2548 2549 isert_dbg("SGL[%d] dma_addr: 0x%llx len: %u\n", 2550 i, (unsigned long long)tmp_sg->dma_address, 2551 tmp_sg->length); 2552 2553 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2554 new_chunk = 0; 2555 continue; 2556 } 2557 new_chunk = 1; 2558 2559 page = chunk_start & PAGE_MASK; 2560 do { 2561 fr_pl[n_pages++] = page; 2562 isert_dbg("Mapped page_list[%d] page_addr: 0x%llx\n", 2563 n_pages - 1, page); 2564 page += PAGE_SIZE; 2565 } while (page < end_addr); 2566 } 2567 2568 return n_pages; 2569 } 2570 2571 static inline void 2572 isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 2573 { 2574 u32 rkey; 2575 2576 memset(inv_wr, 0, sizeof(*inv_wr)); 2577 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 2578 inv_wr->opcode = IB_WR_LOCAL_INV; 2579 inv_wr->ex.invalidate_rkey = mr->rkey; 2580 2581 /* Bump the key */ 2582 rkey = ib_inc_rkey(mr->rkey); 2583 ib_update_fast_reg_key(mr, rkey); 2584 } 2585 2586 static int 2587 isert_fast_reg_mr(struct isert_conn *isert_conn, 2588 struct fast_reg_descriptor *fr_desc, 2589 struct isert_data_buf *mem, 2590 enum isert_indicator ind, 2591 struct ib_sge *sge) 2592 { 2593 struct isert_device *device = isert_conn->device; 2594 struct ib_device *ib_dev = device->ib_device; 2595 struct ib_mr *mr; 2596 struct ib_fast_reg_page_list *frpl; 2597 struct ib_send_wr fr_wr, inv_wr; 2598 struct ib_send_wr *bad_wr, *wr = NULL; 2599 int ret, pagelist_len; 2600 u32 page_off; 2601 2602 if (mem->dma_nents == 1) { 2603 sge->lkey = device->mr->lkey; 2604 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2605 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2606 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2607 sge->addr, sge->length, sge->lkey); 2608 return 0; 2609 } 2610 2611 if (ind == ISERT_DATA_KEY_VALID) { 2612 /* Registering data buffer */ 2613 mr = fr_desc->data_mr; 2614 frpl = fr_desc->data_frpl; 2615 } else { 2616 /* Registering protection buffer */ 2617 mr = fr_desc->pi_ctx->prot_mr; 2618 frpl = fr_desc->pi_ctx->prot_frpl; 2619 } 2620 2621 page_off = mem->offset % PAGE_SIZE; 2622 2623 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2624 fr_desc, mem->nents, mem->offset); 2625 2626 pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 2627 &frpl->page_list[0]); 2628 2629 if (!(fr_desc->ind & ind)) { 2630 isert_inv_rkey(&inv_wr, mr); 2631 wr = &inv_wr; 2632 } 2633 2634 /* Prepare FASTREG WR */ 2635 memset(&fr_wr, 0, sizeof(fr_wr)); 2636 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2637 fr_wr.opcode = IB_WR_FAST_REG_MR; 2638 fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 2639 fr_wr.wr.fast_reg.page_list = frpl; 2640 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2641 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2642 fr_wr.wr.fast_reg.length = mem->len; 2643 fr_wr.wr.fast_reg.rkey = mr->rkey; 2644 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2645 2646 if (!wr) 2647 wr = &fr_wr; 2648 else 2649 wr->next = &fr_wr; 2650 2651 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2652 if (ret) { 2653 isert_err("fast registration failed, ret:%d\n", ret); 2654 return ret; 2655 } 2656 fr_desc->ind &= ~ind; 2657 2658 sge->lkey = mr->lkey; 2659 sge->addr = frpl->page_list[0] + page_off; 2660 sge->length = mem->len; 2661 2662 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2663 sge->addr, sge->length, sge->lkey); 2664 2665 return ret; 2666 } 2667 2668 static inline void 2669 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2670 struct ib_sig_domain *domain) 2671 { 2672 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2673 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2674 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2675 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2676 /* 2677 * At the moment we hard code those, but if in the future 2678 * the target core would like to use it, we will take it 2679 * from se_cmd. 2680 */ 2681 domain->sig.dif.apptag_check_mask = 0xffff; 2682 domain->sig.dif.app_escape = true; 2683 domain->sig.dif.ref_escape = true; 2684 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2685 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2686 domain->sig.dif.ref_remap = true; 2687 }; 2688 2689 static int 2690 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2691 { 2692 switch (se_cmd->prot_op) { 2693 case TARGET_PROT_DIN_INSERT: 2694 case TARGET_PROT_DOUT_STRIP: 2695 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2696 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2697 break; 2698 case TARGET_PROT_DOUT_INSERT: 2699 case TARGET_PROT_DIN_STRIP: 2700 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2701 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2702 break; 2703 case TARGET_PROT_DIN_PASS: 2704 case TARGET_PROT_DOUT_PASS: 2705 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2706 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2707 break; 2708 default: 2709 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2710 return -EINVAL; 2711 } 2712 2713 return 0; 2714 } 2715 2716 static inline u8 2717 isert_set_prot_checks(u8 prot_checks) 2718 { 2719 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2720 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2721 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2722 } 2723 2724 static int 2725 isert_reg_sig_mr(struct isert_conn *isert_conn, 2726 struct se_cmd *se_cmd, 2727 struct isert_rdma_wr *rdma_wr, 2728 struct fast_reg_descriptor *fr_desc) 2729 { 2730 struct ib_send_wr sig_wr, inv_wr; 2731 struct ib_send_wr *bad_wr, *wr = NULL; 2732 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2733 struct ib_sig_attrs sig_attrs; 2734 int ret; 2735 2736 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2737 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2738 if (ret) 2739 goto err; 2740 2741 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2742 2743 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2744 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 2745 wr = &inv_wr; 2746 } 2747 2748 memset(&sig_wr, 0, sizeof(sig_wr)); 2749 sig_wr.opcode = IB_WR_REG_SIG_MR; 2750 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2751 sig_wr.sg_list = &rdma_wr->ib_sg[DATA]; 2752 sig_wr.num_sge = 1; 2753 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 2754 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 2755 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 2756 if (se_cmd->t_prot_sg) 2757 sig_wr.wr.sig_handover.prot = &rdma_wr->ib_sg[PROT]; 2758 2759 if (!wr) 2760 wr = &sig_wr; 2761 else 2762 wr->next = &sig_wr; 2763 2764 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2765 if (ret) { 2766 isert_err("fast registration failed, ret:%d\n", ret); 2767 goto err; 2768 } 2769 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2770 2771 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2772 rdma_wr->ib_sg[SIG].addr = 0; 2773 rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 2774 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2775 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2776 /* 2777 * We have protection guards on the wire 2778 * so we need to set a larget transfer 2779 */ 2780 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 2781 2782 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2783 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2784 rdma_wr->ib_sg[SIG].lkey); 2785 err: 2786 return ret; 2787 } 2788 2789 static int 2790 isert_handle_prot_cmd(struct isert_conn *isert_conn, 2791 struct isert_cmd *isert_cmd, 2792 struct isert_rdma_wr *wr) 2793 { 2794 struct isert_device *device = isert_conn->device; 2795 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2796 int ret; 2797 2798 if (!wr->fr_desc->pi_ctx) { 2799 ret = isert_create_pi_ctx(wr->fr_desc, 2800 device->ib_device, 2801 device->pd); 2802 if (ret) { 2803 isert_err("conn %p failed to allocate pi_ctx\n", 2804 isert_conn); 2805 return ret; 2806 } 2807 } 2808 2809 if (se_cmd->t_prot_sg) { 2810 ret = isert_map_data_buf(isert_conn, isert_cmd, 2811 se_cmd->t_prot_sg, 2812 se_cmd->t_prot_nents, 2813 se_cmd->prot_length, 2814 0, wr->iser_ib_op, &wr->prot); 2815 if (ret) { 2816 isert_err("conn %p failed to map protection buffer\n", 2817 isert_conn); 2818 return ret; 2819 } 2820 2821 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2822 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2823 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2824 if (ret) { 2825 isert_err("conn %p failed to fast reg mr\n", 2826 isert_conn); 2827 goto unmap_prot_cmd; 2828 } 2829 } 2830 2831 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2832 if (ret) { 2833 isert_err("conn %p failed to fast reg mr\n", 2834 isert_conn); 2835 goto unmap_prot_cmd; 2836 } 2837 wr->fr_desc->ind |= ISERT_PROTECTED; 2838 2839 return 0; 2840 2841 unmap_prot_cmd: 2842 if (se_cmd->t_prot_sg) 2843 isert_unmap_data_buf(isert_conn, &wr->prot); 2844 2845 return ret; 2846 } 2847 2848 static int 2849 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2850 struct isert_rdma_wr *wr) 2851 { 2852 struct se_cmd *se_cmd = &cmd->se_cmd; 2853 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2854 struct isert_conn *isert_conn = conn->context; 2855 struct fast_reg_descriptor *fr_desc = NULL; 2856 struct ib_send_wr *send_wr; 2857 struct ib_sge *ib_sg; 2858 u32 offset; 2859 int ret = 0; 2860 unsigned long flags; 2861 2862 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2863 2864 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2865 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2866 se_cmd->t_data_nents, se_cmd->data_length, 2867 offset, wr->iser_ib_op, &wr->data); 2868 if (ret) 2869 return ret; 2870 2871 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2872 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2873 fr_desc = list_first_entry(&isert_conn->fr_pool, 2874 struct fast_reg_descriptor, list); 2875 list_del(&fr_desc->list); 2876 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2877 wr->fr_desc = fr_desc; 2878 } 2879 2880 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2881 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2882 if (ret) 2883 goto unmap_cmd; 2884 2885 if (isert_prot_cmd(isert_conn, se_cmd)) { 2886 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 2887 if (ret) 2888 goto unmap_cmd; 2889 2890 ib_sg = &wr->ib_sg[SIG]; 2891 } else { 2892 ib_sg = &wr->ib_sg[DATA]; 2893 } 2894 2895 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2896 wr->ib_sge = &wr->s_ib_sge; 2897 wr->send_wr_num = 1; 2898 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2899 wr->send_wr = &wr->s_send_wr; 2900 wr->isert_cmd = isert_cmd; 2901 2902 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2903 send_wr->sg_list = &wr->s_ib_sge; 2904 send_wr->num_sge = 1; 2905 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 2906 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2907 send_wr->opcode = IB_WR_RDMA_WRITE; 2908 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2909 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2910 send_wr->send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 2911 0 : IB_SEND_SIGNALED; 2912 } else { 2913 send_wr->opcode = IB_WR_RDMA_READ; 2914 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2915 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2916 send_wr->send_flags = IB_SEND_SIGNALED; 2917 } 2918 2919 return 0; 2920 2921 unmap_cmd: 2922 if (fr_desc) { 2923 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2924 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2925 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2926 } 2927 isert_unmap_data_buf(isert_conn, &wr->data); 2928 2929 return ret; 2930 } 2931 2932 static int 2933 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2934 { 2935 struct se_cmd *se_cmd = &cmd->se_cmd; 2936 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2937 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2938 struct isert_conn *isert_conn = conn->context; 2939 struct isert_device *device = isert_conn->device; 2940 struct ib_send_wr *wr_failed; 2941 int rc; 2942 2943 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2944 isert_cmd, se_cmd->data_length); 2945 2946 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2947 rc = device->reg_rdma_mem(conn, cmd, wr); 2948 if (rc) { 2949 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2950 return rc; 2951 } 2952 2953 if (!isert_prot_cmd(isert_conn, se_cmd)) { 2954 /* 2955 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2956 */ 2957 isert_create_send_desc(isert_conn, isert_cmd, 2958 &isert_cmd->tx_desc); 2959 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2960 &isert_cmd->tx_desc.iscsi_header); 2961 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2962 isert_init_send_wr(isert_conn, isert_cmd, 2963 &isert_cmd->tx_desc.send_wr); 2964 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2965 wr->send_wr_num += 1; 2966 } 2967 2968 rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed); 2969 if (rc) 2970 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2971 2972 if (!isert_prot_cmd(isert_conn, se_cmd)) 2973 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2974 "READ\n", isert_cmd); 2975 else 2976 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2977 isert_cmd); 2978 2979 return 1; 2980 } 2981 2982 static int 2983 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2984 { 2985 struct se_cmd *se_cmd = &cmd->se_cmd; 2986 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2987 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2988 struct isert_conn *isert_conn = conn->context; 2989 struct isert_device *device = isert_conn->device; 2990 struct ib_send_wr *wr_failed; 2991 int rc; 2992 2993 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2994 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2995 wr->iser_ib_op = ISER_IB_RDMA_READ; 2996 rc = device->reg_rdma_mem(conn, cmd, wr); 2997 if (rc) { 2998 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2999 return rc; 3000 } 3001 3002 rc = ib_post_send(isert_conn->qp, wr->send_wr, &wr_failed); 3003 if (rc) 3004 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 3005 3006 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 3007 isert_cmd); 3008 3009 return 0; 3010 } 3011 3012 static int 3013 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3014 { 3015 int ret; 3016 3017 switch (state) { 3018 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 3019 ret = isert_put_nopin(cmd, conn, false); 3020 break; 3021 default: 3022 isert_err("Unknown immediate state: 0x%02x\n", state); 3023 ret = -EINVAL; 3024 break; 3025 } 3026 3027 return ret; 3028 } 3029 3030 static int 3031 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 3032 { 3033 struct isert_conn *isert_conn = conn->context; 3034 int ret; 3035 3036 switch (state) { 3037 case ISTATE_SEND_LOGOUTRSP: 3038 ret = isert_put_logout_rsp(cmd, conn); 3039 if (!ret) 3040 isert_conn->logout_posted = true; 3041 break; 3042 case ISTATE_SEND_NOPIN: 3043 ret = isert_put_nopin(cmd, conn, true); 3044 break; 3045 case ISTATE_SEND_TASKMGTRSP: 3046 ret = isert_put_tm_rsp(cmd, conn); 3047 break; 3048 case ISTATE_SEND_REJECT: 3049 ret = isert_put_reject(cmd, conn); 3050 break; 3051 case ISTATE_SEND_TEXTRSP: 3052 ret = isert_put_text_rsp(cmd, conn); 3053 break; 3054 case ISTATE_SEND_STATUS: 3055 /* 3056 * Special case for sending non GOOD SCSI status from TX thread 3057 * context during pre se_cmd excecution failure. 3058 */ 3059 ret = isert_put_response(conn, cmd); 3060 break; 3061 default: 3062 isert_err("Unknown response state: 0x%02x\n", state); 3063 ret = -EINVAL; 3064 break; 3065 } 3066 3067 return ret; 3068 } 3069 3070 struct rdma_cm_id * 3071 isert_setup_id(struct isert_np *isert_np) 3072 { 3073 struct iscsi_np *np = isert_np->np; 3074 struct rdma_cm_id *id; 3075 struct sockaddr *sa; 3076 int ret; 3077 3078 sa = (struct sockaddr *)&np->np_sockaddr; 3079 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3080 3081 id = rdma_create_id(isert_cma_handler, isert_np, 3082 RDMA_PS_TCP, IB_QPT_RC); 3083 if (IS_ERR(id)) { 3084 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3085 ret = PTR_ERR(id); 3086 goto out; 3087 } 3088 isert_dbg("id %p context %p\n", id, id->context); 3089 3090 ret = rdma_bind_addr(id, sa); 3091 if (ret) { 3092 isert_err("rdma_bind_addr() failed: %d\n", ret); 3093 goto out_id; 3094 } 3095 3096 ret = rdma_listen(id, 0); 3097 if (ret) { 3098 isert_err("rdma_listen() failed: %d\n", ret); 3099 goto out_id; 3100 } 3101 3102 return id; 3103 out_id: 3104 rdma_destroy_id(id); 3105 out: 3106 return ERR_PTR(ret); 3107 } 3108 3109 static int 3110 isert_setup_np(struct iscsi_np *np, 3111 struct __kernel_sockaddr_storage *ksockaddr) 3112 { 3113 struct isert_np *isert_np; 3114 struct rdma_cm_id *isert_lid; 3115 int ret; 3116 3117 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3118 if (!isert_np) { 3119 isert_err("Unable to allocate struct isert_np\n"); 3120 return -ENOMEM; 3121 } 3122 sema_init(&isert_np->np_sem, 0); 3123 mutex_init(&isert_np->np_accept_mutex); 3124 INIT_LIST_HEAD(&isert_np->np_accept_list); 3125 init_completion(&isert_np->np_login_comp); 3126 isert_np->np = np; 3127 3128 /* 3129 * Setup the np->np_sockaddr from the passed sockaddr setup 3130 * in iscsi_target_configfs.c code.. 3131 */ 3132 memcpy(&np->np_sockaddr, ksockaddr, 3133 sizeof(struct __kernel_sockaddr_storage)); 3134 3135 isert_lid = isert_setup_id(isert_np); 3136 if (IS_ERR(isert_lid)) { 3137 ret = PTR_ERR(isert_lid); 3138 goto out; 3139 } 3140 3141 isert_np->np_cm_id = isert_lid; 3142 np->np_context = isert_np; 3143 3144 return 0; 3145 3146 out: 3147 kfree(isert_np); 3148 3149 return ret; 3150 } 3151 3152 static int 3153 isert_rdma_accept(struct isert_conn *isert_conn) 3154 { 3155 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3156 struct rdma_conn_param cp; 3157 int ret; 3158 3159 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3160 cp.initiator_depth = isert_conn->initiator_depth; 3161 cp.retry_count = 7; 3162 cp.rnr_retry_count = 7; 3163 3164 ret = rdma_accept(cm_id, &cp); 3165 if (ret) { 3166 isert_err("rdma_accept() failed with: %d\n", ret); 3167 return ret; 3168 } 3169 3170 return 0; 3171 } 3172 3173 static int 3174 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3175 { 3176 struct isert_conn *isert_conn = conn->context; 3177 int ret; 3178 3179 isert_info("before login_req comp conn: %p\n", isert_conn); 3180 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 3181 if (ret) { 3182 isert_err("isert_conn %p interrupted before got login req\n", 3183 isert_conn); 3184 return ret; 3185 } 3186 reinit_completion(&isert_conn->login_req_comp); 3187 3188 /* 3189 * For login requests after the first PDU, isert_rx_login_req() will 3190 * kick schedule_delayed_work(&conn->login_work) as the packet is 3191 * received, which turns this callback from iscsi_target_do_login_rx() 3192 * into a NOP. 3193 */ 3194 if (!login->first_request) 3195 return 0; 3196 3197 isert_rx_login_req(isert_conn); 3198 3199 isert_info("before login_comp conn: %p\n", conn); 3200 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3201 if (ret) 3202 return ret; 3203 3204 isert_info("processing login->req: %p\n", login->req); 3205 3206 return 0; 3207 } 3208 3209 static void 3210 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3211 struct isert_conn *isert_conn) 3212 { 3213 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3214 struct rdma_route *cm_route = &cm_id->route; 3215 struct sockaddr_in *sock_in; 3216 struct sockaddr_in6 *sock_in6; 3217 3218 conn->login_family = np->np_sockaddr.ss_family; 3219 3220 if (np->np_sockaddr.ss_family == AF_INET6) { 3221 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3222 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3223 &sock_in6->sin6_addr.in6_u); 3224 conn->login_port = ntohs(sock_in6->sin6_port); 3225 3226 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3227 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3228 &sock_in6->sin6_addr.in6_u); 3229 conn->local_port = ntohs(sock_in6->sin6_port); 3230 } else { 3231 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3232 sprintf(conn->login_ip, "%pI4", 3233 &sock_in->sin_addr.s_addr); 3234 conn->login_port = ntohs(sock_in->sin_port); 3235 3236 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3237 sprintf(conn->local_ip, "%pI4", 3238 &sock_in->sin_addr.s_addr); 3239 conn->local_port = ntohs(sock_in->sin_port); 3240 } 3241 } 3242 3243 static int 3244 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3245 { 3246 struct isert_np *isert_np = np->np_context; 3247 struct isert_conn *isert_conn; 3248 int ret; 3249 3250 accept_wait: 3251 ret = down_interruptible(&isert_np->np_sem); 3252 if (ret) 3253 return -ENODEV; 3254 3255 spin_lock_bh(&np->np_thread_lock); 3256 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3257 spin_unlock_bh(&np->np_thread_lock); 3258 isert_dbg("np_thread_state %d\n", 3259 np->np_thread_state); 3260 /** 3261 * No point in stalling here when np_thread 3262 * is in state RESET/SHUTDOWN/EXIT - bail 3263 **/ 3264 return -ENODEV; 3265 } 3266 spin_unlock_bh(&np->np_thread_lock); 3267 3268 mutex_lock(&isert_np->np_accept_mutex); 3269 if (list_empty(&isert_np->np_accept_list)) { 3270 mutex_unlock(&isert_np->np_accept_mutex); 3271 goto accept_wait; 3272 } 3273 isert_conn = list_first_entry(&isert_np->np_accept_list, 3274 struct isert_conn, accept_node); 3275 list_del_init(&isert_conn->accept_node); 3276 mutex_unlock(&isert_np->np_accept_mutex); 3277 3278 conn->context = isert_conn; 3279 isert_conn->conn = conn; 3280 3281 isert_set_conn_info(np, conn, isert_conn); 3282 3283 isert_dbg("Processing isert_conn: %p\n", isert_conn); 3284 3285 return 0; 3286 } 3287 3288 static void 3289 isert_free_np(struct iscsi_np *np) 3290 { 3291 struct isert_np *isert_np = np->np_context; 3292 struct isert_conn *isert_conn, *n; 3293 3294 if (isert_np->np_cm_id) 3295 rdma_destroy_id(isert_np->np_cm_id); 3296 3297 /* 3298 * FIXME: At this point we don't have a good way to insure 3299 * that at this point we don't have hanging connections that 3300 * completed RDMA establishment but didn't start iscsi login 3301 * process. So work-around this by cleaning up what ever piled 3302 * up in np_accept_list. 3303 */ 3304 mutex_lock(&isert_np->np_accept_mutex); 3305 if (!list_empty(&isert_np->np_accept_list)) { 3306 isert_info("Still have isert connections, cleaning up...\n"); 3307 list_for_each_entry_safe(isert_conn, n, 3308 &isert_np->np_accept_list, 3309 accept_node) { 3310 isert_info("cleaning isert_conn %p state (%d)\n", 3311 isert_conn, isert_conn->state); 3312 isert_connect_release(isert_conn); 3313 } 3314 } 3315 mutex_unlock(&isert_np->np_accept_mutex); 3316 3317 np->np_context = NULL; 3318 kfree(isert_np); 3319 } 3320 3321 static void isert_release_work(struct work_struct *work) 3322 { 3323 struct isert_conn *isert_conn = container_of(work, 3324 struct isert_conn, 3325 release_work); 3326 3327 isert_info("Starting release conn %p\n", isert_conn); 3328 3329 wait_for_completion(&isert_conn->wait); 3330 3331 mutex_lock(&isert_conn->mutex); 3332 isert_conn->state = ISER_CONN_DOWN; 3333 mutex_unlock(&isert_conn->mutex); 3334 3335 isert_info("Destroying conn %p\n", isert_conn); 3336 isert_put_conn(isert_conn); 3337 } 3338 3339 static void 3340 isert_wait4logout(struct isert_conn *isert_conn) 3341 { 3342 struct iscsi_conn *conn = isert_conn->conn; 3343 3344 isert_info("conn %p\n", isert_conn); 3345 3346 if (isert_conn->logout_posted) { 3347 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3348 wait_for_completion_timeout(&conn->conn_logout_comp, 3349 SECONDS_FOR_LOGOUT_COMP * HZ); 3350 } 3351 } 3352 3353 static void 3354 isert_wait4cmds(struct iscsi_conn *conn) 3355 { 3356 isert_info("iscsi_conn %p\n", conn); 3357 3358 if (conn->sess) { 3359 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3360 target_wait_for_sess_cmds(conn->sess->se_sess); 3361 } 3362 } 3363 3364 static void 3365 isert_wait4flush(struct isert_conn *isert_conn) 3366 { 3367 struct ib_recv_wr *bad_wr; 3368 3369 isert_info("conn %p\n", isert_conn); 3370 3371 init_completion(&isert_conn->wait_comp_err); 3372 isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3373 /* post an indication that all flush errors were consumed */ 3374 if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { 3375 isert_err("conn %p failed to post beacon", isert_conn); 3376 return; 3377 } 3378 3379 wait_for_completion(&isert_conn->wait_comp_err); 3380 } 3381 3382 static void isert_wait_conn(struct iscsi_conn *conn) 3383 { 3384 struct isert_conn *isert_conn = conn->context; 3385 3386 isert_info("Starting conn %p\n", isert_conn); 3387 3388 mutex_lock(&isert_conn->mutex); 3389 /* 3390 * Only wait for wait_comp_err if the isert_conn made it 3391 * into full feature phase.. 3392 */ 3393 if (isert_conn->state == ISER_CONN_INIT) { 3394 mutex_unlock(&isert_conn->mutex); 3395 return; 3396 } 3397 isert_conn_terminate(isert_conn); 3398 mutex_unlock(&isert_conn->mutex); 3399 3400 isert_wait4cmds(conn); 3401 isert_wait4flush(isert_conn); 3402 isert_wait4logout(isert_conn); 3403 3404 queue_work(isert_release_wq, &isert_conn->release_work); 3405 } 3406 3407 static void isert_free_conn(struct iscsi_conn *conn) 3408 { 3409 struct isert_conn *isert_conn = conn->context; 3410 3411 isert_wait4flush(isert_conn); 3412 isert_put_conn(isert_conn); 3413 } 3414 3415 static struct iscsit_transport iser_target_transport = { 3416 .name = "IB/iSER", 3417 .transport_type = ISCSI_INFINIBAND, 3418 .priv_size = sizeof(struct isert_cmd), 3419 .owner = THIS_MODULE, 3420 .iscsit_setup_np = isert_setup_np, 3421 .iscsit_accept_np = isert_accept_np, 3422 .iscsit_free_np = isert_free_np, 3423 .iscsit_wait_conn = isert_wait_conn, 3424 .iscsit_free_conn = isert_free_conn, 3425 .iscsit_get_login_rx = isert_get_login_rx, 3426 .iscsit_put_login_tx = isert_put_login_tx, 3427 .iscsit_immediate_queue = isert_immediate_queue, 3428 .iscsit_response_queue = isert_response_queue, 3429 .iscsit_get_dataout = isert_get_dataout, 3430 .iscsit_queue_data_in = isert_put_datain, 3431 .iscsit_queue_status = isert_put_response, 3432 .iscsit_aborted_task = isert_aborted_task, 3433 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3434 }; 3435 3436 static int __init isert_init(void) 3437 { 3438 int ret; 3439 3440 isert_comp_wq = alloc_workqueue("isert_comp_wq", 3441 WQ_UNBOUND | WQ_HIGHPRI, 0); 3442 if (!isert_comp_wq) { 3443 isert_err("Unable to allocate isert_comp_wq\n"); 3444 ret = -ENOMEM; 3445 return -ENOMEM; 3446 } 3447 3448 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3449 WQ_UNBOUND_MAX_ACTIVE); 3450 if (!isert_release_wq) { 3451 isert_err("Unable to allocate isert_release_wq\n"); 3452 ret = -ENOMEM; 3453 goto destroy_comp_wq; 3454 } 3455 3456 iscsit_register_transport(&iser_target_transport); 3457 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3458 3459 return 0; 3460 3461 destroy_comp_wq: 3462 destroy_workqueue(isert_comp_wq); 3463 3464 return ret; 3465 } 3466 3467 static void __exit isert_exit(void) 3468 { 3469 flush_scheduled_work(); 3470 destroy_workqueue(isert_release_wq); 3471 destroy_workqueue(isert_comp_wq); 3472 iscsit_unregister_transport(&iser_target_transport); 3473 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3474 } 3475 3476 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3477 MODULE_VERSION("1.0"); 3478 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3479 MODULE_LICENSE("GPL"); 3480 3481 module_init(isert_init); 3482 module_exit(isert_exit); 3483