1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 static int isert_debug_level; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static void 51 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static void 56 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 57 static int 58 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 59 struct isert_rdma_wr *wr); 60 static int 61 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 62 static int 63 isert_rdma_post_recvl(struct isert_conn *isert_conn); 64 static int 65 isert_rdma_accept(struct isert_conn *isert_conn); 66 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 67 68 static void isert_release_work(struct work_struct *work); 69 70 static inline bool 71 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 72 { 73 return (conn->pi_support && 74 cmd->prot_op != TARGET_PROT_NORMAL); 75 } 76 77 78 static void 79 isert_qp_event_callback(struct ib_event *e, void *context) 80 { 81 struct isert_conn *isert_conn = context; 82 83 isert_err("%s (%d): conn %p\n", 84 ib_event_msg(e->event), e->event, isert_conn); 85 86 switch (e->event) { 87 case IB_EVENT_COMM_EST: 88 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 89 break; 90 case IB_EVENT_QP_LAST_WQE_REACHED: 91 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 92 break; 93 default: 94 break; 95 } 96 } 97 98 static int 99 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 100 { 101 int ret; 102 103 ret = ib_query_device(ib_dev, devattr); 104 if (ret) { 105 isert_err("ib_query_device() failed: %d\n", ret); 106 return ret; 107 } 108 isert_dbg("devattr->max_sge: %d\n", devattr->max_sge); 109 isert_dbg("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 110 111 return 0; 112 } 113 114 static struct isert_comp * 115 isert_comp_get(struct isert_conn *isert_conn) 116 { 117 struct isert_device *device = isert_conn->device; 118 struct isert_comp *comp; 119 int i, min = 0; 120 121 mutex_lock(&device_list_mutex); 122 for (i = 0; i < device->comps_used; i++) 123 if (device->comps[i].active_qps < 124 device->comps[min].active_qps) 125 min = i; 126 comp = &device->comps[min]; 127 comp->active_qps++; 128 mutex_unlock(&device_list_mutex); 129 130 isert_info("conn %p, using comp %p min_index: %d\n", 131 isert_conn, comp, min); 132 133 return comp; 134 } 135 136 static void 137 isert_comp_put(struct isert_comp *comp) 138 { 139 mutex_lock(&device_list_mutex); 140 comp->active_qps--; 141 mutex_unlock(&device_list_mutex); 142 } 143 144 static struct ib_qp * 145 isert_create_qp(struct isert_conn *isert_conn, 146 struct isert_comp *comp, 147 struct rdma_cm_id *cma_id) 148 { 149 struct isert_device *device = isert_conn->device; 150 struct ib_qp_init_attr attr; 151 int ret; 152 153 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 154 attr.event_handler = isert_qp_event_callback; 155 attr.qp_context = isert_conn; 156 attr.send_cq = comp->cq; 157 attr.recv_cq = comp->cq; 158 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 159 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 160 /* 161 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 162 * work-around for RDMA_READs with ConnectX-2. 163 * 164 * Also, still make sure to have at least two SGEs for 165 * outgoing control PDU responses. 166 */ 167 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 168 isert_conn->max_sge = attr.cap.max_send_sge; 169 170 attr.cap.max_recv_sge = 1; 171 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 172 attr.qp_type = IB_QPT_RC; 173 if (device->pi_capable) 174 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 175 176 ret = rdma_create_qp(cma_id, device->pd, &attr); 177 if (ret) { 178 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 179 return ERR_PTR(ret); 180 } 181 182 return cma_id->qp; 183 } 184 185 static int 186 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 187 { 188 struct isert_comp *comp; 189 int ret; 190 191 comp = isert_comp_get(isert_conn); 192 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 193 if (IS_ERR(isert_conn->qp)) { 194 ret = PTR_ERR(isert_conn->qp); 195 goto err; 196 } 197 198 return 0; 199 err: 200 isert_comp_put(comp); 201 return ret; 202 } 203 204 static void 205 isert_cq_event_callback(struct ib_event *e, void *context) 206 { 207 isert_dbg("event: %d\n", e->event); 208 } 209 210 static int 211 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 212 { 213 struct isert_device *device = isert_conn->device; 214 struct ib_device *ib_dev = device->ib_device; 215 struct iser_rx_desc *rx_desc; 216 struct ib_sge *rx_sg; 217 u64 dma_addr; 218 int i, j; 219 220 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 221 sizeof(struct iser_rx_desc), GFP_KERNEL); 222 if (!isert_conn->rx_descs) 223 goto fail; 224 225 rx_desc = isert_conn->rx_descs; 226 227 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 228 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 229 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 230 if (ib_dma_mapping_error(ib_dev, dma_addr)) 231 goto dma_map_fail; 232 233 rx_desc->dma_addr = dma_addr; 234 235 rx_sg = &rx_desc->rx_sg; 236 rx_sg->addr = rx_desc->dma_addr; 237 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 238 rx_sg->lkey = device->pd->local_dma_lkey; 239 } 240 241 return 0; 242 243 dma_map_fail: 244 rx_desc = isert_conn->rx_descs; 245 for (j = 0; j < i; j++, rx_desc++) { 246 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 247 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 248 } 249 kfree(isert_conn->rx_descs); 250 isert_conn->rx_descs = NULL; 251 fail: 252 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 253 254 return -ENOMEM; 255 } 256 257 static void 258 isert_free_rx_descriptors(struct isert_conn *isert_conn) 259 { 260 struct ib_device *ib_dev = isert_conn->device->ib_device; 261 struct iser_rx_desc *rx_desc; 262 int i; 263 264 if (!isert_conn->rx_descs) 265 return; 266 267 rx_desc = isert_conn->rx_descs; 268 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 269 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 270 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 271 } 272 273 kfree(isert_conn->rx_descs); 274 isert_conn->rx_descs = NULL; 275 } 276 277 static void isert_cq_work(struct work_struct *); 278 static void isert_cq_callback(struct ib_cq *, void *); 279 280 static void 281 isert_free_comps(struct isert_device *device) 282 { 283 int i; 284 285 for (i = 0; i < device->comps_used; i++) { 286 struct isert_comp *comp = &device->comps[i]; 287 288 if (comp->cq) { 289 cancel_work_sync(&comp->work); 290 ib_destroy_cq(comp->cq); 291 } 292 } 293 kfree(device->comps); 294 } 295 296 static int 297 isert_alloc_comps(struct isert_device *device, 298 struct ib_device_attr *attr) 299 { 300 int i, max_cqe, ret = 0; 301 302 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 303 device->ib_device->num_comp_vectors)); 304 305 isert_info("Using %d CQs, %s supports %d vectors support " 306 "Fast registration %d pi_capable %d\n", 307 device->comps_used, device->ib_device->name, 308 device->ib_device->num_comp_vectors, device->use_fastreg, 309 device->pi_capable); 310 311 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 312 GFP_KERNEL); 313 if (!device->comps) { 314 isert_err("Unable to allocate completion contexts\n"); 315 return -ENOMEM; 316 } 317 318 max_cqe = min(ISER_MAX_CQ_LEN, attr->max_cqe); 319 320 for (i = 0; i < device->comps_used; i++) { 321 struct ib_cq_init_attr cq_attr = {}; 322 struct isert_comp *comp = &device->comps[i]; 323 324 comp->device = device; 325 INIT_WORK(&comp->work, isert_cq_work); 326 cq_attr.cqe = max_cqe; 327 cq_attr.comp_vector = i; 328 comp->cq = ib_create_cq(device->ib_device, 329 isert_cq_callback, 330 isert_cq_event_callback, 331 (void *)comp, 332 &cq_attr); 333 if (IS_ERR(comp->cq)) { 334 isert_err("Unable to allocate cq\n"); 335 ret = PTR_ERR(comp->cq); 336 comp->cq = NULL; 337 goto out_cq; 338 } 339 340 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 341 if (ret) 342 goto out_cq; 343 } 344 345 return 0; 346 out_cq: 347 isert_free_comps(device); 348 return ret; 349 } 350 351 static int 352 isert_create_device_ib_res(struct isert_device *device) 353 { 354 struct ib_device_attr *dev_attr; 355 int ret; 356 357 dev_attr = &device->dev_attr; 358 ret = isert_query_device(device->ib_device, dev_attr); 359 if (ret) 360 return ret; 361 362 /* asign function handlers */ 363 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 364 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 365 device->use_fastreg = 1; 366 device->reg_rdma_mem = isert_reg_rdma; 367 device->unreg_rdma_mem = isert_unreg_rdma; 368 } else { 369 device->use_fastreg = 0; 370 device->reg_rdma_mem = isert_map_rdma; 371 device->unreg_rdma_mem = isert_unmap_cmd; 372 } 373 374 ret = isert_alloc_comps(device, dev_attr); 375 if (ret) 376 return ret; 377 378 device->pd = ib_alloc_pd(device->ib_device); 379 if (IS_ERR(device->pd)) { 380 ret = PTR_ERR(device->pd); 381 isert_err("failed to allocate pd, device %p, ret=%d\n", 382 device, ret); 383 goto out_cq; 384 } 385 386 /* Check signature cap */ 387 device->pi_capable = dev_attr->device_cap_flags & 388 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 389 390 return 0; 391 392 out_cq: 393 isert_free_comps(device); 394 return ret; 395 } 396 397 static void 398 isert_free_device_ib_res(struct isert_device *device) 399 { 400 isert_info("device %p\n", device); 401 402 ib_dealloc_pd(device->pd); 403 isert_free_comps(device); 404 } 405 406 static void 407 isert_device_put(struct isert_device *device) 408 { 409 mutex_lock(&device_list_mutex); 410 device->refcount--; 411 isert_info("device %p refcount %d\n", device, device->refcount); 412 if (!device->refcount) { 413 isert_free_device_ib_res(device); 414 list_del(&device->dev_node); 415 kfree(device); 416 } 417 mutex_unlock(&device_list_mutex); 418 } 419 420 static struct isert_device * 421 isert_device_get(struct rdma_cm_id *cma_id) 422 { 423 struct isert_device *device; 424 int ret; 425 426 mutex_lock(&device_list_mutex); 427 list_for_each_entry(device, &device_list, dev_node) { 428 if (device->ib_device->node_guid == cma_id->device->node_guid) { 429 device->refcount++; 430 isert_info("Found iser device %p refcount %d\n", 431 device, device->refcount); 432 mutex_unlock(&device_list_mutex); 433 return device; 434 } 435 } 436 437 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 438 if (!device) { 439 mutex_unlock(&device_list_mutex); 440 return ERR_PTR(-ENOMEM); 441 } 442 443 INIT_LIST_HEAD(&device->dev_node); 444 445 device->ib_device = cma_id->device; 446 ret = isert_create_device_ib_res(device); 447 if (ret) { 448 kfree(device); 449 mutex_unlock(&device_list_mutex); 450 return ERR_PTR(ret); 451 } 452 453 device->refcount++; 454 list_add_tail(&device->dev_node, &device_list); 455 isert_info("Created a new iser device %p refcount %d\n", 456 device, device->refcount); 457 mutex_unlock(&device_list_mutex); 458 459 return device; 460 } 461 462 static void 463 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 464 { 465 struct fast_reg_descriptor *fr_desc, *tmp; 466 int i = 0; 467 468 if (list_empty(&isert_conn->fr_pool)) 469 return; 470 471 isert_info("Freeing conn %p fastreg pool", isert_conn); 472 473 list_for_each_entry_safe(fr_desc, tmp, 474 &isert_conn->fr_pool, list) { 475 list_del(&fr_desc->list); 476 ib_dereg_mr(fr_desc->data_mr); 477 if (fr_desc->pi_ctx) { 478 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 479 ib_dereg_mr(fr_desc->pi_ctx->sig_mr); 480 kfree(fr_desc->pi_ctx); 481 } 482 kfree(fr_desc); 483 ++i; 484 } 485 486 if (i < isert_conn->fr_pool_size) 487 isert_warn("Pool still has %d regions registered\n", 488 isert_conn->fr_pool_size - i); 489 } 490 491 static int 492 isert_create_pi_ctx(struct fast_reg_descriptor *desc, 493 struct ib_device *device, 494 struct ib_pd *pd) 495 { 496 struct pi_context *pi_ctx; 497 int ret; 498 499 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 500 if (!pi_ctx) { 501 isert_err("Failed to allocate pi context\n"); 502 return -ENOMEM; 503 } 504 505 pi_ctx->prot_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 506 ISCSI_ISER_SG_TABLESIZE); 507 if (IS_ERR(pi_ctx->prot_mr)) { 508 isert_err("Failed to allocate prot frmr err=%ld\n", 509 PTR_ERR(pi_ctx->prot_mr)); 510 ret = PTR_ERR(pi_ctx->prot_mr); 511 goto err_pi_ctx; 512 } 513 desc->ind |= ISERT_PROT_KEY_VALID; 514 515 pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2); 516 if (IS_ERR(pi_ctx->sig_mr)) { 517 isert_err("Failed to allocate signature enabled mr err=%ld\n", 518 PTR_ERR(pi_ctx->sig_mr)); 519 ret = PTR_ERR(pi_ctx->sig_mr); 520 goto err_prot_mr; 521 } 522 523 desc->pi_ctx = pi_ctx; 524 desc->ind |= ISERT_SIG_KEY_VALID; 525 desc->ind &= ~ISERT_PROTECTED; 526 527 return 0; 528 529 err_prot_mr: 530 ib_dereg_mr(pi_ctx->prot_mr); 531 err_pi_ctx: 532 kfree(pi_ctx); 533 534 return ret; 535 } 536 537 static int 538 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 539 struct fast_reg_descriptor *fr_desc) 540 { 541 fr_desc->data_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 542 ISCSI_ISER_SG_TABLESIZE); 543 if (IS_ERR(fr_desc->data_mr)) { 544 isert_err("Failed to allocate data frmr err=%ld\n", 545 PTR_ERR(fr_desc->data_mr)); 546 return PTR_ERR(fr_desc->data_mr); 547 } 548 fr_desc->ind |= ISERT_DATA_KEY_VALID; 549 550 isert_dbg("Created fr_desc %p\n", fr_desc); 551 552 return 0; 553 } 554 555 static int 556 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 557 { 558 struct fast_reg_descriptor *fr_desc; 559 struct isert_device *device = isert_conn->device; 560 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 561 struct se_node_acl *se_nacl = se_sess->se_node_acl; 562 int i, ret, tag_num; 563 /* 564 * Setup the number of FRMRs based upon the number of tags 565 * available to session in iscsi_target_locate_portal(). 566 */ 567 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 568 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 569 570 isert_conn->fr_pool_size = 0; 571 for (i = 0; i < tag_num; i++) { 572 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 573 if (!fr_desc) { 574 isert_err("Failed to allocate fast_reg descriptor\n"); 575 ret = -ENOMEM; 576 goto err; 577 } 578 579 ret = isert_create_fr_desc(device->ib_device, 580 device->pd, fr_desc); 581 if (ret) { 582 isert_err("Failed to create fastreg descriptor err=%d\n", 583 ret); 584 kfree(fr_desc); 585 goto err; 586 } 587 588 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 589 isert_conn->fr_pool_size++; 590 } 591 592 isert_dbg("Creating conn %p fastreg pool size=%d", 593 isert_conn, isert_conn->fr_pool_size); 594 595 return 0; 596 597 err: 598 isert_conn_free_fastreg_pool(isert_conn); 599 return ret; 600 } 601 602 static void 603 isert_init_conn(struct isert_conn *isert_conn) 604 { 605 isert_conn->state = ISER_CONN_INIT; 606 INIT_LIST_HEAD(&isert_conn->node); 607 init_completion(&isert_conn->login_comp); 608 init_completion(&isert_conn->login_req_comp); 609 init_completion(&isert_conn->wait); 610 kref_init(&isert_conn->kref); 611 mutex_init(&isert_conn->mutex); 612 spin_lock_init(&isert_conn->pool_lock); 613 INIT_LIST_HEAD(&isert_conn->fr_pool); 614 INIT_WORK(&isert_conn->release_work, isert_release_work); 615 } 616 617 static void 618 isert_free_login_buf(struct isert_conn *isert_conn) 619 { 620 struct ib_device *ib_dev = isert_conn->device->ib_device; 621 622 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 623 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 624 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 625 ISCSI_DEF_MAX_RECV_SEG_LEN, 626 DMA_FROM_DEVICE); 627 kfree(isert_conn->login_buf); 628 } 629 630 static int 631 isert_alloc_login_buf(struct isert_conn *isert_conn, 632 struct ib_device *ib_dev) 633 { 634 int ret; 635 636 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 637 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 638 if (!isert_conn->login_buf) { 639 isert_err("Unable to allocate isert_conn->login_buf\n"); 640 return -ENOMEM; 641 } 642 643 isert_conn->login_req_buf = isert_conn->login_buf; 644 isert_conn->login_rsp_buf = isert_conn->login_buf + 645 ISCSI_DEF_MAX_RECV_SEG_LEN; 646 647 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 648 isert_conn->login_buf, isert_conn->login_req_buf, 649 isert_conn->login_rsp_buf); 650 651 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 652 (void *)isert_conn->login_req_buf, 653 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 654 655 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 656 if (ret) { 657 isert_err("login_req_dma mapping error: %d\n", ret); 658 isert_conn->login_req_dma = 0; 659 goto out_login_buf; 660 } 661 662 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 663 (void *)isert_conn->login_rsp_buf, 664 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 665 666 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 667 if (ret) { 668 isert_err("login_rsp_dma mapping error: %d\n", ret); 669 isert_conn->login_rsp_dma = 0; 670 goto out_req_dma_map; 671 } 672 673 return 0; 674 675 out_req_dma_map: 676 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 677 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 678 out_login_buf: 679 kfree(isert_conn->login_buf); 680 return ret; 681 } 682 683 static int 684 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 685 { 686 struct isert_np *isert_np = cma_id->context; 687 struct iscsi_np *np = isert_np->np; 688 struct isert_conn *isert_conn; 689 struct isert_device *device; 690 int ret = 0; 691 692 spin_lock_bh(&np->np_thread_lock); 693 if (!np->enabled) { 694 spin_unlock_bh(&np->np_thread_lock); 695 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 696 return rdma_reject(cma_id, NULL, 0); 697 } 698 spin_unlock_bh(&np->np_thread_lock); 699 700 isert_dbg("cma_id: %p, portal: %p\n", 701 cma_id, cma_id->context); 702 703 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 704 if (!isert_conn) 705 return -ENOMEM; 706 707 isert_init_conn(isert_conn); 708 isert_conn->cm_id = cma_id; 709 710 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 711 if (ret) 712 goto out; 713 714 device = isert_device_get(cma_id); 715 if (IS_ERR(device)) { 716 ret = PTR_ERR(device); 717 goto out_rsp_dma_map; 718 } 719 isert_conn->device = device; 720 721 /* Set max inflight RDMA READ requests */ 722 isert_conn->initiator_depth = min_t(u8, 723 event->param.conn.initiator_depth, 724 device->dev_attr.max_qp_init_rd_atom); 725 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 726 727 ret = isert_conn_setup_qp(isert_conn, cma_id); 728 if (ret) 729 goto out_conn_dev; 730 731 ret = isert_rdma_post_recvl(isert_conn); 732 if (ret) 733 goto out_conn_dev; 734 735 ret = isert_rdma_accept(isert_conn); 736 if (ret) 737 goto out_conn_dev; 738 739 mutex_lock(&isert_np->mutex); 740 list_add_tail(&isert_conn->node, &isert_np->accepted); 741 mutex_unlock(&isert_np->mutex); 742 743 return 0; 744 745 out_conn_dev: 746 isert_device_put(device); 747 out_rsp_dma_map: 748 isert_free_login_buf(isert_conn); 749 out: 750 kfree(isert_conn); 751 rdma_reject(cma_id, NULL, 0); 752 return ret; 753 } 754 755 static void 756 isert_connect_release(struct isert_conn *isert_conn) 757 { 758 struct isert_device *device = isert_conn->device; 759 760 isert_dbg("conn %p\n", isert_conn); 761 762 BUG_ON(!device); 763 764 if (device->use_fastreg) 765 isert_conn_free_fastreg_pool(isert_conn); 766 767 isert_free_rx_descriptors(isert_conn); 768 if (isert_conn->cm_id) 769 rdma_destroy_id(isert_conn->cm_id); 770 771 if (isert_conn->qp) { 772 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 773 774 isert_comp_put(comp); 775 ib_destroy_qp(isert_conn->qp); 776 } 777 778 if (isert_conn->login_buf) 779 isert_free_login_buf(isert_conn); 780 781 isert_device_put(device); 782 783 kfree(isert_conn); 784 } 785 786 static void 787 isert_connected_handler(struct rdma_cm_id *cma_id) 788 { 789 struct isert_conn *isert_conn = cma_id->qp->qp_context; 790 struct isert_np *isert_np = cma_id->context; 791 792 isert_info("conn %p\n", isert_conn); 793 794 mutex_lock(&isert_conn->mutex); 795 isert_conn->state = ISER_CONN_UP; 796 kref_get(&isert_conn->kref); 797 mutex_unlock(&isert_conn->mutex); 798 799 mutex_lock(&isert_np->mutex); 800 list_move_tail(&isert_conn->node, &isert_np->pending); 801 mutex_unlock(&isert_np->mutex); 802 803 isert_info("np %p: Allow accept_np to continue\n", isert_np); 804 up(&isert_np->sem); 805 } 806 807 static void 808 isert_release_kref(struct kref *kref) 809 { 810 struct isert_conn *isert_conn = container_of(kref, 811 struct isert_conn, kref); 812 813 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 814 current->pid); 815 816 isert_connect_release(isert_conn); 817 } 818 819 static void 820 isert_put_conn(struct isert_conn *isert_conn) 821 { 822 kref_put(&isert_conn->kref, isert_release_kref); 823 } 824 825 /** 826 * isert_conn_terminate() - Initiate connection termination 827 * @isert_conn: isert connection struct 828 * 829 * Notes: 830 * In case the connection state is FULL_FEATURE, move state 831 * to TEMINATING and start teardown sequence (rdma_disconnect). 832 * In case the connection state is UP, complete flush as well. 833 * 834 * This routine must be called with mutex held. Thus it is 835 * safe to call multiple times. 836 */ 837 static void 838 isert_conn_terminate(struct isert_conn *isert_conn) 839 { 840 int err; 841 842 switch (isert_conn->state) { 843 case ISER_CONN_TERMINATING: 844 break; 845 case ISER_CONN_UP: 846 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 847 isert_info("Terminating conn %p state %d\n", 848 isert_conn, isert_conn->state); 849 isert_conn->state = ISER_CONN_TERMINATING; 850 err = rdma_disconnect(isert_conn->cm_id); 851 if (err) 852 isert_warn("Failed rdma_disconnect isert_conn %p\n", 853 isert_conn); 854 break; 855 default: 856 isert_warn("conn %p teminating in state %d\n", 857 isert_conn, isert_conn->state); 858 } 859 } 860 861 static int 862 isert_np_cma_handler(struct isert_np *isert_np, 863 enum rdma_cm_event_type event) 864 { 865 isert_dbg("%s (%d): isert np %p\n", 866 rdma_event_msg(event), event, isert_np); 867 868 switch (event) { 869 case RDMA_CM_EVENT_DEVICE_REMOVAL: 870 isert_np->cm_id = NULL; 871 break; 872 case RDMA_CM_EVENT_ADDR_CHANGE: 873 isert_np->cm_id = isert_setup_id(isert_np); 874 if (IS_ERR(isert_np->cm_id)) { 875 isert_err("isert np %p setup id failed: %ld\n", 876 isert_np, PTR_ERR(isert_np->cm_id)); 877 isert_np->cm_id = NULL; 878 } 879 break; 880 default: 881 isert_err("isert np %p Unexpected event %d\n", 882 isert_np, event); 883 } 884 885 return -1; 886 } 887 888 static int 889 isert_disconnected_handler(struct rdma_cm_id *cma_id, 890 enum rdma_cm_event_type event) 891 { 892 struct isert_np *isert_np = cma_id->context; 893 struct isert_conn *isert_conn; 894 bool terminating = false; 895 896 if (isert_np->cm_id == cma_id) 897 return isert_np_cma_handler(cma_id->context, event); 898 899 isert_conn = cma_id->qp->qp_context; 900 901 mutex_lock(&isert_conn->mutex); 902 terminating = (isert_conn->state == ISER_CONN_TERMINATING); 903 isert_conn_terminate(isert_conn); 904 mutex_unlock(&isert_conn->mutex); 905 906 isert_info("conn %p completing wait\n", isert_conn); 907 complete(&isert_conn->wait); 908 909 if (terminating) 910 goto out; 911 912 mutex_lock(&isert_np->mutex); 913 if (!list_empty(&isert_conn->node)) { 914 list_del_init(&isert_conn->node); 915 isert_put_conn(isert_conn); 916 queue_work(isert_release_wq, &isert_conn->release_work); 917 } 918 mutex_unlock(&isert_np->mutex); 919 920 out: 921 return 0; 922 } 923 924 static int 925 isert_connect_error(struct rdma_cm_id *cma_id) 926 { 927 struct isert_conn *isert_conn = cma_id->qp->qp_context; 928 929 list_del_init(&isert_conn->node); 930 isert_conn->cm_id = NULL; 931 isert_put_conn(isert_conn); 932 933 return -1; 934 } 935 936 static int 937 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 938 { 939 int ret = 0; 940 941 isert_info("%s (%d): status %d id %p np %p\n", 942 rdma_event_msg(event->event), event->event, 943 event->status, cma_id, cma_id->context); 944 945 switch (event->event) { 946 case RDMA_CM_EVENT_CONNECT_REQUEST: 947 ret = isert_connect_request(cma_id, event); 948 if (ret) 949 isert_err("failed handle connect request %d\n", ret); 950 break; 951 case RDMA_CM_EVENT_ESTABLISHED: 952 isert_connected_handler(cma_id); 953 break; 954 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 955 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 956 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 957 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 958 ret = isert_disconnected_handler(cma_id, event->event); 959 break; 960 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 961 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 962 case RDMA_CM_EVENT_CONNECT_ERROR: 963 ret = isert_connect_error(cma_id); 964 break; 965 default: 966 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 967 break; 968 } 969 970 return ret; 971 } 972 973 static int 974 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 975 { 976 struct ib_recv_wr *rx_wr, *rx_wr_failed; 977 int i, ret; 978 struct iser_rx_desc *rx_desc; 979 980 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 981 rx_desc = &isert_conn->rx_descs[i]; 982 rx_wr->wr_id = (uintptr_t)rx_desc; 983 rx_wr->sg_list = &rx_desc->rx_sg; 984 rx_wr->num_sge = 1; 985 rx_wr->next = rx_wr + 1; 986 } 987 rx_wr--; 988 rx_wr->next = NULL; /* mark end of work requests list */ 989 990 isert_conn->post_recv_buf_count += count; 991 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 992 &rx_wr_failed); 993 if (ret) { 994 isert_err("ib_post_recv() failed with ret: %d\n", ret); 995 isert_conn->post_recv_buf_count -= count; 996 } 997 998 return ret; 999 } 1000 1001 static int 1002 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 1003 { 1004 struct ib_recv_wr *rx_wr_failed, rx_wr; 1005 int ret; 1006 1007 rx_wr.wr_id = (uintptr_t)rx_desc; 1008 rx_wr.sg_list = &rx_desc->rx_sg; 1009 rx_wr.num_sge = 1; 1010 rx_wr.next = NULL; 1011 1012 isert_conn->post_recv_buf_count++; 1013 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 1014 if (ret) { 1015 isert_err("ib_post_recv() failed with ret: %d\n", ret); 1016 isert_conn->post_recv_buf_count--; 1017 } 1018 1019 return ret; 1020 } 1021 1022 static int 1023 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1024 { 1025 struct ib_device *ib_dev = isert_conn->cm_id->device; 1026 struct ib_send_wr send_wr, *send_wr_failed; 1027 int ret; 1028 1029 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1030 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1031 1032 send_wr.next = NULL; 1033 send_wr.wr_id = (uintptr_t)tx_desc; 1034 send_wr.sg_list = tx_desc->tx_sg; 1035 send_wr.num_sge = tx_desc->num_sge; 1036 send_wr.opcode = IB_WR_SEND; 1037 send_wr.send_flags = IB_SEND_SIGNALED; 1038 1039 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1040 if (ret) 1041 isert_err("ib_post_send() failed, ret: %d\n", ret); 1042 1043 return ret; 1044 } 1045 1046 static void 1047 isert_create_send_desc(struct isert_conn *isert_conn, 1048 struct isert_cmd *isert_cmd, 1049 struct iser_tx_desc *tx_desc) 1050 { 1051 struct isert_device *device = isert_conn->device; 1052 struct ib_device *ib_dev = device->ib_device; 1053 1054 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1055 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1056 1057 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 1058 tx_desc->iser_header.flags = ISER_VER; 1059 1060 tx_desc->num_sge = 1; 1061 tx_desc->isert_cmd = isert_cmd; 1062 1063 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 1064 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1065 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1066 } 1067 } 1068 1069 static int 1070 isert_init_tx_hdrs(struct isert_conn *isert_conn, 1071 struct iser_tx_desc *tx_desc) 1072 { 1073 struct isert_device *device = isert_conn->device; 1074 struct ib_device *ib_dev = device->ib_device; 1075 u64 dma_addr; 1076 1077 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1078 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1079 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1080 isert_err("ib_dma_mapping_error() failed\n"); 1081 return -ENOMEM; 1082 } 1083 1084 tx_desc->dma_addr = dma_addr; 1085 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1086 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1087 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1088 1089 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 1090 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 1091 tx_desc->tx_sg[0].lkey); 1092 1093 return 0; 1094 } 1095 1096 static void 1097 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1098 struct ib_send_wr *send_wr) 1099 { 1100 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 1101 1102 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1103 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1104 send_wr->opcode = IB_WR_SEND; 1105 send_wr->sg_list = &tx_desc->tx_sg[0]; 1106 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 1107 send_wr->send_flags = IB_SEND_SIGNALED; 1108 } 1109 1110 static int 1111 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1112 { 1113 struct ib_recv_wr rx_wr, *rx_wr_fail; 1114 struct ib_sge sge; 1115 int ret; 1116 1117 memset(&sge, 0, sizeof(struct ib_sge)); 1118 sge.addr = isert_conn->login_req_dma; 1119 sge.length = ISER_RX_LOGIN_SIZE; 1120 sge.lkey = isert_conn->device->pd->local_dma_lkey; 1121 1122 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1123 sge.addr, sge.length, sge.lkey); 1124 1125 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1126 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1127 rx_wr.sg_list = &sge; 1128 rx_wr.num_sge = 1; 1129 1130 isert_conn->post_recv_buf_count++; 1131 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 1132 if (ret) { 1133 isert_err("ib_post_recv() failed: %d\n", ret); 1134 isert_conn->post_recv_buf_count--; 1135 } 1136 1137 return ret; 1138 } 1139 1140 static int 1141 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1142 u32 length) 1143 { 1144 struct isert_conn *isert_conn = conn->context; 1145 struct isert_device *device = isert_conn->device; 1146 struct ib_device *ib_dev = device->ib_device; 1147 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1148 int ret; 1149 1150 isert_create_send_desc(isert_conn, NULL, tx_desc); 1151 1152 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1153 sizeof(struct iscsi_hdr)); 1154 1155 isert_init_tx_hdrs(isert_conn, tx_desc); 1156 1157 if (length > 0) { 1158 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1159 1160 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1161 length, DMA_TO_DEVICE); 1162 1163 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1164 1165 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1166 length, DMA_TO_DEVICE); 1167 1168 tx_dsg->addr = isert_conn->login_rsp_dma; 1169 tx_dsg->length = length; 1170 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1171 tx_desc->num_sge = 2; 1172 } 1173 if (!login->login_failed) { 1174 if (login->login_complete) { 1175 if (!conn->sess->sess_ops->SessionType && 1176 isert_conn->device->use_fastreg) { 1177 ret = isert_conn_create_fastreg_pool(isert_conn); 1178 if (ret) { 1179 isert_err("Conn: %p failed to create" 1180 " fastreg pool\n", isert_conn); 1181 return ret; 1182 } 1183 } 1184 1185 ret = isert_alloc_rx_descriptors(isert_conn); 1186 if (ret) 1187 return ret; 1188 1189 ret = isert_post_recvm(isert_conn, 1190 ISERT_QP_MAX_RECV_DTOS); 1191 if (ret) 1192 return ret; 1193 1194 /* Now we are in FULL_FEATURE phase */ 1195 mutex_lock(&isert_conn->mutex); 1196 isert_conn->state = ISER_CONN_FULL_FEATURE; 1197 mutex_unlock(&isert_conn->mutex); 1198 goto post_send; 1199 } 1200 1201 ret = isert_rdma_post_recvl(isert_conn); 1202 if (ret) 1203 return ret; 1204 } 1205 post_send: 1206 ret = isert_post_send(isert_conn, tx_desc); 1207 if (ret) 1208 return ret; 1209 1210 return 0; 1211 } 1212 1213 static void 1214 isert_rx_login_req(struct isert_conn *isert_conn) 1215 { 1216 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 1217 int rx_buflen = isert_conn->login_req_len; 1218 struct iscsi_conn *conn = isert_conn->conn; 1219 struct iscsi_login *login = conn->conn_login; 1220 int size; 1221 1222 isert_info("conn %p\n", isert_conn); 1223 1224 WARN_ON_ONCE(!login); 1225 1226 if (login->first_request) { 1227 struct iscsi_login_req *login_req = 1228 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1229 /* 1230 * Setup the initial iscsi_login values from the leading 1231 * login request PDU. 1232 */ 1233 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1234 login->current_stage = 1235 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1236 >> 2; 1237 login->version_min = login_req->min_version; 1238 login->version_max = login_req->max_version; 1239 memcpy(login->isid, login_req->isid, 6); 1240 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1241 login->init_task_tag = login_req->itt; 1242 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1243 login->cid = be16_to_cpu(login_req->cid); 1244 login->tsih = be16_to_cpu(login_req->tsih); 1245 } 1246 1247 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1248 1249 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1250 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1251 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1252 MAX_KEY_VALUE_PAIRS); 1253 memcpy(login->req_buf, &rx_desc->data[0], size); 1254 1255 if (login->first_request) { 1256 complete(&isert_conn->login_comp); 1257 return; 1258 } 1259 schedule_delayed_work(&conn->login_work, 0); 1260 } 1261 1262 static struct iscsi_cmd 1263 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1264 { 1265 struct isert_conn *isert_conn = conn->context; 1266 struct isert_cmd *isert_cmd; 1267 struct iscsi_cmd *cmd; 1268 1269 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1270 if (!cmd) { 1271 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1272 return NULL; 1273 } 1274 isert_cmd = iscsit_priv_cmd(cmd); 1275 isert_cmd->conn = isert_conn; 1276 isert_cmd->iscsi_cmd = cmd; 1277 isert_cmd->rx_desc = rx_desc; 1278 1279 return cmd; 1280 } 1281 1282 static int 1283 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1284 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1285 struct iser_rx_desc *rx_desc, unsigned char *buf) 1286 { 1287 struct iscsi_conn *conn = isert_conn->conn; 1288 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1289 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1290 bool dump_payload = false; 1291 unsigned int data_len; 1292 1293 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1294 if (rc < 0) 1295 return rc; 1296 1297 imm_data = cmd->immediate_data; 1298 imm_data_len = cmd->first_burst_len; 1299 unsol_data = cmd->unsolicited_data; 1300 data_len = cmd->se_cmd.data_length; 1301 1302 if (imm_data && imm_data_len == data_len) 1303 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1304 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1305 if (rc < 0) { 1306 return 0; 1307 } else if (rc > 0) { 1308 dump_payload = true; 1309 goto sequence_cmd; 1310 } 1311 1312 if (!imm_data) 1313 return 0; 1314 1315 if (imm_data_len != data_len) { 1316 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1317 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1318 &rx_desc->data[0], imm_data_len); 1319 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1320 sg_nents, imm_data_len); 1321 } else { 1322 sg_init_table(&isert_cmd->sg, 1); 1323 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1324 cmd->se_cmd.t_data_nents = 1; 1325 sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 1326 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1327 imm_data_len); 1328 } 1329 1330 cmd->write_data_done += imm_data_len; 1331 1332 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1333 spin_lock_bh(&cmd->istate_lock); 1334 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1335 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1336 spin_unlock_bh(&cmd->istate_lock); 1337 } 1338 1339 sequence_cmd: 1340 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1341 1342 if (!rc && dump_payload == false && unsol_data) 1343 iscsit_set_unsoliticed_dataout(cmd); 1344 else if (dump_payload && imm_data) 1345 target_put_sess_cmd(&cmd->se_cmd); 1346 1347 return 0; 1348 } 1349 1350 static int 1351 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1352 struct iser_rx_desc *rx_desc, unsigned char *buf) 1353 { 1354 struct scatterlist *sg_start; 1355 struct iscsi_conn *conn = isert_conn->conn; 1356 struct iscsi_cmd *cmd = NULL; 1357 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1358 u32 unsol_data_len = ntoh24(hdr->dlength); 1359 int rc, sg_nents, sg_off, page_off; 1360 1361 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1362 if (rc < 0) 1363 return rc; 1364 else if (!cmd) 1365 return 0; 1366 /* 1367 * FIXME: Unexpected unsolicited_data out 1368 */ 1369 if (!cmd->unsolicited_data) { 1370 isert_err("Received unexpected solicited data payload\n"); 1371 dump_stack(); 1372 return -1; 1373 } 1374 1375 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1376 "write_data_done: %u, data_length: %u\n", 1377 unsol_data_len, cmd->write_data_done, 1378 cmd->se_cmd.data_length); 1379 1380 sg_off = cmd->write_data_done / PAGE_SIZE; 1381 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1382 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1383 page_off = cmd->write_data_done % PAGE_SIZE; 1384 /* 1385 * FIXME: Non page-aligned unsolicited_data out 1386 */ 1387 if (page_off) { 1388 isert_err("unexpected non-page aligned data payload\n"); 1389 dump_stack(); 1390 return -1; 1391 } 1392 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1393 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1394 sg_nents, &rx_desc->data[0], unsol_data_len); 1395 1396 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1397 unsol_data_len); 1398 1399 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1400 if (rc < 0) 1401 return rc; 1402 1403 /* 1404 * multiple data-outs on the same command can arrive - 1405 * so post the buffer before hand 1406 */ 1407 rc = isert_post_recv(isert_conn, rx_desc); 1408 if (rc) { 1409 isert_err("ib_post_recv failed with %d\n", rc); 1410 return rc; 1411 } 1412 return 0; 1413 } 1414 1415 static int 1416 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1417 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1418 unsigned char *buf) 1419 { 1420 struct iscsi_conn *conn = isert_conn->conn; 1421 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1422 int rc; 1423 1424 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1425 if (rc < 0) 1426 return rc; 1427 /* 1428 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1429 */ 1430 1431 return iscsit_process_nop_out(conn, cmd, hdr); 1432 } 1433 1434 static int 1435 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1436 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1437 struct iscsi_text *hdr) 1438 { 1439 struct iscsi_conn *conn = isert_conn->conn; 1440 u32 payload_length = ntoh24(hdr->dlength); 1441 int rc; 1442 unsigned char *text_in = NULL; 1443 1444 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1445 if (rc < 0) 1446 return rc; 1447 1448 if (payload_length) { 1449 text_in = kzalloc(payload_length, GFP_KERNEL); 1450 if (!text_in) { 1451 isert_err("Unable to allocate text_in of payload_length: %u\n", 1452 payload_length); 1453 return -ENOMEM; 1454 } 1455 } 1456 cmd->text_in_ptr = text_in; 1457 1458 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1459 1460 return iscsit_process_text_cmd(conn, cmd, hdr); 1461 } 1462 1463 static int 1464 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1465 uint32_t read_stag, uint64_t read_va, 1466 uint32_t write_stag, uint64_t write_va) 1467 { 1468 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1469 struct iscsi_conn *conn = isert_conn->conn; 1470 struct iscsi_cmd *cmd; 1471 struct isert_cmd *isert_cmd; 1472 int ret = -EINVAL; 1473 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1474 1475 if (conn->sess->sess_ops->SessionType && 1476 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1477 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1478 " ignoring\n", opcode); 1479 return 0; 1480 } 1481 1482 switch (opcode) { 1483 case ISCSI_OP_SCSI_CMD: 1484 cmd = isert_allocate_cmd(conn, rx_desc); 1485 if (!cmd) 1486 break; 1487 1488 isert_cmd = iscsit_priv_cmd(cmd); 1489 isert_cmd->read_stag = read_stag; 1490 isert_cmd->read_va = read_va; 1491 isert_cmd->write_stag = write_stag; 1492 isert_cmd->write_va = write_va; 1493 1494 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1495 rx_desc, (unsigned char *)hdr); 1496 break; 1497 case ISCSI_OP_NOOP_OUT: 1498 cmd = isert_allocate_cmd(conn, rx_desc); 1499 if (!cmd) 1500 break; 1501 1502 isert_cmd = iscsit_priv_cmd(cmd); 1503 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1504 rx_desc, (unsigned char *)hdr); 1505 break; 1506 case ISCSI_OP_SCSI_DATA_OUT: 1507 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1508 (unsigned char *)hdr); 1509 break; 1510 case ISCSI_OP_SCSI_TMFUNC: 1511 cmd = isert_allocate_cmd(conn, rx_desc); 1512 if (!cmd) 1513 break; 1514 1515 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1516 (unsigned char *)hdr); 1517 break; 1518 case ISCSI_OP_LOGOUT: 1519 cmd = isert_allocate_cmd(conn, rx_desc); 1520 if (!cmd) 1521 break; 1522 1523 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1524 break; 1525 case ISCSI_OP_TEXT: 1526 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1527 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1528 else 1529 cmd = isert_allocate_cmd(conn, rx_desc); 1530 1531 if (!cmd) 1532 break; 1533 1534 isert_cmd = iscsit_priv_cmd(cmd); 1535 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1536 rx_desc, (struct iscsi_text *)hdr); 1537 break; 1538 default: 1539 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1540 dump_stack(); 1541 break; 1542 } 1543 1544 return ret; 1545 } 1546 1547 static void 1548 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1549 { 1550 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1551 uint64_t read_va = 0, write_va = 0; 1552 uint32_t read_stag = 0, write_stag = 0; 1553 1554 switch (iser_hdr->flags & 0xF0) { 1555 case ISCSI_CTRL: 1556 if (iser_hdr->flags & ISER_RSV) { 1557 read_stag = be32_to_cpu(iser_hdr->read_stag); 1558 read_va = be64_to_cpu(iser_hdr->read_va); 1559 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1560 read_stag, (unsigned long long)read_va); 1561 } 1562 if (iser_hdr->flags & ISER_WSV) { 1563 write_stag = be32_to_cpu(iser_hdr->write_stag); 1564 write_va = be64_to_cpu(iser_hdr->write_va); 1565 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1566 write_stag, (unsigned long long)write_va); 1567 } 1568 1569 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1570 break; 1571 case ISER_HELLO: 1572 isert_err("iSER Hello message\n"); 1573 break; 1574 default: 1575 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1576 break; 1577 } 1578 1579 isert_rx_opcode(isert_conn, rx_desc, 1580 read_stag, read_va, write_stag, write_va); 1581 } 1582 1583 static void 1584 isert_rcv_completion(struct iser_rx_desc *desc, 1585 struct isert_conn *isert_conn, 1586 u32 xfer_len) 1587 { 1588 struct ib_device *ib_dev = isert_conn->cm_id->device; 1589 struct iscsi_hdr *hdr; 1590 u64 rx_dma; 1591 int rx_buflen; 1592 1593 if ((char *)desc == isert_conn->login_req_buf) { 1594 rx_dma = isert_conn->login_req_dma; 1595 rx_buflen = ISER_RX_LOGIN_SIZE; 1596 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1597 rx_dma, rx_buflen); 1598 } else { 1599 rx_dma = desc->dma_addr; 1600 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1601 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1602 rx_dma, rx_buflen); 1603 } 1604 1605 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1606 1607 hdr = &desc->iscsi_header; 1608 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1609 hdr->opcode, hdr->itt, hdr->flags, 1610 (int)(xfer_len - ISER_HEADERS_LEN)); 1611 1612 if ((char *)desc == isert_conn->login_req_buf) { 1613 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 1614 if (isert_conn->conn) { 1615 struct iscsi_login *login = isert_conn->conn->conn_login; 1616 1617 if (login && !login->first_request) 1618 isert_rx_login_req(isert_conn); 1619 } 1620 mutex_lock(&isert_conn->mutex); 1621 complete(&isert_conn->login_req_comp); 1622 mutex_unlock(&isert_conn->mutex); 1623 } else { 1624 isert_rx_do_work(desc, isert_conn); 1625 } 1626 1627 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1628 DMA_FROM_DEVICE); 1629 1630 isert_conn->post_recv_buf_count--; 1631 } 1632 1633 static int 1634 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1635 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1636 enum iser_ib_op_code op, struct isert_data_buf *data) 1637 { 1638 struct ib_device *ib_dev = isert_conn->cm_id->device; 1639 1640 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1641 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1642 1643 data->len = length - offset; 1644 data->offset = offset; 1645 data->sg_off = data->offset / PAGE_SIZE; 1646 1647 data->sg = &sg[data->sg_off]; 1648 data->nents = min_t(unsigned int, nents - data->sg_off, 1649 ISCSI_ISER_SG_TABLESIZE); 1650 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1651 PAGE_SIZE); 1652 1653 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1654 data->dma_dir); 1655 if (unlikely(!data->dma_nents)) { 1656 isert_err("Cmd: unable to dma map SGs %p\n", sg); 1657 return -EINVAL; 1658 } 1659 1660 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1661 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1662 1663 return 0; 1664 } 1665 1666 static void 1667 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1668 { 1669 struct ib_device *ib_dev = isert_conn->cm_id->device; 1670 1671 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1672 memset(data, 0, sizeof(*data)); 1673 } 1674 1675 1676 1677 static void 1678 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1679 { 1680 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1681 1682 isert_dbg("Cmd %p\n", isert_cmd); 1683 1684 if (wr->data.sg) { 1685 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1686 isert_unmap_data_buf(isert_conn, &wr->data); 1687 } 1688 1689 if (wr->rdma_wr) { 1690 isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1691 kfree(wr->rdma_wr); 1692 wr->rdma_wr = NULL; 1693 } 1694 1695 if (wr->ib_sge) { 1696 isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1697 kfree(wr->ib_sge); 1698 wr->ib_sge = NULL; 1699 } 1700 } 1701 1702 static void 1703 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1704 { 1705 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1706 1707 isert_dbg("Cmd %p\n", isert_cmd); 1708 1709 if (wr->fr_desc) { 1710 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 1711 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1712 isert_unmap_data_buf(isert_conn, &wr->prot); 1713 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1714 } 1715 spin_lock_bh(&isert_conn->pool_lock); 1716 list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); 1717 spin_unlock_bh(&isert_conn->pool_lock); 1718 wr->fr_desc = NULL; 1719 } 1720 1721 if (wr->data.sg) { 1722 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1723 isert_unmap_data_buf(isert_conn, &wr->data); 1724 } 1725 1726 wr->ib_sge = NULL; 1727 wr->rdma_wr = NULL; 1728 } 1729 1730 static void 1731 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1732 { 1733 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1734 struct isert_conn *isert_conn = isert_cmd->conn; 1735 struct iscsi_conn *conn = isert_conn->conn; 1736 struct isert_device *device = isert_conn->device; 1737 struct iscsi_text_rsp *hdr; 1738 1739 isert_dbg("Cmd %p\n", isert_cmd); 1740 1741 switch (cmd->iscsi_opcode) { 1742 case ISCSI_OP_SCSI_CMD: 1743 spin_lock_bh(&conn->cmd_lock); 1744 if (!list_empty(&cmd->i_conn_node)) 1745 list_del_init(&cmd->i_conn_node); 1746 spin_unlock_bh(&conn->cmd_lock); 1747 1748 if (cmd->data_direction == DMA_TO_DEVICE) { 1749 iscsit_stop_dataout_timer(cmd); 1750 /* 1751 * Check for special case during comp_err where 1752 * WRITE_PENDING has been handed off from core, 1753 * but requires an extra target_put_sess_cmd() 1754 * before transport_generic_free_cmd() below. 1755 */ 1756 if (comp_err && 1757 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1758 struct se_cmd *se_cmd = &cmd->se_cmd; 1759 1760 target_put_sess_cmd(se_cmd); 1761 } 1762 } 1763 1764 device->unreg_rdma_mem(isert_cmd, isert_conn); 1765 transport_generic_free_cmd(&cmd->se_cmd, 0); 1766 break; 1767 case ISCSI_OP_SCSI_TMFUNC: 1768 spin_lock_bh(&conn->cmd_lock); 1769 if (!list_empty(&cmd->i_conn_node)) 1770 list_del_init(&cmd->i_conn_node); 1771 spin_unlock_bh(&conn->cmd_lock); 1772 1773 transport_generic_free_cmd(&cmd->se_cmd, 0); 1774 break; 1775 case ISCSI_OP_REJECT: 1776 case ISCSI_OP_NOOP_OUT: 1777 case ISCSI_OP_TEXT: 1778 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1779 /* If the continue bit is on, keep the command alive */ 1780 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1781 break; 1782 1783 spin_lock_bh(&conn->cmd_lock); 1784 if (!list_empty(&cmd->i_conn_node)) 1785 list_del_init(&cmd->i_conn_node); 1786 spin_unlock_bh(&conn->cmd_lock); 1787 1788 /* 1789 * Handle special case for REJECT when iscsi_add_reject*() has 1790 * overwritten the original iscsi_opcode assignment, and the 1791 * associated cmd->se_cmd needs to be released. 1792 */ 1793 if (cmd->se_cmd.se_tfo != NULL) { 1794 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1795 cmd->iscsi_opcode); 1796 transport_generic_free_cmd(&cmd->se_cmd, 0); 1797 break; 1798 } 1799 /* 1800 * Fall-through 1801 */ 1802 default: 1803 iscsit_release_cmd(cmd); 1804 break; 1805 } 1806 } 1807 1808 static void 1809 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1810 { 1811 if (tx_desc->dma_addr != 0) { 1812 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1813 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1814 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1815 tx_desc->dma_addr = 0; 1816 } 1817 } 1818 1819 static void 1820 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1821 struct ib_device *ib_dev, bool comp_err) 1822 { 1823 if (isert_cmd->pdu_buf_dma != 0) { 1824 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1825 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1826 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1827 isert_cmd->pdu_buf_dma = 0; 1828 } 1829 1830 isert_unmap_tx_desc(tx_desc, ib_dev); 1831 isert_put_cmd(isert_cmd, comp_err); 1832 } 1833 1834 static int 1835 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1836 { 1837 struct ib_mr_status mr_status; 1838 int ret; 1839 1840 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1841 if (ret) { 1842 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1843 goto fail_mr_status; 1844 } 1845 1846 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1847 u64 sec_offset_err; 1848 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1849 1850 switch (mr_status.sig_err.err_type) { 1851 case IB_SIG_BAD_GUARD: 1852 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1853 break; 1854 case IB_SIG_BAD_REFTAG: 1855 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1856 break; 1857 case IB_SIG_BAD_APPTAG: 1858 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1859 break; 1860 } 1861 sec_offset_err = mr_status.sig_err.sig_err_offset; 1862 do_div(sec_offset_err, block_size); 1863 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1864 1865 isert_err("PI error found type %d at sector 0x%llx " 1866 "expected 0x%x vs actual 0x%x\n", 1867 mr_status.sig_err.err_type, 1868 (unsigned long long)se_cmd->bad_sector, 1869 mr_status.sig_err.expected, 1870 mr_status.sig_err.actual); 1871 ret = 1; 1872 } 1873 1874 fail_mr_status: 1875 return ret; 1876 } 1877 1878 static void 1879 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1880 struct isert_cmd *isert_cmd) 1881 { 1882 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1883 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1884 struct se_cmd *se_cmd = &cmd->se_cmd; 1885 struct isert_conn *isert_conn = isert_cmd->conn; 1886 struct isert_device *device = isert_conn->device; 1887 int ret = 0; 1888 1889 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1890 ret = isert_check_pi_status(se_cmd, 1891 wr->fr_desc->pi_ctx->sig_mr); 1892 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1893 } 1894 1895 device->unreg_rdma_mem(isert_cmd, isert_conn); 1896 wr->rdma_wr_num = 0; 1897 if (ret) 1898 transport_send_check_condition_and_sense(se_cmd, 1899 se_cmd->pi_err, 0); 1900 else 1901 isert_put_response(isert_conn->conn, cmd); 1902 } 1903 1904 static void 1905 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1906 struct isert_cmd *isert_cmd) 1907 { 1908 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1909 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1910 struct se_cmd *se_cmd = &cmd->se_cmd; 1911 struct isert_conn *isert_conn = isert_cmd->conn; 1912 struct isert_device *device = isert_conn->device; 1913 int ret = 0; 1914 1915 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1916 ret = isert_check_pi_status(se_cmd, 1917 wr->fr_desc->pi_ctx->sig_mr); 1918 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1919 } 1920 1921 iscsit_stop_dataout_timer(cmd); 1922 device->unreg_rdma_mem(isert_cmd, isert_conn); 1923 cmd->write_data_done = wr->data.len; 1924 wr->rdma_wr_num = 0; 1925 1926 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1927 spin_lock_bh(&cmd->istate_lock); 1928 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1929 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1930 spin_unlock_bh(&cmd->istate_lock); 1931 1932 if (ret) { 1933 target_put_sess_cmd(se_cmd); 1934 transport_send_check_condition_and_sense(se_cmd, 1935 se_cmd->pi_err, 0); 1936 } else { 1937 target_execute_cmd(se_cmd); 1938 } 1939 } 1940 1941 static void 1942 isert_do_control_comp(struct work_struct *work) 1943 { 1944 struct isert_cmd *isert_cmd = container_of(work, 1945 struct isert_cmd, comp_work); 1946 struct isert_conn *isert_conn = isert_cmd->conn; 1947 struct ib_device *ib_dev = isert_conn->cm_id->device; 1948 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1949 1950 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1951 1952 switch (cmd->i_state) { 1953 case ISTATE_SEND_TASKMGTRSP: 1954 iscsit_tmr_post_handler(cmd, cmd->conn); 1955 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1956 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1957 cmd->i_state = ISTATE_SENT_STATUS; 1958 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1959 ib_dev, false); 1960 break; 1961 case ISTATE_SEND_LOGOUTRSP: 1962 iscsit_logout_post_handler(cmd, cmd->conn); 1963 break; 1964 default: 1965 isert_err("Unknown i_state %d\n", cmd->i_state); 1966 dump_stack(); 1967 break; 1968 } 1969 } 1970 1971 static void 1972 isert_response_completion(struct iser_tx_desc *tx_desc, 1973 struct isert_cmd *isert_cmd, 1974 struct isert_conn *isert_conn, 1975 struct ib_device *ib_dev) 1976 { 1977 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1978 1979 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1980 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1981 cmd->i_state == ISTATE_SEND_REJECT || 1982 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1983 isert_unmap_tx_desc(tx_desc, ib_dev); 1984 1985 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1986 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1987 return; 1988 } 1989 1990 cmd->i_state = ISTATE_SENT_STATUS; 1991 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1992 } 1993 1994 static void 1995 isert_snd_completion(struct iser_tx_desc *tx_desc, 1996 struct isert_conn *isert_conn) 1997 { 1998 struct ib_device *ib_dev = isert_conn->cm_id->device; 1999 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2000 struct isert_rdma_wr *wr; 2001 2002 if (!isert_cmd) { 2003 isert_unmap_tx_desc(tx_desc, ib_dev); 2004 return; 2005 } 2006 wr = &isert_cmd->rdma_wr; 2007 2008 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 2009 2010 switch (wr->iser_ib_op) { 2011 case ISER_IB_SEND: 2012 isert_response_completion(tx_desc, isert_cmd, 2013 isert_conn, ib_dev); 2014 break; 2015 case ISER_IB_RDMA_WRITE: 2016 isert_completion_rdma_write(tx_desc, isert_cmd); 2017 break; 2018 case ISER_IB_RDMA_READ: 2019 isert_completion_rdma_read(tx_desc, isert_cmd); 2020 break; 2021 default: 2022 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 2023 dump_stack(); 2024 break; 2025 } 2026 } 2027 2028 /** 2029 * is_isert_tx_desc() - Indicate if the completion wr_id 2030 * is a TX descriptor or not. 2031 * @isert_conn: iser connection 2032 * @wr_id: completion WR identifier 2033 * 2034 * Since we cannot rely on wc opcode in FLUSH errors 2035 * we must work around it by checking if the wr_id address 2036 * falls in the iser connection rx_descs buffer. If so 2037 * it is an RX descriptor, otherwize it is a TX. 2038 */ 2039 static inline bool 2040 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 2041 { 2042 void *start = isert_conn->rx_descs; 2043 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); 2044 2045 if (wr_id >= start && wr_id < start + len) 2046 return false; 2047 2048 return true; 2049 } 2050 2051 static void 2052 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 2053 { 2054 if (wc->wr_id == ISER_BEACON_WRID) { 2055 isert_info("conn %p completing wait_comp_err\n", 2056 isert_conn); 2057 complete(&isert_conn->wait_comp_err); 2058 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 2059 struct ib_device *ib_dev = isert_conn->cm_id->device; 2060 struct isert_cmd *isert_cmd; 2061 struct iser_tx_desc *desc; 2062 2063 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2064 isert_cmd = desc->isert_cmd; 2065 if (!isert_cmd) 2066 isert_unmap_tx_desc(desc, ib_dev); 2067 else 2068 isert_completion_put(desc, isert_cmd, ib_dev, true); 2069 } else { 2070 isert_conn->post_recv_buf_count--; 2071 if (!isert_conn->post_recv_buf_count) 2072 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2073 } 2074 } 2075 2076 static void 2077 isert_handle_wc(struct ib_wc *wc) 2078 { 2079 struct isert_conn *isert_conn; 2080 struct iser_tx_desc *tx_desc; 2081 struct iser_rx_desc *rx_desc; 2082 2083 isert_conn = wc->qp->qp_context; 2084 if (likely(wc->status == IB_WC_SUCCESS)) { 2085 if (wc->opcode == IB_WC_RECV) { 2086 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 2087 isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); 2088 } else { 2089 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2090 isert_snd_completion(tx_desc, isert_conn); 2091 } 2092 } else { 2093 if (wc->status != IB_WC_WR_FLUSH_ERR) 2094 isert_err("%s (%d): wr id %llx vend_err %x\n", 2095 ib_wc_status_msg(wc->status), wc->status, 2096 wc->wr_id, wc->vendor_err); 2097 else 2098 isert_dbg("%s (%d): wr id %llx\n", 2099 ib_wc_status_msg(wc->status), wc->status, 2100 wc->wr_id); 2101 2102 if (wc->wr_id != ISER_FASTREG_LI_WRID) 2103 isert_cq_comp_err(isert_conn, wc); 2104 } 2105 } 2106 2107 static void 2108 isert_cq_work(struct work_struct *work) 2109 { 2110 enum { isert_poll_budget = 65536 }; 2111 struct isert_comp *comp = container_of(work, struct isert_comp, 2112 work); 2113 struct ib_wc *const wcs = comp->wcs; 2114 int i, n, completed = 0; 2115 2116 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 2117 for (i = 0; i < n; i++) 2118 isert_handle_wc(&wcs[i]); 2119 2120 completed += n; 2121 if (completed >= isert_poll_budget) 2122 break; 2123 } 2124 2125 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2126 } 2127 2128 static void 2129 isert_cq_callback(struct ib_cq *cq, void *context) 2130 { 2131 struct isert_comp *comp = context; 2132 2133 queue_work(isert_comp_wq, &comp->work); 2134 } 2135 2136 static int 2137 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2138 { 2139 struct ib_send_wr *wr_failed; 2140 int ret; 2141 2142 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2143 if (ret) { 2144 isert_err("ib_post_recv failed with %d\n", ret); 2145 return ret; 2146 } 2147 2148 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2149 &wr_failed); 2150 if (ret) { 2151 isert_err("ib_post_send failed with %d\n", ret); 2152 return ret; 2153 } 2154 return ret; 2155 } 2156 2157 static int 2158 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2159 { 2160 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2161 struct isert_conn *isert_conn = conn->context; 2162 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2163 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2164 &isert_cmd->tx_desc.iscsi_header; 2165 2166 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2167 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2168 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2169 /* 2170 * Attach SENSE DATA payload to iSCSI Response PDU 2171 */ 2172 if (cmd->se_cmd.sense_buffer && 2173 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2174 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2175 struct isert_device *device = isert_conn->device; 2176 struct ib_device *ib_dev = device->ib_device; 2177 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2178 u32 padding, pdu_len; 2179 2180 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2181 cmd->sense_buffer); 2182 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2183 2184 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2185 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2186 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2187 2188 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2189 (void *)cmd->sense_buffer, pdu_len, 2190 DMA_TO_DEVICE); 2191 2192 isert_cmd->pdu_buf_len = pdu_len; 2193 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2194 tx_dsg->length = pdu_len; 2195 tx_dsg->lkey = device->pd->local_dma_lkey; 2196 isert_cmd->tx_desc.num_sge = 2; 2197 } 2198 2199 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2200 2201 isert_dbg("Posting SCSI Response\n"); 2202 2203 return isert_post_response(isert_conn, isert_cmd); 2204 } 2205 2206 static void 2207 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2208 { 2209 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2210 struct isert_conn *isert_conn = conn->context; 2211 struct isert_device *device = isert_conn->device; 2212 2213 spin_lock_bh(&conn->cmd_lock); 2214 if (!list_empty(&cmd->i_conn_node)) 2215 list_del_init(&cmd->i_conn_node); 2216 spin_unlock_bh(&conn->cmd_lock); 2217 2218 if (cmd->data_direction == DMA_TO_DEVICE) 2219 iscsit_stop_dataout_timer(cmd); 2220 2221 device->unreg_rdma_mem(isert_cmd, isert_conn); 2222 } 2223 2224 static enum target_prot_op 2225 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2226 { 2227 struct isert_conn *isert_conn = conn->context; 2228 struct isert_device *device = isert_conn->device; 2229 2230 if (conn->tpg->tpg_attrib.t10_pi) { 2231 if (device->pi_capable) { 2232 isert_info("conn %p PI offload enabled\n", isert_conn); 2233 isert_conn->pi_support = true; 2234 return TARGET_PROT_ALL; 2235 } 2236 } 2237 2238 isert_info("conn %p PI offload disabled\n", isert_conn); 2239 isert_conn->pi_support = false; 2240 2241 return TARGET_PROT_NORMAL; 2242 } 2243 2244 static int 2245 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2246 bool nopout_response) 2247 { 2248 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2249 struct isert_conn *isert_conn = conn->context; 2250 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2251 2252 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2253 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2254 &isert_cmd->tx_desc.iscsi_header, 2255 nopout_response); 2256 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2257 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2258 2259 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2260 2261 return isert_post_response(isert_conn, isert_cmd); 2262 } 2263 2264 static int 2265 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2266 { 2267 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2268 struct isert_conn *isert_conn = conn->context; 2269 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2270 2271 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2272 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2273 &isert_cmd->tx_desc.iscsi_header); 2274 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2275 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2276 2277 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2278 2279 return isert_post_response(isert_conn, isert_cmd); 2280 } 2281 2282 static int 2283 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2284 { 2285 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2286 struct isert_conn *isert_conn = conn->context; 2287 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2288 2289 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2290 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2291 &isert_cmd->tx_desc.iscsi_header); 2292 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2293 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2294 2295 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2296 2297 return isert_post_response(isert_conn, isert_cmd); 2298 } 2299 2300 static int 2301 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2302 { 2303 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2304 struct isert_conn *isert_conn = conn->context; 2305 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2306 struct isert_device *device = isert_conn->device; 2307 struct ib_device *ib_dev = device->ib_device; 2308 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2309 struct iscsi_reject *hdr = 2310 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2311 2312 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2313 iscsit_build_reject(cmd, conn, hdr); 2314 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2315 2316 hton24(hdr->dlength, ISCSI_HDR_LEN); 2317 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2318 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2319 DMA_TO_DEVICE); 2320 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2321 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2322 tx_dsg->length = ISCSI_HDR_LEN; 2323 tx_dsg->lkey = device->pd->local_dma_lkey; 2324 isert_cmd->tx_desc.num_sge = 2; 2325 2326 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2327 2328 isert_dbg("conn %p Posting Reject\n", isert_conn); 2329 2330 return isert_post_response(isert_conn, isert_cmd); 2331 } 2332 2333 static int 2334 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2335 { 2336 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2337 struct isert_conn *isert_conn = conn->context; 2338 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2339 struct iscsi_text_rsp *hdr = 2340 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2341 u32 txt_rsp_len; 2342 int rc; 2343 2344 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2345 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2346 if (rc < 0) 2347 return rc; 2348 2349 txt_rsp_len = rc; 2350 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2351 2352 if (txt_rsp_len) { 2353 struct isert_device *device = isert_conn->device; 2354 struct ib_device *ib_dev = device->ib_device; 2355 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2356 void *txt_rsp_buf = cmd->buf_ptr; 2357 2358 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2359 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2360 2361 isert_cmd->pdu_buf_len = txt_rsp_len; 2362 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2363 tx_dsg->length = txt_rsp_len; 2364 tx_dsg->lkey = device->pd->local_dma_lkey; 2365 isert_cmd->tx_desc.num_sge = 2; 2366 } 2367 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2368 2369 isert_dbg("conn %p Text Response\n", isert_conn); 2370 2371 return isert_post_response(isert_conn, isert_cmd); 2372 } 2373 2374 static int 2375 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2376 struct ib_sge *ib_sge, struct ib_rdma_wr *rdma_wr, 2377 u32 data_left, u32 offset) 2378 { 2379 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2380 struct scatterlist *sg_start, *tmp_sg; 2381 struct isert_device *device = isert_conn->device; 2382 struct ib_device *ib_dev = device->ib_device; 2383 u32 sg_off, page_off; 2384 int i = 0, sg_nents; 2385 2386 sg_off = offset / PAGE_SIZE; 2387 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2388 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2389 page_off = offset % PAGE_SIZE; 2390 2391 rdma_wr->wr.sg_list = ib_sge; 2392 rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; 2393 /* 2394 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2395 */ 2396 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2397 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 2398 "page_off: %u\n", 2399 (unsigned long long)tmp_sg->dma_address, 2400 tmp_sg->length, page_off); 2401 2402 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2403 ib_sge->length = min_t(u32, data_left, 2404 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2405 ib_sge->lkey = device->pd->local_dma_lkey; 2406 2407 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 2408 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2409 page_off = 0; 2410 data_left -= ib_sge->length; 2411 if (!data_left) 2412 break; 2413 ib_sge++; 2414 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2415 } 2416 2417 rdma_wr->wr.num_sge = ++i; 2418 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2419 rdma_wr->wr.sg_list, rdma_wr->wr.num_sge); 2420 2421 return rdma_wr->wr.num_sge; 2422 } 2423 2424 static int 2425 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2426 struct isert_rdma_wr *wr) 2427 { 2428 struct se_cmd *se_cmd = &cmd->se_cmd; 2429 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2430 struct isert_conn *isert_conn = conn->context; 2431 struct isert_data_buf *data = &wr->data; 2432 struct ib_rdma_wr *rdma_wr; 2433 struct ib_sge *ib_sge; 2434 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2435 int ret = 0, i, ib_sge_cnt; 2436 2437 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2438 2439 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2440 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2441 se_cmd->t_data_nents, se_cmd->data_length, 2442 offset, wr->iser_ib_op, &wr->data); 2443 if (ret) 2444 return ret; 2445 2446 data_left = data->len; 2447 offset = data->offset; 2448 2449 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2450 if (!ib_sge) { 2451 isert_warn("Unable to allocate ib_sge\n"); 2452 ret = -ENOMEM; 2453 goto unmap_cmd; 2454 } 2455 wr->ib_sge = ib_sge; 2456 2457 wr->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2458 wr->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * wr->rdma_wr_num, 2459 GFP_KERNEL); 2460 if (!wr->rdma_wr) { 2461 isert_dbg("Unable to allocate wr->rdma_wr\n"); 2462 ret = -ENOMEM; 2463 goto unmap_cmd; 2464 } 2465 2466 wr->isert_cmd = isert_cmd; 2467 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2468 2469 for (i = 0; i < wr->rdma_wr_num; i++) { 2470 rdma_wr = &isert_cmd->rdma_wr.rdma_wr[i]; 2471 data_len = min(data_left, rdma_write_max); 2472 2473 rdma_wr->wr.send_flags = 0; 2474 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2475 rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2476 rdma_wr->remote_addr = isert_cmd->read_va + offset; 2477 rdma_wr->rkey = isert_cmd->read_stag; 2478 if (i + 1 == wr->rdma_wr_num) 2479 rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr; 2480 else 2481 rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; 2482 } else { 2483 rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2484 rdma_wr->remote_addr = isert_cmd->write_va + va_offset; 2485 rdma_wr->rkey = isert_cmd->write_stag; 2486 if (i + 1 == wr->rdma_wr_num) 2487 rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 2488 else 2489 rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; 2490 } 2491 2492 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2493 rdma_wr, data_len, offset); 2494 ib_sge += ib_sge_cnt; 2495 2496 offset += data_len; 2497 va_offset += data_len; 2498 data_left -= data_len; 2499 } 2500 2501 return 0; 2502 unmap_cmd: 2503 isert_unmap_data_buf(isert_conn, data); 2504 2505 return ret; 2506 } 2507 2508 static inline void 2509 isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 2510 { 2511 u32 rkey; 2512 2513 memset(inv_wr, 0, sizeof(*inv_wr)); 2514 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 2515 inv_wr->opcode = IB_WR_LOCAL_INV; 2516 inv_wr->ex.invalidate_rkey = mr->rkey; 2517 2518 /* Bump the key */ 2519 rkey = ib_inc_rkey(mr->rkey); 2520 ib_update_fast_reg_key(mr, rkey); 2521 } 2522 2523 static int 2524 isert_fast_reg_mr(struct isert_conn *isert_conn, 2525 struct fast_reg_descriptor *fr_desc, 2526 struct isert_data_buf *mem, 2527 enum isert_indicator ind, 2528 struct ib_sge *sge) 2529 { 2530 struct isert_device *device = isert_conn->device; 2531 struct ib_device *ib_dev = device->ib_device; 2532 struct ib_mr *mr; 2533 struct ib_reg_wr reg_wr; 2534 struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 2535 int ret, n; 2536 2537 if (mem->dma_nents == 1) { 2538 sge->lkey = device->pd->local_dma_lkey; 2539 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2540 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2541 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2542 sge->addr, sge->length, sge->lkey); 2543 return 0; 2544 } 2545 2546 if (ind == ISERT_DATA_KEY_VALID) 2547 /* Registering data buffer */ 2548 mr = fr_desc->data_mr; 2549 else 2550 /* Registering protection buffer */ 2551 mr = fr_desc->pi_ctx->prot_mr; 2552 2553 if (!(fr_desc->ind & ind)) { 2554 isert_inv_rkey(&inv_wr, mr); 2555 wr = &inv_wr; 2556 } 2557 2558 n = ib_map_mr_sg(mr, mem->sg, mem->nents, PAGE_SIZE); 2559 if (unlikely(n != mem->nents)) { 2560 isert_err("failed to map mr sg (%d/%d)\n", 2561 n, mem->nents); 2562 return n < 0 ? n : -EINVAL; 2563 } 2564 2565 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2566 fr_desc, mem->nents, mem->offset); 2567 2568 reg_wr.wr.next = NULL; 2569 reg_wr.wr.opcode = IB_WR_REG_MR; 2570 reg_wr.wr.wr_id = ISER_FASTREG_LI_WRID; 2571 reg_wr.wr.send_flags = 0; 2572 reg_wr.wr.num_sge = 0; 2573 reg_wr.mr = mr; 2574 reg_wr.key = mr->lkey; 2575 reg_wr.access = IB_ACCESS_LOCAL_WRITE; 2576 2577 if (!wr) 2578 wr = ®_wr.wr; 2579 else 2580 wr->next = ®_wr.wr; 2581 2582 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2583 if (ret) { 2584 isert_err("fast registration failed, ret:%d\n", ret); 2585 return ret; 2586 } 2587 fr_desc->ind &= ~ind; 2588 2589 sge->lkey = mr->lkey; 2590 sge->addr = mr->iova; 2591 sge->length = mr->length; 2592 2593 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2594 sge->addr, sge->length, sge->lkey); 2595 2596 return ret; 2597 } 2598 2599 static inline void 2600 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2601 struct ib_sig_domain *domain) 2602 { 2603 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2604 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2605 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2606 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2607 /* 2608 * At the moment we hard code those, but if in the future 2609 * the target core would like to use it, we will take it 2610 * from se_cmd. 2611 */ 2612 domain->sig.dif.apptag_check_mask = 0xffff; 2613 domain->sig.dif.app_escape = true; 2614 domain->sig.dif.ref_escape = true; 2615 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2616 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2617 domain->sig.dif.ref_remap = true; 2618 }; 2619 2620 static int 2621 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2622 { 2623 switch (se_cmd->prot_op) { 2624 case TARGET_PROT_DIN_INSERT: 2625 case TARGET_PROT_DOUT_STRIP: 2626 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2627 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2628 break; 2629 case TARGET_PROT_DOUT_INSERT: 2630 case TARGET_PROT_DIN_STRIP: 2631 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2632 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2633 break; 2634 case TARGET_PROT_DIN_PASS: 2635 case TARGET_PROT_DOUT_PASS: 2636 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2637 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2638 break; 2639 default: 2640 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2641 return -EINVAL; 2642 } 2643 2644 return 0; 2645 } 2646 2647 static inline u8 2648 isert_set_prot_checks(u8 prot_checks) 2649 { 2650 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2651 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2652 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2653 } 2654 2655 static int 2656 isert_reg_sig_mr(struct isert_conn *isert_conn, 2657 struct se_cmd *se_cmd, 2658 struct isert_rdma_wr *rdma_wr, 2659 struct fast_reg_descriptor *fr_desc) 2660 { 2661 struct ib_sig_handover_wr sig_wr; 2662 struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 2663 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2664 struct ib_sig_attrs sig_attrs; 2665 int ret; 2666 2667 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2668 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2669 if (ret) 2670 goto err; 2671 2672 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2673 2674 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2675 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 2676 wr = &inv_wr; 2677 } 2678 2679 memset(&sig_wr, 0, sizeof(sig_wr)); 2680 sig_wr.wr.opcode = IB_WR_REG_SIG_MR; 2681 sig_wr.wr.wr_id = ISER_FASTREG_LI_WRID; 2682 sig_wr.wr.sg_list = &rdma_wr->ib_sg[DATA]; 2683 sig_wr.wr.num_sge = 1; 2684 sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; 2685 sig_wr.sig_attrs = &sig_attrs; 2686 sig_wr.sig_mr = pi_ctx->sig_mr; 2687 if (se_cmd->t_prot_sg) 2688 sig_wr.prot = &rdma_wr->ib_sg[PROT]; 2689 2690 if (!wr) 2691 wr = &sig_wr.wr; 2692 else 2693 wr->next = &sig_wr.wr; 2694 2695 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2696 if (ret) { 2697 isert_err("fast registration failed, ret:%d\n", ret); 2698 goto err; 2699 } 2700 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2701 2702 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2703 rdma_wr->ib_sg[SIG].addr = 0; 2704 rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 2705 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2706 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2707 /* 2708 * We have protection guards on the wire 2709 * so we need to set a larget transfer 2710 */ 2711 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 2712 2713 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2714 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2715 rdma_wr->ib_sg[SIG].lkey); 2716 err: 2717 return ret; 2718 } 2719 2720 static int 2721 isert_handle_prot_cmd(struct isert_conn *isert_conn, 2722 struct isert_cmd *isert_cmd, 2723 struct isert_rdma_wr *wr) 2724 { 2725 struct isert_device *device = isert_conn->device; 2726 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2727 int ret; 2728 2729 if (!wr->fr_desc->pi_ctx) { 2730 ret = isert_create_pi_ctx(wr->fr_desc, 2731 device->ib_device, 2732 device->pd); 2733 if (ret) { 2734 isert_err("conn %p failed to allocate pi_ctx\n", 2735 isert_conn); 2736 return ret; 2737 } 2738 } 2739 2740 if (se_cmd->t_prot_sg) { 2741 ret = isert_map_data_buf(isert_conn, isert_cmd, 2742 se_cmd->t_prot_sg, 2743 se_cmd->t_prot_nents, 2744 se_cmd->prot_length, 2745 0, wr->iser_ib_op, &wr->prot); 2746 if (ret) { 2747 isert_err("conn %p failed to map protection buffer\n", 2748 isert_conn); 2749 return ret; 2750 } 2751 2752 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2753 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2754 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2755 if (ret) { 2756 isert_err("conn %p failed to fast reg mr\n", 2757 isert_conn); 2758 goto unmap_prot_cmd; 2759 } 2760 } 2761 2762 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2763 if (ret) { 2764 isert_err("conn %p failed to fast reg mr\n", 2765 isert_conn); 2766 goto unmap_prot_cmd; 2767 } 2768 wr->fr_desc->ind |= ISERT_PROTECTED; 2769 2770 return 0; 2771 2772 unmap_prot_cmd: 2773 if (se_cmd->t_prot_sg) 2774 isert_unmap_data_buf(isert_conn, &wr->prot); 2775 2776 return ret; 2777 } 2778 2779 static int 2780 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2781 struct isert_rdma_wr *wr) 2782 { 2783 struct se_cmd *se_cmd = &cmd->se_cmd; 2784 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2785 struct isert_conn *isert_conn = conn->context; 2786 struct fast_reg_descriptor *fr_desc = NULL; 2787 struct ib_rdma_wr *rdma_wr; 2788 struct ib_sge *ib_sg; 2789 u32 offset; 2790 int ret = 0; 2791 unsigned long flags; 2792 2793 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2794 2795 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2796 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2797 se_cmd->t_data_nents, se_cmd->data_length, 2798 offset, wr->iser_ib_op, &wr->data); 2799 if (ret) 2800 return ret; 2801 2802 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2803 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2804 fr_desc = list_first_entry(&isert_conn->fr_pool, 2805 struct fast_reg_descriptor, list); 2806 list_del(&fr_desc->list); 2807 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2808 wr->fr_desc = fr_desc; 2809 } 2810 2811 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2812 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2813 if (ret) 2814 goto unmap_cmd; 2815 2816 if (isert_prot_cmd(isert_conn, se_cmd)) { 2817 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 2818 if (ret) 2819 goto unmap_cmd; 2820 2821 ib_sg = &wr->ib_sg[SIG]; 2822 } else { 2823 ib_sg = &wr->ib_sg[DATA]; 2824 } 2825 2826 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2827 wr->ib_sge = &wr->s_ib_sge; 2828 wr->rdma_wr_num = 1; 2829 memset(&wr->s_rdma_wr, 0, sizeof(wr->s_rdma_wr)); 2830 wr->rdma_wr = &wr->s_rdma_wr; 2831 wr->isert_cmd = isert_cmd; 2832 2833 rdma_wr = &isert_cmd->rdma_wr.s_rdma_wr; 2834 rdma_wr->wr.sg_list = &wr->s_ib_sge; 2835 rdma_wr->wr.num_sge = 1; 2836 rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; 2837 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2838 rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2839 rdma_wr->remote_addr = isert_cmd->read_va; 2840 rdma_wr->rkey = isert_cmd->read_stag; 2841 rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 2842 0 : IB_SEND_SIGNALED; 2843 } else { 2844 rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2845 rdma_wr->remote_addr = isert_cmd->write_va; 2846 rdma_wr->rkey = isert_cmd->write_stag; 2847 rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 2848 } 2849 2850 return 0; 2851 2852 unmap_cmd: 2853 if (fr_desc) { 2854 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2855 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2856 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2857 } 2858 isert_unmap_data_buf(isert_conn, &wr->data); 2859 2860 return ret; 2861 } 2862 2863 static int 2864 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2865 { 2866 struct se_cmd *se_cmd = &cmd->se_cmd; 2867 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2868 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2869 struct isert_conn *isert_conn = conn->context; 2870 struct isert_device *device = isert_conn->device; 2871 struct ib_send_wr *wr_failed; 2872 int rc; 2873 2874 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2875 isert_cmd, se_cmd->data_length); 2876 2877 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2878 rc = device->reg_rdma_mem(conn, cmd, wr); 2879 if (rc) { 2880 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2881 return rc; 2882 } 2883 2884 if (!isert_prot_cmd(isert_conn, se_cmd)) { 2885 /* 2886 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2887 */ 2888 isert_create_send_desc(isert_conn, isert_cmd, 2889 &isert_cmd->tx_desc); 2890 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2891 &isert_cmd->tx_desc.iscsi_header); 2892 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2893 isert_init_send_wr(isert_conn, isert_cmd, 2894 &isert_cmd->tx_desc.send_wr); 2895 isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr; 2896 wr->rdma_wr_num += 1; 2897 2898 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2899 if (rc) { 2900 isert_err("ib_post_recv failed with %d\n", rc); 2901 return rc; 2902 } 2903 } 2904 2905 rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); 2906 if (rc) 2907 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2908 2909 if (!isert_prot_cmd(isert_conn, se_cmd)) 2910 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2911 "READ\n", isert_cmd); 2912 else 2913 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2914 isert_cmd); 2915 2916 return 1; 2917 } 2918 2919 static int 2920 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2921 { 2922 struct se_cmd *se_cmd = &cmd->se_cmd; 2923 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2924 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2925 struct isert_conn *isert_conn = conn->context; 2926 struct isert_device *device = isert_conn->device; 2927 struct ib_send_wr *wr_failed; 2928 int rc; 2929 2930 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2931 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2932 wr->iser_ib_op = ISER_IB_RDMA_READ; 2933 rc = device->reg_rdma_mem(conn, cmd, wr); 2934 if (rc) { 2935 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2936 return rc; 2937 } 2938 2939 rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); 2940 if (rc) 2941 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2942 2943 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2944 isert_cmd); 2945 2946 return 0; 2947 } 2948 2949 static int 2950 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2951 { 2952 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2953 int ret = 0; 2954 2955 switch (state) { 2956 case ISTATE_REMOVE: 2957 spin_lock_bh(&conn->cmd_lock); 2958 list_del_init(&cmd->i_conn_node); 2959 spin_unlock_bh(&conn->cmd_lock); 2960 isert_put_cmd(isert_cmd, true); 2961 break; 2962 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2963 ret = isert_put_nopin(cmd, conn, false); 2964 break; 2965 default: 2966 isert_err("Unknown immediate state: 0x%02x\n", state); 2967 ret = -EINVAL; 2968 break; 2969 } 2970 2971 return ret; 2972 } 2973 2974 static int 2975 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2976 { 2977 struct isert_conn *isert_conn = conn->context; 2978 int ret; 2979 2980 switch (state) { 2981 case ISTATE_SEND_LOGOUTRSP: 2982 ret = isert_put_logout_rsp(cmd, conn); 2983 if (!ret) 2984 isert_conn->logout_posted = true; 2985 break; 2986 case ISTATE_SEND_NOPIN: 2987 ret = isert_put_nopin(cmd, conn, true); 2988 break; 2989 case ISTATE_SEND_TASKMGTRSP: 2990 ret = isert_put_tm_rsp(cmd, conn); 2991 break; 2992 case ISTATE_SEND_REJECT: 2993 ret = isert_put_reject(cmd, conn); 2994 break; 2995 case ISTATE_SEND_TEXTRSP: 2996 ret = isert_put_text_rsp(cmd, conn); 2997 break; 2998 case ISTATE_SEND_STATUS: 2999 /* 3000 * Special case for sending non GOOD SCSI status from TX thread 3001 * context during pre se_cmd excecution failure. 3002 */ 3003 ret = isert_put_response(conn, cmd); 3004 break; 3005 default: 3006 isert_err("Unknown response state: 0x%02x\n", state); 3007 ret = -EINVAL; 3008 break; 3009 } 3010 3011 return ret; 3012 } 3013 3014 struct rdma_cm_id * 3015 isert_setup_id(struct isert_np *isert_np) 3016 { 3017 struct iscsi_np *np = isert_np->np; 3018 struct rdma_cm_id *id; 3019 struct sockaddr *sa; 3020 int ret; 3021 3022 sa = (struct sockaddr *)&np->np_sockaddr; 3023 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3024 3025 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 3026 RDMA_PS_TCP, IB_QPT_RC); 3027 if (IS_ERR(id)) { 3028 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3029 ret = PTR_ERR(id); 3030 goto out; 3031 } 3032 isert_dbg("id %p context %p\n", id, id->context); 3033 3034 ret = rdma_bind_addr(id, sa); 3035 if (ret) { 3036 isert_err("rdma_bind_addr() failed: %d\n", ret); 3037 goto out_id; 3038 } 3039 3040 ret = rdma_listen(id, 0); 3041 if (ret) { 3042 isert_err("rdma_listen() failed: %d\n", ret); 3043 goto out_id; 3044 } 3045 3046 return id; 3047 out_id: 3048 rdma_destroy_id(id); 3049 out: 3050 return ERR_PTR(ret); 3051 } 3052 3053 static int 3054 isert_setup_np(struct iscsi_np *np, 3055 struct sockaddr_storage *ksockaddr) 3056 { 3057 struct isert_np *isert_np; 3058 struct rdma_cm_id *isert_lid; 3059 int ret; 3060 3061 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3062 if (!isert_np) { 3063 isert_err("Unable to allocate struct isert_np\n"); 3064 return -ENOMEM; 3065 } 3066 sema_init(&isert_np->sem, 0); 3067 mutex_init(&isert_np->mutex); 3068 INIT_LIST_HEAD(&isert_np->accepted); 3069 INIT_LIST_HEAD(&isert_np->pending); 3070 isert_np->np = np; 3071 3072 /* 3073 * Setup the np->np_sockaddr from the passed sockaddr setup 3074 * in iscsi_target_configfs.c code.. 3075 */ 3076 memcpy(&np->np_sockaddr, ksockaddr, 3077 sizeof(struct sockaddr_storage)); 3078 3079 isert_lid = isert_setup_id(isert_np); 3080 if (IS_ERR(isert_lid)) { 3081 ret = PTR_ERR(isert_lid); 3082 goto out; 3083 } 3084 3085 isert_np->cm_id = isert_lid; 3086 np->np_context = isert_np; 3087 3088 return 0; 3089 3090 out: 3091 kfree(isert_np); 3092 3093 return ret; 3094 } 3095 3096 static int 3097 isert_rdma_accept(struct isert_conn *isert_conn) 3098 { 3099 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3100 struct rdma_conn_param cp; 3101 int ret; 3102 3103 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3104 cp.initiator_depth = isert_conn->initiator_depth; 3105 cp.retry_count = 7; 3106 cp.rnr_retry_count = 7; 3107 3108 ret = rdma_accept(cm_id, &cp); 3109 if (ret) { 3110 isert_err("rdma_accept() failed with: %d\n", ret); 3111 return ret; 3112 } 3113 3114 return 0; 3115 } 3116 3117 static int 3118 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3119 { 3120 struct isert_conn *isert_conn = conn->context; 3121 int ret; 3122 3123 isert_info("before login_req comp conn: %p\n", isert_conn); 3124 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 3125 if (ret) { 3126 isert_err("isert_conn %p interrupted before got login req\n", 3127 isert_conn); 3128 return ret; 3129 } 3130 reinit_completion(&isert_conn->login_req_comp); 3131 3132 /* 3133 * For login requests after the first PDU, isert_rx_login_req() will 3134 * kick schedule_delayed_work(&conn->login_work) as the packet is 3135 * received, which turns this callback from iscsi_target_do_login_rx() 3136 * into a NOP. 3137 */ 3138 if (!login->first_request) 3139 return 0; 3140 3141 isert_rx_login_req(isert_conn); 3142 3143 isert_info("before login_comp conn: %p\n", conn); 3144 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3145 if (ret) 3146 return ret; 3147 3148 isert_info("processing login->req: %p\n", login->req); 3149 3150 return 0; 3151 } 3152 3153 static void 3154 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3155 struct isert_conn *isert_conn) 3156 { 3157 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3158 struct rdma_route *cm_route = &cm_id->route; 3159 3160 conn->login_family = np->np_sockaddr.ss_family; 3161 3162 conn->login_sockaddr = cm_route->addr.dst_addr; 3163 conn->local_sockaddr = cm_route->addr.src_addr; 3164 } 3165 3166 static int 3167 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3168 { 3169 struct isert_np *isert_np = np->np_context; 3170 struct isert_conn *isert_conn; 3171 int ret; 3172 3173 accept_wait: 3174 ret = down_interruptible(&isert_np->sem); 3175 if (ret) 3176 return -ENODEV; 3177 3178 spin_lock_bh(&np->np_thread_lock); 3179 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3180 spin_unlock_bh(&np->np_thread_lock); 3181 isert_dbg("np_thread_state %d\n", 3182 np->np_thread_state); 3183 /** 3184 * No point in stalling here when np_thread 3185 * is in state RESET/SHUTDOWN/EXIT - bail 3186 **/ 3187 return -ENODEV; 3188 } 3189 spin_unlock_bh(&np->np_thread_lock); 3190 3191 mutex_lock(&isert_np->mutex); 3192 if (list_empty(&isert_np->pending)) { 3193 mutex_unlock(&isert_np->mutex); 3194 goto accept_wait; 3195 } 3196 isert_conn = list_first_entry(&isert_np->pending, 3197 struct isert_conn, node); 3198 list_del_init(&isert_conn->node); 3199 mutex_unlock(&isert_np->mutex); 3200 3201 conn->context = isert_conn; 3202 isert_conn->conn = conn; 3203 3204 isert_set_conn_info(np, conn, isert_conn); 3205 3206 isert_dbg("Processing isert_conn: %p\n", isert_conn); 3207 3208 return 0; 3209 } 3210 3211 static void 3212 isert_free_np(struct iscsi_np *np) 3213 { 3214 struct isert_np *isert_np = np->np_context; 3215 struct isert_conn *isert_conn, *n; 3216 3217 if (isert_np->cm_id) 3218 rdma_destroy_id(isert_np->cm_id); 3219 3220 /* 3221 * FIXME: At this point we don't have a good way to insure 3222 * that at this point we don't have hanging connections that 3223 * completed RDMA establishment but didn't start iscsi login 3224 * process. So work-around this by cleaning up what ever piled 3225 * up in accepted and pending lists. 3226 */ 3227 mutex_lock(&isert_np->mutex); 3228 if (!list_empty(&isert_np->pending)) { 3229 isert_info("Still have isert pending connections\n"); 3230 list_for_each_entry_safe(isert_conn, n, 3231 &isert_np->pending, 3232 node) { 3233 isert_info("cleaning isert_conn %p state (%d)\n", 3234 isert_conn, isert_conn->state); 3235 isert_connect_release(isert_conn); 3236 } 3237 } 3238 3239 if (!list_empty(&isert_np->accepted)) { 3240 isert_info("Still have isert accepted connections\n"); 3241 list_for_each_entry_safe(isert_conn, n, 3242 &isert_np->accepted, 3243 node) { 3244 isert_info("cleaning isert_conn %p state (%d)\n", 3245 isert_conn, isert_conn->state); 3246 isert_connect_release(isert_conn); 3247 } 3248 } 3249 mutex_unlock(&isert_np->mutex); 3250 3251 np->np_context = NULL; 3252 kfree(isert_np); 3253 } 3254 3255 static void isert_release_work(struct work_struct *work) 3256 { 3257 struct isert_conn *isert_conn = container_of(work, 3258 struct isert_conn, 3259 release_work); 3260 3261 isert_info("Starting release conn %p\n", isert_conn); 3262 3263 wait_for_completion(&isert_conn->wait); 3264 3265 mutex_lock(&isert_conn->mutex); 3266 isert_conn->state = ISER_CONN_DOWN; 3267 mutex_unlock(&isert_conn->mutex); 3268 3269 isert_info("Destroying conn %p\n", isert_conn); 3270 isert_put_conn(isert_conn); 3271 } 3272 3273 static void 3274 isert_wait4logout(struct isert_conn *isert_conn) 3275 { 3276 struct iscsi_conn *conn = isert_conn->conn; 3277 3278 isert_info("conn %p\n", isert_conn); 3279 3280 if (isert_conn->logout_posted) { 3281 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3282 wait_for_completion_timeout(&conn->conn_logout_comp, 3283 SECONDS_FOR_LOGOUT_COMP * HZ); 3284 } 3285 } 3286 3287 static void 3288 isert_wait4cmds(struct iscsi_conn *conn) 3289 { 3290 isert_info("iscsi_conn %p\n", conn); 3291 3292 if (conn->sess) { 3293 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3294 target_wait_for_sess_cmds(conn->sess->se_sess); 3295 } 3296 } 3297 3298 static void 3299 isert_wait4flush(struct isert_conn *isert_conn) 3300 { 3301 struct ib_recv_wr *bad_wr; 3302 3303 isert_info("conn %p\n", isert_conn); 3304 3305 init_completion(&isert_conn->wait_comp_err); 3306 isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3307 /* post an indication that all flush errors were consumed */ 3308 if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { 3309 isert_err("conn %p failed to post beacon", isert_conn); 3310 return; 3311 } 3312 3313 wait_for_completion(&isert_conn->wait_comp_err); 3314 } 3315 3316 /** 3317 * isert_put_unsol_pending_cmds() - Drop commands waiting for 3318 * unsolicitate dataout 3319 * @conn: iscsi connection 3320 * 3321 * We might still have commands that are waiting for unsolicited 3322 * dataouts messages. We must put the extra reference on those 3323 * before blocking on the target_wait_for_session_cmds 3324 */ 3325 static void 3326 isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 3327 { 3328 struct iscsi_cmd *cmd, *tmp; 3329 static LIST_HEAD(drop_cmd_list); 3330 3331 spin_lock_bh(&conn->cmd_lock); 3332 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 3333 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 3334 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 3335 (cmd->write_data_done < cmd->se_cmd.data_length)) 3336 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 3337 } 3338 spin_unlock_bh(&conn->cmd_lock); 3339 3340 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 3341 list_del_init(&cmd->i_conn_node); 3342 if (cmd->i_state != ISTATE_REMOVE) { 3343 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 3344 3345 isert_info("conn %p dropping cmd %p\n", conn, cmd); 3346 isert_put_cmd(isert_cmd, true); 3347 } 3348 } 3349 } 3350 3351 static void isert_wait_conn(struct iscsi_conn *conn) 3352 { 3353 struct isert_conn *isert_conn = conn->context; 3354 3355 isert_info("Starting conn %p\n", isert_conn); 3356 3357 mutex_lock(&isert_conn->mutex); 3358 /* 3359 * Only wait for wait_comp_err if the isert_conn made it 3360 * into full feature phase.. 3361 */ 3362 if (isert_conn->state == ISER_CONN_INIT) { 3363 mutex_unlock(&isert_conn->mutex); 3364 return; 3365 } 3366 isert_conn_terminate(isert_conn); 3367 mutex_unlock(&isert_conn->mutex); 3368 3369 isert_wait4flush(isert_conn); 3370 isert_put_unsol_pending_cmds(conn); 3371 isert_wait4cmds(conn); 3372 isert_wait4logout(isert_conn); 3373 3374 queue_work(isert_release_wq, &isert_conn->release_work); 3375 } 3376 3377 static void isert_free_conn(struct iscsi_conn *conn) 3378 { 3379 struct isert_conn *isert_conn = conn->context; 3380 3381 isert_wait4flush(isert_conn); 3382 isert_put_conn(isert_conn); 3383 } 3384 3385 static struct iscsit_transport iser_target_transport = { 3386 .name = "IB/iSER", 3387 .transport_type = ISCSI_INFINIBAND, 3388 .priv_size = sizeof(struct isert_cmd), 3389 .owner = THIS_MODULE, 3390 .iscsit_setup_np = isert_setup_np, 3391 .iscsit_accept_np = isert_accept_np, 3392 .iscsit_free_np = isert_free_np, 3393 .iscsit_wait_conn = isert_wait_conn, 3394 .iscsit_free_conn = isert_free_conn, 3395 .iscsit_get_login_rx = isert_get_login_rx, 3396 .iscsit_put_login_tx = isert_put_login_tx, 3397 .iscsit_immediate_queue = isert_immediate_queue, 3398 .iscsit_response_queue = isert_response_queue, 3399 .iscsit_get_dataout = isert_get_dataout, 3400 .iscsit_queue_data_in = isert_put_datain, 3401 .iscsit_queue_status = isert_put_response, 3402 .iscsit_aborted_task = isert_aborted_task, 3403 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3404 }; 3405 3406 static int __init isert_init(void) 3407 { 3408 int ret; 3409 3410 isert_comp_wq = alloc_workqueue("isert_comp_wq", 3411 WQ_UNBOUND | WQ_HIGHPRI, 0); 3412 if (!isert_comp_wq) { 3413 isert_err("Unable to allocate isert_comp_wq\n"); 3414 ret = -ENOMEM; 3415 return -ENOMEM; 3416 } 3417 3418 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3419 WQ_UNBOUND_MAX_ACTIVE); 3420 if (!isert_release_wq) { 3421 isert_err("Unable to allocate isert_release_wq\n"); 3422 ret = -ENOMEM; 3423 goto destroy_comp_wq; 3424 } 3425 3426 iscsit_register_transport(&iser_target_transport); 3427 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3428 3429 return 0; 3430 3431 destroy_comp_wq: 3432 destroy_workqueue(isert_comp_wq); 3433 3434 return ret; 3435 } 3436 3437 static void __exit isert_exit(void) 3438 { 3439 flush_scheduled_work(); 3440 destroy_workqueue(isert_release_wq); 3441 destroy_workqueue(isert_comp_wq); 3442 iscsit_unregister_transport(&iser_target_transport); 3443 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3444 } 3445 3446 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3447 MODULE_VERSION("1.0"); 3448 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3449 MODULE_LICENSE("GPL"); 3450 3451 module_init(isert_init); 3452 module_exit(isert_exit); 3453