1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "ib_isert.h" 33 34 #define ISERT_MAX_CONN 8 35 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 38 ISERT_MAX_CONN) 39 40 static int isert_debug_level; 41 module_param_named(debug_level, isert_debug_level, int, 0644); 42 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 43 44 static DEFINE_MUTEX(device_list_mutex); 45 static LIST_HEAD(device_list); 46 static struct workqueue_struct *isert_comp_wq; 47 static struct workqueue_struct *isert_release_wq; 48 49 static void 50 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51 static int 52 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 53 struct isert_rdma_wr *wr); 54 static void 55 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 56 static int 57 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 58 struct isert_rdma_wr *wr); 59 static int 60 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 61 static int 62 isert_rdma_post_recvl(struct isert_conn *isert_conn); 63 static int 64 isert_rdma_accept(struct isert_conn *isert_conn); 65 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 66 67 static void isert_release_work(struct work_struct *work); 68 69 static inline bool 70 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 71 { 72 return (conn->pi_support && 73 cmd->prot_op != TARGET_PROT_NORMAL); 74 } 75 76 77 static void 78 isert_qp_event_callback(struct ib_event *e, void *context) 79 { 80 struct isert_conn *isert_conn = context; 81 82 isert_err("%s (%d): conn %p\n", 83 ib_event_msg(e->event), e->event, isert_conn); 84 85 switch (e->event) { 86 case IB_EVENT_COMM_EST: 87 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 88 break; 89 case IB_EVENT_QP_LAST_WQE_REACHED: 90 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 91 break; 92 default: 93 break; 94 } 95 } 96 97 static struct isert_comp * 98 isert_comp_get(struct isert_conn *isert_conn) 99 { 100 struct isert_device *device = isert_conn->device; 101 struct isert_comp *comp; 102 int i, min = 0; 103 104 mutex_lock(&device_list_mutex); 105 for (i = 0; i < device->comps_used; i++) 106 if (device->comps[i].active_qps < 107 device->comps[min].active_qps) 108 min = i; 109 comp = &device->comps[min]; 110 comp->active_qps++; 111 mutex_unlock(&device_list_mutex); 112 113 isert_info("conn %p, using comp %p min_index: %d\n", 114 isert_conn, comp, min); 115 116 return comp; 117 } 118 119 static void 120 isert_comp_put(struct isert_comp *comp) 121 { 122 mutex_lock(&device_list_mutex); 123 comp->active_qps--; 124 mutex_unlock(&device_list_mutex); 125 } 126 127 static struct ib_qp * 128 isert_create_qp(struct isert_conn *isert_conn, 129 struct isert_comp *comp, 130 struct rdma_cm_id *cma_id) 131 { 132 struct isert_device *device = isert_conn->device; 133 struct ib_qp_init_attr attr; 134 int ret; 135 136 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 137 attr.event_handler = isert_qp_event_callback; 138 attr.qp_context = isert_conn; 139 attr.send_cq = comp->cq; 140 attr.recv_cq = comp->cq; 141 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 142 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 143 attr.cap.max_send_sge = device->ib_device->attrs.max_sge; 144 isert_conn->max_sge = min(device->ib_device->attrs.max_sge, 145 device->ib_device->attrs.max_sge_rd); 146 attr.cap.max_recv_sge = 1; 147 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 148 attr.qp_type = IB_QPT_RC; 149 if (device->pi_capable) 150 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 151 152 ret = rdma_create_qp(cma_id, device->pd, &attr); 153 if (ret) { 154 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 155 return ERR_PTR(ret); 156 } 157 158 return cma_id->qp; 159 } 160 161 static int 162 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 163 { 164 struct isert_comp *comp; 165 int ret; 166 167 comp = isert_comp_get(isert_conn); 168 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 169 if (IS_ERR(isert_conn->qp)) { 170 ret = PTR_ERR(isert_conn->qp); 171 goto err; 172 } 173 174 return 0; 175 err: 176 isert_comp_put(comp); 177 return ret; 178 } 179 180 static void 181 isert_cq_event_callback(struct ib_event *e, void *context) 182 { 183 isert_dbg("event: %d\n", e->event); 184 } 185 186 static int 187 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 188 { 189 struct isert_device *device = isert_conn->device; 190 struct ib_device *ib_dev = device->ib_device; 191 struct iser_rx_desc *rx_desc; 192 struct ib_sge *rx_sg; 193 u64 dma_addr; 194 int i, j; 195 196 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 197 sizeof(struct iser_rx_desc), GFP_KERNEL); 198 if (!isert_conn->rx_descs) 199 goto fail; 200 201 rx_desc = isert_conn->rx_descs; 202 203 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 204 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 205 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 206 if (ib_dma_mapping_error(ib_dev, dma_addr)) 207 goto dma_map_fail; 208 209 rx_desc->dma_addr = dma_addr; 210 211 rx_sg = &rx_desc->rx_sg; 212 rx_sg->addr = rx_desc->dma_addr; 213 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 214 rx_sg->lkey = device->pd->local_dma_lkey; 215 } 216 217 return 0; 218 219 dma_map_fail: 220 rx_desc = isert_conn->rx_descs; 221 for (j = 0; j < i; j++, rx_desc++) { 222 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 223 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 224 } 225 kfree(isert_conn->rx_descs); 226 isert_conn->rx_descs = NULL; 227 fail: 228 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 229 230 return -ENOMEM; 231 } 232 233 static void 234 isert_free_rx_descriptors(struct isert_conn *isert_conn) 235 { 236 struct ib_device *ib_dev = isert_conn->device->ib_device; 237 struct iser_rx_desc *rx_desc; 238 int i; 239 240 if (!isert_conn->rx_descs) 241 return; 242 243 rx_desc = isert_conn->rx_descs; 244 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 245 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 246 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 247 } 248 249 kfree(isert_conn->rx_descs); 250 isert_conn->rx_descs = NULL; 251 } 252 253 static void isert_cq_work(struct work_struct *); 254 static void isert_cq_callback(struct ib_cq *, void *); 255 256 static void 257 isert_free_comps(struct isert_device *device) 258 { 259 int i; 260 261 for (i = 0; i < device->comps_used; i++) { 262 struct isert_comp *comp = &device->comps[i]; 263 264 if (comp->cq) { 265 cancel_work_sync(&comp->work); 266 ib_destroy_cq(comp->cq); 267 } 268 } 269 kfree(device->comps); 270 } 271 272 static int 273 isert_alloc_comps(struct isert_device *device) 274 { 275 int i, max_cqe, ret = 0; 276 277 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 278 device->ib_device->num_comp_vectors)); 279 280 isert_info("Using %d CQs, %s supports %d vectors support " 281 "Fast registration %d pi_capable %d\n", 282 device->comps_used, device->ib_device->name, 283 device->ib_device->num_comp_vectors, device->use_fastreg, 284 device->pi_capable); 285 286 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 287 GFP_KERNEL); 288 if (!device->comps) { 289 isert_err("Unable to allocate completion contexts\n"); 290 return -ENOMEM; 291 } 292 293 max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 294 295 for (i = 0; i < device->comps_used; i++) { 296 struct ib_cq_init_attr cq_attr = {}; 297 struct isert_comp *comp = &device->comps[i]; 298 299 comp->device = device; 300 INIT_WORK(&comp->work, isert_cq_work); 301 cq_attr.cqe = max_cqe; 302 cq_attr.comp_vector = i; 303 comp->cq = ib_create_cq(device->ib_device, 304 isert_cq_callback, 305 isert_cq_event_callback, 306 (void *)comp, 307 &cq_attr); 308 if (IS_ERR(comp->cq)) { 309 isert_err("Unable to allocate cq\n"); 310 ret = PTR_ERR(comp->cq); 311 comp->cq = NULL; 312 goto out_cq; 313 } 314 315 ret = ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 316 if (ret) 317 goto out_cq; 318 } 319 320 return 0; 321 out_cq: 322 isert_free_comps(device); 323 return ret; 324 } 325 326 static int 327 isert_create_device_ib_res(struct isert_device *device) 328 { 329 struct ib_device *ib_dev = device->ib_device; 330 int ret; 331 332 isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge); 333 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 334 335 /* asign function handlers */ 336 if (ib_dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 337 ib_dev->attrs.device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 338 device->use_fastreg = 1; 339 device->reg_rdma_mem = isert_reg_rdma; 340 device->unreg_rdma_mem = isert_unreg_rdma; 341 } else { 342 device->use_fastreg = 0; 343 device->reg_rdma_mem = isert_map_rdma; 344 device->unreg_rdma_mem = isert_unmap_cmd; 345 } 346 347 ret = isert_alloc_comps(device); 348 if (ret) 349 goto out; 350 351 device->pd = ib_alloc_pd(ib_dev); 352 if (IS_ERR(device->pd)) { 353 ret = PTR_ERR(device->pd); 354 isert_err("failed to allocate pd, device %p, ret=%d\n", 355 device, ret); 356 goto out_cq; 357 } 358 359 /* Check signature cap */ 360 device->pi_capable = ib_dev->attrs.device_cap_flags & 361 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 362 363 return 0; 364 365 out_cq: 366 isert_free_comps(device); 367 out: 368 if (ret > 0) 369 ret = -EINVAL; 370 return ret; 371 } 372 373 static void 374 isert_free_device_ib_res(struct isert_device *device) 375 { 376 isert_info("device %p\n", device); 377 378 ib_dealloc_pd(device->pd); 379 isert_free_comps(device); 380 } 381 382 static void 383 isert_device_put(struct isert_device *device) 384 { 385 mutex_lock(&device_list_mutex); 386 device->refcount--; 387 isert_info("device %p refcount %d\n", device, device->refcount); 388 if (!device->refcount) { 389 isert_free_device_ib_res(device); 390 list_del(&device->dev_node); 391 kfree(device); 392 } 393 mutex_unlock(&device_list_mutex); 394 } 395 396 static struct isert_device * 397 isert_device_get(struct rdma_cm_id *cma_id) 398 { 399 struct isert_device *device; 400 int ret; 401 402 mutex_lock(&device_list_mutex); 403 list_for_each_entry(device, &device_list, dev_node) { 404 if (device->ib_device->node_guid == cma_id->device->node_guid) { 405 device->refcount++; 406 isert_info("Found iser device %p refcount %d\n", 407 device, device->refcount); 408 mutex_unlock(&device_list_mutex); 409 return device; 410 } 411 } 412 413 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 414 if (!device) { 415 mutex_unlock(&device_list_mutex); 416 return ERR_PTR(-ENOMEM); 417 } 418 419 INIT_LIST_HEAD(&device->dev_node); 420 421 device->ib_device = cma_id->device; 422 ret = isert_create_device_ib_res(device); 423 if (ret) { 424 kfree(device); 425 mutex_unlock(&device_list_mutex); 426 return ERR_PTR(ret); 427 } 428 429 device->refcount++; 430 list_add_tail(&device->dev_node, &device_list); 431 isert_info("Created a new iser device %p refcount %d\n", 432 device, device->refcount); 433 mutex_unlock(&device_list_mutex); 434 435 return device; 436 } 437 438 static void 439 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 440 { 441 struct fast_reg_descriptor *fr_desc, *tmp; 442 int i = 0; 443 444 if (list_empty(&isert_conn->fr_pool)) 445 return; 446 447 isert_info("Freeing conn %p fastreg pool", isert_conn); 448 449 list_for_each_entry_safe(fr_desc, tmp, 450 &isert_conn->fr_pool, list) { 451 list_del(&fr_desc->list); 452 ib_dereg_mr(fr_desc->data_mr); 453 if (fr_desc->pi_ctx) { 454 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 455 ib_dereg_mr(fr_desc->pi_ctx->sig_mr); 456 kfree(fr_desc->pi_ctx); 457 } 458 kfree(fr_desc); 459 ++i; 460 } 461 462 if (i < isert_conn->fr_pool_size) 463 isert_warn("Pool still has %d regions registered\n", 464 isert_conn->fr_pool_size - i); 465 } 466 467 static int 468 isert_create_pi_ctx(struct fast_reg_descriptor *desc, 469 struct ib_device *device, 470 struct ib_pd *pd) 471 { 472 struct pi_context *pi_ctx; 473 int ret; 474 475 pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL); 476 if (!pi_ctx) { 477 isert_err("Failed to allocate pi context\n"); 478 return -ENOMEM; 479 } 480 481 pi_ctx->prot_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 482 ISCSI_ISER_SG_TABLESIZE); 483 if (IS_ERR(pi_ctx->prot_mr)) { 484 isert_err("Failed to allocate prot frmr err=%ld\n", 485 PTR_ERR(pi_ctx->prot_mr)); 486 ret = PTR_ERR(pi_ctx->prot_mr); 487 goto err_pi_ctx; 488 } 489 desc->ind |= ISERT_PROT_KEY_VALID; 490 491 pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2); 492 if (IS_ERR(pi_ctx->sig_mr)) { 493 isert_err("Failed to allocate signature enabled mr err=%ld\n", 494 PTR_ERR(pi_ctx->sig_mr)); 495 ret = PTR_ERR(pi_ctx->sig_mr); 496 goto err_prot_mr; 497 } 498 499 desc->pi_ctx = pi_ctx; 500 desc->ind |= ISERT_SIG_KEY_VALID; 501 desc->ind &= ~ISERT_PROTECTED; 502 503 return 0; 504 505 err_prot_mr: 506 ib_dereg_mr(pi_ctx->prot_mr); 507 err_pi_ctx: 508 kfree(pi_ctx); 509 510 return ret; 511 } 512 513 static int 514 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 515 struct fast_reg_descriptor *fr_desc) 516 { 517 fr_desc->data_mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 518 ISCSI_ISER_SG_TABLESIZE); 519 if (IS_ERR(fr_desc->data_mr)) { 520 isert_err("Failed to allocate data frmr err=%ld\n", 521 PTR_ERR(fr_desc->data_mr)); 522 return PTR_ERR(fr_desc->data_mr); 523 } 524 fr_desc->ind |= ISERT_DATA_KEY_VALID; 525 526 isert_dbg("Created fr_desc %p\n", fr_desc); 527 528 return 0; 529 } 530 531 static int 532 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 533 { 534 struct fast_reg_descriptor *fr_desc; 535 struct isert_device *device = isert_conn->device; 536 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 537 struct se_node_acl *se_nacl = se_sess->se_node_acl; 538 int i, ret, tag_num; 539 /* 540 * Setup the number of FRMRs based upon the number of tags 541 * available to session in iscsi_target_locate_portal(). 542 */ 543 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 544 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 545 546 isert_conn->fr_pool_size = 0; 547 for (i = 0; i < tag_num; i++) { 548 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 549 if (!fr_desc) { 550 isert_err("Failed to allocate fast_reg descriptor\n"); 551 ret = -ENOMEM; 552 goto err; 553 } 554 555 ret = isert_create_fr_desc(device->ib_device, 556 device->pd, fr_desc); 557 if (ret) { 558 isert_err("Failed to create fastreg descriptor err=%d\n", 559 ret); 560 kfree(fr_desc); 561 goto err; 562 } 563 564 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 565 isert_conn->fr_pool_size++; 566 } 567 568 isert_dbg("Creating conn %p fastreg pool size=%d", 569 isert_conn, isert_conn->fr_pool_size); 570 571 return 0; 572 573 err: 574 isert_conn_free_fastreg_pool(isert_conn); 575 return ret; 576 } 577 578 static void 579 isert_init_conn(struct isert_conn *isert_conn) 580 { 581 isert_conn->state = ISER_CONN_INIT; 582 INIT_LIST_HEAD(&isert_conn->node); 583 init_completion(&isert_conn->login_comp); 584 init_completion(&isert_conn->login_req_comp); 585 init_completion(&isert_conn->wait); 586 kref_init(&isert_conn->kref); 587 mutex_init(&isert_conn->mutex); 588 spin_lock_init(&isert_conn->pool_lock); 589 INIT_LIST_HEAD(&isert_conn->fr_pool); 590 INIT_WORK(&isert_conn->release_work, isert_release_work); 591 } 592 593 static void 594 isert_free_login_buf(struct isert_conn *isert_conn) 595 { 596 struct ib_device *ib_dev = isert_conn->device->ib_device; 597 598 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 599 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 600 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 601 ISCSI_DEF_MAX_RECV_SEG_LEN, 602 DMA_FROM_DEVICE); 603 kfree(isert_conn->login_buf); 604 } 605 606 static int 607 isert_alloc_login_buf(struct isert_conn *isert_conn, 608 struct ib_device *ib_dev) 609 { 610 int ret; 611 612 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 613 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 614 if (!isert_conn->login_buf) { 615 isert_err("Unable to allocate isert_conn->login_buf\n"); 616 return -ENOMEM; 617 } 618 619 isert_conn->login_req_buf = isert_conn->login_buf; 620 isert_conn->login_rsp_buf = isert_conn->login_buf + 621 ISCSI_DEF_MAX_RECV_SEG_LEN; 622 623 isert_dbg("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 624 isert_conn->login_buf, isert_conn->login_req_buf, 625 isert_conn->login_rsp_buf); 626 627 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 628 (void *)isert_conn->login_req_buf, 629 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 630 631 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 632 if (ret) { 633 isert_err("login_req_dma mapping error: %d\n", ret); 634 isert_conn->login_req_dma = 0; 635 goto out_login_buf; 636 } 637 638 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 639 (void *)isert_conn->login_rsp_buf, 640 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 641 642 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 643 if (ret) { 644 isert_err("login_rsp_dma mapping error: %d\n", ret); 645 isert_conn->login_rsp_dma = 0; 646 goto out_req_dma_map; 647 } 648 649 return 0; 650 651 out_req_dma_map: 652 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 653 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 654 out_login_buf: 655 kfree(isert_conn->login_buf); 656 return ret; 657 } 658 659 static void 660 isert_set_nego_params(struct isert_conn *isert_conn, 661 struct rdma_conn_param *param) 662 { 663 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 664 665 /* Set max inflight RDMA READ requests */ 666 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 667 attr->max_qp_init_rd_atom); 668 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 669 670 if (param->private_data) { 671 u8 flags = *(u8 *)param->private_data; 672 673 /* 674 * use remote invalidation if the both initiator 675 * and the HCA support it 676 */ 677 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 678 (attr->device_cap_flags & 679 IB_DEVICE_MEM_MGT_EXTENSIONS); 680 if (isert_conn->snd_w_inv) 681 isert_info("Using remote invalidation\n"); 682 } 683 } 684 685 static int 686 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 687 { 688 struct isert_np *isert_np = cma_id->context; 689 struct iscsi_np *np = isert_np->np; 690 struct isert_conn *isert_conn; 691 struct isert_device *device; 692 int ret = 0; 693 694 spin_lock_bh(&np->np_thread_lock); 695 if (!np->enabled) { 696 spin_unlock_bh(&np->np_thread_lock); 697 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 698 return rdma_reject(cma_id, NULL, 0); 699 } 700 spin_unlock_bh(&np->np_thread_lock); 701 702 isert_dbg("cma_id: %p, portal: %p\n", 703 cma_id, cma_id->context); 704 705 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 706 if (!isert_conn) 707 return -ENOMEM; 708 709 isert_init_conn(isert_conn); 710 isert_conn->cm_id = cma_id; 711 712 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 713 if (ret) 714 goto out; 715 716 device = isert_device_get(cma_id); 717 if (IS_ERR(device)) { 718 ret = PTR_ERR(device); 719 goto out_rsp_dma_map; 720 } 721 isert_conn->device = device; 722 723 isert_set_nego_params(isert_conn, &event->param.conn); 724 725 ret = isert_conn_setup_qp(isert_conn, cma_id); 726 if (ret) 727 goto out_conn_dev; 728 729 ret = isert_rdma_post_recvl(isert_conn); 730 if (ret) 731 goto out_conn_dev; 732 733 ret = isert_rdma_accept(isert_conn); 734 if (ret) 735 goto out_conn_dev; 736 737 mutex_lock(&isert_np->mutex); 738 list_add_tail(&isert_conn->node, &isert_np->accepted); 739 mutex_unlock(&isert_np->mutex); 740 741 return 0; 742 743 out_conn_dev: 744 isert_device_put(device); 745 out_rsp_dma_map: 746 isert_free_login_buf(isert_conn); 747 out: 748 kfree(isert_conn); 749 rdma_reject(cma_id, NULL, 0); 750 return ret; 751 } 752 753 static void 754 isert_connect_release(struct isert_conn *isert_conn) 755 { 756 struct isert_device *device = isert_conn->device; 757 758 isert_dbg("conn %p\n", isert_conn); 759 760 BUG_ON(!device); 761 762 if (device->use_fastreg) 763 isert_conn_free_fastreg_pool(isert_conn); 764 765 isert_free_rx_descriptors(isert_conn); 766 if (isert_conn->cm_id) 767 rdma_destroy_id(isert_conn->cm_id); 768 769 if (isert_conn->qp) { 770 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 771 772 isert_comp_put(comp); 773 ib_destroy_qp(isert_conn->qp); 774 } 775 776 if (isert_conn->login_buf) 777 isert_free_login_buf(isert_conn); 778 779 isert_device_put(device); 780 781 kfree(isert_conn); 782 } 783 784 static void 785 isert_connected_handler(struct rdma_cm_id *cma_id) 786 { 787 struct isert_conn *isert_conn = cma_id->qp->qp_context; 788 struct isert_np *isert_np = cma_id->context; 789 790 isert_info("conn %p\n", isert_conn); 791 792 mutex_lock(&isert_conn->mutex); 793 isert_conn->state = ISER_CONN_UP; 794 kref_get(&isert_conn->kref); 795 mutex_unlock(&isert_conn->mutex); 796 797 mutex_lock(&isert_np->mutex); 798 list_move_tail(&isert_conn->node, &isert_np->pending); 799 mutex_unlock(&isert_np->mutex); 800 801 isert_info("np %p: Allow accept_np to continue\n", isert_np); 802 up(&isert_np->sem); 803 } 804 805 static void 806 isert_release_kref(struct kref *kref) 807 { 808 struct isert_conn *isert_conn = container_of(kref, 809 struct isert_conn, kref); 810 811 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 812 current->pid); 813 814 isert_connect_release(isert_conn); 815 } 816 817 static void 818 isert_put_conn(struct isert_conn *isert_conn) 819 { 820 kref_put(&isert_conn->kref, isert_release_kref); 821 } 822 823 /** 824 * isert_conn_terminate() - Initiate connection termination 825 * @isert_conn: isert connection struct 826 * 827 * Notes: 828 * In case the connection state is FULL_FEATURE, move state 829 * to TEMINATING and start teardown sequence (rdma_disconnect). 830 * In case the connection state is UP, complete flush as well. 831 * 832 * This routine must be called with mutex held. Thus it is 833 * safe to call multiple times. 834 */ 835 static void 836 isert_conn_terminate(struct isert_conn *isert_conn) 837 { 838 int err; 839 840 switch (isert_conn->state) { 841 case ISER_CONN_TERMINATING: 842 break; 843 case ISER_CONN_UP: 844 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 845 isert_info("Terminating conn %p state %d\n", 846 isert_conn, isert_conn->state); 847 isert_conn->state = ISER_CONN_TERMINATING; 848 err = rdma_disconnect(isert_conn->cm_id); 849 if (err) 850 isert_warn("Failed rdma_disconnect isert_conn %p\n", 851 isert_conn); 852 break; 853 default: 854 isert_warn("conn %p teminating in state %d\n", 855 isert_conn, isert_conn->state); 856 } 857 } 858 859 static int 860 isert_np_cma_handler(struct isert_np *isert_np, 861 enum rdma_cm_event_type event) 862 { 863 isert_dbg("%s (%d): isert np %p\n", 864 rdma_event_msg(event), event, isert_np); 865 866 switch (event) { 867 case RDMA_CM_EVENT_DEVICE_REMOVAL: 868 isert_np->cm_id = NULL; 869 break; 870 case RDMA_CM_EVENT_ADDR_CHANGE: 871 isert_np->cm_id = isert_setup_id(isert_np); 872 if (IS_ERR(isert_np->cm_id)) { 873 isert_err("isert np %p setup id failed: %ld\n", 874 isert_np, PTR_ERR(isert_np->cm_id)); 875 isert_np->cm_id = NULL; 876 } 877 break; 878 default: 879 isert_err("isert np %p Unexpected event %d\n", 880 isert_np, event); 881 } 882 883 return -1; 884 } 885 886 static int 887 isert_disconnected_handler(struct rdma_cm_id *cma_id, 888 enum rdma_cm_event_type event) 889 { 890 struct isert_np *isert_np = cma_id->context; 891 struct isert_conn *isert_conn; 892 bool terminating = false; 893 894 if (isert_np->cm_id == cma_id) 895 return isert_np_cma_handler(cma_id->context, event); 896 897 isert_conn = cma_id->qp->qp_context; 898 899 mutex_lock(&isert_conn->mutex); 900 terminating = (isert_conn->state == ISER_CONN_TERMINATING); 901 isert_conn_terminate(isert_conn); 902 mutex_unlock(&isert_conn->mutex); 903 904 isert_info("conn %p completing wait\n", isert_conn); 905 complete(&isert_conn->wait); 906 907 if (terminating) 908 goto out; 909 910 mutex_lock(&isert_np->mutex); 911 if (!list_empty(&isert_conn->node)) { 912 list_del_init(&isert_conn->node); 913 isert_put_conn(isert_conn); 914 queue_work(isert_release_wq, &isert_conn->release_work); 915 } 916 mutex_unlock(&isert_np->mutex); 917 918 out: 919 return 0; 920 } 921 922 static int 923 isert_connect_error(struct rdma_cm_id *cma_id) 924 { 925 struct isert_conn *isert_conn = cma_id->qp->qp_context; 926 927 list_del_init(&isert_conn->node); 928 isert_conn->cm_id = NULL; 929 isert_put_conn(isert_conn); 930 931 return -1; 932 } 933 934 static int 935 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 936 { 937 int ret = 0; 938 939 isert_info("%s (%d): status %d id %p np %p\n", 940 rdma_event_msg(event->event), event->event, 941 event->status, cma_id, cma_id->context); 942 943 switch (event->event) { 944 case RDMA_CM_EVENT_CONNECT_REQUEST: 945 ret = isert_connect_request(cma_id, event); 946 if (ret) 947 isert_err("failed handle connect request %d\n", ret); 948 break; 949 case RDMA_CM_EVENT_ESTABLISHED: 950 isert_connected_handler(cma_id); 951 break; 952 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 953 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 954 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 955 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 956 ret = isert_disconnected_handler(cma_id, event->event); 957 break; 958 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 959 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 960 case RDMA_CM_EVENT_CONNECT_ERROR: 961 ret = isert_connect_error(cma_id); 962 break; 963 default: 964 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 965 break; 966 } 967 968 return ret; 969 } 970 971 static int 972 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 973 { 974 struct ib_recv_wr *rx_wr, *rx_wr_failed; 975 int i, ret; 976 struct iser_rx_desc *rx_desc; 977 978 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 979 rx_desc = &isert_conn->rx_descs[i]; 980 rx_wr->wr_id = (uintptr_t)rx_desc; 981 rx_wr->sg_list = &rx_desc->rx_sg; 982 rx_wr->num_sge = 1; 983 rx_wr->next = rx_wr + 1; 984 } 985 rx_wr--; 986 rx_wr->next = NULL; /* mark end of work requests list */ 987 988 isert_conn->post_recv_buf_count += count; 989 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 990 &rx_wr_failed); 991 if (ret) { 992 isert_err("ib_post_recv() failed with ret: %d\n", ret); 993 isert_conn->post_recv_buf_count -= count; 994 } 995 996 return ret; 997 } 998 999 static int 1000 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 1001 { 1002 struct ib_recv_wr *rx_wr_failed, rx_wr; 1003 int ret; 1004 1005 rx_wr.wr_id = (uintptr_t)rx_desc; 1006 rx_wr.sg_list = &rx_desc->rx_sg; 1007 rx_wr.num_sge = 1; 1008 rx_wr.next = NULL; 1009 1010 isert_conn->post_recv_buf_count++; 1011 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 1012 if (ret) { 1013 isert_err("ib_post_recv() failed with ret: %d\n", ret); 1014 isert_conn->post_recv_buf_count--; 1015 } 1016 1017 return ret; 1018 } 1019 1020 static int 1021 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 1022 { 1023 struct ib_device *ib_dev = isert_conn->cm_id->device; 1024 struct ib_send_wr send_wr, *send_wr_failed; 1025 int ret; 1026 1027 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 1028 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1029 1030 send_wr.next = NULL; 1031 send_wr.wr_id = (uintptr_t)tx_desc; 1032 send_wr.sg_list = tx_desc->tx_sg; 1033 send_wr.num_sge = tx_desc->num_sge; 1034 send_wr.opcode = IB_WR_SEND; 1035 send_wr.send_flags = IB_SEND_SIGNALED; 1036 1037 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 1038 if (ret) 1039 isert_err("ib_post_send() failed, ret: %d\n", ret); 1040 1041 return ret; 1042 } 1043 1044 static void 1045 isert_create_send_desc(struct isert_conn *isert_conn, 1046 struct isert_cmd *isert_cmd, 1047 struct iser_tx_desc *tx_desc) 1048 { 1049 struct isert_device *device = isert_conn->device; 1050 struct ib_device *ib_dev = device->ib_device; 1051 1052 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 1053 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1054 1055 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 1056 tx_desc->iser_header.flags = ISCSI_CTRL; 1057 1058 tx_desc->num_sge = 1; 1059 tx_desc->isert_cmd = isert_cmd; 1060 1061 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 1062 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1063 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 1064 } 1065 } 1066 1067 static int 1068 isert_init_tx_hdrs(struct isert_conn *isert_conn, 1069 struct iser_tx_desc *tx_desc) 1070 { 1071 struct isert_device *device = isert_conn->device; 1072 struct ib_device *ib_dev = device->ib_device; 1073 u64 dma_addr; 1074 1075 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 1076 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1077 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 1078 isert_err("ib_dma_mapping_error() failed\n"); 1079 return -ENOMEM; 1080 } 1081 1082 tx_desc->dma_addr = dma_addr; 1083 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 1084 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 1085 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 1086 1087 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 1088 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 1089 tx_desc->tx_sg[0].lkey); 1090 1091 return 0; 1092 } 1093 1094 static void 1095 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1096 struct ib_send_wr *send_wr) 1097 { 1098 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 1099 1100 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 1101 send_wr->wr_id = (uintptr_t)&isert_cmd->tx_desc; 1102 1103 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 1104 send_wr->opcode = IB_WR_SEND_WITH_INV; 1105 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 1106 } else { 1107 send_wr->opcode = IB_WR_SEND; 1108 } 1109 1110 send_wr->sg_list = &tx_desc->tx_sg[0]; 1111 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 1112 send_wr->send_flags = IB_SEND_SIGNALED; 1113 } 1114 1115 static int 1116 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1117 { 1118 struct ib_recv_wr rx_wr, *rx_wr_fail; 1119 struct ib_sge sge; 1120 int ret; 1121 1122 memset(&sge, 0, sizeof(struct ib_sge)); 1123 sge.addr = isert_conn->login_req_dma; 1124 sge.length = ISER_RX_LOGIN_SIZE; 1125 sge.lkey = isert_conn->device->pd->local_dma_lkey; 1126 1127 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 1128 sge.addr, sge.length, sge.lkey); 1129 1130 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1131 rx_wr.wr_id = (uintptr_t)isert_conn->login_req_buf; 1132 rx_wr.sg_list = &sge; 1133 rx_wr.num_sge = 1; 1134 1135 isert_conn->post_recv_buf_count++; 1136 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 1137 if (ret) { 1138 isert_err("ib_post_recv() failed: %d\n", ret); 1139 isert_conn->post_recv_buf_count--; 1140 } 1141 1142 return ret; 1143 } 1144 1145 static int 1146 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1147 u32 length) 1148 { 1149 struct isert_conn *isert_conn = conn->context; 1150 struct isert_device *device = isert_conn->device; 1151 struct ib_device *ib_dev = device->ib_device; 1152 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 1153 int ret; 1154 1155 isert_create_send_desc(isert_conn, NULL, tx_desc); 1156 1157 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1158 sizeof(struct iscsi_hdr)); 1159 1160 isert_init_tx_hdrs(isert_conn, tx_desc); 1161 1162 if (length > 0) { 1163 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1164 1165 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1166 length, DMA_TO_DEVICE); 1167 1168 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1169 1170 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1171 length, DMA_TO_DEVICE); 1172 1173 tx_dsg->addr = isert_conn->login_rsp_dma; 1174 tx_dsg->length = length; 1175 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1176 tx_desc->num_sge = 2; 1177 } 1178 if (!login->login_failed) { 1179 if (login->login_complete) { 1180 if (!conn->sess->sess_ops->SessionType && 1181 isert_conn->device->use_fastreg) { 1182 ret = isert_conn_create_fastreg_pool(isert_conn); 1183 if (ret) { 1184 isert_err("Conn: %p failed to create" 1185 " fastreg pool\n", isert_conn); 1186 return ret; 1187 } 1188 } 1189 1190 ret = isert_alloc_rx_descriptors(isert_conn); 1191 if (ret) 1192 return ret; 1193 1194 ret = isert_post_recvm(isert_conn, 1195 ISERT_QP_MAX_RECV_DTOS); 1196 if (ret) 1197 return ret; 1198 1199 /* Now we are in FULL_FEATURE phase */ 1200 mutex_lock(&isert_conn->mutex); 1201 isert_conn->state = ISER_CONN_FULL_FEATURE; 1202 mutex_unlock(&isert_conn->mutex); 1203 goto post_send; 1204 } 1205 1206 ret = isert_rdma_post_recvl(isert_conn); 1207 if (ret) 1208 return ret; 1209 } 1210 post_send: 1211 ret = isert_post_send(isert_conn, tx_desc); 1212 if (ret) 1213 return ret; 1214 1215 return 0; 1216 } 1217 1218 static void 1219 isert_rx_login_req(struct isert_conn *isert_conn) 1220 { 1221 struct iser_rx_desc *rx_desc = (void *)isert_conn->login_req_buf; 1222 int rx_buflen = isert_conn->login_req_len; 1223 struct iscsi_conn *conn = isert_conn->conn; 1224 struct iscsi_login *login = conn->conn_login; 1225 int size; 1226 1227 isert_info("conn %p\n", isert_conn); 1228 1229 WARN_ON_ONCE(!login); 1230 1231 if (login->first_request) { 1232 struct iscsi_login_req *login_req = 1233 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1234 /* 1235 * Setup the initial iscsi_login values from the leading 1236 * login request PDU. 1237 */ 1238 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1239 login->current_stage = 1240 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1241 >> 2; 1242 login->version_min = login_req->min_version; 1243 login->version_max = login_req->max_version; 1244 memcpy(login->isid, login_req->isid, 6); 1245 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1246 login->init_task_tag = login_req->itt; 1247 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1248 login->cid = be16_to_cpu(login_req->cid); 1249 login->tsih = be16_to_cpu(login_req->tsih); 1250 } 1251 1252 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1253 1254 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1255 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1256 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1257 MAX_KEY_VALUE_PAIRS); 1258 memcpy(login->req_buf, &rx_desc->data[0], size); 1259 1260 if (login->first_request) { 1261 complete(&isert_conn->login_comp); 1262 return; 1263 } 1264 schedule_delayed_work(&conn->login_work, 0); 1265 } 1266 1267 static struct iscsi_cmd 1268 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1269 { 1270 struct isert_conn *isert_conn = conn->context; 1271 struct isert_cmd *isert_cmd; 1272 struct iscsi_cmd *cmd; 1273 1274 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1275 if (!cmd) { 1276 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1277 return NULL; 1278 } 1279 isert_cmd = iscsit_priv_cmd(cmd); 1280 isert_cmd->conn = isert_conn; 1281 isert_cmd->iscsi_cmd = cmd; 1282 isert_cmd->rx_desc = rx_desc; 1283 1284 return cmd; 1285 } 1286 1287 static int 1288 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1289 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1290 struct iser_rx_desc *rx_desc, unsigned char *buf) 1291 { 1292 struct iscsi_conn *conn = isert_conn->conn; 1293 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1294 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1295 bool dump_payload = false; 1296 unsigned int data_len; 1297 1298 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1299 if (rc < 0) 1300 return rc; 1301 1302 imm_data = cmd->immediate_data; 1303 imm_data_len = cmd->first_burst_len; 1304 unsol_data = cmd->unsolicited_data; 1305 data_len = cmd->se_cmd.data_length; 1306 1307 if (imm_data && imm_data_len == data_len) 1308 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1309 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1310 if (rc < 0) { 1311 return 0; 1312 } else if (rc > 0) { 1313 dump_payload = true; 1314 goto sequence_cmd; 1315 } 1316 1317 if (!imm_data) 1318 return 0; 1319 1320 if (imm_data_len != data_len) { 1321 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1322 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1323 &rx_desc->data[0], imm_data_len); 1324 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1325 sg_nents, imm_data_len); 1326 } else { 1327 sg_init_table(&isert_cmd->sg, 1); 1328 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1329 cmd->se_cmd.t_data_nents = 1; 1330 sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 1331 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1332 imm_data_len); 1333 } 1334 1335 cmd->write_data_done += imm_data_len; 1336 1337 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1338 spin_lock_bh(&cmd->istate_lock); 1339 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1340 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1341 spin_unlock_bh(&cmd->istate_lock); 1342 } 1343 1344 sequence_cmd: 1345 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1346 1347 if (!rc && dump_payload == false && unsol_data) 1348 iscsit_set_unsoliticed_dataout(cmd); 1349 else if (dump_payload && imm_data) 1350 target_put_sess_cmd(&cmd->se_cmd); 1351 1352 return 0; 1353 } 1354 1355 static int 1356 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1357 struct iser_rx_desc *rx_desc, unsigned char *buf) 1358 { 1359 struct scatterlist *sg_start; 1360 struct iscsi_conn *conn = isert_conn->conn; 1361 struct iscsi_cmd *cmd = NULL; 1362 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1363 u32 unsol_data_len = ntoh24(hdr->dlength); 1364 int rc, sg_nents, sg_off, page_off; 1365 1366 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1367 if (rc < 0) 1368 return rc; 1369 else if (!cmd) 1370 return 0; 1371 /* 1372 * FIXME: Unexpected unsolicited_data out 1373 */ 1374 if (!cmd->unsolicited_data) { 1375 isert_err("Received unexpected solicited data payload\n"); 1376 dump_stack(); 1377 return -1; 1378 } 1379 1380 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1381 "write_data_done: %u, data_length: %u\n", 1382 unsol_data_len, cmd->write_data_done, 1383 cmd->se_cmd.data_length); 1384 1385 sg_off = cmd->write_data_done / PAGE_SIZE; 1386 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1387 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1388 page_off = cmd->write_data_done % PAGE_SIZE; 1389 /* 1390 * FIXME: Non page-aligned unsolicited_data out 1391 */ 1392 if (page_off) { 1393 isert_err("unexpected non-page aligned data payload\n"); 1394 dump_stack(); 1395 return -1; 1396 } 1397 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1398 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1399 sg_nents, &rx_desc->data[0], unsol_data_len); 1400 1401 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1402 unsol_data_len); 1403 1404 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1405 if (rc < 0) 1406 return rc; 1407 1408 /* 1409 * multiple data-outs on the same command can arrive - 1410 * so post the buffer before hand 1411 */ 1412 rc = isert_post_recv(isert_conn, rx_desc); 1413 if (rc) { 1414 isert_err("ib_post_recv failed with %d\n", rc); 1415 return rc; 1416 } 1417 return 0; 1418 } 1419 1420 static int 1421 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1422 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1423 unsigned char *buf) 1424 { 1425 struct iscsi_conn *conn = isert_conn->conn; 1426 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1427 int rc; 1428 1429 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1430 if (rc < 0) 1431 return rc; 1432 /* 1433 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1434 */ 1435 1436 return iscsit_process_nop_out(conn, cmd, hdr); 1437 } 1438 1439 static int 1440 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1441 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1442 struct iscsi_text *hdr) 1443 { 1444 struct iscsi_conn *conn = isert_conn->conn; 1445 u32 payload_length = ntoh24(hdr->dlength); 1446 int rc; 1447 unsigned char *text_in = NULL; 1448 1449 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1450 if (rc < 0) 1451 return rc; 1452 1453 if (payload_length) { 1454 text_in = kzalloc(payload_length, GFP_KERNEL); 1455 if (!text_in) { 1456 isert_err("Unable to allocate text_in of payload_length: %u\n", 1457 payload_length); 1458 return -ENOMEM; 1459 } 1460 } 1461 cmd->text_in_ptr = text_in; 1462 1463 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1464 1465 return iscsit_process_text_cmd(conn, cmd, hdr); 1466 } 1467 1468 static int 1469 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1470 uint32_t read_stag, uint64_t read_va, 1471 uint32_t write_stag, uint64_t write_va) 1472 { 1473 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1474 struct iscsi_conn *conn = isert_conn->conn; 1475 struct iscsi_cmd *cmd; 1476 struct isert_cmd *isert_cmd; 1477 int ret = -EINVAL; 1478 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1479 1480 if (conn->sess->sess_ops->SessionType && 1481 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1482 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1483 " ignoring\n", opcode); 1484 return 0; 1485 } 1486 1487 switch (opcode) { 1488 case ISCSI_OP_SCSI_CMD: 1489 cmd = isert_allocate_cmd(conn, rx_desc); 1490 if (!cmd) 1491 break; 1492 1493 isert_cmd = iscsit_priv_cmd(cmd); 1494 isert_cmd->read_stag = read_stag; 1495 isert_cmd->read_va = read_va; 1496 isert_cmd->write_stag = write_stag; 1497 isert_cmd->write_va = write_va; 1498 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1499 1500 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1501 rx_desc, (unsigned char *)hdr); 1502 break; 1503 case ISCSI_OP_NOOP_OUT: 1504 cmd = isert_allocate_cmd(conn, rx_desc); 1505 if (!cmd) 1506 break; 1507 1508 isert_cmd = iscsit_priv_cmd(cmd); 1509 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1510 rx_desc, (unsigned char *)hdr); 1511 break; 1512 case ISCSI_OP_SCSI_DATA_OUT: 1513 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1514 (unsigned char *)hdr); 1515 break; 1516 case ISCSI_OP_SCSI_TMFUNC: 1517 cmd = isert_allocate_cmd(conn, rx_desc); 1518 if (!cmd) 1519 break; 1520 1521 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1522 (unsigned char *)hdr); 1523 break; 1524 case ISCSI_OP_LOGOUT: 1525 cmd = isert_allocate_cmd(conn, rx_desc); 1526 if (!cmd) 1527 break; 1528 1529 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1530 break; 1531 case ISCSI_OP_TEXT: 1532 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1533 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1534 else 1535 cmd = isert_allocate_cmd(conn, rx_desc); 1536 1537 if (!cmd) 1538 break; 1539 1540 isert_cmd = iscsit_priv_cmd(cmd); 1541 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1542 rx_desc, (struct iscsi_text *)hdr); 1543 break; 1544 default: 1545 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1546 dump_stack(); 1547 break; 1548 } 1549 1550 return ret; 1551 } 1552 1553 static void 1554 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1555 { 1556 struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1557 uint64_t read_va = 0, write_va = 0; 1558 uint32_t read_stag = 0, write_stag = 0; 1559 1560 switch (iser_ctrl->flags & 0xF0) { 1561 case ISCSI_CTRL: 1562 if (iser_ctrl->flags & ISER_RSV) { 1563 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1564 read_va = be64_to_cpu(iser_ctrl->read_va); 1565 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1566 read_stag, (unsigned long long)read_va); 1567 } 1568 if (iser_ctrl->flags & ISER_WSV) { 1569 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1570 write_va = be64_to_cpu(iser_ctrl->write_va); 1571 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1572 write_stag, (unsigned long long)write_va); 1573 } 1574 1575 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1576 break; 1577 case ISER_HELLO: 1578 isert_err("iSER Hello message\n"); 1579 break; 1580 default: 1581 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1582 break; 1583 } 1584 1585 isert_rx_opcode(isert_conn, rx_desc, 1586 read_stag, read_va, write_stag, write_va); 1587 } 1588 1589 static void 1590 isert_rcv_completion(struct iser_rx_desc *desc, 1591 struct isert_conn *isert_conn, 1592 u32 xfer_len) 1593 { 1594 struct ib_device *ib_dev = isert_conn->cm_id->device; 1595 struct iscsi_hdr *hdr; 1596 u64 rx_dma; 1597 int rx_buflen; 1598 1599 if ((char *)desc == isert_conn->login_req_buf) { 1600 rx_dma = isert_conn->login_req_dma; 1601 rx_buflen = ISER_RX_LOGIN_SIZE; 1602 isert_dbg("login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1603 rx_dma, rx_buflen); 1604 } else { 1605 rx_dma = desc->dma_addr; 1606 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1607 isert_dbg("req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1608 rx_dma, rx_buflen); 1609 } 1610 1611 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1612 1613 hdr = &desc->iscsi_header; 1614 isert_dbg("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1615 hdr->opcode, hdr->itt, hdr->flags, 1616 (int)(xfer_len - ISER_HEADERS_LEN)); 1617 1618 if ((char *)desc == isert_conn->login_req_buf) { 1619 isert_conn->login_req_len = xfer_len - ISER_HEADERS_LEN; 1620 if (isert_conn->conn) { 1621 struct iscsi_login *login = isert_conn->conn->conn_login; 1622 1623 if (login && !login->first_request) 1624 isert_rx_login_req(isert_conn); 1625 } 1626 mutex_lock(&isert_conn->mutex); 1627 complete(&isert_conn->login_req_comp); 1628 mutex_unlock(&isert_conn->mutex); 1629 } else { 1630 isert_rx_do_work(desc, isert_conn); 1631 } 1632 1633 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1634 DMA_FROM_DEVICE); 1635 1636 isert_conn->post_recv_buf_count--; 1637 } 1638 1639 static int 1640 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1641 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1642 enum iser_ib_op_code op, struct isert_data_buf *data) 1643 { 1644 struct ib_device *ib_dev = isert_conn->cm_id->device; 1645 1646 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1647 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1648 1649 data->len = length - offset; 1650 data->offset = offset; 1651 data->sg_off = data->offset / PAGE_SIZE; 1652 1653 data->sg = &sg[data->sg_off]; 1654 data->nents = min_t(unsigned int, nents - data->sg_off, 1655 ISCSI_ISER_SG_TABLESIZE); 1656 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1657 PAGE_SIZE); 1658 1659 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1660 data->dma_dir); 1661 if (unlikely(!data->dma_nents)) { 1662 isert_err("Cmd: unable to dma map SGs %p\n", sg); 1663 return -EINVAL; 1664 } 1665 1666 isert_dbg("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1667 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1668 1669 return 0; 1670 } 1671 1672 static void 1673 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1674 { 1675 struct ib_device *ib_dev = isert_conn->cm_id->device; 1676 1677 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1678 memset(data, 0, sizeof(*data)); 1679 } 1680 1681 1682 1683 static void 1684 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1685 { 1686 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1687 1688 isert_dbg("Cmd %p\n", isert_cmd); 1689 1690 if (wr->data.sg) { 1691 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1692 isert_unmap_data_buf(isert_conn, &wr->data); 1693 } 1694 1695 if (wr->rdma_wr) { 1696 isert_dbg("Cmd %p free send_wr\n", isert_cmd); 1697 kfree(wr->rdma_wr); 1698 wr->rdma_wr = NULL; 1699 } 1700 1701 if (wr->ib_sge) { 1702 isert_dbg("Cmd %p free ib_sge\n", isert_cmd); 1703 kfree(wr->ib_sge); 1704 wr->ib_sge = NULL; 1705 } 1706 } 1707 1708 static void 1709 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1710 { 1711 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1712 1713 isert_dbg("Cmd %p\n", isert_cmd); 1714 1715 if (wr->fr_desc) { 1716 isert_dbg("Cmd %p free fr_desc %p\n", isert_cmd, wr->fr_desc); 1717 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1718 isert_unmap_data_buf(isert_conn, &wr->prot); 1719 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1720 } 1721 spin_lock_bh(&isert_conn->pool_lock); 1722 list_add_tail(&wr->fr_desc->list, &isert_conn->fr_pool); 1723 spin_unlock_bh(&isert_conn->pool_lock); 1724 wr->fr_desc = NULL; 1725 } 1726 1727 if (wr->data.sg) { 1728 isert_dbg("Cmd %p unmap_sg op\n", isert_cmd); 1729 isert_unmap_data_buf(isert_conn, &wr->data); 1730 } 1731 1732 wr->ib_sge = NULL; 1733 wr->rdma_wr = NULL; 1734 } 1735 1736 static void 1737 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1738 { 1739 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1740 struct isert_conn *isert_conn = isert_cmd->conn; 1741 struct iscsi_conn *conn = isert_conn->conn; 1742 struct isert_device *device = isert_conn->device; 1743 struct iscsi_text_rsp *hdr; 1744 1745 isert_dbg("Cmd %p\n", isert_cmd); 1746 1747 switch (cmd->iscsi_opcode) { 1748 case ISCSI_OP_SCSI_CMD: 1749 spin_lock_bh(&conn->cmd_lock); 1750 if (!list_empty(&cmd->i_conn_node)) 1751 list_del_init(&cmd->i_conn_node); 1752 spin_unlock_bh(&conn->cmd_lock); 1753 1754 if (cmd->data_direction == DMA_TO_DEVICE) { 1755 iscsit_stop_dataout_timer(cmd); 1756 /* 1757 * Check for special case during comp_err where 1758 * WRITE_PENDING has been handed off from core, 1759 * but requires an extra target_put_sess_cmd() 1760 * before transport_generic_free_cmd() below. 1761 */ 1762 if (comp_err && 1763 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1764 struct se_cmd *se_cmd = &cmd->se_cmd; 1765 1766 target_put_sess_cmd(se_cmd); 1767 } 1768 } 1769 1770 device->unreg_rdma_mem(isert_cmd, isert_conn); 1771 transport_generic_free_cmd(&cmd->se_cmd, 0); 1772 break; 1773 case ISCSI_OP_SCSI_TMFUNC: 1774 spin_lock_bh(&conn->cmd_lock); 1775 if (!list_empty(&cmd->i_conn_node)) 1776 list_del_init(&cmd->i_conn_node); 1777 spin_unlock_bh(&conn->cmd_lock); 1778 1779 transport_generic_free_cmd(&cmd->se_cmd, 0); 1780 break; 1781 case ISCSI_OP_REJECT: 1782 case ISCSI_OP_NOOP_OUT: 1783 case ISCSI_OP_TEXT: 1784 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1785 /* If the continue bit is on, keep the command alive */ 1786 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1787 break; 1788 1789 spin_lock_bh(&conn->cmd_lock); 1790 if (!list_empty(&cmd->i_conn_node)) 1791 list_del_init(&cmd->i_conn_node); 1792 spin_unlock_bh(&conn->cmd_lock); 1793 1794 /* 1795 * Handle special case for REJECT when iscsi_add_reject*() has 1796 * overwritten the original iscsi_opcode assignment, and the 1797 * associated cmd->se_cmd needs to be released. 1798 */ 1799 if (cmd->se_cmd.se_tfo != NULL) { 1800 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1801 cmd->iscsi_opcode); 1802 transport_generic_free_cmd(&cmd->se_cmd, 0); 1803 break; 1804 } 1805 /* 1806 * Fall-through 1807 */ 1808 default: 1809 iscsit_release_cmd(cmd); 1810 break; 1811 } 1812 } 1813 1814 static void 1815 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1816 { 1817 if (tx_desc->dma_addr != 0) { 1818 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1819 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1820 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1821 tx_desc->dma_addr = 0; 1822 } 1823 } 1824 1825 static void 1826 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1827 struct ib_device *ib_dev, bool comp_err) 1828 { 1829 if (isert_cmd->pdu_buf_dma != 0) { 1830 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1831 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1832 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1833 isert_cmd->pdu_buf_dma = 0; 1834 } 1835 1836 isert_unmap_tx_desc(tx_desc, ib_dev); 1837 isert_put_cmd(isert_cmd, comp_err); 1838 } 1839 1840 static int 1841 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1842 { 1843 struct ib_mr_status mr_status; 1844 int ret; 1845 1846 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1847 if (ret) { 1848 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1849 goto fail_mr_status; 1850 } 1851 1852 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1853 u64 sec_offset_err; 1854 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1855 1856 switch (mr_status.sig_err.err_type) { 1857 case IB_SIG_BAD_GUARD: 1858 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1859 break; 1860 case IB_SIG_BAD_REFTAG: 1861 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1862 break; 1863 case IB_SIG_BAD_APPTAG: 1864 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1865 break; 1866 } 1867 sec_offset_err = mr_status.sig_err.sig_err_offset; 1868 do_div(sec_offset_err, block_size); 1869 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1870 1871 isert_err("PI error found type %d at sector 0x%llx " 1872 "expected 0x%x vs actual 0x%x\n", 1873 mr_status.sig_err.err_type, 1874 (unsigned long long)se_cmd->bad_sector, 1875 mr_status.sig_err.expected, 1876 mr_status.sig_err.actual); 1877 ret = 1; 1878 } 1879 1880 fail_mr_status: 1881 return ret; 1882 } 1883 1884 static void 1885 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1886 struct isert_cmd *isert_cmd) 1887 { 1888 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1889 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1890 struct se_cmd *se_cmd = &cmd->se_cmd; 1891 struct isert_conn *isert_conn = isert_cmd->conn; 1892 struct isert_device *device = isert_conn->device; 1893 int ret = 0; 1894 1895 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1896 ret = isert_check_pi_status(se_cmd, 1897 wr->fr_desc->pi_ctx->sig_mr); 1898 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1899 } 1900 1901 device->unreg_rdma_mem(isert_cmd, isert_conn); 1902 wr->rdma_wr_num = 0; 1903 if (ret) 1904 transport_send_check_condition_and_sense(se_cmd, 1905 se_cmd->pi_err, 0); 1906 else 1907 isert_put_response(isert_conn->conn, cmd); 1908 } 1909 1910 static void 1911 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1912 struct isert_cmd *isert_cmd) 1913 { 1914 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1915 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1916 struct se_cmd *se_cmd = &cmd->se_cmd; 1917 struct isert_conn *isert_conn = isert_cmd->conn; 1918 struct isert_device *device = isert_conn->device; 1919 int ret = 0; 1920 1921 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1922 ret = isert_check_pi_status(se_cmd, 1923 wr->fr_desc->pi_ctx->sig_mr); 1924 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1925 } 1926 1927 iscsit_stop_dataout_timer(cmd); 1928 device->unreg_rdma_mem(isert_cmd, isert_conn); 1929 cmd->write_data_done = wr->data.len; 1930 wr->rdma_wr_num = 0; 1931 1932 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1933 spin_lock_bh(&cmd->istate_lock); 1934 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1935 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1936 spin_unlock_bh(&cmd->istate_lock); 1937 1938 if (ret) { 1939 target_put_sess_cmd(se_cmd); 1940 transport_send_check_condition_and_sense(se_cmd, 1941 se_cmd->pi_err, 0); 1942 } else { 1943 target_execute_cmd(se_cmd); 1944 } 1945 } 1946 1947 static void 1948 isert_do_control_comp(struct work_struct *work) 1949 { 1950 struct isert_cmd *isert_cmd = container_of(work, 1951 struct isert_cmd, comp_work); 1952 struct isert_conn *isert_conn = isert_cmd->conn; 1953 struct ib_device *ib_dev = isert_conn->cm_id->device; 1954 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1955 1956 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1957 1958 switch (cmd->i_state) { 1959 case ISTATE_SEND_TASKMGTRSP: 1960 iscsit_tmr_post_handler(cmd, cmd->conn); 1961 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1962 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1963 cmd->i_state = ISTATE_SENT_STATUS; 1964 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1965 ib_dev, false); 1966 break; 1967 case ISTATE_SEND_LOGOUTRSP: 1968 iscsit_logout_post_handler(cmd, cmd->conn); 1969 break; 1970 default: 1971 isert_err("Unknown i_state %d\n", cmd->i_state); 1972 dump_stack(); 1973 break; 1974 } 1975 } 1976 1977 static void 1978 isert_response_completion(struct iser_tx_desc *tx_desc, 1979 struct isert_cmd *isert_cmd, 1980 struct isert_conn *isert_conn, 1981 struct ib_device *ib_dev) 1982 { 1983 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1984 1985 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1986 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1987 cmd->i_state == ISTATE_SEND_REJECT || 1988 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1989 isert_unmap_tx_desc(tx_desc, ib_dev); 1990 1991 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1992 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1993 return; 1994 } 1995 1996 cmd->i_state = ISTATE_SENT_STATUS; 1997 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1998 } 1999 2000 static void 2001 isert_snd_completion(struct iser_tx_desc *tx_desc, 2002 struct isert_conn *isert_conn) 2003 { 2004 struct ib_device *ib_dev = isert_conn->cm_id->device; 2005 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 2006 struct isert_rdma_wr *wr; 2007 2008 if (!isert_cmd) { 2009 isert_unmap_tx_desc(tx_desc, ib_dev); 2010 return; 2011 } 2012 wr = &isert_cmd->rdma_wr; 2013 2014 isert_dbg("Cmd %p iser_ib_op %d\n", isert_cmd, wr->iser_ib_op); 2015 2016 switch (wr->iser_ib_op) { 2017 case ISER_IB_SEND: 2018 isert_response_completion(tx_desc, isert_cmd, 2019 isert_conn, ib_dev); 2020 break; 2021 case ISER_IB_RDMA_WRITE: 2022 isert_completion_rdma_write(tx_desc, isert_cmd); 2023 break; 2024 case ISER_IB_RDMA_READ: 2025 isert_completion_rdma_read(tx_desc, isert_cmd); 2026 break; 2027 default: 2028 isert_err("Unknown wr->iser_ib_op: 0x%x\n", wr->iser_ib_op); 2029 dump_stack(); 2030 break; 2031 } 2032 } 2033 2034 /** 2035 * is_isert_tx_desc() - Indicate if the completion wr_id 2036 * is a TX descriptor or not. 2037 * @isert_conn: iser connection 2038 * @wr_id: completion WR identifier 2039 * 2040 * Since we cannot rely on wc opcode in FLUSH errors 2041 * we must work around it by checking if the wr_id address 2042 * falls in the iser connection rx_descs buffer. If so 2043 * it is an RX descriptor, otherwize it is a TX. 2044 */ 2045 static inline bool 2046 is_isert_tx_desc(struct isert_conn *isert_conn, void *wr_id) 2047 { 2048 void *start = isert_conn->rx_descs; 2049 int len = ISERT_QP_MAX_RECV_DTOS * sizeof(*isert_conn->rx_descs); 2050 2051 if (wr_id >= start && wr_id < start + len) 2052 return false; 2053 2054 return true; 2055 } 2056 2057 static void 2058 isert_cq_comp_err(struct isert_conn *isert_conn, struct ib_wc *wc) 2059 { 2060 if (wc->wr_id == ISER_BEACON_WRID) { 2061 isert_info("conn %p completing wait_comp_err\n", 2062 isert_conn); 2063 complete(&isert_conn->wait_comp_err); 2064 } else if (is_isert_tx_desc(isert_conn, (void *)(uintptr_t)wc->wr_id)) { 2065 struct ib_device *ib_dev = isert_conn->cm_id->device; 2066 struct isert_cmd *isert_cmd; 2067 struct iser_tx_desc *desc; 2068 2069 desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2070 isert_cmd = desc->isert_cmd; 2071 if (!isert_cmd) 2072 isert_unmap_tx_desc(desc, ib_dev); 2073 else 2074 isert_completion_put(desc, isert_cmd, ib_dev, true); 2075 } else { 2076 isert_conn->post_recv_buf_count--; 2077 if (!isert_conn->post_recv_buf_count) 2078 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2079 } 2080 } 2081 2082 static void 2083 isert_handle_wc(struct ib_wc *wc) 2084 { 2085 struct isert_conn *isert_conn; 2086 struct iser_tx_desc *tx_desc; 2087 struct iser_rx_desc *rx_desc; 2088 2089 isert_conn = wc->qp->qp_context; 2090 if (likely(wc->status == IB_WC_SUCCESS)) { 2091 if (wc->opcode == IB_WC_RECV) { 2092 rx_desc = (struct iser_rx_desc *)(uintptr_t)wc->wr_id; 2093 isert_rcv_completion(rx_desc, isert_conn, wc->byte_len); 2094 } else { 2095 tx_desc = (struct iser_tx_desc *)(uintptr_t)wc->wr_id; 2096 isert_snd_completion(tx_desc, isert_conn); 2097 } 2098 } else { 2099 if (wc->status != IB_WC_WR_FLUSH_ERR) 2100 isert_err("%s (%d): wr id %llx vend_err %x\n", 2101 ib_wc_status_msg(wc->status), wc->status, 2102 wc->wr_id, wc->vendor_err); 2103 else 2104 isert_dbg("%s (%d): wr id %llx\n", 2105 ib_wc_status_msg(wc->status), wc->status, 2106 wc->wr_id); 2107 2108 if (wc->wr_id != ISER_FASTREG_LI_WRID) 2109 isert_cq_comp_err(isert_conn, wc); 2110 } 2111 } 2112 2113 static void 2114 isert_cq_work(struct work_struct *work) 2115 { 2116 enum { isert_poll_budget = 65536 }; 2117 struct isert_comp *comp = container_of(work, struct isert_comp, 2118 work); 2119 struct ib_wc *const wcs = comp->wcs; 2120 int i, n, completed = 0; 2121 2122 while ((n = ib_poll_cq(comp->cq, ARRAY_SIZE(comp->wcs), wcs)) > 0) { 2123 for (i = 0; i < n; i++) 2124 isert_handle_wc(&wcs[i]); 2125 2126 completed += n; 2127 if (completed >= isert_poll_budget) 2128 break; 2129 } 2130 2131 ib_req_notify_cq(comp->cq, IB_CQ_NEXT_COMP); 2132 } 2133 2134 static void 2135 isert_cq_callback(struct ib_cq *cq, void *context) 2136 { 2137 struct isert_comp *comp = context; 2138 2139 queue_work(isert_comp_wq, &comp->work); 2140 } 2141 2142 static int 2143 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2144 { 2145 struct ib_send_wr *wr_failed; 2146 int ret; 2147 2148 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2149 if (ret) { 2150 isert_err("ib_post_recv failed with %d\n", ret); 2151 return ret; 2152 } 2153 2154 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 2155 &wr_failed); 2156 if (ret) { 2157 isert_err("ib_post_send failed with %d\n", ret); 2158 return ret; 2159 } 2160 return ret; 2161 } 2162 2163 static int 2164 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2165 { 2166 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2167 struct isert_conn *isert_conn = conn->context; 2168 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2169 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2170 &isert_cmd->tx_desc.iscsi_header; 2171 2172 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2173 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2174 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2175 /* 2176 * Attach SENSE DATA payload to iSCSI Response PDU 2177 */ 2178 if (cmd->se_cmd.sense_buffer && 2179 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2180 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2181 struct isert_device *device = isert_conn->device; 2182 struct ib_device *ib_dev = device->ib_device; 2183 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2184 u32 padding, pdu_len; 2185 2186 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2187 cmd->sense_buffer); 2188 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2189 2190 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2191 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2192 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2193 2194 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2195 (void *)cmd->sense_buffer, pdu_len, 2196 DMA_TO_DEVICE); 2197 2198 isert_cmd->pdu_buf_len = pdu_len; 2199 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2200 tx_dsg->length = pdu_len; 2201 tx_dsg->lkey = device->pd->local_dma_lkey; 2202 isert_cmd->tx_desc.num_sge = 2; 2203 } 2204 2205 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2206 2207 isert_dbg("Posting SCSI Response\n"); 2208 2209 return isert_post_response(isert_conn, isert_cmd); 2210 } 2211 2212 static void 2213 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2214 { 2215 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2216 struct isert_conn *isert_conn = conn->context; 2217 struct isert_device *device = isert_conn->device; 2218 2219 spin_lock_bh(&conn->cmd_lock); 2220 if (!list_empty(&cmd->i_conn_node)) 2221 list_del_init(&cmd->i_conn_node); 2222 spin_unlock_bh(&conn->cmd_lock); 2223 2224 if (cmd->data_direction == DMA_TO_DEVICE) 2225 iscsit_stop_dataout_timer(cmd); 2226 2227 device->unreg_rdma_mem(isert_cmd, isert_conn); 2228 } 2229 2230 static enum target_prot_op 2231 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2232 { 2233 struct isert_conn *isert_conn = conn->context; 2234 struct isert_device *device = isert_conn->device; 2235 2236 if (conn->tpg->tpg_attrib.t10_pi) { 2237 if (device->pi_capable) { 2238 isert_info("conn %p PI offload enabled\n", isert_conn); 2239 isert_conn->pi_support = true; 2240 return TARGET_PROT_ALL; 2241 } 2242 } 2243 2244 isert_info("conn %p PI offload disabled\n", isert_conn); 2245 isert_conn->pi_support = false; 2246 2247 return TARGET_PROT_NORMAL; 2248 } 2249 2250 static int 2251 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2252 bool nopout_response) 2253 { 2254 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2255 struct isert_conn *isert_conn = conn->context; 2256 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2257 2258 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2259 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2260 &isert_cmd->tx_desc.iscsi_header, 2261 nopout_response); 2262 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2263 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2264 2265 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 2266 2267 return isert_post_response(isert_conn, isert_cmd); 2268 } 2269 2270 static int 2271 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2272 { 2273 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2274 struct isert_conn *isert_conn = conn->context; 2275 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2276 2277 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2278 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2279 &isert_cmd->tx_desc.iscsi_header); 2280 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2281 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2282 2283 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 2284 2285 return isert_post_response(isert_conn, isert_cmd); 2286 } 2287 2288 static int 2289 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2290 { 2291 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2292 struct isert_conn *isert_conn = conn->context; 2293 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2294 2295 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2296 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2297 &isert_cmd->tx_desc.iscsi_header); 2298 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2299 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2300 2301 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 2302 2303 return isert_post_response(isert_conn, isert_cmd); 2304 } 2305 2306 static int 2307 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2308 { 2309 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2310 struct isert_conn *isert_conn = conn->context; 2311 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2312 struct isert_device *device = isert_conn->device; 2313 struct ib_device *ib_dev = device->ib_device; 2314 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2315 struct iscsi_reject *hdr = 2316 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2317 2318 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2319 iscsit_build_reject(cmd, conn, hdr); 2320 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2321 2322 hton24(hdr->dlength, ISCSI_HDR_LEN); 2323 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2324 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2325 DMA_TO_DEVICE); 2326 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2327 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2328 tx_dsg->length = ISCSI_HDR_LEN; 2329 tx_dsg->lkey = device->pd->local_dma_lkey; 2330 isert_cmd->tx_desc.num_sge = 2; 2331 2332 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2333 2334 isert_dbg("conn %p Posting Reject\n", isert_conn); 2335 2336 return isert_post_response(isert_conn, isert_cmd); 2337 } 2338 2339 static int 2340 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2341 { 2342 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2343 struct isert_conn *isert_conn = conn->context; 2344 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2345 struct iscsi_text_rsp *hdr = 2346 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2347 u32 txt_rsp_len; 2348 int rc; 2349 2350 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2351 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2352 if (rc < 0) 2353 return rc; 2354 2355 txt_rsp_len = rc; 2356 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2357 2358 if (txt_rsp_len) { 2359 struct isert_device *device = isert_conn->device; 2360 struct ib_device *ib_dev = device->ib_device; 2361 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2362 void *txt_rsp_buf = cmd->buf_ptr; 2363 2364 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2365 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2366 2367 isert_cmd->pdu_buf_len = txt_rsp_len; 2368 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2369 tx_dsg->length = txt_rsp_len; 2370 tx_dsg->lkey = device->pd->local_dma_lkey; 2371 isert_cmd->tx_desc.num_sge = 2; 2372 } 2373 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2374 2375 isert_dbg("conn %p Text Response\n", isert_conn); 2376 2377 return isert_post_response(isert_conn, isert_cmd); 2378 } 2379 2380 static int 2381 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2382 struct ib_sge *ib_sge, struct ib_rdma_wr *rdma_wr, 2383 u32 data_left, u32 offset) 2384 { 2385 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2386 struct scatterlist *sg_start, *tmp_sg; 2387 struct isert_device *device = isert_conn->device; 2388 struct ib_device *ib_dev = device->ib_device; 2389 u32 sg_off, page_off; 2390 int i = 0, sg_nents; 2391 2392 sg_off = offset / PAGE_SIZE; 2393 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2394 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2395 page_off = offset % PAGE_SIZE; 2396 2397 rdma_wr->wr.sg_list = ib_sge; 2398 rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; 2399 /* 2400 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2401 */ 2402 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2403 isert_dbg("RDMA from SGL dma_addr: 0x%llx dma_len: %u, " 2404 "page_off: %u\n", 2405 (unsigned long long)tmp_sg->dma_address, 2406 tmp_sg->length, page_off); 2407 2408 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2409 ib_sge->length = min_t(u32, data_left, 2410 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2411 ib_sge->lkey = device->pd->local_dma_lkey; 2412 2413 isert_dbg("RDMA ib_sge: addr: 0x%llx length: %u lkey: %x\n", 2414 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2415 page_off = 0; 2416 data_left -= ib_sge->length; 2417 if (!data_left) 2418 break; 2419 ib_sge++; 2420 isert_dbg("Incrementing ib_sge pointer to %p\n", ib_sge); 2421 } 2422 2423 rdma_wr->wr.num_sge = ++i; 2424 isert_dbg("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2425 rdma_wr->wr.sg_list, rdma_wr->wr.num_sge); 2426 2427 return rdma_wr->wr.num_sge; 2428 } 2429 2430 static int 2431 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2432 struct isert_rdma_wr *wr) 2433 { 2434 struct se_cmd *se_cmd = &cmd->se_cmd; 2435 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2436 struct isert_conn *isert_conn = conn->context; 2437 struct isert_data_buf *data = &wr->data; 2438 struct ib_rdma_wr *rdma_wr; 2439 struct ib_sge *ib_sge; 2440 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2441 int ret = 0, i, ib_sge_cnt; 2442 2443 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2444 2445 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2446 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2447 se_cmd->t_data_nents, se_cmd->data_length, 2448 offset, wr->iser_ib_op, &wr->data); 2449 if (ret) 2450 return ret; 2451 2452 data_left = data->len; 2453 offset = data->offset; 2454 2455 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2456 if (!ib_sge) { 2457 isert_warn("Unable to allocate ib_sge\n"); 2458 ret = -ENOMEM; 2459 goto unmap_cmd; 2460 } 2461 wr->ib_sge = ib_sge; 2462 2463 wr->rdma_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2464 wr->rdma_wr = kzalloc(sizeof(struct ib_rdma_wr) * wr->rdma_wr_num, 2465 GFP_KERNEL); 2466 if (!wr->rdma_wr) { 2467 isert_dbg("Unable to allocate wr->rdma_wr\n"); 2468 ret = -ENOMEM; 2469 goto unmap_cmd; 2470 } 2471 2472 wr->isert_cmd = isert_cmd; 2473 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2474 2475 for (i = 0; i < wr->rdma_wr_num; i++) { 2476 rdma_wr = &isert_cmd->rdma_wr.rdma_wr[i]; 2477 data_len = min(data_left, rdma_write_max); 2478 2479 rdma_wr->wr.send_flags = 0; 2480 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2481 rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2482 rdma_wr->remote_addr = isert_cmd->read_va + offset; 2483 rdma_wr->rkey = isert_cmd->read_stag; 2484 if (i + 1 == wr->rdma_wr_num) 2485 rdma_wr->wr.next = &isert_cmd->tx_desc.send_wr; 2486 else 2487 rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; 2488 } else { 2489 rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2490 rdma_wr->remote_addr = isert_cmd->write_va + va_offset; 2491 rdma_wr->rkey = isert_cmd->write_stag; 2492 if (i + 1 == wr->rdma_wr_num) 2493 rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 2494 else 2495 rdma_wr->wr.next = &wr->rdma_wr[i + 1].wr; 2496 } 2497 2498 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2499 rdma_wr, data_len, offset); 2500 ib_sge += ib_sge_cnt; 2501 2502 offset += data_len; 2503 va_offset += data_len; 2504 data_left -= data_len; 2505 } 2506 2507 return 0; 2508 unmap_cmd: 2509 isert_unmap_data_buf(isert_conn, data); 2510 2511 return ret; 2512 } 2513 2514 static inline void 2515 isert_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 2516 { 2517 u32 rkey; 2518 2519 memset(inv_wr, 0, sizeof(*inv_wr)); 2520 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 2521 inv_wr->opcode = IB_WR_LOCAL_INV; 2522 inv_wr->ex.invalidate_rkey = mr->rkey; 2523 2524 /* Bump the key */ 2525 rkey = ib_inc_rkey(mr->rkey); 2526 ib_update_fast_reg_key(mr, rkey); 2527 } 2528 2529 static int 2530 isert_fast_reg_mr(struct isert_conn *isert_conn, 2531 struct fast_reg_descriptor *fr_desc, 2532 struct isert_data_buf *mem, 2533 enum isert_indicator ind, 2534 struct ib_sge *sge) 2535 { 2536 struct isert_device *device = isert_conn->device; 2537 struct ib_device *ib_dev = device->ib_device; 2538 struct ib_mr *mr; 2539 struct ib_reg_wr reg_wr; 2540 struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 2541 int ret, n; 2542 2543 if (mem->dma_nents == 1) { 2544 sge->lkey = device->pd->local_dma_lkey; 2545 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2546 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2547 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2548 sge->addr, sge->length, sge->lkey); 2549 return 0; 2550 } 2551 2552 if (ind == ISERT_DATA_KEY_VALID) 2553 /* Registering data buffer */ 2554 mr = fr_desc->data_mr; 2555 else 2556 /* Registering protection buffer */ 2557 mr = fr_desc->pi_ctx->prot_mr; 2558 2559 if (!(fr_desc->ind & ind)) { 2560 isert_inv_rkey(&inv_wr, mr); 2561 wr = &inv_wr; 2562 } 2563 2564 n = ib_map_mr_sg(mr, mem->sg, mem->nents, PAGE_SIZE); 2565 if (unlikely(n != mem->nents)) { 2566 isert_err("failed to map mr sg (%d/%d)\n", 2567 n, mem->nents); 2568 return n < 0 ? n : -EINVAL; 2569 } 2570 2571 isert_dbg("Use fr_desc %p sg_nents %d offset %u\n", 2572 fr_desc, mem->nents, mem->offset); 2573 2574 reg_wr.wr.next = NULL; 2575 reg_wr.wr.opcode = IB_WR_REG_MR; 2576 reg_wr.wr.wr_id = ISER_FASTREG_LI_WRID; 2577 reg_wr.wr.send_flags = 0; 2578 reg_wr.wr.num_sge = 0; 2579 reg_wr.mr = mr; 2580 reg_wr.key = mr->lkey; 2581 reg_wr.access = IB_ACCESS_LOCAL_WRITE; 2582 2583 if (!wr) 2584 wr = ®_wr.wr; 2585 else 2586 wr->next = ®_wr.wr; 2587 2588 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2589 if (ret) { 2590 isert_err("fast registration failed, ret:%d\n", ret); 2591 return ret; 2592 } 2593 fr_desc->ind &= ~ind; 2594 2595 sge->lkey = mr->lkey; 2596 sge->addr = mr->iova; 2597 sge->length = mr->length; 2598 2599 isert_dbg("sge: addr: 0x%llx length: %u lkey: %x\n", 2600 sge->addr, sge->length, sge->lkey); 2601 2602 return ret; 2603 } 2604 2605 static inline void 2606 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2607 struct ib_sig_domain *domain) 2608 { 2609 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2610 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2611 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2612 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2613 /* 2614 * At the moment we hard code those, but if in the future 2615 * the target core would like to use it, we will take it 2616 * from se_cmd. 2617 */ 2618 domain->sig.dif.apptag_check_mask = 0xffff; 2619 domain->sig.dif.app_escape = true; 2620 domain->sig.dif.ref_escape = true; 2621 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2622 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2623 domain->sig.dif.ref_remap = true; 2624 }; 2625 2626 static int 2627 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2628 { 2629 switch (se_cmd->prot_op) { 2630 case TARGET_PROT_DIN_INSERT: 2631 case TARGET_PROT_DOUT_STRIP: 2632 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2633 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2634 break; 2635 case TARGET_PROT_DOUT_INSERT: 2636 case TARGET_PROT_DIN_STRIP: 2637 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2638 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2639 break; 2640 case TARGET_PROT_DIN_PASS: 2641 case TARGET_PROT_DOUT_PASS: 2642 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2643 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2644 break; 2645 default: 2646 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2647 return -EINVAL; 2648 } 2649 2650 return 0; 2651 } 2652 2653 static inline u8 2654 isert_set_prot_checks(u8 prot_checks) 2655 { 2656 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2657 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2658 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2659 } 2660 2661 static int 2662 isert_reg_sig_mr(struct isert_conn *isert_conn, 2663 struct se_cmd *se_cmd, 2664 struct isert_rdma_wr *rdma_wr, 2665 struct fast_reg_descriptor *fr_desc) 2666 { 2667 struct ib_sig_handover_wr sig_wr; 2668 struct ib_send_wr inv_wr, *bad_wr, *wr = NULL; 2669 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2670 struct ib_sig_attrs sig_attrs; 2671 int ret; 2672 2673 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2674 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2675 if (ret) 2676 goto err; 2677 2678 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2679 2680 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2681 isert_inv_rkey(&inv_wr, pi_ctx->sig_mr); 2682 wr = &inv_wr; 2683 } 2684 2685 memset(&sig_wr, 0, sizeof(sig_wr)); 2686 sig_wr.wr.opcode = IB_WR_REG_SIG_MR; 2687 sig_wr.wr.wr_id = ISER_FASTREG_LI_WRID; 2688 sig_wr.wr.sg_list = &rdma_wr->ib_sg[DATA]; 2689 sig_wr.wr.num_sge = 1; 2690 sig_wr.access_flags = IB_ACCESS_LOCAL_WRITE; 2691 sig_wr.sig_attrs = &sig_attrs; 2692 sig_wr.sig_mr = pi_ctx->sig_mr; 2693 if (se_cmd->t_prot_sg) 2694 sig_wr.prot = &rdma_wr->ib_sg[PROT]; 2695 2696 if (!wr) 2697 wr = &sig_wr.wr; 2698 else 2699 wr->next = &sig_wr.wr; 2700 2701 ret = ib_post_send(isert_conn->qp, wr, &bad_wr); 2702 if (ret) { 2703 isert_err("fast registration failed, ret:%d\n", ret); 2704 goto err; 2705 } 2706 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2707 2708 rdma_wr->ib_sg[SIG].lkey = pi_ctx->sig_mr->lkey; 2709 rdma_wr->ib_sg[SIG].addr = 0; 2710 rdma_wr->ib_sg[SIG].length = se_cmd->data_length; 2711 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2712 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2713 /* 2714 * We have protection guards on the wire 2715 * so we need to set a larget transfer 2716 */ 2717 rdma_wr->ib_sg[SIG].length += se_cmd->prot_length; 2718 2719 isert_dbg("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2720 rdma_wr->ib_sg[SIG].addr, rdma_wr->ib_sg[SIG].length, 2721 rdma_wr->ib_sg[SIG].lkey); 2722 err: 2723 return ret; 2724 } 2725 2726 static int 2727 isert_handle_prot_cmd(struct isert_conn *isert_conn, 2728 struct isert_cmd *isert_cmd, 2729 struct isert_rdma_wr *wr) 2730 { 2731 struct isert_device *device = isert_conn->device; 2732 struct se_cmd *se_cmd = &isert_cmd->iscsi_cmd->se_cmd; 2733 int ret; 2734 2735 if (!wr->fr_desc->pi_ctx) { 2736 ret = isert_create_pi_ctx(wr->fr_desc, 2737 device->ib_device, 2738 device->pd); 2739 if (ret) { 2740 isert_err("conn %p failed to allocate pi_ctx\n", 2741 isert_conn); 2742 return ret; 2743 } 2744 } 2745 2746 if (se_cmd->t_prot_sg) { 2747 ret = isert_map_data_buf(isert_conn, isert_cmd, 2748 se_cmd->t_prot_sg, 2749 se_cmd->t_prot_nents, 2750 se_cmd->prot_length, 2751 0, wr->iser_ib_op, &wr->prot); 2752 if (ret) { 2753 isert_err("conn %p failed to map protection buffer\n", 2754 isert_conn); 2755 return ret; 2756 } 2757 2758 memset(&wr->ib_sg[PROT], 0, sizeof(wr->ib_sg[PROT])); 2759 ret = isert_fast_reg_mr(isert_conn, wr->fr_desc, &wr->prot, 2760 ISERT_PROT_KEY_VALID, &wr->ib_sg[PROT]); 2761 if (ret) { 2762 isert_err("conn %p failed to fast reg mr\n", 2763 isert_conn); 2764 goto unmap_prot_cmd; 2765 } 2766 } 2767 2768 ret = isert_reg_sig_mr(isert_conn, se_cmd, wr, wr->fr_desc); 2769 if (ret) { 2770 isert_err("conn %p failed to fast reg mr\n", 2771 isert_conn); 2772 goto unmap_prot_cmd; 2773 } 2774 wr->fr_desc->ind |= ISERT_PROTECTED; 2775 2776 return 0; 2777 2778 unmap_prot_cmd: 2779 if (se_cmd->t_prot_sg) 2780 isert_unmap_data_buf(isert_conn, &wr->prot); 2781 2782 return ret; 2783 } 2784 2785 static int 2786 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2787 struct isert_rdma_wr *wr) 2788 { 2789 struct se_cmd *se_cmd = &cmd->se_cmd; 2790 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2791 struct isert_conn *isert_conn = conn->context; 2792 struct fast_reg_descriptor *fr_desc = NULL; 2793 struct ib_rdma_wr *rdma_wr; 2794 struct ib_sge *ib_sg; 2795 u32 offset; 2796 int ret = 0; 2797 unsigned long flags; 2798 2799 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2800 2801 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2802 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2803 se_cmd->t_data_nents, se_cmd->data_length, 2804 offset, wr->iser_ib_op, &wr->data); 2805 if (ret) 2806 return ret; 2807 2808 if (wr->data.dma_nents != 1 || isert_prot_cmd(isert_conn, se_cmd)) { 2809 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2810 fr_desc = list_first_entry(&isert_conn->fr_pool, 2811 struct fast_reg_descriptor, list); 2812 list_del(&fr_desc->list); 2813 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2814 wr->fr_desc = fr_desc; 2815 } 2816 2817 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2818 ISERT_DATA_KEY_VALID, &wr->ib_sg[DATA]); 2819 if (ret) 2820 goto unmap_cmd; 2821 2822 if (isert_prot_cmd(isert_conn, se_cmd)) { 2823 ret = isert_handle_prot_cmd(isert_conn, isert_cmd, wr); 2824 if (ret) 2825 goto unmap_cmd; 2826 2827 ib_sg = &wr->ib_sg[SIG]; 2828 } else { 2829 ib_sg = &wr->ib_sg[DATA]; 2830 } 2831 2832 memcpy(&wr->s_ib_sge, ib_sg, sizeof(*ib_sg)); 2833 wr->ib_sge = &wr->s_ib_sge; 2834 wr->rdma_wr_num = 1; 2835 memset(&wr->s_rdma_wr, 0, sizeof(wr->s_rdma_wr)); 2836 wr->rdma_wr = &wr->s_rdma_wr; 2837 wr->isert_cmd = isert_cmd; 2838 2839 rdma_wr = &isert_cmd->rdma_wr.s_rdma_wr; 2840 rdma_wr->wr.sg_list = &wr->s_ib_sge; 2841 rdma_wr->wr.num_sge = 1; 2842 rdma_wr->wr.wr_id = (uintptr_t)&isert_cmd->tx_desc; 2843 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2844 rdma_wr->wr.opcode = IB_WR_RDMA_WRITE; 2845 rdma_wr->remote_addr = isert_cmd->read_va; 2846 rdma_wr->rkey = isert_cmd->read_stag; 2847 rdma_wr->wr.send_flags = !isert_prot_cmd(isert_conn, se_cmd) ? 2848 0 : IB_SEND_SIGNALED; 2849 } else { 2850 rdma_wr->wr.opcode = IB_WR_RDMA_READ; 2851 rdma_wr->remote_addr = isert_cmd->write_va; 2852 rdma_wr->rkey = isert_cmd->write_stag; 2853 rdma_wr->wr.send_flags = IB_SEND_SIGNALED; 2854 } 2855 2856 return 0; 2857 2858 unmap_cmd: 2859 if (fr_desc) { 2860 spin_lock_irqsave(&isert_conn->pool_lock, flags); 2861 list_add_tail(&fr_desc->list, &isert_conn->fr_pool); 2862 spin_unlock_irqrestore(&isert_conn->pool_lock, flags); 2863 } 2864 isert_unmap_data_buf(isert_conn, &wr->data); 2865 2866 return ret; 2867 } 2868 2869 static int 2870 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2871 { 2872 struct se_cmd *se_cmd = &cmd->se_cmd; 2873 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2874 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2875 struct isert_conn *isert_conn = conn->context; 2876 struct isert_device *device = isert_conn->device; 2877 struct ib_send_wr *wr_failed; 2878 int rc; 2879 2880 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2881 isert_cmd, se_cmd->data_length); 2882 2883 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2884 rc = device->reg_rdma_mem(conn, cmd, wr); 2885 if (rc) { 2886 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2887 return rc; 2888 } 2889 2890 if (!isert_prot_cmd(isert_conn, se_cmd)) { 2891 /* 2892 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2893 */ 2894 isert_create_send_desc(isert_conn, isert_cmd, 2895 &isert_cmd->tx_desc); 2896 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2897 &isert_cmd->tx_desc.iscsi_header); 2898 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2899 isert_init_send_wr(isert_conn, isert_cmd, 2900 &isert_cmd->tx_desc.send_wr); 2901 isert_cmd->rdma_wr.s_rdma_wr.wr.next = &isert_cmd->tx_desc.send_wr; 2902 wr->rdma_wr_num += 1; 2903 2904 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2905 if (rc) { 2906 isert_err("ib_post_recv failed with %d\n", rc); 2907 return rc; 2908 } 2909 } 2910 2911 rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); 2912 if (rc) 2913 isert_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2914 2915 if (!isert_prot_cmd(isert_conn, se_cmd)) 2916 isert_dbg("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2917 "READ\n", isert_cmd); 2918 else 2919 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2920 isert_cmd); 2921 2922 return 1; 2923 } 2924 2925 static int 2926 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2927 { 2928 struct se_cmd *se_cmd = &cmd->se_cmd; 2929 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2930 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2931 struct isert_conn *isert_conn = conn->context; 2932 struct isert_device *device = isert_conn->device; 2933 struct ib_send_wr *wr_failed; 2934 int rc; 2935 2936 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2937 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2938 wr->iser_ib_op = ISER_IB_RDMA_READ; 2939 rc = device->reg_rdma_mem(conn, cmd, wr); 2940 if (rc) { 2941 isert_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2942 return rc; 2943 } 2944 2945 rc = ib_post_send(isert_conn->qp, &wr->rdma_wr->wr, &wr_failed); 2946 if (rc) 2947 isert_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2948 2949 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2950 isert_cmd); 2951 2952 return 0; 2953 } 2954 2955 static int 2956 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2957 { 2958 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2959 int ret = 0; 2960 2961 switch (state) { 2962 case ISTATE_REMOVE: 2963 spin_lock_bh(&conn->cmd_lock); 2964 list_del_init(&cmd->i_conn_node); 2965 spin_unlock_bh(&conn->cmd_lock); 2966 isert_put_cmd(isert_cmd, true); 2967 break; 2968 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2969 ret = isert_put_nopin(cmd, conn, false); 2970 break; 2971 default: 2972 isert_err("Unknown immediate state: 0x%02x\n", state); 2973 ret = -EINVAL; 2974 break; 2975 } 2976 2977 return ret; 2978 } 2979 2980 static int 2981 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2982 { 2983 struct isert_conn *isert_conn = conn->context; 2984 int ret; 2985 2986 switch (state) { 2987 case ISTATE_SEND_LOGOUTRSP: 2988 ret = isert_put_logout_rsp(cmd, conn); 2989 if (!ret) 2990 isert_conn->logout_posted = true; 2991 break; 2992 case ISTATE_SEND_NOPIN: 2993 ret = isert_put_nopin(cmd, conn, true); 2994 break; 2995 case ISTATE_SEND_TASKMGTRSP: 2996 ret = isert_put_tm_rsp(cmd, conn); 2997 break; 2998 case ISTATE_SEND_REJECT: 2999 ret = isert_put_reject(cmd, conn); 3000 break; 3001 case ISTATE_SEND_TEXTRSP: 3002 ret = isert_put_text_rsp(cmd, conn); 3003 break; 3004 case ISTATE_SEND_STATUS: 3005 /* 3006 * Special case for sending non GOOD SCSI status from TX thread 3007 * context during pre se_cmd excecution failure. 3008 */ 3009 ret = isert_put_response(conn, cmd); 3010 break; 3011 default: 3012 isert_err("Unknown response state: 0x%02x\n", state); 3013 ret = -EINVAL; 3014 break; 3015 } 3016 3017 return ret; 3018 } 3019 3020 struct rdma_cm_id * 3021 isert_setup_id(struct isert_np *isert_np) 3022 { 3023 struct iscsi_np *np = isert_np->np; 3024 struct rdma_cm_id *id; 3025 struct sockaddr *sa; 3026 int ret; 3027 3028 sa = (struct sockaddr *)&np->np_sockaddr; 3029 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 3030 3031 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 3032 RDMA_PS_TCP, IB_QPT_RC); 3033 if (IS_ERR(id)) { 3034 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 3035 ret = PTR_ERR(id); 3036 goto out; 3037 } 3038 isert_dbg("id %p context %p\n", id, id->context); 3039 3040 ret = rdma_bind_addr(id, sa); 3041 if (ret) { 3042 isert_err("rdma_bind_addr() failed: %d\n", ret); 3043 goto out_id; 3044 } 3045 3046 ret = rdma_listen(id, 0); 3047 if (ret) { 3048 isert_err("rdma_listen() failed: %d\n", ret); 3049 goto out_id; 3050 } 3051 3052 return id; 3053 out_id: 3054 rdma_destroy_id(id); 3055 out: 3056 return ERR_PTR(ret); 3057 } 3058 3059 static int 3060 isert_setup_np(struct iscsi_np *np, 3061 struct sockaddr_storage *ksockaddr) 3062 { 3063 struct isert_np *isert_np; 3064 struct rdma_cm_id *isert_lid; 3065 int ret; 3066 3067 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3068 if (!isert_np) { 3069 isert_err("Unable to allocate struct isert_np\n"); 3070 return -ENOMEM; 3071 } 3072 sema_init(&isert_np->sem, 0); 3073 mutex_init(&isert_np->mutex); 3074 INIT_LIST_HEAD(&isert_np->accepted); 3075 INIT_LIST_HEAD(&isert_np->pending); 3076 isert_np->np = np; 3077 3078 /* 3079 * Setup the np->np_sockaddr from the passed sockaddr setup 3080 * in iscsi_target_configfs.c code.. 3081 */ 3082 memcpy(&np->np_sockaddr, ksockaddr, 3083 sizeof(struct sockaddr_storage)); 3084 3085 isert_lid = isert_setup_id(isert_np); 3086 if (IS_ERR(isert_lid)) { 3087 ret = PTR_ERR(isert_lid); 3088 goto out; 3089 } 3090 3091 isert_np->cm_id = isert_lid; 3092 np->np_context = isert_np; 3093 3094 return 0; 3095 3096 out: 3097 kfree(isert_np); 3098 3099 return ret; 3100 } 3101 3102 static int 3103 isert_rdma_accept(struct isert_conn *isert_conn) 3104 { 3105 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3106 struct rdma_conn_param cp; 3107 int ret; 3108 struct iser_cm_hdr rsp_hdr; 3109 3110 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3111 cp.initiator_depth = isert_conn->initiator_depth; 3112 cp.retry_count = 7; 3113 cp.rnr_retry_count = 7; 3114 3115 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 3116 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 3117 if (!isert_conn->snd_w_inv) 3118 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 3119 cp.private_data = (void *)&rsp_hdr; 3120 cp.private_data_len = sizeof(rsp_hdr); 3121 3122 ret = rdma_accept(cm_id, &cp); 3123 if (ret) { 3124 isert_err("rdma_accept() failed with: %d\n", ret); 3125 return ret; 3126 } 3127 3128 return 0; 3129 } 3130 3131 static int 3132 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3133 { 3134 struct isert_conn *isert_conn = conn->context; 3135 int ret; 3136 3137 isert_info("before login_req comp conn: %p\n", isert_conn); 3138 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 3139 if (ret) { 3140 isert_err("isert_conn %p interrupted before got login req\n", 3141 isert_conn); 3142 return ret; 3143 } 3144 reinit_completion(&isert_conn->login_req_comp); 3145 3146 /* 3147 * For login requests after the first PDU, isert_rx_login_req() will 3148 * kick schedule_delayed_work(&conn->login_work) as the packet is 3149 * received, which turns this callback from iscsi_target_do_login_rx() 3150 * into a NOP. 3151 */ 3152 if (!login->first_request) 3153 return 0; 3154 3155 isert_rx_login_req(isert_conn); 3156 3157 isert_info("before login_comp conn: %p\n", conn); 3158 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 3159 if (ret) 3160 return ret; 3161 3162 isert_info("processing login->req: %p\n", login->req); 3163 3164 return 0; 3165 } 3166 3167 static void 3168 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3169 struct isert_conn *isert_conn) 3170 { 3171 struct rdma_cm_id *cm_id = isert_conn->cm_id; 3172 struct rdma_route *cm_route = &cm_id->route; 3173 3174 conn->login_family = np->np_sockaddr.ss_family; 3175 3176 conn->login_sockaddr = cm_route->addr.dst_addr; 3177 conn->local_sockaddr = cm_route->addr.src_addr; 3178 } 3179 3180 static int 3181 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3182 { 3183 struct isert_np *isert_np = np->np_context; 3184 struct isert_conn *isert_conn; 3185 int ret; 3186 3187 accept_wait: 3188 ret = down_interruptible(&isert_np->sem); 3189 if (ret) 3190 return -ENODEV; 3191 3192 spin_lock_bh(&np->np_thread_lock); 3193 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3194 spin_unlock_bh(&np->np_thread_lock); 3195 isert_dbg("np_thread_state %d\n", 3196 np->np_thread_state); 3197 /** 3198 * No point in stalling here when np_thread 3199 * is in state RESET/SHUTDOWN/EXIT - bail 3200 **/ 3201 return -ENODEV; 3202 } 3203 spin_unlock_bh(&np->np_thread_lock); 3204 3205 mutex_lock(&isert_np->mutex); 3206 if (list_empty(&isert_np->pending)) { 3207 mutex_unlock(&isert_np->mutex); 3208 goto accept_wait; 3209 } 3210 isert_conn = list_first_entry(&isert_np->pending, 3211 struct isert_conn, node); 3212 list_del_init(&isert_conn->node); 3213 mutex_unlock(&isert_np->mutex); 3214 3215 conn->context = isert_conn; 3216 isert_conn->conn = conn; 3217 3218 isert_set_conn_info(np, conn, isert_conn); 3219 3220 isert_dbg("Processing isert_conn: %p\n", isert_conn); 3221 3222 return 0; 3223 } 3224 3225 static void 3226 isert_free_np(struct iscsi_np *np) 3227 { 3228 struct isert_np *isert_np = np->np_context; 3229 struct isert_conn *isert_conn, *n; 3230 3231 if (isert_np->cm_id) 3232 rdma_destroy_id(isert_np->cm_id); 3233 3234 /* 3235 * FIXME: At this point we don't have a good way to insure 3236 * that at this point we don't have hanging connections that 3237 * completed RDMA establishment but didn't start iscsi login 3238 * process. So work-around this by cleaning up what ever piled 3239 * up in accepted and pending lists. 3240 */ 3241 mutex_lock(&isert_np->mutex); 3242 if (!list_empty(&isert_np->pending)) { 3243 isert_info("Still have isert pending connections\n"); 3244 list_for_each_entry_safe(isert_conn, n, 3245 &isert_np->pending, 3246 node) { 3247 isert_info("cleaning isert_conn %p state (%d)\n", 3248 isert_conn, isert_conn->state); 3249 isert_connect_release(isert_conn); 3250 } 3251 } 3252 3253 if (!list_empty(&isert_np->accepted)) { 3254 isert_info("Still have isert accepted connections\n"); 3255 list_for_each_entry_safe(isert_conn, n, 3256 &isert_np->accepted, 3257 node) { 3258 isert_info("cleaning isert_conn %p state (%d)\n", 3259 isert_conn, isert_conn->state); 3260 isert_connect_release(isert_conn); 3261 } 3262 } 3263 mutex_unlock(&isert_np->mutex); 3264 3265 np->np_context = NULL; 3266 kfree(isert_np); 3267 } 3268 3269 static void isert_release_work(struct work_struct *work) 3270 { 3271 struct isert_conn *isert_conn = container_of(work, 3272 struct isert_conn, 3273 release_work); 3274 3275 isert_info("Starting release conn %p\n", isert_conn); 3276 3277 wait_for_completion(&isert_conn->wait); 3278 3279 mutex_lock(&isert_conn->mutex); 3280 isert_conn->state = ISER_CONN_DOWN; 3281 mutex_unlock(&isert_conn->mutex); 3282 3283 isert_info("Destroying conn %p\n", isert_conn); 3284 isert_put_conn(isert_conn); 3285 } 3286 3287 static void 3288 isert_wait4logout(struct isert_conn *isert_conn) 3289 { 3290 struct iscsi_conn *conn = isert_conn->conn; 3291 3292 isert_info("conn %p\n", isert_conn); 3293 3294 if (isert_conn->logout_posted) { 3295 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 3296 wait_for_completion_timeout(&conn->conn_logout_comp, 3297 SECONDS_FOR_LOGOUT_COMP * HZ); 3298 } 3299 } 3300 3301 static void 3302 isert_wait4cmds(struct iscsi_conn *conn) 3303 { 3304 isert_info("iscsi_conn %p\n", conn); 3305 3306 if (conn->sess) { 3307 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 3308 target_wait_for_sess_cmds(conn->sess->se_sess); 3309 } 3310 } 3311 3312 static void 3313 isert_wait4flush(struct isert_conn *isert_conn) 3314 { 3315 struct ib_recv_wr *bad_wr; 3316 3317 isert_info("conn %p\n", isert_conn); 3318 3319 init_completion(&isert_conn->wait_comp_err); 3320 isert_conn->beacon.wr_id = ISER_BEACON_WRID; 3321 /* post an indication that all flush errors were consumed */ 3322 if (ib_post_recv(isert_conn->qp, &isert_conn->beacon, &bad_wr)) { 3323 isert_err("conn %p failed to post beacon", isert_conn); 3324 return; 3325 } 3326 3327 wait_for_completion(&isert_conn->wait_comp_err); 3328 } 3329 3330 /** 3331 * isert_put_unsol_pending_cmds() - Drop commands waiting for 3332 * unsolicitate dataout 3333 * @conn: iscsi connection 3334 * 3335 * We might still have commands that are waiting for unsolicited 3336 * dataouts messages. We must put the extra reference on those 3337 * before blocking on the target_wait_for_session_cmds 3338 */ 3339 static void 3340 isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 3341 { 3342 struct iscsi_cmd *cmd, *tmp; 3343 static LIST_HEAD(drop_cmd_list); 3344 3345 spin_lock_bh(&conn->cmd_lock); 3346 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 3347 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 3348 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 3349 (cmd->write_data_done < cmd->se_cmd.data_length)) 3350 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 3351 } 3352 spin_unlock_bh(&conn->cmd_lock); 3353 3354 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 3355 list_del_init(&cmd->i_conn_node); 3356 if (cmd->i_state != ISTATE_REMOVE) { 3357 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 3358 3359 isert_info("conn %p dropping cmd %p\n", conn, cmd); 3360 isert_put_cmd(isert_cmd, true); 3361 } 3362 } 3363 } 3364 3365 static void isert_wait_conn(struct iscsi_conn *conn) 3366 { 3367 struct isert_conn *isert_conn = conn->context; 3368 3369 isert_info("Starting conn %p\n", isert_conn); 3370 3371 mutex_lock(&isert_conn->mutex); 3372 /* 3373 * Only wait for wait_comp_err if the isert_conn made it 3374 * into full feature phase.. 3375 */ 3376 if (isert_conn->state == ISER_CONN_INIT) { 3377 mutex_unlock(&isert_conn->mutex); 3378 return; 3379 } 3380 isert_conn_terminate(isert_conn); 3381 mutex_unlock(&isert_conn->mutex); 3382 3383 isert_wait4flush(isert_conn); 3384 isert_put_unsol_pending_cmds(conn); 3385 isert_wait4cmds(conn); 3386 isert_wait4logout(isert_conn); 3387 3388 queue_work(isert_release_wq, &isert_conn->release_work); 3389 } 3390 3391 static void isert_free_conn(struct iscsi_conn *conn) 3392 { 3393 struct isert_conn *isert_conn = conn->context; 3394 3395 isert_wait4flush(isert_conn); 3396 isert_put_conn(isert_conn); 3397 } 3398 3399 static struct iscsit_transport iser_target_transport = { 3400 .name = "IB/iSER", 3401 .transport_type = ISCSI_INFINIBAND, 3402 .priv_size = sizeof(struct isert_cmd), 3403 .owner = THIS_MODULE, 3404 .iscsit_setup_np = isert_setup_np, 3405 .iscsit_accept_np = isert_accept_np, 3406 .iscsit_free_np = isert_free_np, 3407 .iscsit_wait_conn = isert_wait_conn, 3408 .iscsit_free_conn = isert_free_conn, 3409 .iscsit_get_login_rx = isert_get_login_rx, 3410 .iscsit_put_login_tx = isert_put_login_tx, 3411 .iscsit_immediate_queue = isert_immediate_queue, 3412 .iscsit_response_queue = isert_response_queue, 3413 .iscsit_get_dataout = isert_get_dataout, 3414 .iscsit_queue_data_in = isert_put_datain, 3415 .iscsit_queue_status = isert_put_response, 3416 .iscsit_aborted_task = isert_aborted_task, 3417 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3418 }; 3419 3420 static int __init isert_init(void) 3421 { 3422 int ret; 3423 3424 isert_comp_wq = alloc_workqueue("isert_comp_wq", 3425 WQ_UNBOUND | WQ_HIGHPRI, 0); 3426 if (!isert_comp_wq) { 3427 isert_err("Unable to allocate isert_comp_wq\n"); 3428 ret = -ENOMEM; 3429 return -ENOMEM; 3430 } 3431 3432 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 3433 WQ_UNBOUND_MAX_ACTIVE); 3434 if (!isert_release_wq) { 3435 isert_err("Unable to allocate isert_release_wq\n"); 3436 ret = -ENOMEM; 3437 goto destroy_comp_wq; 3438 } 3439 3440 iscsit_register_transport(&iser_target_transport); 3441 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3442 3443 return 0; 3444 3445 destroy_comp_wq: 3446 destroy_workqueue(isert_comp_wq); 3447 3448 return ret; 3449 } 3450 3451 static void __exit isert_exit(void) 3452 { 3453 flush_scheduled_work(); 3454 destroy_workqueue(isert_release_wq); 3455 destroy_workqueue(isert_comp_wq); 3456 iscsit_unregister_transport(&iser_target_transport); 3457 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 3458 } 3459 3460 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3461 MODULE_VERSION("1.0"); 3462 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3463 MODULE_LICENSE("GPL"); 3464 3465 module_init(isert_init); 3466 module_exit(isert_exit); 3467