1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <linux/llist.h> 26 #include <rdma/ib_verbs.h> 27 #include <rdma/rdma_cm.h> 28 #include <target/target_core_base.h> 29 #include <target/target_core_fabric.h> 30 #include <target/iscsi/iscsi_transport.h> 31 #include <linux/semaphore.h> 32 33 #include "isert_proto.h" 34 #include "ib_isert.h" 35 36 #define ISERT_MAX_CONN 8 37 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 39 40 static DEFINE_MUTEX(device_list_mutex); 41 static LIST_HEAD(device_list); 42 static struct workqueue_struct *isert_rx_wq; 43 static struct workqueue_struct *isert_comp_wq; 44 45 static void 46 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 47 static int 48 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 49 struct isert_rdma_wr *wr); 50 static void 51 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static int 56 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 57 58 static void 59 isert_qp_event_callback(struct ib_event *e, void *context) 60 { 61 struct isert_conn *isert_conn = (struct isert_conn *)context; 62 63 pr_err("isert_qp_event_callback event: %d\n", e->event); 64 switch (e->event) { 65 case IB_EVENT_COMM_EST: 66 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 67 break; 68 case IB_EVENT_QP_LAST_WQE_REACHED: 69 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 70 break; 71 default: 72 break; 73 } 74 } 75 76 static int 77 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 78 { 79 int ret; 80 81 ret = ib_query_device(ib_dev, devattr); 82 if (ret) { 83 pr_err("ib_query_device() failed: %d\n", ret); 84 return ret; 85 } 86 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 87 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 88 89 return 0; 90 } 91 92 static int 93 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id, 94 u8 protection) 95 { 96 struct isert_device *device = isert_conn->conn_device; 97 struct ib_qp_init_attr attr; 98 int ret, index, min_index = 0; 99 100 mutex_lock(&device_list_mutex); 101 for (index = 0; index < device->cqs_used; index++) 102 if (device->cq_active_qps[index] < 103 device->cq_active_qps[min_index]) 104 min_index = index; 105 device->cq_active_qps[min_index]++; 106 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 107 mutex_unlock(&device_list_mutex); 108 109 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 110 attr.event_handler = isert_qp_event_callback; 111 attr.qp_context = isert_conn; 112 attr.send_cq = device->dev_tx_cq[min_index]; 113 attr.recv_cq = device->dev_rx_cq[min_index]; 114 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 115 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 116 /* 117 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 118 * work-around for RDMA_READs with ConnectX-2. 119 * 120 * Also, still make sure to have at least two SGEs for 121 * outgoing control PDU responses. 122 */ 123 attr.cap.max_send_sge = max(2, device->dev_attr.max_sge - 2); 124 isert_conn->max_sge = attr.cap.max_send_sge; 125 126 attr.cap.max_recv_sge = 1; 127 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 128 attr.qp_type = IB_QPT_RC; 129 if (protection) 130 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 131 132 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 133 cma_id->device); 134 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 135 isert_conn->conn_pd->device); 136 137 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 138 if (ret) { 139 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 140 return ret; 141 } 142 isert_conn->conn_qp = cma_id->qp; 143 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 144 145 return 0; 146 } 147 148 static void 149 isert_cq_event_callback(struct ib_event *e, void *context) 150 { 151 pr_debug("isert_cq_event_callback event: %d\n", e->event); 152 } 153 154 static int 155 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 156 { 157 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 158 struct iser_rx_desc *rx_desc; 159 struct ib_sge *rx_sg; 160 u64 dma_addr; 161 int i, j; 162 163 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 164 sizeof(struct iser_rx_desc), GFP_KERNEL); 165 if (!isert_conn->conn_rx_descs) 166 goto fail; 167 168 rx_desc = isert_conn->conn_rx_descs; 169 170 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 171 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 172 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 173 if (ib_dma_mapping_error(ib_dev, dma_addr)) 174 goto dma_map_fail; 175 176 rx_desc->dma_addr = dma_addr; 177 178 rx_sg = &rx_desc->rx_sg; 179 rx_sg->addr = rx_desc->dma_addr; 180 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 181 rx_sg->lkey = isert_conn->conn_mr->lkey; 182 } 183 184 isert_conn->conn_rx_desc_head = 0; 185 return 0; 186 187 dma_map_fail: 188 rx_desc = isert_conn->conn_rx_descs; 189 for (j = 0; j < i; j++, rx_desc++) { 190 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 191 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 192 } 193 kfree(isert_conn->conn_rx_descs); 194 isert_conn->conn_rx_descs = NULL; 195 fail: 196 return -ENOMEM; 197 } 198 199 static void 200 isert_free_rx_descriptors(struct isert_conn *isert_conn) 201 { 202 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 203 struct iser_rx_desc *rx_desc; 204 int i; 205 206 if (!isert_conn->conn_rx_descs) 207 return; 208 209 rx_desc = isert_conn->conn_rx_descs; 210 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 211 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 212 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 213 } 214 215 kfree(isert_conn->conn_rx_descs); 216 isert_conn->conn_rx_descs = NULL; 217 } 218 219 static void isert_cq_tx_work(struct work_struct *); 220 static void isert_cq_tx_callback(struct ib_cq *, void *); 221 static void isert_cq_rx_work(struct work_struct *); 222 static void isert_cq_rx_callback(struct ib_cq *, void *); 223 224 static int 225 isert_create_device_ib_res(struct isert_device *device) 226 { 227 struct ib_device *ib_dev = device->ib_device; 228 struct isert_cq_desc *cq_desc; 229 struct ib_device_attr *dev_attr; 230 int ret = 0, i, j; 231 int max_rx_cqe, max_tx_cqe; 232 233 dev_attr = &device->dev_attr; 234 ret = isert_query_device(ib_dev, dev_attr); 235 if (ret) 236 return ret; 237 238 max_rx_cqe = min(ISER_MAX_RX_CQ_LEN, dev_attr->max_cqe); 239 max_tx_cqe = min(ISER_MAX_TX_CQ_LEN, dev_attr->max_cqe); 240 241 /* asign function handlers */ 242 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 243 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 244 device->use_fastreg = 1; 245 device->reg_rdma_mem = isert_reg_rdma; 246 device->unreg_rdma_mem = isert_unreg_rdma; 247 } else { 248 device->use_fastreg = 0; 249 device->reg_rdma_mem = isert_map_rdma; 250 device->unreg_rdma_mem = isert_unmap_cmd; 251 } 252 253 /* Check signature cap */ 254 device->pi_capable = dev_attr->device_cap_flags & 255 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 256 257 device->cqs_used = min_t(int, num_online_cpus(), 258 device->ib_device->num_comp_vectors); 259 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 260 pr_debug("Using %d CQs, device %s supports %d vectors support " 261 "Fast registration %d pi_capable %d\n", 262 device->cqs_used, device->ib_device->name, 263 device->ib_device->num_comp_vectors, device->use_fastreg, 264 device->pi_capable); 265 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 266 device->cqs_used, GFP_KERNEL); 267 if (!device->cq_desc) { 268 pr_err("Unable to allocate device->cq_desc\n"); 269 return -ENOMEM; 270 } 271 cq_desc = device->cq_desc; 272 273 for (i = 0; i < device->cqs_used; i++) { 274 cq_desc[i].device = device; 275 cq_desc[i].cq_index = i; 276 277 INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 278 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 279 isert_cq_rx_callback, 280 isert_cq_event_callback, 281 (void *)&cq_desc[i], 282 max_rx_cqe, i); 283 if (IS_ERR(device->dev_rx_cq[i])) { 284 ret = PTR_ERR(device->dev_rx_cq[i]); 285 device->dev_rx_cq[i] = NULL; 286 goto out_cq; 287 } 288 289 INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 290 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 291 isert_cq_tx_callback, 292 isert_cq_event_callback, 293 (void *)&cq_desc[i], 294 max_tx_cqe, i); 295 if (IS_ERR(device->dev_tx_cq[i])) { 296 ret = PTR_ERR(device->dev_tx_cq[i]); 297 device->dev_tx_cq[i] = NULL; 298 goto out_cq; 299 } 300 301 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 302 if (ret) 303 goto out_cq; 304 305 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 306 if (ret) 307 goto out_cq; 308 } 309 310 return 0; 311 312 out_cq: 313 for (j = 0; j < i; j++) { 314 cq_desc = &device->cq_desc[j]; 315 316 if (device->dev_rx_cq[j]) { 317 cancel_work_sync(&cq_desc->cq_rx_work); 318 ib_destroy_cq(device->dev_rx_cq[j]); 319 } 320 if (device->dev_tx_cq[j]) { 321 cancel_work_sync(&cq_desc->cq_tx_work); 322 ib_destroy_cq(device->dev_tx_cq[j]); 323 } 324 } 325 kfree(device->cq_desc); 326 327 return ret; 328 } 329 330 static void 331 isert_free_device_ib_res(struct isert_device *device) 332 { 333 struct isert_cq_desc *cq_desc; 334 int i; 335 336 for (i = 0; i < device->cqs_used; i++) { 337 cq_desc = &device->cq_desc[i]; 338 339 cancel_work_sync(&cq_desc->cq_rx_work); 340 cancel_work_sync(&cq_desc->cq_tx_work); 341 ib_destroy_cq(device->dev_rx_cq[i]); 342 ib_destroy_cq(device->dev_tx_cq[i]); 343 device->dev_rx_cq[i] = NULL; 344 device->dev_tx_cq[i] = NULL; 345 } 346 347 kfree(device->cq_desc); 348 } 349 350 static void 351 isert_device_try_release(struct isert_device *device) 352 { 353 mutex_lock(&device_list_mutex); 354 device->refcount--; 355 if (!device->refcount) { 356 isert_free_device_ib_res(device); 357 list_del(&device->dev_node); 358 kfree(device); 359 } 360 mutex_unlock(&device_list_mutex); 361 } 362 363 static struct isert_device * 364 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 365 { 366 struct isert_device *device; 367 int ret; 368 369 mutex_lock(&device_list_mutex); 370 list_for_each_entry(device, &device_list, dev_node) { 371 if (device->ib_device->node_guid == cma_id->device->node_guid) { 372 device->refcount++; 373 mutex_unlock(&device_list_mutex); 374 return device; 375 } 376 } 377 378 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 379 if (!device) { 380 mutex_unlock(&device_list_mutex); 381 return ERR_PTR(-ENOMEM); 382 } 383 384 INIT_LIST_HEAD(&device->dev_node); 385 386 device->ib_device = cma_id->device; 387 ret = isert_create_device_ib_res(device); 388 if (ret) { 389 kfree(device); 390 mutex_unlock(&device_list_mutex); 391 return ERR_PTR(ret); 392 } 393 394 device->refcount++; 395 list_add_tail(&device->dev_node, &device_list); 396 mutex_unlock(&device_list_mutex); 397 398 return device; 399 } 400 401 static void 402 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 403 { 404 struct fast_reg_descriptor *fr_desc, *tmp; 405 int i = 0; 406 407 if (list_empty(&isert_conn->conn_fr_pool)) 408 return; 409 410 pr_debug("Freeing conn %p fastreg pool", isert_conn); 411 412 list_for_each_entry_safe(fr_desc, tmp, 413 &isert_conn->conn_fr_pool, list) { 414 list_del(&fr_desc->list); 415 ib_free_fast_reg_page_list(fr_desc->data_frpl); 416 ib_dereg_mr(fr_desc->data_mr); 417 if (fr_desc->pi_ctx) { 418 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 419 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 420 ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 421 kfree(fr_desc->pi_ctx); 422 } 423 kfree(fr_desc); 424 ++i; 425 } 426 427 if (i < isert_conn->conn_fr_pool_size) 428 pr_warn("Pool still has %d regions registered\n", 429 isert_conn->conn_fr_pool_size - i); 430 } 431 432 static int 433 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 434 struct fast_reg_descriptor *fr_desc, u8 protection) 435 { 436 int ret; 437 438 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 439 ISCSI_ISER_SG_TABLESIZE); 440 if (IS_ERR(fr_desc->data_frpl)) { 441 pr_err("Failed to allocate data frpl err=%ld\n", 442 PTR_ERR(fr_desc->data_frpl)); 443 return PTR_ERR(fr_desc->data_frpl); 444 } 445 446 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 447 if (IS_ERR(fr_desc->data_mr)) { 448 pr_err("Failed to allocate data frmr err=%ld\n", 449 PTR_ERR(fr_desc->data_mr)); 450 ret = PTR_ERR(fr_desc->data_mr); 451 goto err_data_frpl; 452 } 453 pr_debug("Create fr_desc %p page_list %p\n", 454 fr_desc, fr_desc->data_frpl->page_list); 455 fr_desc->ind |= ISERT_DATA_KEY_VALID; 456 457 if (protection) { 458 struct ib_mr_init_attr mr_init_attr = {0}; 459 struct pi_context *pi_ctx; 460 461 fr_desc->pi_ctx = kzalloc(sizeof(*fr_desc->pi_ctx), GFP_KERNEL); 462 if (!fr_desc->pi_ctx) { 463 pr_err("Failed to allocate pi context\n"); 464 ret = -ENOMEM; 465 goto err_data_mr; 466 } 467 pi_ctx = fr_desc->pi_ctx; 468 469 pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(ib_device, 470 ISCSI_ISER_SG_TABLESIZE); 471 if (IS_ERR(pi_ctx->prot_frpl)) { 472 pr_err("Failed to allocate prot frpl err=%ld\n", 473 PTR_ERR(pi_ctx->prot_frpl)); 474 ret = PTR_ERR(pi_ctx->prot_frpl); 475 goto err_pi_ctx; 476 } 477 478 pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 479 if (IS_ERR(pi_ctx->prot_mr)) { 480 pr_err("Failed to allocate prot frmr err=%ld\n", 481 PTR_ERR(pi_ctx->prot_mr)); 482 ret = PTR_ERR(pi_ctx->prot_mr); 483 goto err_prot_frpl; 484 } 485 fr_desc->ind |= ISERT_PROT_KEY_VALID; 486 487 mr_init_attr.max_reg_descriptors = 2; 488 mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 489 pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 490 if (IS_ERR(pi_ctx->sig_mr)) { 491 pr_err("Failed to allocate signature enabled mr err=%ld\n", 492 PTR_ERR(pi_ctx->sig_mr)); 493 ret = PTR_ERR(pi_ctx->sig_mr); 494 goto err_prot_mr; 495 } 496 fr_desc->ind |= ISERT_SIG_KEY_VALID; 497 } 498 fr_desc->ind &= ~ISERT_PROTECTED; 499 500 return 0; 501 err_prot_mr: 502 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 503 err_prot_frpl: 504 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 505 err_pi_ctx: 506 kfree(fr_desc->pi_ctx); 507 err_data_mr: 508 ib_dereg_mr(fr_desc->data_mr); 509 err_data_frpl: 510 ib_free_fast_reg_page_list(fr_desc->data_frpl); 511 512 return ret; 513 } 514 515 static int 516 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn, u8 pi_support) 517 { 518 struct fast_reg_descriptor *fr_desc; 519 struct isert_device *device = isert_conn->conn_device; 520 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 521 struct se_node_acl *se_nacl = se_sess->se_node_acl; 522 int i, ret, tag_num; 523 /* 524 * Setup the number of FRMRs based upon the number of tags 525 * available to session in iscsi_target_locate_portal(). 526 */ 527 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 528 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 529 530 isert_conn->conn_fr_pool_size = 0; 531 for (i = 0; i < tag_num; i++) { 532 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 533 if (!fr_desc) { 534 pr_err("Failed to allocate fast_reg descriptor\n"); 535 ret = -ENOMEM; 536 goto err; 537 } 538 539 ret = isert_create_fr_desc(device->ib_device, 540 isert_conn->conn_pd, fr_desc, 541 pi_support); 542 if (ret) { 543 pr_err("Failed to create fastreg descriptor err=%d\n", 544 ret); 545 kfree(fr_desc); 546 goto err; 547 } 548 549 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 550 isert_conn->conn_fr_pool_size++; 551 } 552 553 pr_debug("Creating conn %p fastreg pool size=%d", 554 isert_conn, isert_conn->conn_fr_pool_size); 555 556 return 0; 557 558 err: 559 isert_conn_free_fastreg_pool(isert_conn); 560 return ret; 561 } 562 563 static int 564 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 565 { 566 struct iscsi_np *np = cma_id->context; 567 struct isert_np *isert_np = np->np_context; 568 struct isert_conn *isert_conn; 569 struct isert_device *device; 570 struct ib_device *ib_dev = cma_id->device; 571 int ret = 0; 572 u8 pi_support; 573 574 spin_lock_bh(&np->np_thread_lock); 575 if (!np->enabled) { 576 spin_unlock_bh(&np->np_thread_lock); 577 pr_debug("iscsi_np is not enabled, reject connect request\n"); 578 return rdma_reject(cma_id, NULL, 0); 579 } 580 spin_unlock_bh(&np->np_thread_lock); 581 582 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 583 cma_id, cma_id->context); 584 585 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 586 if (!isert_conn) { 587 pr_err("Unable to allocate isert_conn\n"); 588 return -ENOMEM; 589 } 590 isert_conn->state = ISER_CONN_INIT; 591 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 592 init_completion(&isert_conn->conn_login_comp); 593 init_completion(&isert_conn->conn_wait); 594 init_completion(&isert_conn->conn_wait_comp_err); 595 kref_init(&isert_conn->conn_kref); 596 mutex_init(&isert_conn->conn_mutex); 597 spin_lock_init(&isert_conn->conn_lock); 598 INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 599 600 cma_id->context = isert_conn; 601 isert_conn->conn_cm_id = cma_id; 602 603 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 604 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 605 if (!isert_conn->login_buf) { 606 pr_err("Unable to allocate isert_conn->login_buf\n"); 607 ret = -ENOMEM; 608 goto out; 609 } 610 611 isert_conn->login_req_buf = isert_conn->login_buf; 612 isert_conn->login_rsp_buf = isert_conn->login_buf + 613 ISCSI_DEF_MAX_RECV_SEG_LEN; 614 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 615 isert_conn->login_buf, isert_conn->login_req_buf, 616 isert_conn->login_rsp_buf); 617 618 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 619 (void *)isert_conn->login_req_buf, 620 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 621 622 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 623 if (ret) { 624 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 625 ret); 626 isert_conn->login_req_dma = 0; 627 goto out_login_buf; 628 } 629 630 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 631 (void *)isert_conn->login_rsp_buf, 632 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 633 634 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 635 if (ret) { 636 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 637 ret); 638 isert_conn->login_rsp_dma = 0; 639 goto out_req_dma_map; 640 } 641 642 device = isert_device_find_by_ib_dev(cma_id); 643 if (IS_ERR(device)) { 644 ret = PTR_ERR(device); 645 goto out_rsp_dma_map; 646 } 647 648 /* Set max inflight RDMA READ requests */ 649 isert_conn->initiator_depth = min_t(u8, 650 event->param.conn.initiator_depth, 651 device->dev_attr.max_qp_init_rd_atom); 652 pr_debug("Using initiator_depth: %u\n", isert_conn->initiator_depth); 653 654 isert_conn->conn_device = device; 655 isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 656 if (IS_ERR(isert_conn->conn_pd)) { 657 ret = PTR_ERR(isert_conn->conn_pd); 658 pr_err("ib_alloc_pd failed for conn %p: ret=%d\n", 659 isert_conn, ret); 660 goto out_pd; 661 } 662 663 isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 664 IB_ACCESS_LOCAL_WRITE); 665 if (IS_ERR(isert_conn->conn_mr)) { 666 ret = PTR_ERR(isert_conn->conn_mr); 667 pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 668 isert_conn, ret); 669 goto out_mr; 670 } 671 672 pi_support = np->tpg_np->tpg->tpg_attrib.t10_pi; 673 if (pi_support && !device->pi_capable) { 674 pr_err("Protection information requested but not supported, " 675 "rejecting connect request\n"); 676 ret = rdma_reject(cma_id, NULL, 0); 677 goto out_mr; 678 } 679 680 ret = isert_conn_setup_qp(isert_conn, cma_id, pi_support); 681 if (ret) 682 goto out_conn_dev; 683 684 mutex_lock(&isert_np->np_accept_mutex); 685 list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list); 686 mutex_unlock(&isert_np->np_accept_mutex); 687 688 pr_debug("isert_connect_request() up np_sem np: %p\n", np); 689 up(&isert_np->np_sem); 690 return 0; 691 692 out_conn_dev: 693 ib_dereg_mr(isert_conn->conn_mr); 694 out_mr: 695 ib_dealloc_pd(isert_conn->conn_pd); 696 out_pd: 697 isert_device_try_release(device); 698 out_rsp_dma_map: 699 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 700 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 701 out_req_dma_map: 702 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 703 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 704 out_login_buf: 705 kfree(isert_conn->login_buf); 706 out: 707 kfree(isert_conn); 708 return ret; 709 } 710 711 static void 712 isert_connect_release(struct isert_conn *isert_conn) 713 { 714 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 715 struct isert_device *device = isert_conn->conn_device; 716 int cq_index; 717 718 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 719 720 if (device && device->use_fastreg) 721 isert_conn_free_fastreg_pool(isert_conn); 722 723 if (isert_conn->conn_qp) { 724 cq_index = ((struct isert_cq_desc *) 725 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 726 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 727 isert_conn->conn_device->cq_active_qps[cq_index]--; 728 729 rdma_destroy_qp(isert_conn->conn_cm_id); 730 } 731 732 isert_free_rx_descriptors(isert_conn); 733 rdma_destroy_id(isert_conn->conn_cm_id); 734 735 ib_dereg_mr(isert_conn->conn_mr); 736 ib_dealloc_pd(isert_conn->conn_pd); 737 738 if (isert_conn->login_buf) { 739 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 740 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 741 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 742 ISCSI_DEF_MAX_RECV_SEG_LEN, 743 DMA_FROM_DEVICE); 744 kfree(isert_conn->login_buf); 745 } 746 kfree(isert_conn); 747 748 if (device) 749 isert_device_try_release(device); 750 751 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 752 } 753 754 static void 755 isert_connected_handler(struct rdma_cm_id *cma_id) 756 { 757 struct isert_conn *isert_conn = cma_id->context; 758 759 kref_get(&isert_conn->conn_kref); 760 } 761 762 static void 763 isert_release_conn_kref(struct kref *kref) 764 { 765 struct isert_conn *isert_conn = container_of(kref, 766 struct isert_conn, conn_kref); 767 768 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 769 current->comm, current->pid); 770 771 isert_connect_release(isert_conn); 772 } 773 774 static void 775 isert_put_conn(struct isert_conn *isert_conn) 776 { 777 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 778 } 779 780 static void 781 isert_disconnect_work(struct work_struct *work) 782 { 783 struct isert_conn *isert_conn = container_of(work, 784 struct isert_conn, conn_logout_work); 785 786 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 787 mutex_lock(&isert_conn->conn_mutex); 788 if (isert_conn->state == ISER_CONN_UP) 789 isert_conn->state = ISER_CONN_TERMINATING; 790 791 if (isert_conn->post_recv_buf_count == 0 && 792 atomic_read(&isert_conn->post_send_buf_count) == 0) { 793 mutex_unlock(&isert_conn->conn_mutex); 794 goto wake_up; 795 } 796 if (!isert_conn->conn_cm_id) { 797 mutex_unlock(&isert_conn->conn_mutex); 798 isert_put_conn(isert_conn); 799 return; 800 } 801 802 if (isert_conn->disconnect) { 803 /* Send DREQ/DREP towards our initiator */ 804 rdma_disconnect(isert_conn->conn_cm_id); 805 } 806 807 mutex_unlock(&isert_conn->conn_mutex); 808 809 wake_up: 810 complete(&isert_conn->conn_wait); 811 } 812 813 static int 814 isert_disconnected_handler(struct rdma_cm_id *cma_id, bool disconnect) 815 { 816 struct isert_conn *isert_conn; 817 818 if (!cma_id->qp) { 819 struct isert_np *isert_np = cma_id->context; 820 821 isert_np->np_cm_id = NULL; 822 return -1; 823 } 824 825 isert_conn = (struct isert_conn *)cma_id->context; 826 827 isert_conn->disconnect = disconnect; 828 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 829 schedule_work(&isert_conn->conn_logout_work); 830 831 return 0; 832 } 833 834 static int 835 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 836 { 837 int ret = 0; 838 bool disconnect = false; 839 840 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 841 event->event, event->status, cma_id->context, cma_id); 842 843 switch (event->event) { 844 case RDMA_CM_EVENT_CONNECT_REQUEST: 845 ret = isert_connect_request(cma_id, event); 846 if (ret) 847 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 848 event->event, ret); 849 break; 850 case RDMA_CM_EVENT_ESTABLISHED: 851 isert_connected_handler(cma_id); 852 break; 853 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 854 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 855 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 856 disconnect = true; 857 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 858 ret = isert_disconnected_handler(cma_id, disconnect); 859 break; 860 case RDMA_CM_EVENT_CONNECT_ERROR: 861 default: 862 pr_err("Unhandled RDMA CMA event: %d\n", event->event); 863 break; 864 } 865 866 return ret; 867 } 868 869 static int 870 isert_post_recv(struct isert_conn *isert_conn, u32 count) 871 { 872 struct ib_recv_wr *rx_wr, *rx_wr_failed; 873 int i, ret; 874 unsigned int rx_head = isert_conn->conn_rx_desc_head; 875 struct iser_rx_desc *rx_desc; 876 877 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 878 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 879 rx_wr->wr_id = (unsigned long)rx_desc; 880 rx_wr->sg_list = &rx_desc->rx_sg; 881 rx_wr->num_sge = 1; 882 rx_wr->next = rx_wr + 1; 883 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 884 } 885 886 rx_wr--; 887 rx_wr->next = NULL; /* mark end of work requests list */ 888 889 isert_conn->post_recv_buf_count += count; 890 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 891 &rx_wr_failed); 892 if (ret) { 893 pr_err("ib_post_recv() failed with ret: %d\n", ret); 894 isert_conn->post_recv_buf_count -= count; 895 } else { 896 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 897 isert_conn->conn_rx_desc_head = rx_head; 898 } 899 return ret; 900 } 901 902 static int 903 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 904 { 905 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 906 struct ib_send_wr send_wr, *send_wr_failed; 907 int ret; 908 909 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 910 ISER_HEADERS_LEN, DMA_TO_DEVICE); 911 912 send_wr.next = NULL; 913 send_wr.wr_id = (unsigned long)tx_desc; 914 send_wr.sg_list = tx_desc->tx_sg; 915 send_wr.num_sge = tx_desc->num_sge; 916 send_wr.opcode = IB_WR_SEND; 917 send_wr.send_flags = IB_SEND_SIGNALED; 918 919 atomic_inc(&isert_conn->post_send_buf_count); 920 921 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 922 if (ret) { 923 pr_err("ib_post_send() failed, ret: %d\n", ret); 924 atomic_dec(&isert_conn->post_send_buf_count); 925 } 926 927 return ret; 928 } 929 930 static void 931 isert_create_send_desc(struct isert_conn *isert_conn, 932 struct isert_cmd *isert_cmd, 933 struct iser_tx_desc *tx_desc) 934 { 935 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 936 937 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 938 ISER_HEADERS_LEN, DMA_TO_DEVICE); 939 940 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 941 tx_desc->iser_header.flags = ISER_VER; 942 943 tx_desc->num_sge = 1; 944 tx_desc->isert_cmd = isert_cmd; 945 946 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 947 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 948 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 949 } 950 } 951 952 static int 953 isert_init_tx_hdrs(struct isert_conn *isert_conn, 954 struct iser_tx_desc *tx_desc) 955 { 956 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 957 u64 dma_addr; 958 959 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 960 ISER_HEADERS_LEN, DMA_TO_DEVICE); 961 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 962 pr_err("ib_dma_mapping_error() failed\n"); 963 return -ENOMEM; 964 } 965 966 tx_desc->dma_addr = dma_addr; 967 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 968 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 969 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 970 971 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 972 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 973 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 974 975 return 0; 976 } 977 978 static void 979 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 980 struct ib_send_wr *send_wr, bool coalesce) 981 { 982 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 983 984 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 985 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 986 send_wr->opcode = IB_WR_SEND; 987 send_wr->sg_list = &tx_desc->tx_sg[0]; 988 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 989 /* 990 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 991 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 992 */ 993 mutex_lock(&isert_conn->conn_mutex); 994 if (coalesce && isert_conn->state == ISER_CONN_UP && 995 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 996 tx_desc->llnode_active = true; 997 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 998 mutex_unlock(&isert_conn->conn_mutex); 999 return; 1000 } 1001 isert_conn->conn_comp_batch = 0; 1002 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 1003 mutex_unlock(&isert_conn->conn_mutex); 1004 1005 send_wr->send_flags = IB_SEND_SIGNALED; 1006 } 1007 1008 static int 1009 isert_rdma_post_recvl(struct isert_conn *isert_conn) 1010 { 1011 struct ib_recv_wr rx_wr, *rx_wr_fail; 1012 struct ib_sge sge; 1013 int ret; 1014 1015 memset(&sge, 0, sizeof(struct ib_sge)); 1016 sge.addr = isert_conn->login_req_dma; 1017 sge.length = ISER_RX_LOGIN_SIZE; 1018 sge.lkey = isert_conn->conn_mr->lkey; 1019 1020 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 1021 sge.addr, sge.length, sge.lkey); 1022 1023 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1024 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 1025 rx_wr.sg_list = &sge; 1026 rx_wr.num_sge = 1; 1027 1028 isert_conn->post_recv_buf_count++; 1029 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 1030 if (ret) { 1031 pr_err("ib_post_recv() failed: %d\n", ret); 1032 isert_conn->post_recv_buf_count--; 1033 } 1034 1035 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 1036 return ret; 1037 } 1038 1039 static int 1040 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1041 u32 length) 1042 { 1043 struct isert_conn *isert_conn = conn->context; 1044 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1045 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 1046 int ret; 1047 1048 isert_create_send_desc(isert_conn, NULL, tx_desc); 1049 1050 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1051 sizeof(struct iscsi_hdr)); 1052 1053 isert_init_tx_hdrs(isert_conn, tx_desc); 1054 1055 if (length > 0) { 1056 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1057 1058 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1059 length, DMA_TO_DEVICE); 1060 1061 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1062 1063 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1064 length, DMA_TO_DEVICE); 1065 1066 tx_dsg->addr = isert_conn->login_rsp_dma; 1067 tx_dsg->length = length; 1068 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1069 tx_desc->num_sge = 2; 1070 } 1071 if (!login->login_failed) { 1072 if (login->login_complete) { 1073 if (!conn->sess->sess_ops->SessionType && 1074 isert_conn->conn_device->use_fastreg) { 1075 /* Normal Session and fastreg is used */ 1076 u8 pi_support = login->np->tpg_np->tpg->tpg_attrib.t10_pi; 1077 1078 ret = isert_conn_create_fastreg_pool(isert_conn, 1079 pi_support); 1080 if (ret) { 1081 pr_err("Conn: %p failed to create" 1082 " fastreg pool\n", isert_conn); 1083 return ret; 1084 } 1085 } 1086 1087 ret = isert_alloc_rx_descriptors(isert_conn); 1088 if (ret) 1089 return ret; 1090 1091 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1092 if (ret) 1093 return ret; 1094 1095 isert_conn->state = ISER_CONN_UP; 1096 goto post_send; 1097 } 1098 1099 ret = isert_rdma_post_recvl(isert_conn); 1100 if (ret) 1101 return ret; 1102 } 1103 post_send: 1104 ret = isert_post_send(isert_conn, tx_desc); 1105 if (ret) 1106 return ret; 1107 1108 return 0; 1109 } 1110 1111 static void 1112 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 1113 struct isert_conn *isert_conn) 1114 { 1115 struct iscsi_conn *conn = isert_conn->conn; 1116 struct iscsi_login *login = conn->conn_login; 1117 int size; 1118 1119 if (!login) { 1120 pr_err("conn->conn_login is NULL\n"); 1121 dump_stack(); 1122 return; 1123 } 1124 1125 if (login->first_request) { 1126 struct iscsi_login_req *login_req = 1127 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1128 /* 1129 * Setup the initial iscsi_login values from the leading 1130 * login request PDU. 1131 */ 1132 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1133 login->current_stage = 1134 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1135 >> 2; 1136 login->version_min = login_req->min_version; 1137 login->version_max = login_req->max_version; 1138 memcpy(login->isid, login_req->isid, 6); 1139 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1140 login->init_task_tag = login_req->itt; 1141 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1142 login->cid = be16_to_cpu(login_req->cid); 1143 login->tsih = be16_to_cpu(login_req->tsih); 1144 } 1145 1146 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1147 1148 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1149 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1150 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1151 memcpy(login->req_buf, &rx_desc->data[0], size); 1152 1153 if (login->first_request) { 1154 complete(&isert_conn->conn_login_comp); 1155 return; 1156 } 1157 schedule_delayed_work(&conn->login_work, 0); 1158 } 1159 1160 static struct iscsi_cmd 1161 *isert_allocate_cmd(struct iscsi_conn *conn) 1162 { 1163 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1164 struct isert_cmd *isert_cmd; 1165 struct iscsi_cmd *cmd; 1166 1167 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1168 if (!cmd) { 1169 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1170 return NULL; 1171 } 1172 isert_cmd = iscsit_priv_cmd(cmd); 1173 isert_cmd->conn = isert_conn; 1174 isert_cmd->iscsi_cmd = cmd; 1175 1176 return cmd; 1177 } 1178 1179 static int 1180 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1181 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1182 struct iser_rx_desc *rx_desc, unsigned char *buf) 1183 { 1184 struct iscsi_conn *conn = isert_conn->conn; 1185 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1186 struct scatterlist *sg; 1187 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1188 bool dump_payload = false; 1189 1190 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1191 if (rc < 0) 1192 return rc; 1193 1194 imm_data = cmd->immediate_data; 1195 imm_data_len = cmd->first_burst_len; 1196 unsol_data = cmd->unsolicited_data; 1197 1198 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1199 if (rc < 0) { 1200 return 0; 1201 } else if (rc > 0) { 1202 dump_payload = true; 1203 goto sequence_cmd; 1204 } 1205 1206 if (!imm_data) 1207 return 0; 1208 1209 sg = &cmd->se_cmd.t_data_sg[0]; 1210 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1211 1212 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1213 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1214 1215 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1216 1217 cmd->write_data_done += imm_data_len; 1218 1219 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1220 spin_lock_bh(&cmd->istate_lock); 1221 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1222 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1223 spin_unlock_bh(&cmd->istate_lock); 1224 } 1225 1226 sequence_cmd: 1227 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1228 1229 if (!rc && dump_payload == false && unsol_data) 1230 iscsit_set_unsoliticed_dataout(cmd); 1231 else if (dump_payload && imm_data) 1232 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); 1233 1234 return 0; 1235 } 1236 1237 static int 1238 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1239 struct iser_rx_desc *rx_desc, unsigned char *buf) 1240 { 1241 struct scatterlist *sg_start; 1242 struct iscsi_conn *conn = isert_conn->conn; 1243 struct iscsi_cmd *cmd = NULL; 1244 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1245 u32 unsol_data_len = ntoh24(hdr->dlength); 1246 int rc, sg_nents, sg_off, page_off; 1247 1248 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1249 if (rc < 0) 1250 return rc; 1251 else if (!cmd) 1252 return 0; 1253 /* 1254 * FIXME: Unexpected unsolicited_data out 1255 */ 1256 if (!cmd->unsolicited_data) { 1257 pr_err("Received unexpected solicited data payload\n"); 1258 dump_stack(); 1259 return -1; 1260 } 1261 1262 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1263 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1264 1265 sg_off = cmd->write_data_done / PAGE_SIZE; 1266 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1267 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1268 page_off = cmd->write_data_done % PAGE_SIZE; 1269 /* 1270 * FIXME: Non page-aligned unsolicited_data out 1271 */ 1272 if (page_off) { 1273 pr_err("Received unexpected non-page aligned data payload\n"); 1274 dump_stack(); 1275 return -1; 1276 } 1277 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1278 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1279 1280 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1281 unsol_data_len); 1282 1283 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1284 if (rc < 0) 1285 return rc; 1286 1287 return 0; 1288 } 1289 1290 static int 1291 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1292 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1293 unsigned char *buf) 1294 { 1295 struct iscsi_conn *conn = isert_conn->conn; 1296 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1297 int rc; 1298 1299 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1300 if (rc < 0) 1301 return rc; 1302 /* 1303 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1304 */ 1305 1306 return iscsit_process_nop_out(conn, cmd, hdr); 1307 } 1308 1309 static int 1310 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1311 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1312 struct iscsi_text *hdr) 1313 { 1314 struct iscsi_conn *conn = isert_conn->conn; 1315 u32 payload_length = ntoh24(hdr->dlength); 1316 int rc; 1317 unsigned char *text_in; 1318 1319 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1320 if (rc < 0) 1321 return rc; 1322 1323 text_in = kzalloc(payload_length, GFP_KERNEL); 1324 if (!text_in) { 1325 pr_err("Unable to allocate text_in of payload_length: %u\n", 1326 payload_length); 1327 return -ENOMEM; 1328 } 1329 cmd->text_in_ptr = text_in; 1330 1331 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1332 1333 return iscsit_process_text_cmd(conn, cmd, hdr); 1334 } 1335 1336 static int 1337 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1338 uint32_t read_stag, uint64_t read_va, 1339 uint32_t write_stag, uint64_t write_va) 1340 { 1341 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1342 struct iscsi_conn *conn = isert_conn->conn; 1343 struct iscsi_session *sess = conn->sess; 1344 struct iscsi_cmd *cmd; 1345 struct isert_cmd *isert_cmd; 1346 int ret = -EINVAL; 1347 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1348 1349 if (sess->sess_ops->SessionType && 1350 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1351 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1352 " ignoring\n", opcode); 1353 return 0; 1354 } 1355 1356 switch (opcode) { 1357 case ISCSI_OP_SCSI_CMD: 1358 cmd = isert_allocate_cmd(conn); 1359 if (!cmd) 1360 break; 1361 1362 isert_cmd = iscsit_priv_cmd(cmd); 1363 isert_cmd->read_stag = read_stag; 1364 isert_cmd->read_va = read_va; 1365 isert_cmd->write_stag = write_stag; 1366 isert_cmd->write_va = write_va; 1367 1368 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1369 rx_desc, (unsigned char *)hdr); 1370 break; 1371 case ISCSI_OP_NOOP_OUT: 1372 cmd = isert_allocate_cmd(conn); 1373 if (!cmd) 1374 break; 1375 1376 isert_cmd = iscsit_priv_cmd(cmd); 1377 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1378 rx_desc, (unsigned char *)hdr); 1379 break; 1380 case ISCSI_OP_SCSI_DATA_OUT: 1381 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1382 (unsigned char *)hdr); 1383 break; 1384 case ISCSI_OP_SCSI_TMFUNC: 1385 cmd = isert_allocate_cmd(conn); 1386 if (!cmd) 1387 break; 1388 1389 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1390 (unsigned char *)hdr); 1391 break; 1392 case ISCSI_OP_LOGOUT: 1393 cmd = isert_allocate_cmd(conn); 1394 if (!cmd) 1395 break; 1396 1397 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1398 if (ret > 0) 1399 wait_for_completion_timeout(&conn->conn_logout_comp, 1400 SECONDS_FOR_LOGOUT_COMP * 1401 HZ); 1402 break; 1403 case ISCSI_OP_TEXT: 1404 cmd = isert_allocate_cmd(conn); 1405 if (!cmd) 1406 break; 1407 1408 isert_cmd = iscsit_priv_cmd(cmd); 1409 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1410 rx_desc, (struct iscsi_text *)hdr); 1411 break; 1412 default: 1413 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1414 dump_stack(); 1415 break; 1416 } 1417 1418 return ret; 1419 } 1420 1421 static void 1422 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1423 { 1424 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1425 uint64_t read_va = 0, write_va = 0; 1426 uint32_t read_stag = 0, write_stag = 0; 1427 int rc; 1428 1429 switch (iser_hdr->flags & 0xF0) { 1430 case ISCSI_CTRL: 1431 if (iser_hdr->flags & ISER_RSV) { 1432 read_stag = be32_to_cpu(iser_hdr->read_stag); 1433 read_va = be64_to_cpu(iser_hdr->read_va); 1434 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1435 read_stag, (unsigned long long)read_va); 1436 } 1437 if (iser_hdr->flags & ISER_WSV) { 1438 write_stag = be32_to_cpu(iser_hdr->write_stag); 1439 write_va = be64_to_cpu(iser_hdr->write_va); 1440 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1441 write_stag, (unsigned long long)write_va); 1442 } 1443 1444 pr_debug("ISER ISCSI_CTRL PDU\n"); 1445 break; 1446 case ISER_HELLO: 1447 pr_err("iSER Hello message\n"); 1448 break; 1449 default: 1450 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1451 break; 1452 } 1453 1454 rc = isert_rx_opcode(isert_conn, rx_desc, 1455 read_stag, read_va, write_stag, write_va); 1456 } 1457 1458 static void 1459 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1460 unsigned long xfer_len) 1461 { 1462 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1463 struct iscsi_hdr *hdr; 1464 u64 rx_dma; 1465 int rx_buflen, outstanding; 1466 1467 if ((char *)desc == isert_conn->login_req_buf) { 1468 rx_dma = isert_conn->login_req_dma; 1469 rx_buflen = ISER_RX_LOGIN_SIZE; 1470 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1471 rx_dma, rx_buflen); 1472 } else { 1473 rx_dma = desc->dma_addr; 1474 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1475 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1476 rx_dma, rx_buflen); 1477 } 1478 1479 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1480 1481 hdr = &desc->iscsi_header; 1482 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1483 hdr->opcode, hdr->itt, hdr->flags, 1484 (int)(xfer_len - ISER_HEADERS_LEN)); 1485 1486 if ((char *)desc == isert_conn->login_req_buf) 1487 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1488 isert_conn); 1489 else 1490 isert_rx_do_work(desc, isert_conn); 1491 1492 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1493 DMA_FROM_DEVICE); 1494 1495 isert_conn->post_recv_buf_count--; 1496 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1497 isert_conn->post_recv_buf_count); 1498 1499 if ((char *)desc == isert_conn->login_req_buf) 1500 return; 1501 1502 outstanding = isert_conn->post_recv_buf_count; 1503 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1504 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1505 ISERT_MIN_POSTED_RX); 1506 err = isert_post_recv(isert_conn, count); 1507 if (err) { 1508 pr_err("isert_post_recv() count: %d failed, %d\n", 1509 count, err); 1510 } 1511 } 1512 } 1513 1514 static int 1515 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1516 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1517 enum iser_ib_op_code op, struct isert_data_buf *data) 1518 { 1519 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1520 1521 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1522 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1523 1524 data->len = length - offset; 1525 data->offset = offset; 1526 data->sg_off = data->offset / PAGE_SIZE; 1527 1528 data->sg = &sg[data->sg_off]; 1529 data->nents = min_t(unsigned int, nents - data->sg_off, 1530 ISCSI_ISER_SG_TABLESIZE); 1531 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1532 PAGE_SIZE); 1533 1534 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1535 data->dma_dir); 1536 if (unlikely(!data->dma_nents)) { 1537 pr_err("Cmd: unable to dma map SGs %p\n", sg); 1538 return -EINVAL; 1539 } 1540 1541 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1542 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1543 1544 return 0; 1545 } 1546 1547 static void 1548 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1549 { 1550 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1551 1552 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1553 memset(data, 0, sizeof(*data)); 1554 } 1555 1556 1557 1558 static void 1559 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1560 { 1561 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1562 1563 pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1564 1565 if (wr->data.sg) { 1566 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1567 isert_unmap_data_buf(isert_conn, &wr->data); 1568 } 1569 1570 if (wr->send_wr) { 1571 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1572 kfree(wr->send_wr); 1573 wr->send_wr = NULL; 1574 } 1575 1576 if (wr->ib_sge) { 1577 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 1578 kfree(wr->ib_sge); 1579 wr->ib_sge = NULL; 1580 } 1581 } 1582 1583 static void 1584 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1585 { 1586 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1587 LIST_HEAD(unmap_list); 1588 1589 pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd); 1590 1591 if (wr->fr_desc) { 1592 pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n", 1593 isert_cmd, wr->fr_desc); 1594 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1595 isert_unmap_data_buf(isert_conn, &wr->prot); 1596 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1597 } 1598 spin_lock_bh(&isert_conn->conn_lock); 1599 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 1600 spin_unlock_bh(&isert_conn->conn_lock); 1601 wr->fr_desc = NULL; 1602 } 1603 1604 if (wr->data.sg) { 1605 pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1606 isert_unmap_data_buf(isert_conn, &wr->data); 1607 } 1608 1609 wr->ib_sge = NULL; 1610 wr->send_wr = NULL; 1611 } 1612 1613 static void 1614 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1615 { 1616 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1617 struct isert_conn *isert_conn = isert_cmd->conn; 1618 struct iscsi_conn *conn = isert_conn->conn; 1619 struct isert_device *device = isert_conn->conn_device; 1620 1621 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1622 1623 switch (cmd->iscsi_opcode) { 1624 case ISCSI_OP_SCSI_CMD: 1625 spin_lock_bh(&conn->cmd_lock); 1626 if (!list_empty(&cmd->i_conn_node)) 1627 list_del_init(&cmd->i_conn_node); 1628 spin_unlock_bh(&conn->cmd_lock); 1629 1630 if (cmd->data_direction == DMA_TO_DEVICE) { 1631 iscsit_stop_dataout_timer(cmd); 1632 /* 1633 * Check for special case during comp_err where 1634 * WRITE_PENDING has been handed off from core, 1635 * but requires an extra target_put_sess_cmd() 1636 * before transport_generic_free_cmd() below. 1637 */ 1638 if (comp_err && 1639 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1640 struct se_cmd *se_cmd = &cmd->se_cmd; 1641 1642 target_put_sess_cmd(se_cmd->se_sess, se_cmd); 1643 } 1644 } 1645 1646 device->unreg_rdma_mem(isert_cmd, isert_conn); 1647 transport_generic_free_cmd(&cmd->se_cmd, 0); 1648 break; 1649 case ISCSI_OP_SCSI_TMFUNC: 1650 spin_lock_bh(&conn->cmd_lock); 1651 if (!list_empty(&cmd->i_conn_node)) 1652 list_del_init(&cmd->i_conn_node); 1653 spin_unlock_bh(&conn->cmd_lock); 1654 1655 transport_generic_free_cmd(&cmd->se_cmd, 0); 1656 break; 1657 case ISCSI_OP_REJECT: 1658 case ISCSI_OP_NOOP_OUT: 1659 case ISCSI_OP_TEXT: 1660 spin_lock_bh(&conn->cmd_lock); 1661 if (!list_empty(&cmd->i_conn_node)) 1662 list_del_init(&cmd->i_conn_node); 1663 spin_unlock_bh(&conn->cmd_lock); 1664 1665 /* 1666 * Handle special case for REJECT when iscsi_add_reject*() has 1667 * overwritten the original iscsi_opcode assignment, and the 1668 * associated cmd->se_cmd needs to be released. 1669 */ 1670 if (cmd->se_cmd.se_tfo != NULL) { 1671 pr_debug("Calling transport_generic_free_cmd from" 1672 " isert_put_cmd for 0x%02x\n", 1673 cmd->iscsi_opcode); 1674 transport_generic_free_cmd(&cmd->se_cmd, 0); 1675 break; 1676 } 1677 /* 1678 * Fall-through 1679 */ 1680 default: 1681 iscsit_release_cmd(cmd); 1682 break; 1683 } 1684 } 1685 1686 static void 1687 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1688 { 1689 if (tx_desc->dma_addr != 0) { 1690 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1691 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1692 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1693 tx_desc->dma_addr = 0; 1694 } 1695 } 1696 1697 static void 1698 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1699 struct ib_device *ib_dev, bool comp_err) 1700 { 1701 if (isert_cmd->pdu_buf_dma != 0) { 1702 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1703 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1704 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1705 isert_cmd->pdu_buf_dma = 0; 1706 } 1707 1708 isert_unmap_tx_desc(tx_desc, ib_dev); 1709 isert_put_cmd(isert_cmd, comp_err); 1710 } 1711 1712 static int 1713 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1714 { 1715 struct ib_mr_status mr_status; 1716 int ret; 1717 1718 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1719 if (ret) { 1720 pr_err("ib_check_mr_status failed, ret %d\n", ret); 1721 goto fail_mr_status; 1722 } 1723 1724 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1725 u64 sec_offset_err; 1726 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1727 1728 switch (mr_status.sig_err.err_type) { 1729 case IB_SIG_BAD_GUARD: 1730 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1731 break; 1732 case IB_SIG_BAD_REFTAG: 1733 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1734 break; 1735 case IB_SIG_BAD_APPTAG: 1736 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1737 break; 1738 } 1739 sec_offset_err = mr_status.sig_err.sig_err_offset; 1740 do_div(sec_offset_err, block_size); 1741 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1742 1743 pr_err("isert: PI error found type %d at sector 0x%llx " 1744 "expected 0x%x vs actual 0x%x\n", 1745 mr_status.sig_err.err_type, 1746 (unsigned long long)se_cmd->bad_sector, 1747 mr_status.sig_err.expected, 1748 mr_status.sig_err.actual); 1749 ret = 1; 1750 } 1751 1752 fail_mr_status: 1753 return ret; 1754 } 1755 1756 static void 1757 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1758 struct isert_cmd *isert_cmd) 1759 { 1760 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1761 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1762 struct se_cmd *se_cmd = &cmd->se_cmd; 1763 struct isert_conn *isert_conn = isert_cmd->conn; 1764 struct isert_device *device = isert_conn->conn_device; 1765 int ret = 0; 1766 1767 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1768 ret = isert_check_pi_status(se_cmd, 1769 wr->fr_desc->pi_ctx->sig_mr); 1770 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1771 } 1772 1773 device->unreg_rdma_mem(isert_cmd, isert_conn); 1774 wr->send_wr_num = 0; 1775 if (ret) 1776 transport_send_check_condition_and_sense(se_cmd, 1777 se_cmd->pi_err, 0); 1778 else 1779 isert_put_response(isert_conn->conn, cmd); 1780 } 1781 1782 static void 1783 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1784 struct isert_cmd *isert_cmd) 1785 { 1786 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1787 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1788 struct se_cmd *se_cmd = &cmd->se_cmd; 1789 struct isert_conn *isert_conn = isert_cmd->conn; 1790 struct isert_device *device = isert_conn->conn_device; 1791 int ret = 0; 1792 1793 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1794 ret = isert_check_pi_status(se_cmd, 1795 wr->fr_desc->pi_ctx->sig_mr); 1796 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1797 } 1798 1799 iscsit_stop_dataout_timer(cmd); 1800 device->unreg_rdma_mem(isert_cmd, isert_conn); 1801 cmd->write_data_done = wr->data.len; 1802 wr->send_wr_num = 0; 1803 1804 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1805 spin_lock_bh(&cmd->istate_lock); 1806 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1807 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1808 spin_unlock_bh(&cmd->istate_lock); 1809 1810 if (ret) 1811 transport_send_check_condition_and_sense(se_cmd, 1812 se_cmd->pi_err, 0); 1813 else 1814 target_execute_cmd(se_cmd); 1815 } 1816 1817 static void 1818 isert_do_control_comp(struct work_struct *work) 1819 { 1820 struct isert_cmd *isert_cmd = container_of(work, 1821 struct isert_cmd, comp_work); 1822 struct isert_conn *isert_conn = isert_cmd->conn; 1823 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1824 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1825 1826 switch (cmd->i_state) { 1827 case ISTATE_SEND_TASKMGTRSP: 1828 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1829 1830 atomic_dec(&isert_conn->post_send_buf_count); 1831 iscsit_tmr_post_handler(cmd, cmd->conn); 1832 1833 cmd->i_state = ISTATE_SENT_STATUS; 1834 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1835 break; 1836 case ISTATE_SEND_REJECT: 1837 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1838 atomic_dec(&isert_conn->post_send_buf_count); 1839 1840 cmd->i_state = ISTATE_SENT_STATUS; 1841 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1842 break; 1843 case ISTATE_SEND_LOGOUTRSP: 1844 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1845 1846 atomic_dec(&isert_conn->post_send_buf_count); 1847 iscsit_logout_post_handler(cmd, cmd->conn); 1848 break; 1849 case ISTATE_SEND_TEXTRSP: 1850 atomic_dec(&isert_conn->post_send_buf_count); 1851 cmd->i_state = ISTATE_SENT_STATUS; 1852 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1853 break; 1854 default: 1855 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1856 dump_stack(); 1857 break; 1858 } 1859 } 1860 1861 static void 1862 isert_response_completion(struct iser_tx_desc *tx_desc, 1863 struct isert_cmd *isert_cmd, 1864 struct isert_conn *isert_conn, 1865 struct ib_device *ib_dev) 1866 { 1867 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1868 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1869 1870 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1871 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1872 cmd->i_state == ISTATE_SEND_REJECT || 1873 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1874 isert_unmap_tx_desc(tx_desc, ib_dev); 1875 1876 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1877 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1878 return; 1879 } 1880 1881 /** 1882 * If send_wr_num is 0 this means that we got 1883 * RDMA completion and we cleared it and we should 1884 * simply decrement the response post. else the 1885 * response is incorporated in send_wr_num, just 1886 * sub it. 1887 **/ 1888 if (wr->send_wr_num) 1889 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1890 else 1891 atomic_dec(&isert_conn->post_send_buf_count); 1892 1893 cmd->i_state = ISTATE_SENT_STATUS; 1894 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1895 } 1896 1897 static void 1898 __isert_send_completion(struct iser_tx_desc *tx_desc, 1899 struct isert_conn *isert_conn) 1900 { 1901 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1902 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1903 struct isert_rdma_wr *wr; 1904 1905 if (!isert_cmd) { 1906 atomic_dec(&isert_conn->post_send_buf_count); 1907 isert_unmap_tx_desc(tx_desc, ib_dev); 1908 return; 1909 } 1910 wr = &isert_cmd->rdma_wr; 1911 1912 switch (wr->iser_ib_op) { 1913 case ISER_IB_RECV: 1914 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1915 dump_stack(); 1916 break; 1917 case ISER_IB_SEND: 1918 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1919 isert_response_completion(tx_desc, isert_cmd, 1920 isert_conn, ib_dev); 1921 break; 1922 case ISER_IB_RDMA_WRITE: 1923 pr_debug("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1924 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1925 isert_completion_rdma_write(tx_desc, isert_cmd); 1926 break; 1927 case ISER_IB_RDMA_READ: 1928 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1929 1930 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1931 isert_completion_rdma_read(tx_desc, isert_cmd); 1932 break; 1933 default: 1934 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1935 dump_stack(); 1936 break; 1937 } 1938 } 1939 1940 static void 1941 isert_send_completion(struct iser_tx_desc *tx_desc, 1942 struct isert_conn *isert_conn) 1943 { 1944 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1945 struct iser_tx_desc *t; 1946 /* 1947 * Drain coalesced completion llist starting from comp_llnode_batch 1948 * setup in isert_init_send_wr(), and then complete trailing tx_desc. 1949 */ 1950 while (llnode) { 1951 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1952 llnode = llist_next(llnode); 1953 __isert_send_completion(t, isert_conn); 1954 } 1955 __isert_send_completion(tx_desc, isert_conn); 1956 } 1957 1958 static void 1959 isert_cq_drain_comp_llist(struct isert_conn *isert_conn, struct ib_device *ib_dev) 1960 { 1961 struct llist_node *llnode; 1962 struct isert_rdma_wr *wr; 1963 struct iser_tx_desc *t; 1964 1965 mutex_lock(&isert_conn->conn_mutex); 1966 llnode = llist_del_all(&isert_conn->conn_comp_llist); 1967 isert_conn->conn_comp_batch = 0; 1968 mutex_unlock(&isert_conn->conn_mutex); 1969 1970 while (llnode) { 1971 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1972 llnode = llist_next(llnode); 1973 wr = &t->isert_cmd->rdma_wr; 1974 1975 /** 1976 * If send_wr_num is 0 this means that we got 1977 * RDMA completion and we cleared it and we should 1978 * simply decrement the response post. else the 1979 * response is incorporated in send_wr_num, just 1980 * sub it. 1981 **/ 1982 if (wr->send_wr_num) 1983 atomic_sub(wr->send_wr_num, 1984 &isert_conn->post_send_buf_count); 1985 else 1986 atomic_dec(&isert_conn->post_send_buf_count); 1987 1988 isert_completion_put(t, t->isert_cmd, ib_dev, true); 1989 } 1990 } 1991 1992 static void 1993 isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1994 { 1995 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1996 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1997 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1998 struct isert_rdma_wr *wr; 1999 struct iser_tx_desc *t; 2000 2001 while (llnode) { 2002 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 2003 llnode = llist_next(llnode); 2004 wr = &t->isert_cmd->rdma_wr; 2005 2006 /** 2007 * If send_wr_num is 0 this means that we got 2008 * RDMA completion and we cleared it and we should 2009 * simply decrement the response post. else the 2010 * response is incorporated in send_wr_num, just 2011 * sub it. 2012 **/ 2013 if (wr->send_wr_num) 2014 atomic_sub(wr->send_wr_num, 2015 &isert_conn->post_send_buf_count); 2016 else 2017 atomic_dec(&isert_conn->post_send_buf_count); 2018 2019 isert_completion_put(t, t->isert_cmd, ib_dev, true); 2020 } 2021 tx_desc->comp_llnode_batch = NULL; 2022 2023 if (!isert_cmd) 2024 isert_unmap_tx_desc(tx_desc, ib_dev); 2025 else 2026 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 2027 } 2028 2029 static void 2030 isert_cq_rx_comp_err(struct isert_conn *isert_conn) 2031 { 2032 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2033 struct iscsi_conn *conn = isert_conn->conn; 2034 2035 if (isert_conn->post_recv_buf_count) 2036 return; 2037 2038 isert_cq_drain_comp_llist(isert_conn, ib_dev); 2039 2040 if (conn->sess) { 2041 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2042 target_wait_for_sess_cmds(conn->sess->se_sess); 2043 } 2044 2045 while (atomic_read(&isert_conn->post_send_buf_count)) 2046 msleep(3000); 2047 2048 mutex_lock(&isert_conn->conn_mutex); 2049 isert_conn->state = ISER_CONN_DOWN; 2050 mutex_unlock(&isert_conn->conn_mutex); 2051 2052 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2053 2054 complete(&isert_conn->conn_wait_comp_err); 2055 } 2056 2057 static void 2058 isert_cq_tx_work(struct work_struct *work) 2059 { 2060 struct isert_cq_desc *cq_desc = container_of(work, 2061 struct isert_cq_desc, cq_tx_work); 2062 struct isert_device *device = cq_desc->device; 2063 int cq_index = cq_desc->cq_index; 2064 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 2065 struct isert_conn *isert_conn; 2066 struct iser_tx_desc *tx_desc; 2067 struct ib_wc wc; 2068 2069 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 2070 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 2071 isert_conn = wc.qp->qp_context; 2072 2073 if (wc.status == IB_WC_SUCCESS) { 2074 isert_send_completion(tx_desc, isert_conn); 2075 } else { 2076 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2077 pr_debug("TX wc.status: 0x%08x\n", wc.status); 2078 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 2079 2080 if (wc.wr_id != ISER_FASTREG_LI_WRID) { 2081 if (tx_desc->llnode_active) 2082 continue; 2083 2084 atomic_dec(&isert_conn->post_send_buf_count); 2085 isert_cq_tx_comp_err(tx_desc, isert_conn); 2086 } 2087 } 2088 } 2089 2090 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 2091 } 2092 2093 static void 2094 isert_cq_tx_callback(struct ib_cq *cq, void *context) 2095 { 2096 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2097 2098 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 2099 } 2100 2101 static void 2102 isert_cq_rx_work(struct work_struct *work) 2103 { 2104 struct isert_cq_desc *cq_desc = container_of(work, 2105 struct isert_cq_desc, cq_rx_work); 2106 struct isert_device *device = cq_desc->device; 2107 int cq_index = cq_desc->cq_index; 2108 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 2109 struct isert_conn *isert_conn; 2110 struct iser_rx_desc *rx_desc; 2111 struct ib_wc wc; 2112 unsigned long xfer_len; 2113 2114 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 2115 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 2116 isert_conn = wc.qp->qp_context; 2117 2118 if (wc.status == IB_WC_SUCCESS) { 2119 xfer_len = (unsigned long)wc.byte_len; 2120 isert_rx_completion(rx_desc, isert_conn, xfer_len); 2121 } else { 2122 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2123 if (wc.status != IB_WC_WR_FLUSH_ERR) { 2124 pr_debug("RX wc.status: 0x%08x\n", wc.status); 2125 pr_debug("RX wc.vendor_err: 0x%08x\n", 2126 wc.vendor_err); 2127 } 2128 isert_conn->post_recv_buf_count--; 2129 isert_cq_rx_comp_err(isert_conn); 2130 } 2131 } 2132 2133 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 2134 } 2135 2136 static void 2137 isert_cq_rx_callback(struct ib_cq *cq, void *context) 2138 { 2139 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2140 2141 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 2142 } 2143 2144 static int 2145 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2146 { 2147 struct ib_send_wr *wr_failed; 2148 int ret; 2149 2150 atomic_inc(&isert_conn->post_send_buf_count); 2151 2152 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 2153 &wr_failed); 2154 if (ret) { 2155 pr_err("ib_post_send failed with %d\n", ret); 2156 atomic_dec(&isert_conn->post_send_buf_count); 2157 return ret; 2158 } 2159 return ret; 2160 } 2161 2162 static int 2163 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2164 { 2165 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2166 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2167 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2168 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2169 &isert_cmd->tx_desc.iscsi_header; 2170 2171 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2172 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2173 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2174 /* 2175 * Attach SENSE DATA payload to iSCSI Response PDU 2176 */ 2177 if (cmd->se_cmd.sense_buffer && 2178 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2179 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2180 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2181 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2182 u32 padding, pdu_len; 2183 2184 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2185 cmd->sense_buffer); 2186 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2187 2188 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2189 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2190 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2191 2192 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2193 (void *)cmd->sense_buffer, pdu_len, 2194 DMA_TO_DEVICE); 2195 2196 isert_cmd->pdu_buf_len = pdu_len; 2197 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2198 tx_dsg->length = pdu_len; 2199 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2200 isert_cmd->tx_desc.num_sge = 2; 2201 } 2202 2203 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2204 2205 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2206 2207 return isert_post_response(isert_conn, isert_cmd); 2208 } 2209 2210 static void 2211 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2212 { 2213 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2214 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2215 struct isert_device *device = isert_conn->conn_device; 2216 2217 spin_lock_bh(&conn->cmd_lock); 2218 if (!list_empty(&cmd->i_conn_node)) 2219 list_del_init(&cmd->i_conn_node); 2220 spin_unlock_bh(&conn->cmd_lock); 2221 2222 if (cmd->data_direction == DMA_TO_DEVICE) 2223 iscsit_stop_dataout_timer(cmd); 2224 2225 device->unreg_rdma_mem(isert_cmd, isert_conn); 2226 } 2227 2228 static enum target_prot_op 2229 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2230 { 2231 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2232 struct isert_device *device = isert_conn->conn_device; 2233 2234 if (device->pi_capable) 2235 return TARGET_PROT_ALL; 2236 2237 return TARGET_PROT_NORMAL; 2238 } 2239 2240 static int 2241 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2242 bool nopout_response) 2243 { 2244 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2245 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2246 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2247 2248 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2249 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2250 &isert_cmd->tx_desc.iscsi_header, 2251 nopout_response); 2252 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2253 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2254 2255 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2256 2257 return isert_post_response(isert_conn, isert_cmd); 2258 } 2259 2260 static int 2261 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2262 { 2263 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2264 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2265 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2266 2267 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2268 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2269 &isert_cmd->tx_desc.iscsi_header); 2270 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2271 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2272 2273 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2274 2275 return isert_post_response(isert_conn, isert_cmd); 2276 } 2277 2278 static int 2279 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2280 { 2281 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2282 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2283 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2284 2285 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2286 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2287 &isert_cmd->tx_desc.iscsi_header); 2288 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2289 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2290 2291 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2292 2293 return isert_post_response(isert_conn, isert_cmd); 2294 } 2295 2296 static int 2297 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2298 { 2299 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2300 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2301 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2302 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2303 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2304 struct iscsi_reject *hdr = 2305 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2306 2307 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2308 iscsit_build_reject(cmd, conn, hdr); 2309 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2310 2311 hton24(hdr->dlength, ISCSI_HDR_LEN); 2312 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2313 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2314 DMA_TO_DEVICE); 2315 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2316 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2317 tx_dsg->length = ISCSI_HDR_LEN; 2318 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2319 isert_cmd->tx_desc.num_sge = 2; 2320 2321 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2322 2323 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2324 2325 return isert_post_response(isert_conn, isert_cmd); 2326 } 2327 2328 static int 2329 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2330 { 2331 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2332 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2333 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2334 struct iscsi_text_rsp *hdr = 2335 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2336 u32 txt_rsp_len; 2337 int rc; 2338 2339 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2340 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2341 if (rc < 0) 2342 return rc; 2343 2344 txt_rsp_len = rc; 2345 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2346 2347 if (txt_rsp_len) { 2348 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2349 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2350 void *txt_rsp_buf = cmd->buf_ptr; 2351 2352 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2353 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2354 2355 isert_cmd->pdu_buf_len = txt_rsp_len; 2356 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2357 tx_dsg->length = txt_rsp_len; 2358 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2359 isert_cmd->tx_desc.num_sge = 2; 2360 } 2361 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2362 2363 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2364 2365 return isert_post_response(isert_conn, isert_cmd); 2366 } 2367 2368 static int 2369 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2370 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2371 u32 data_left, u32 offset) 2372 { 2373 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2374 struct scatterlist *sg_start, *tmp_sg; 2375 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2376 u32 sg_off, page_off; 2377 int i = 0, sg_nents; 2378 2379 sg_off = offset / PAGE_SIZE; 2380 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2381 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2382 page_off = offset % PAGE_SIZE; 2383 2384 send_wr->sg_list = ib_sge; 2385 send_wr->num_sge = sg_nents; 2386 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2387 /* 2388 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2389 */ 2390 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2391 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2392 (unsigned long long)tmp_sg->dma_address, 2393 tmp_sg->length, page_off); 2394 2395 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2396 ib_sge->length = min_t(u32, data_left, 2397 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2398 ib_sge->lkey = isert_conn->conn_mr->lkey; 2399 2400 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2401 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2402 page_off = 0; 2403 data_left -= ib_sge->length; 2404 ib_sge++; 2405 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2406 } 2407 2408 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2409 send_wr->sg_list, send_wr->num_sge); 2410 2411 return sg_nents; 2412 } 2413 2414 static int 2415 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2416 struct isert_rdma_wr *wr) 2417 { 2418 struct se_cmd *se_cmd = &cmd->se_cmd; 2419 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2420 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2421 struct isert_data_buf *data = &wr->data; 2422 struct ib_send_wr *send_wr; 2423 struct ib_sge *ib_sge; 2424 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2425 int ret = 0, i, ib_sge_cnt; 2426 2427 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2428 2429 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2430 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2431 se_cmd->t_data_nents, se_cmd->data_length, 2432 offset, wr->iser_ib_op, &wr->data); 2433 if (ret) 2434 return ret; 2435 2436 data_left = data->len; 2437 offset = data->offset; 2438 2439 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2440 if (!ib_sge) { 2441 pr_warn("Unable to allocate ib_sge\n"); 2442 ret = -ENOMEM; 2443 goto unmap_cmd; 2444 } 2445 wr->ib_sge = ib_sge; 2446 2447 wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2448 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2449 GFP_KERNEL); 2450 if (!wr->send_wr) { 2451 pr_debug("Unable to allocate wr->send_wr\n"); 2452 ret = -ENOMEM; 2453 goto unmap_cmd; 2454 } 2455 2456 wr->isert_cmd = isert_cmd; 2457 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2458 2459 for (i = 0; i < wr->send_wr_num; i++) { 2460 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2461 data_len = min(data_left, rdma_write_max); 2462 2463 send_wr->send_flags = 0; 2464 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2465 send_wr->opcode = IB_WR_RDMA_WRITE; 2466 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2467 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2468 if (i + 1 == wr->send_wr_num) 2469 send_wr->next = &isert_cmd->tx_desc.send_wr; 2470 else 2471 send_wr->next = &wr->send_wr[i + 1]; 2472 } else { 2473 send_wr->opcode = IB_WR_RDMA_READ; 2474 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2475 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2476 if (i + 1 == wr->send_wr_num) 2477 send_wr->send_flags = IB_SEND_SIGNALED; 2478 else 2479 send_wr->next = &wr->send_wr[i + 1]; 2480 } 2481 2482 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2483 send_wr, data_len, offset); 2484 ib_sge += ib_sge_cnt; 2485 2486 offset += data_len; 2487 va_offset += data_len; 2488 data_left -= data_len; 2489 } 2490 2491 return 0; 2492 unmap_cmd: 2493 isert_unmap_data_buf(isert_conn, data); 2494 2495 return ret; 2496 } 2497 2498 static int 2499 isert_map_fr_pagelist(struct ib_device *ib_dev, 2500 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2501 { 2502 u64 start_addr, end_addr, page, chunk_start = 0; 2503 struct scatterlist *tmp_sg; 2504 int i = 0, new_chunk, last_ent, n_pages; 2505 2506 n_pages = 0; 2507 new_chunk = 1; 2508 last_ent = sg_nents - 1; 2509 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2510 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2511 if (new_chunk) 2512 chunk_start = start_addr; 2513 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2514 2515 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 2516 i, (unsigned long long)tmp_sg->dma_address, 2517 tmp_sg->length); 2518 2519 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2520 new_chunk = 0; 2521 continue; 2522 } 2523 new_chunk = 1; 2524 2525 page = chunk_start & PAGE_MASK; 2526 do { 2527 fr_pl[n_pages++] = page; 2528 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 2529 n_pages - 1, page); 2530 page += PAGE_SIZE; 2531 } while (page < end_addr); 2532 } 2533 2534 return n_pages; 2535 } 2536 2537 static int 2538 isert_fast_reg_mr(struct isert_conn *isert_conn, 2539 struct fast_reg_descriptor *fr_desc, 2540 struct isert_data_buf *mem, 2541 enum isert_indicator ind, 2542 struct ib_sge *sge) 2543 { 2544 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2545 struct ib_mr *mr; 2546 struct ib_fast_reg_page_list *frpl; 2547 struct ib_send_wr fr_wr, inv_wr; 2548 struct ib_send_wr *bad_wr, *wr = NULL; 2549 int ret, pagelist_len; 2550 u32 page_off; 2551 u8 key; 2552 2553 if (mem->dma_nents == 1) { 2554 sge->lkey = isert_conn->conn_mr->lkey; 2555 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2556 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2557 pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 2558 __func__, __LINE__, sge->addr, sge->length, 2559 sge->lkey); 2560 return 0; 2561 } 2562 2563 if (ind == ISERT_DATA_KEY_VALID) { 2564 /* Registering data buffer */ 2565 mr = fr_desc->data_mr; 2566 frpl = fr_desc->data_frpl; 2567 } else { 2568 /* Registering protection buffer */ 2569 mr = fr_desc->pi_ctx->prot_mr; 2570 frpl = fr_desc->pi_ctx->prot_frpl; 2571 } 2572 2573 page_off = mem->offset % PAGE_SIZE; 2574 2575 pr_debug("Use fr_desc %p sg_nents %d offset %u\n", 2576 fr_desc, mem->nents, mem->offset); 2577 2578 pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 2579 &frpl->page_list[0]); 2580 2581 if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) { 2582 memset(&inv_wr, 0, sizeof(inv_wr)); 2583 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 2584 inv_wr.opcode = IB_WR_LOCAL_INV; 2585 inv_wr.ex.invalidate_rkey = mr->rkey; 2586 wr = &inv_wr; 2587 /* Bump the key */ 2588 key = (u8)(mr->rkey & 0x000000FF); 2589 ib_update_fast_reg_key(mr, ++key); 2590 } 2591 2592 /* Prepare FASTREG WR */ 2593 memset(&fr_wr, 0, sizeof(fr_wr)); 2594 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2595 fr_wr.opcode = IB_WR_FAST_REG_MR; 2596 fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 2597 fr_wr.wr.fast_reg.page_list = frpl; 2598 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2599 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2600 fr_wr.wr.fast_reg.length = mem->len; 2601 fr_wr.wr.fast_reg.rkey = mr->rkey; 2602 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2603 2604 if (!wr) 2605 wr = &fr_wr; 2606 else 2607 wr->next = &fr_wr; 2608 2609 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2610 if (ret) { 2611 pr_err("fast registration failed, ret:%d\n", ret); 2612 return ret; 2613 } 2614 fr_desc->ind &= ~ind; 2615 2616 sge->lkey = mr->lkey; 2617 sge->addr = frpl->page_list[0] + page_off; 2618 sge->length = mem->len; 2619 2620 pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 2621 __func__, __LINE__, sge->addr, sge->length, 2622 sge->lkey); 2623 2624 return ret; 2625 } 2626 2627 static inline void 2628 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2629 struct ib_sig_domain *domain) 2630 { 2631 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2632 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2633 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2634 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2635 /* 2636 * At the moment we hard code those, but if in the future 2637 * the target core would like to use it, we will take it 2638 * from se_cmd. 2639 */ 2640 domain->sig.dif.apptag_check_mask = 0xffff; 2641 domain->sig.dif.app_escape = true; 2642 domain->sig.dif.ref_escape = true; 2643 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2644 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2645 domain->sig.dif.ref_remap = true; 2646 }; 2647 2648 static int 2649 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2650 { 2651 switch (se_cmd->prot_op) { 2652 case TARGET_PROT_DIN_INSERT: 2653 case TARGET_PROT_DOUT_STRIP: 2654 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2655 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2656 break; 2657 case TARGET_PROT_DOUT_INSERT: 2658 case TARGET_PROT_DIN_STRIP: 2659 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2660 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2661 break; 2662 case TARGET_PROT_DIN_PASS: 2663 case TARGET_PROT_DOUT_PASS: 2664 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2665 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2666 break; 2667 default: 2668 pr_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2669 return -EINVAL; 2670 } 2671 2672 return 0; 2673 } 2674 2675 static inline u8 2676 isert_set_prot_checks(u8 prot_checks) 2677 { 2678 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2679 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2680 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2681 } 2682 2683 static int 2684 isert_reg_sig_mr(struct isert_conn *isert_conn, struct se_cmd *se_cmd, 2685 struct fast_reg_descriptor *fr_desc, 2686 struct ib_sge *data_sge, struct ib_sge *prot_sge, 2687 struct ib_sge *sig_sge) 2688 { 2689 struct ib_send_wr sig_wr, inv_wr; 2690 struct ib_send_wr *bad_wr, *wr = NULL; 2691 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2692 struct ib_sig_attrs sig_attrs; 2693 int ret; 2694 u32 key; 2695 2696 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2697 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2698 if (ret) 2699 goto err; 2700 2701 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2702 2703 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2704 memset(&inv_wr, 0, sizeof(inv_wr)); 2705 inv_wr.opcode = IB_WR_LOCAL_INV; 2706 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 2707 inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; 2708 wr = &inv_wr; 2709 /* Bump the key */ 2710 key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); 2711 ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); 2712 } 2713 2714 memset(&sig_wr, 0, sizeof(sig_wr)); 2715 sig_wr.opcode = IB_WR_REG_SIG_MR; 2716 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2717 sig_wr.sg_list = data_sge; 2718 sig_wr.num_sge = 1; 2719 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 2720 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 2721 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 2722 if (se_cmd->t_prot_sg) 2723 sig_wr.wr.sig_handover.prot = prot_sge; 2724 2725 if (!wr) 2726 wr = &sig_wr; 2727 else 2728 wr->next = &sig_wr; 2729 2730 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2731 if (ret) { 2732 pr_err("fast registration failed, ret:%d\n", ret); 2733 goto err; 2734 } 2735 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2736 2737 sig_sge->lkey = pi_ctx->sig_mr->lkey; 2738 sig_sge->addr = 0; 2739 sig_sge->length = se_cmd->data_length; 2740 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2741 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2742 /* 2743 * We have protection guards on the wire 2744 * so we need to set a larget transfer 2745 */ 2746 sig_sge->length += se_cmd->prot_length; 2747 2748 pr_debug("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2749 sig_sge->addr, sig_sge->length, 2750 sig_sge->lkey); 2751 err: 2752 return ret; 2753 } 2754 2755 static int 2756 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2757 struct isert_rdma_wr *wr) 2758 { 2759 struct se_cmd *se_cmd = &cmd->se_cmd; 2760 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2761 struct isert_conn *isert_conn = conn->context; 2762 struct ib_sge data_sge; 2763 struct ib_send_wr *send_wr; 2764 struct fast_reg_descriptor *fr_desc = NULL; 2765 u32 offset; 2766 int ret = 0; 2767 unsigned long flags; 2768 2769 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2770 2771 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2772 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2773 se_cmd->t_data_nents, se_cmd->data_length, 2774 offset, wr->iser_ib_op, &wr->data); 2775 if (ret) 2776 return ret; 2777 2778 if (wr->data.dma_nents != 1 || 2779 se_cmd->prot_op != TARGET_PROT_NORMAL) { 2780 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2781 fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2782 struct fast_reg_descriptor, list); 2783 list_del(&fr_desc->list); 2784 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2785 wr->fr_desc = fr_desc; 2786 } 2787 2788 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2789 ISERT_DATA_KEY_VALID, &data_sge); 2790 if (ret) 2791 goto unmap_cmd; 2792 2793 if (se_cmd->prot_op != TARGET_PROT_NORMAL) { 2794 struct ib_sge prot_sge, sig_sge; 2795 2796 if (se_cmd->t_prot_sg) { 2797 ret = isert_map_data_buf(isert_conn, isert_cmd, 2798 se_cmd->t_prot_sg, 2799 se_cmd->t_prot_nents, 2800 se_cmd->prot_length, 2801 0, wr->iser_ib_op, &wr->prot); 2802 if (ret) 2803 goto unmap_cmd; 2804 2805 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->prot, 2806 ISERT_PROT_KEY_VALID, &prot_sge); 2807 if (ret) 2808 goto unmap_prot_cmd; 2809 } 2810 2811 ret = isert_reg_sig_mr(isert_conn, se_cmd, fr_desc, 2812 &data_sge, &prot_sge, &sig_sge); 2813 if (ret) 2814 goto unmap_prot_cmd; 2815 2816 fr_desc->ind |= ISERT_PROTECTED; 2817 memcpy(&wr->s_ib_sge, &sig_sge, sizeof(sig_sge)); 2818 } else 2819 memcpy(&wr->s_ib_sge, &data_sge, sizeof(data_sge)); 2820 2821 wr->ib_sge = &wr->s_ib_sge; 2822 wr->send_wr_num = 1; 2823 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2824 wr->send_wr = &wr->s_send_wr; 2825 wr->isert_cmd = isert_cmd; 2826 2827 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2828 send_wr->sg_list = &wr->s_ib_sge; 2829 send_wr->num_sge = 1; 2830 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2831 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2832 send_wr->opcode = IB_WR_RDMA_WRITE; 2833 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2834 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2835 send_wr->send_flags = se_cmd->prot_op == TARGET_PROT_NORMAL ? 2836 0 : IB_SEND_SIGNALED; 2837 } else { 2838 send_wr->opcode = IB_WR_RDMA_READ; 2839 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2840 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2841 send_wr->send_flags = IB_SEND_SIGNALED; 2842 } 2843 2844 return 0; 2845 unmap_prot_cmd: 2846 if (se_cmd->t_prot_sg) 2847 isert_unmap_data_buf(isert_conn, &wr->prot); 2848 unmap_cmd: 2849 if (fr_desc) { 2850 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2851 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2852 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2853 } 2854 isert_unmap_data_buf(isert_conn, &wr->data); 2855 2856 return ret; 2857 } 2858 2859 static int 2860 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2861 { 2862 struct se_cmd *se_cmd = &cmd->se_cmd; 2863 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2864 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2865 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2866 struct isert_device *device = isert_conn->conn_device; 2867 struct ib_send_wr *wr_failed; 2868 int rc; 2869 2870 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 2871 isert_cmd, se_cmd->data_length); 2872 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2873 rc = device->reg_rdma_mem(conn, cmd, wr); 2874 if (rc) { 2875 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2876 return rc; 2877 } 2878 2879 if (se_cmd->prot_op == TARGET_PROT_NORMAL) { 2880 /* 2881 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2882 */ 2883 isert_create_send_desc(isert_conn, isert_cmd, 2884 &isert_cmd->tx_desc); 2885 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2886 &isert_cmd->tx_desc.iscsi_header); 2887 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2888 isert_init_send_wr(isert_conn, isert_cmd, 2889 &isert_cmd->tx_desc.send_wr, false); 2890 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2891 wr->send_wr_num += 1; 2892 } 2893 2894 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2895 2896 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2897 if (rc) { 2898 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2899 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2900 } 2901 2902 if (se_cmd->prot_op == TARGET_PROT_NORMAL) 2903 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2904 "READ\n", isert_cmd); 2905 else 2906 pr_debug("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2907 isert_cmd); 2908 2909 return 1; 2910 } 2911 2912 static int 2913 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2914 { 2915 struct se_cmd *se_cmd = &cmd->se_cmd; 2916 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2917 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2918 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2919 struct isert_device *device = isert_conn->conn_device; 2920 struct ib_send_wr *wr_failed; 2921 int rc; 2922 2923 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2924 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2925 wr->iser_ib_op = ISER_IB_RDMA_READ; 2926 rc = device->reg_rdma_mem(conn, cmd, wr); 2927 if (rc) { 2928 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2929 return rc; 2930 } 2931 2932 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2933 2934 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2935 if (rc) { 2936 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2937 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2938 } 2939 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2940 isert_cmd); 2941 2942 return 0; 2943 } 2944 2945 static int 2946 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2947 { 2948 int ret; 2949 2950 switch (state) { 2951 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2952 ret = isert_put_nopin(cmd, conn, false); 2953 break; 2954 default: 2955 pr_err("Unknown immediate state: 0x%02x\n", state); 2956 ret = -EINVAL; 2957 break; 2958 } 2959 2960 return ret; 2961 } 2962 2963 static int 2964 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2965 { 2966 int ret; 2967 2968 switch (state) { 2969 case ISTATE_SEND_LOGOUTRSP: 2970 ret = isert_put_logout_rsp(cmd, conn); 2971 if (!ret) { 2972 pr_debug("Returning iSER Logout -EAGAIN\n"); 2973 ret = -EAGAIN; 2974 } 2975 break; 2976 case ISTATE_SEND_NOPIN: 2977 ret = isert_put_nopin(cmd, conn, true); 2978 break; 2979 case ISTATE_SEND_TASKMGTRSP: 2980 ret = isert_put_tm_rsp(cmd, conn); 2981 break; 2982 case ISTATE_SEND_REJECT: 2983 ret = isert_put_reject(cmd, conn); 2984 break; 2985 case ISTATE_SEND_TEXTRSP: 2986 ret = isert_put_text_rsp(cmd, conn); 2987 break; 2988 case ISTATE_SEND_STATUS: 2989 /* 2990 * Special case for sending non GOOD SCSI status from TX thread 2991 * context during pre se_cmd excecution failure. 2992 */ 2993 ret = isert_put_response(conn, cmd); 2994 break; 2995 default: 2996 pr_err("Unknown response state: 0x%02x\n", state); 2997 ret = -EINVAL; 2998 break; 2999 } 3000 3001 return ret; 3002 } 3003 3004 static int 3005 isert_setup_np(struct iscsi_np *np, 3006 struct __kernel_sockaddr_storage *ksockaddr) 3007 { 3008 struct isert_np *isert_np; 3009 struct rdma_cm_id *isert_lid; 3010 struct sockaddr *sa; 3011 int ret; 3012 3013 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3014 if (!isert_np) { 3015 pr_err("Unable to allocate struct isert_np\n"); 3016 return -ENOMEM; 3017 } 3018 sema_init(&isert_np->np_sem, 0); 3019 mutex_init(&isert_np->np_accept_mutex); 3020 INIT_LIST_HEAD(&isert_np->np_accept_list); 3021 init_completion(&isert_np->np_login_comp); 3022 3023 sa = (struct sockaddr *)ksockaddr; 3024 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 3025 /* 3026 * Setup the np->np_sockaddr from the passed sockaddr setup 3027 * in iscsi_target_configfs.c code.. 3028 */ 3029 memcpy(&np->np_sockaddr, ksockaddr, 3030 sizeof(struct __kernel_sockaddr_storage)); 3031 3032 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 3033 IB_QPT_RC); 3034 if (IS_ERR(isert_lid)) { 3035 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 3036 PTR_ERR(isert_lid)); 3037 ret = PTR_ERR(isert_lid); 3038 goto out; 3039 } 3040 3041 ret = rdma_bind_addr(isert_lid, sa); 3042 if (ret) { 3043 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 3044 goto out_lid; 3045 } 3046 3047 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 3048 if (ret) { 3049 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 3050 goto out_lid; 3051 } 3052 3053 isert_np->np_cm_id = isert_lid; 3054 np->np_context = isert_np; 3055 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 3056 3057 return 0; 3058 3059 out_lid: 3060 rdma_destroy_id(isert_lid); 3061 out: 3062 kfree(isert_np); 3063 return ret; 3064 } 3065 3066 static int 3067 isert_rdma_accept(struct isert_conn *isert_conn) 3068 { 3069 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3070 struct rdma_conn_param cp; 3071 int ret; 3072 3073 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3074 cp.initiator_depth = isert_conn->initiator_depth; 3075 cp.retry_count = 7; 3076 cp.rnr_retry_count = 7; 3077 3078 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 3079 3080 ret = rdma_accept(cm_id, &cp); 3081 if (ret) { 3082 pr_err("rdma_accept() failed with: %d\n", ret); 3083 return ret; 3084 } 3085 3086 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 3087 3088 return 0; 3089 } 3090 3091 static int 3092 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3093 { 3094 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3095 int ret; 3096 3097 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 3098 /* 3099 * For login requests after the first PDU, isert_rx_login_req() will 3100 * kick schedule_delayed_work(&conn->login_work) as the packet is 3101 * received, which turns this callback from iscsi_target_do_login_rx() 3102 * into a NOP. 3103 */ 3104 if (!login->first_request) 3105 return 0; 3106 3107 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 3108 if (ret) 3109 return ret; 3110 3111 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 3112 return 0; 3113 } 3114 3115 static void 3116 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3117 struct isert_conn *isert_conn) 3118 { 3119 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3120 struct rdma_route *cm_route = &cm_id->route; 3121 struct sockaddr_in *sock_in; 3122 struct sockaddr_in6 *sock_in6; 3123 3124 conn->login_family = np->np_sockaddr.ss_family; 3125 3126 if (np->np_sockaddr.ss_family == AF_INET6) { 3127 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3128 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3129 &sock_in6->sin6_addr.in6_u); 3130 conn->login_port = ntohs(sock_in6->sin6_port); 3131 3132 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3133 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3134 &sock_in6->sin6_addr.in6_u); 3135 conn->local_port = ntohs(sock_in6->sin6_port); 3136 } else { 3137 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3138 sprintf(conn->login_ip, "%pI4", 3139 &sock_in->sin_addr.s_addr); 3140 conn->login_port = ntohs(sock_in->sin_port); 3141 3142 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3143 sprintf(conn->local_ip, "%pI4", 3144 &sock_in->sin_addr.s_addr); 3145 conn->local_port = ntohs(sock_in->sin_port); 3146 } 3147 } 3148 3149 static int 3150 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3151 { 3152 struct isert_np *isert_np = (struct isert_np *)np->np_context; 3153 struct isert_conn *isert_conn; 3154 int max_accept = 0, ret; 3155 3156 accept_wait: 3157 ret = down_interruptible(&isert_np->np_sem); 3158 if (ret || max_accept > 5) 3159 return -ENODEV; 3160 3161 spin_lock_bh(&np->np_thread_lock); 3162 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3163 spin_unlock_bh(&np->np_thread_lock); 3164 pr_debug("np_thread_state %d for isert_accept_np\n", 3165 np->np_thread_state); 3166 /** 3167 * No point in stalling here when np_thread 3168 * is in state RESET/SHUTDOWN/EXIT - bail 3169 **/ 3170 return -ENODEV; 3171 } 3172 spin_unlock_bh(&np->np_thread_lock); 3173 3174 mutex_lock(&isert_np->np_accept_mutex); 3175 if (list_empty(&isert_np->np_accept_list)) { 3176 mutex_unlock(&isert_np->np_accept_mutex); 3177 max_accept++; 3178 goto accept_wait; 3179 } 3180 isert_conn = list_first_entry(&isert_np->np_accept_list, 3181 struct isert_conn, conn_accept_node); 3182 list_del_init(&isert_conn->conn_accept_node); 3183 mutex_unlock(&isert_np->np_accept_mutex); 3184 3185 conn->context = isert_conn; 3186 isert_conn->conn = conn; 3187 max_accept = 0; 3188 3189 ret = isert_rdma_post_recvl(isert_conn); 3190 if (ret) 3191 return ret; 3192 3193 ret = isert_rdma_accept(isert_conn); 3194 if (ret) 3195 return ret; 3196 3197 isert_set_conn_info(np, conn, isert_conn); 3198 3199 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 3200 return 0; 3201 } 3202 3203 static void 3204 isert_free_np(struct iscsi_np *np) 3205 { 3206 struct isert_np *isert_np = (struct isert_np *)np->np_context; 3207 3208 if (isert_np->np_cm_id) 3209 rdma_destroy_id(isert_np->np_cm_id); 3210 3211 np->np_context = NULL; 3212 kfree(isert_np); 3213 } 3214 3215 static void isert_wait_conn(struct iscsi_conn *conn) 3216 { 3217 struct isert_conn *isert_conn = conn->context; 3218 3219 pr_debug("isert_wait_conn: Starting \n"); 3220 3221 mutex_lock(&isert_conn->conn_mutex); 3222 if (isert_conn->conn_cm_id && !isert_conn->disconnect) { 3223 pr_debug("Calling rdma_disconnect from isert_wait_conn\n"); 3224 rdma_disconnect(isert_conn->conn_cm_id); 3225 } 3226 /* 3227 * Only wait for conn_wait_comp_err if the isert_conn made it 3228 * into full feature phase.. 3229 */ 3230 if (isert_conn->state == ISER_CONN_INIT) { 3231 mutex_unlock(&isert_conn->conn_mutex); 3232 return; 3233 } 3234 if (isert_conn->state == ISER_CONN_UP) 3235 isert_conn->state = ISER_CONN_TERMINATING; 3236 mutex_unlock(&isert_conn->conn_mutex); 3237 3238 wait_for_completion(&isert_conn->conn_wait_comp_err); 3239 3240 wait_for_completion(&isert_conn->conn_wait); 3241 isert_put_conn(isert_conn); 3242 } 3243 3244 static void isert_free_conn(struct iscsi_conn *conn) 3245 { 3246 struct isert_conn *isert_conn = conn->context; 3247 3248 isert_put_conn(isert_conn); 3249 } 3250 3251 static struct iscsit_transport iser_target_transport = { 3252 .name = "IB/iSER", 3253 .transport_type = ISCSI_INFINIBAND, 3254 .priv_size = sizeof(struct isert_cmd), 3255 .owner = THIS_MODULE, 3256 .iscsit_setup_np = isert_setup_np, 3257 .iscsit_accept_np = isert_accept_np, 3258 .iscsit_free_np = isert_free_np, 3259 .iscsit_wait_conn = isert_wait_conn, 3260 .iscsit_free_conn = isert_free_conn, 3261 .iscsit_get_login_rx = isert_get_login_rx, 3262 .iscsit_put_login_tx = isert_put_login_tx, 3263 .iscsit_immediate_queue = isert_immediate_queue, 3264 .iscsit_response_queue = isert_response_queue, 3265 .iscsit_get_dataout = isert_get_dataout, 3266 .iscsit_queue_data_in = isert_put_datain, 3267 .iscsit_queue_status = isert_put_response, 3268 .iscsit_aborted_task = isert_aborted_task, 3269 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3270 }; 3271 3272 static int __init isert_init(void) 3273 { 3274 int ret; 3275 3276 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 3277 if (!isert_rx_wq) { 3278 pr_err("Unable to allocate isert_rx_wq\n"); 3279 return -ENOMEM; 3280 } 3281 3282 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 3283 if (!isert_comp_wq) { 3284 pr_err("Unable to allocate isert_comp_wq\n"); 3285 ret = -ENOMEM; 3286 goto destroy_rx_wq; 3287 } 3288 3289 iscsit_register_transport(&iser_target_transport); 3290 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3291 return 0; 3292 3293 destroy_rx_wq: 3294 destroy_workqueue(isert_rx_wq); 3295 return ret; 3296 } 3297 3298 static void __exit isert_exit(void) 3299 { 3300 flush_scheduled_work(); 3301 destroy_workqueue(isert_comp_wq); 3302 destroy_workqueue(isert_rx_wq); 3303 iscsit_unregister_transport(&iser_target_transport); 3304 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 3305 } 3306 3307 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3308 MODULE_VERSION("0.1"); 3309 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3310 MODULE_LICENSE("GPL"); 3311 3312 module_init(isert_init); 3313 module_exit(isert_exit); 3314