1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <linux/llist.h> 26 #include <rdma/ib_verbs.h> 27 #include <rdma/rdma_cm.h> 28 #include <target/target_core_base.h> 29 #include <target/target_core_fabric.h> 30 #include <target/iscsi/iscsi_transport.h> 31 #include <linux/semaphore.h> 32 33 #include "isert_proto.h" 34 #include "ib_isert.h" 35 36 #define ISERT_MAX_CONN 8 37 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 38 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 39 40 static DEFINE_MUTEX(device_list_mutex); 41 static LIST_HEAD(device_list); 42 static struct workqueue_struct *isert_rx_wq; 43 static struct workqueue_struct *isert_comp_wq; 44 45 static void 46 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 47 static int 48 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 49 struct isert_rdma_wr *wr); 50 static void 51 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 52 static int 53 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 54 struct isert_rdma_wr *wr); 55 static int 56 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 57 58 static void 59 isert_qp_event_callback(struct ib_event *e, void *context) 60 { 61 struct isert_conn *isert_conn = (struct isert_conn *)context; 62 63 pr_err("isert_qp_event_callback event: %d\n", e->event); 64 switch (e->event) { 65 case IB_EVENT_COMM_EST: 66 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 67 break; 68 case IB_EVENT_QP_LAST_WQE_REACHED: 69 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 70 break; 71 default: 72 break; 73 } 74 } 75 76 static int 77 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 78 { 79 int ret; 80 81 ret = ib_query_device(ib_dev, devattr); 82 if (ret) { 83 pr_err("ib_query_device() failed: %d\n", ret); 84 return ret; 85 } 86 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 87 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 88 89 return 0; 90 } 91 92 static int 93 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id, 94 u8 protection) 95 { 96 struct isert_device *device = isert_conn->conn_device; 97 struct ib_qp_init_attr attr; 98 int ret, index, min_index = 0; 99 100 mutex_lock(&device_list_mutex); 101 for (index = 0; index < device->cqs_used; index++) 102 if (device->cq_active_qps[index] < 103 device->cq_active_qps[min_index]) 104 min_index = index; 105 device->cq_active_qps[min_index]++; 106 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 107 mutex_unlock(&device_list_mutex); 108 109 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 110 attr.event_handler = isert_qp_event_callback; 111 attr.qp_context = isert_conn; 112 attr.send_cq = device->dev_tx_cq[min_index]; 113 attr.recv_cq = device->dev_rx_cq[min_index]; 114 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 115 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 116 /* 117 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 118 * work-around for RDMA_READ.. 119 */ 120 attr.cap.max_send_sge = device->dev_attr.max_sge - 2; 121 isert_conn->max_sge = attr.cap.max_send_sge; 122 123 attr.cap.max_recv_sge = 1; 124 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 125 attr.qp_type = IB_QPT_RC; 126 if (protection) 127 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 128 129 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 130 cma_id->device); 131 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 132 isert_conn->conn_pd->device); 133 134 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 135 if (ret) { 136 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 137 return ret; 138 } 139 isert_conn->conn_qp = cma_id->qp; 140 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 141 142 return 0; 143 } 144 145 static void 146 isert_cq_event_callback(struct ib_event *e, void *context) 147 { 148 pr_debug("isert_cq_event_callback event: %d\n", e->event); 149 } 150 151 static int 152 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 153 { 154 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 155 struct iser_rx_desc *rx_desc; 156 struct ib_sge *rx_sg; 157 u64 dma_addr; 158 int i, j; 159 160 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 161 sizeof(struct iser_rx_desc), GFP_KERNEL); 162 if (!isert_conn->conn_rx_descs) 163 goto fail; 164 165 rx_desc = isert_conn->conn_rx_descs; 166 167 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 168 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 169 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 170 if (ib_dma_mapping_error(ib_dev, dma_addr)) 171 goto dma_map_fail; 172 173 rx_desc->dma_addr = dma_addr; 174 175 rx_sg = &rx_desc->rx_sg; 176 rx_sg->addr = rx_desc->dma_addr; 177 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 178 rx_sg->lkey = isert_conn->conn_mr->lkey; 179 } 180 181 isert_conn->conn_rx_desc_head = 0; 182 return 0; 183 184 dma_map_fail: 185 rx_desc = isert_conn->conn_rx_descs; 186 for (j = 0; j < i; j++, rx_desc++) { 187 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 188 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 189 } 190 kfree(isert_conn->conn_rx_descs); 191 isert_conn->conn_rx_descs = NULL; 192 fail: 193 return -ENOMEM; 194 } 195 196 static void 197 isert_free_rx_descriptors(struct isert_conn *isert_conn) 198 { 199 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 200 struct iser_rx_desc *rx_desc; 201 int i; 202 203 if (!isert_conn->conn_rx_descs) 204 return; 205 206 rx_desc = isert_conn->conn_rx_descs; 207 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 208 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 209 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 210 } 211 212 kfree(isert_conn->conn_rx_descs); 213 isert_conn->conn_rx_descs = NULL; 214 } 215 216 static void isert_cq_tx_work(struct work_struct *); 217 static void isert_cq_tx_callback(struct ib_cq *, void *); 218 static void isert_cq_rx_work(struct work_struct *); 219 static void isert_cq_rx_callback(struct ib_cq *, void *); 220 221 static int 222 isert_create_device_ib_res(struct isert_device *device) 223 { 224 struct ib_device *ib_dev = device->ib_device; 225 struct isert_cq_desc *cq_desc; 226 struct ib_device_attr *dev_attr; 227 int ret = 0, i, j; 228 229 dev_attr = &device->dev_attr; 230 ret = isert_query_device(ib_dev, dev_attr); 231 if (ret) 232 return ret; 233 234 /* asign function handlers */ 235 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS && 236 dev_attr->device_cap_flags & IB_DEVICE_SIGNATURE_HANDOVER) { 237 device->use_fastreg = 1; 238 device->reg_rdma_mem = isert_reg_rdma; 239 device->unreg_rdma_mem = isert_unreg_rdma; 240 } else { 241 device->use_fastreg = 0; 242 device->reg_rdma_mem = isert_map_rdma; 243 device->unreg_rdma_mem = isert_unmap_cmd; 244 } 245 246 /* Check signature cap */ 247 device->pi_capable = dev_attr->device_cap_flags & 248 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 249 250 device->cqs_used = min_t(int, num_online_cpus(), 251 device->ib_device->num_comp_vectors); 252 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 253 pr_debug("Using %d CQs, device %s supports %d vectors support " 254 "Fast registration %d pi_capable %d\n", 255 device->cqs_used, device->ib_device->name, 256 device->ib_device->num_comp_vectors, device->use_fastreg, 257 device->pi_capable); 258 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 259 device->cqs_used, GFP_KERNEL); 260 if (!device->cq_desc) { 261 pr_err("Unable to allocate device->cq_desc\n"); 262 return -ENOMEM; 263 } 264 cq_desc = device->cq_desc; 265 266 for (i = 0; i < device->cqs_used; i++) { 267 cq_desc[i].device = device; 268 cq_desc[i].cq_index = i; 269 270 INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 271 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 272 isert_cq_rx_callback, 273 isert_cq_event_callback, 274 (void *)&cq_desc[i], 275 ISER_MAX_RX_CQ_LEN, i); 276 if (IS_ERR(device->dev_rx_cq[i])) { 277 ret = PTR_ERR(device->dev_rx_cq[i]); 278 device->dev_rx_cq[i] = NULL; 279 goto out_cq; 280 } 281 282 INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 283 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 284 isert_cq_tx_callback, 285 isert_cq_event_callback, 286 (void *)&cq_desc[i], 287 ISER_MAX_TX_CQ_LEN, i); 288 if (IS_ERR(device->dev_tx_cq[i])) { 289 ret = PTR_ERR(device->dev_tx_cq[i]); 290 device->dev_tx_cq[i] = NULL; 291 goto out_cq; 292 } 293 294 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 295 if (ret) 296 goto out_cq; 297 298 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 299 if (ret) 300 goto out_cq; 301 } 302 303 return 0; 304 305 out_cq: 306 for (j = 0; j < i; j++) { 307 cq_desc = &device->cq_desc[j]; 308 309 if (device->dev_rx_cq[j]) { 310 cancel_work_sync(&cq_desc->cq_rx_work); 311 ib_destroy_cq(device->dev_rx_cq[j]); 312 } 313 if (device->dev_tx_cq[j]) { 314 cancel_work_sync(&cq_desc->cq_tx_work); 315 ib_destroy_cq(device->dev_tx_cq[j]); 316 } 317 } 318 kfree(device->cq_desc); 319 320 return ret; 321 } 322 323 static void 324 isert_free_device_ib_res(struct isert_device *device) 325 { 326 struct isert_cq_desc *cq_desc; 327 int i; 328 329 for (i = 0; i < device->cqs_used; i++) { 330 cq_desc = &device->cq_desc[i]; 331 332 cancel_work_sync(&cq_desc->cq_rx_work); 333 cancel_work_sync(&cq_desc->cq_tx_work); 334 ib_destroy_cq(device->dev_rx_cq[i]); 335 ib_destroy_cq(device->dev_tx_cq[i]); 336 device->dev_rx_cq[i] = NULL; 337 device->dev_tx_cq[i] = NULL; 338 } 339 340 kfree(device->cq_desc); 341 } 342 343 static void 344 isert_device_try_release(struct isert_device *device) 345 { 346 mutex_lock(&device_list_mutex); 347 device->refcount--; 348 if (!device->refcount) { 349 isert_free_device_ib_res(device); 350 list_del(&device->dev_node); 351 kfree(device); 352 } 353 mutex_unlock(&device_list_mutex); 354 } 355 356 static struct isert_device * 357 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 358 { 359 struct isert_device *device; 360 int ret; 361 362 mutex_lock(&device_list_mutex); 363 list_for_each_entry(device, &device_list, dev_node) { 364 if (device->ib_device->node_guid == cma_id->device->node_guid) { 365 device->refcount++; 366 mutex_unlock(&device_list_mutex); 367 return device; 368 } 369 } 370 371 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 372 if (!device) { 373 mutex_unlock(&device_list_mutex); 374 return ERR_PTR(-ENOMEM); 375 } 376 377 INIT_LIST_HEAD(&device->dev_node); 378 379 device->ib_device = cma_id->device; 380 ret = isert_create_device_ib_res(device); 381 if (ret) { 382 kfree(device); 383 mutex_unlock(&device_list_mutex); 384 return ERR_PTR(ret); 385 } 386 387 device->refcount++; 388 list_add_tail(&device->dev_node, &device_list); 389 mutex_unlock(&device_list_mutex); 390 391 return device; 392 } 393 394 static void 395 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 396 { 397 struct fast_reg_descriptor *fr_desc, *tmp; 398 int i = 0; 399 400 if (list_empty(&isert_conn->conn_fr_pool)) 401 return; 402 403 pr_debug("Freeing conn %p fastreg pool", isert_conn); 404 405 list_for_each_entry_safe(fr_desc, tmp, 406 &isert_conn->conn_fr_pool, list) { 407 list_del(&fr_desc->list); 408 ib_free_fast_reg_page_list(fr_desc->data_frpl); 409 ib_dereg_mr(fr_desc->data_mr); 410 if (fr_desc->pi_ctx) { 411 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 412 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 413 ib_destroy_mr(fr_desc->pi_ctx->sig_mr); 414 kfree(fr_desc->pi_ctx); 415 } 416 kfree(fr_desc); 417 ++i; 418 } 419 420 if (i < isert_conn->conn_fr_pool_size) 421 pr_warn("Pool still has %d regions registered\n", 422 isert_conn->conn_fr_pool_size - i); 423 } 424 425 static int 426 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 427 struct fast_reg_descriptor *fr_desc, u8 protection) 428 { 429 int ret; 430 431 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 432 ISCSI_ISER_SG_TABLESIZE); 433 if (IS_ERR(fr_desc->data_frpl)) { 434 pr_err("Failed to allocate data frpl err=%ld\n", 435 PTR_ERR(fr_desc->data_frpl)); 436 return PTR_ERR(fr_desc->data_frpl); 437 } 438 439 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 440 if (IS_ERR(fr_desc->data_mr)) { 441 pr_err("Failed to allocate data frmr err=%ld\n", 442 PTR_ERR(fr_desc->data_mr)); 443 ret = PTR_ERR(fr_desc->data_mr); 444 goto err_data_frpl; 445 } 446 pr_debug("Create fr_desc %p page_list %p\n", 447 fr_desc, fr_desc->data_frpl->page_list); 448 fr_desc->ind |= ISERT_DATA_KEY_VALID; 449 450 if (protection) { 451 struct ib_mr_init_attr mr_init_attr = {0}; 452 struct pi_context *pi_ctx; 453 454 fr_desc->pi_ctx = kzalloc(sizeof(*fr_desc->pi_ctx), GFP_KERNEL); 455 if (!fr_desc->pi_ctx) { 456 pr_err("Failed to allocate pi context\n"); 457 ret = -ENOMEM; 458 goto err_data_mr; 459 } 460 pi_ctx = fr_desc->pi_ctx; 461 462 pi_ctx->prot_frpl = ib_alloc_fast_reg_page_list(ib_device, 463 ISCSI_ISER_SG_TABLESIZE); 464 if (IS_ERR(pi_ctx->prot_frpl)) { 465 pr_err("Failed to allocate prot frpl err=%ld\n", 466 PTR_ERR(pi_ctx->prot_frpl)); 467 ret = PTR_ERR(pi_ctx->prot_frpl); 468 goto err_pi_ctx; 469 } 470 471 pi_ctx->prot_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 472 if (IS_ERR(pi_ctx->prot_mr)) { 473 pr_err("Failed to allocate prot frmr err=%ld\n", 474 PTR_ERR(pi_ctx->prot_mr)); 475 ret = PTR_ERR(pi_ctx->prot_mr); 476 goto err_prot_frpl; 477 } 478 fr_desc->ind |= ISERT_PROT_KEY_VALID; 479 480 mr_init_attr.max_reg_descriptors = 2; 481 mr_init_attr.flags |= IB_MR_SIGNATURE_EN; 482 pi_ctx->sig_mr = ib_create_mr(pd, &mr_init_attr); 483 if (IS_ERR(pi_ctx->sig_mr)) { 484 pr_err("Failed to allocate signature enabled mr err=%ld\n", 485 PTR_ERR(pi_ctx->sig_mr)); 486 ret = PTR_ERR(pi_ctx->sig_mr); 487 goto err_prot_mr; 488 } 489 fr_desc->ind |= ISERT_SIG_KEY_VALID; 490 } 491 fr_desc->ind &= ~ISERT_PROTECTED; 492 493 return 0; 494 err_prot_mr: 495 ib_dereg_mr(fr_desc->pi_ctx->prot_mr); 496 err_prot_frpl: 497 ib_free_fast_reg_page_list(fr_desc->pi_ctx->prot_frpl); 498 err_pi_ctx: 499 kfree(fr_desc->pi_ctx); 500 err_data_mr: 501 ib_dereg_mr(fr_desc->data_mr); 502 err_data_frpl: 503 ib_free_fast_reg_page_list(fr_desc->data_frpl); 504 505 return ret; 506 } 507 508 static int 509 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn, u8 pi_support) 510 { 511 struct fast_reg_descriptor *fr_desc; 512 struct isert_device *device = isert_conn->conn_device; 513 struct se_session *se_sess = isert_conn->conn->sess->se_sess; 514 struct se_node_acl *se_nacl = se_sess->se_node_acl; 515 int i, ret, tag_num; 516 /* 517 * Setup the number of FRMRs based upon the number of tags 518 * available to session in iscsi_target_locate_portal(). 519 */ 520 tag_num = max_t(u32, ISCSIT_MIN_TAGS, se_nacl->queue_depth); 521 tag_num = (tag_num * 2) + ISCSIT_EXTRA_TAGS; 522 523 isert_conn->conn_fr_pool_size = 0; 524 for (i = 0; i < tag_num; i++) { 525 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 526 if (!fr_desc) { 527 pr_err("Failed to allocate fast_reg descriptor\n"); 528 ret = -ENOMEM; 529 goto err; 530 } 531 532 ret = isert_create_fr_desc(device->ib_device, 533 isert_conn->conn_pd, fr_desc, 534 pi_support); 535 if (ret) { 536 pr_err("Failed to create fastreg descriptor err=%d\n", 537 ret); 538 kfree(fr_desc); 539 goto err; 540 } 541 542 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 543 isert_conn->conn_fr_pool_size++; 544 } 545 546 pr_debug("Creating conn %p fastreg pool size=%d", 547 isert_conn, isert_conn->conn_fr_pool_size); 548 549 return 0; 550 551 err: 552 isert_conn_free_fastreg_pool(isert_conn); 553 return ret; 554 } 555 556 static int 557 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 558 { 559 struct iscsi_np *np = cma_id->context; 560 struct isert_np *isert_np = np->np_context; 561 struct isert_conn *isert_conn; 562 struct isert_device *device; 563 struct ib_device *ib_dev = cma_id->device; 564 int ret = 0; 565 u8 pi_support; 566 567 spin_lock_bh(&np->np_thread_lock); 568 if (!np->enabled) { 569 spin_unlock_bh(&np->np_thread_lock); 570 pr_debug("iscsi_np is not enabled, reject connect request\n"); 571 return rdma_reject(cma_id, NULL, 0); 572 } 573 spin_unlock_bh(&np->np_thread_lock); 574 575 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 576 cma_id, cma_id->context); 577 578 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 579 if (!isert_conn) { 580 pr_err("Unable to allocate isert_conn\n"); 581 return -ENOMEM; 582 } 583 isert_conn->state = ISER_CONN_INIT; 584 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 585 init_completion(&isert_conn->conn_login_comp); 586 init_completion(&isert_conn->conn_wait); 587 init_completion(&isert_conn->conn_wait_comp_err); 588 kref_init(&isert_conn->conn_kref); 589 kref_get(&isert_conn->conn_kref); 590 mutex_init(&isert_conn->conn_mutex); 591 spin_lock_init(&isert_conn->conn_lock); 592 INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 593 594 cma_id->context = isert_conn; 595 isert_conn->conn_cm_id = cma_id; 596 isert_conn->responder_resources = event->param.conn.responder_resources; 597 isert_conn->initiator_depth = event->param.conn.initiator_depth; 598 pr_debug("Using responder_resources: %u initiator_depth: %u\n", 599 isert_conn->responder_resources, isert_conn->initiator_depth); 600 601 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 602 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 603 if (!isert_conn->login_buf) { 604 pr_err("Unable to allocate isert_conn->login_buf\n"); 605 ret = -ENOMEM; 606 goto out; 607 } 608 609 isert_conn->login_req_buf = isert_conn->login_buf; 610 isert_conn->login_rsp_buf = isert_conn->login_buf + 611 ISCSI_DEF_MAX_RECV_SEG_LEN; 612 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 613 isert_conn->login_buf, isert_conn->login_req_buf, 614 isert_conn->login_rsp_buf); 615 616 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 617 (void *)isert_conn->login_req_buf, 618 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 619 620 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 621 if (ret) { 622 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 623 ret); 624 isert_conn->login_req_dma = 0; 625 goto out_login_buf; 626 } 627 628 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 629 (void *)isert_conn->login_rsp_buf, 630 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 631 632 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 633 if (ret) { 634 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 635 ret); 636 isert_conn->login_rsp_dma = 0; 637 goto out_req_dma_map; 638 } 639 640 device = isert_device_find_by_ib_dev(cma_id); 641 if (IS_ERR(device)) { 642 ret = PTR_ERR(device); 643 goto out_rsp_dma_map; 644 } 645 646 isert_conn->conn_device = device; 647 isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 648 if (IS_ERR(isert_conn->conn_pd)) { 649 ret = PTR_ERR(isert_conn->conn_pd); 650 pr_err("ib_alloc_pd failed for conn %p: ret=%d\n", 651 isert_conn, ret); 652 goto out_pd; 653 } 654 655 isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 656 IB_ACCESS_LOCAL_WRITE); 657 if (IS_ERR(isert_conn->conn_mr)) { 658 ret = PTR_ERR(isert_conn->conn_mr); 659 pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 660 isert_conn, ret); 661 goto out_mr; 662 } 663 664 pi_support = np->tpg_np->tpg->tpg_attrib.t10_pi; 665 if (pi_support && !device->pi_capable) { 666 pr_err("Protection information requested but not supported, " 667 "rejecting connect request\n"); 668 ret = rdma_reject(cma_id, NULL, 0); 669 goto out_mr; 670 } 671 672 ret = isert_conn_setup_qp(isert_conn, cma_id, pi_support); 673 if (ret) 674 goto out_conn_dev; 675 676 mutex_lock(&isert_np->np_accept_mutex); 677 list_add_tail(&isert_conn->conn_accept_node, &isert_np->np_accept_list); 678 mutex_unlock(&isert_np->np_accept_mutex); 679 680 pr_debug("isert_connect_request() up np_sem np: %p\n", np); 681 up(&isert_np->np_sem); 682 return 0; 683 684 out_conn_dev: 685 ib_dereg_mr(isert_conn->conn_mr); 686 out_mr: 687 ib_dealloc_pd(isert_conn->conn_pd); 688 out_pd: 689 isert_device_try_release(device); 690 out_rsp_dma_map: 691 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 692 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 693 out_req_dma_map: 694 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 695 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 696 out_login_buf: 697 kfree(isert_conn->login_buf); 698 out: 699 kfree(isert_conn); 700 return ret; 701 } 702 703 static void 704 isert_connect_release(struct isert_conn *isert_conn) 705 { 706 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 707 struct isert_device *device = isert_conn->conn_device; 708 int cq_index; 709 710 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 711 712 if (device && device->use_fastreg) 713 isert_conn_free_fastreg_pool(isert_conn); 714 715 if (isert_conn->conn_qp) { 716 cq_index = ((struct isert_cq_desc *) 717 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 718 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 719 isert_conn->conn_device->cq_active_qps[cq_index]--; 720 721 rdma_destroy_qp(isert_conn->conn_cm_id); 722 } 723 724 isert_free_rx_descriptors(isert_conn); 725 rdma_destroy_id(isert_conn->conn_cm_id); 726 727 ib_dereg_mr(isert_conn->conn_mr); 728 ib_dealloc_pd(isert_conn->conn_pd); 729 730 if (isert_conn->login_buf) { 731 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 732 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 733 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 734 ISCSI_DEF_MAX_RECV_SEG_LEN, 735 DMA_FROM_DEVICE); 736 kfree(isert_conn->login_buf); 737 } 738 kfree(isert_conn); 739 740 if (device) 741 isert_device_try_release(device); 742 743 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 744 } 745 746 static void 747 isert_connected_handler(struct rdma_cm_id *cma_id) 748 { 749 return; 750 } 751 752 static void 753 isert_release_conn_kref(struct kref *kref) 754 { 755 struct isert_conn *isert_conn = container_of(kref, 756 struct isert_conn, conn_kref); 757 758 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 759 current->comm, current->pid); 760 761 isert_connect_release(isert_conn); 762 } 763 764 static void 765 isert_put_conn(struct isert_conn *isert_conn) 766 { 767 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 768 } 769 770 static void 771 isert_disconnect_work(struct work_struct *work) 772 { 773 struct isert_conn *isert_conn = container_of(work, 774 struct isert_conn, conn_logout_work); 775 776 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 777 mutex_lock(&isert_conn->conn_mutex); 778 if (isert_conn->state == ISER_CONN_UP) 779 isert_conn->state = ISER_CONN_TERMINATING; 780 781 if (isert_conn->post_recv_buf_count == 0 && 782 atomic_read(&isert_conn->post_send_buf_count) == 0) { 783 mutex_unlock(&isert_conn->conn_mutex); 784 goto wake_up; 785 } 786 if (!isert_conn->conn_cm_id) { 787 mutex_unlock(&isert_conn->conn_mutex); 788 isert_put_conn(isert_conn); 789 return; 790 } 791 792 if (isert_conn->disconnect) { 793 /* Send DREQ/DREP towards our initiator */ 794 rdma_disconnect(isert_conn->conn_cm_id); 795 } 796 797 mutex_unlock(&isert_conn->conn_mutex); 798 799 wake_up: 800 complete(&isert_conn->conn_wait); 801 isert_put_conn(isert_conn); 802 } 803 804 static void 805 isert_disconnected_handler(struct rdma_cm_id *cma_id, bool disconnect) 806 { 807 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 808 809 isert_conn->disconnect = disconnect; 810 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 811 schedule_work(&isert_conn->conn_logout_work); 812 } 813 814 static int 815 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 816 { 817 int ret = 0; 818 bool disconnect = false; 819 820 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 821 event->event, event->status, cma_id->context, cma_id); 822 823 switch (event->event) { 824 case RDMA_CM_EVENT_CONNECT_REQUEST: 825 ret = isert_connect_request(cma_id, event); 826 break; 827 case RDMA_CM_EVENT_ESTABLISHED: 828 isert_connected_handler(cma_id); 829 break; 830 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 831 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 832 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 833 disconnect = true; 834 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 835 isert_disconnected_handler(cma_id, disconnect); 836 break; 837 case RDMA_CM_EVENT_CONNECT_ERROR: 838 default: 839 pr_err("Unhandled RDMA CMA event: %d\n", event->event); 840 break; 841 } 842 843 if (ret != 0) { 844 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 845 event->event, ret); 846 dump_stack(); 847 } 848 849 return ret; 850 } 851 852 static int 853 isert_post_recv(struct isert_conn *isert_conn, u32 count) 854 { 855 struct ib_recv_wr *rx_wr, *rx_wr_failed; 856 int i, ret; 857 unsigned int rx_head = isert_conn->conn_rx_desc_head; 858 struct iser_rx_desc *rx_desc; 859 860 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 861 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 862 rx_wr->wr_id = (unsigned long)rx_desc; 863 rx_wr->sg_list = &rx_desc->rx_sg; 864 rx_wr->num_sge = 1; 865 rx_wr->next = rx_wr + 1; 866 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 867 } 868 869 rx_wr--; 870 rx_wr->next = NULL; /* mark end of work requests list */ 871 872 isert_conn->post_recv_buf_count += count; 873 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 874 &rx_wr_failed); 875 if (ret) { 876 pr_err("ib_post_recv() failed with ret: %d\n", ret); 877 isert_conn->post_recv_buf_count -= count; 878 } else { 879 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 880 isert_conn->conn_rx_desc_head = rx_head; 881 } 882 return ret; 883 } 884 885 static int 886 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 887 { 888 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 889 struct ib_send_wr send_wr, *send_wr_failed; 890 int ret; 891 892 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 893 ISER_HEADERS_LEN, DMA_TO_DEVICE); 894 895 send_wr.next = NULL; 896 send_wr.wr_id = (unsigned long)tx_desc; 897 send_wr.sg_list = tx_desc->tx_sg; 898 send_wr.num_sge = tx_desc->num_sge; 899 send_wr.opcode = IB_WR_SEND; 900 send_wr.send_flags = IB_SEND_SIGNALED; 901 902 atomic_inc(&isert_conn->post_send_buf_count); 903 904 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 905 if (ret) { 906 pr_err("ib_post_send() failed, ret: %d\n", ret); 907 atomic_dec(&isert_conn->post_send_buf_count); 908 } 909 910 return ret; 911 } 912 913 static void 914 isert_create_send_desc(struct isert_conn *isert_conn, 915 struct isert_cmd *isert_cmd, 916 struct iser_tx_desc *tx_desc) 917 { 918 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 919 920 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 921 ISER_HEADERS_LEN, DMA_TO_DEVICE); 922 923 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 924 tx_desc->iser_header.flags = ISER_VER; 925 926 tx_desc->num_sge = 1; 927 tx_desc->isert_cmd = isert_cmd; 928 929 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 930 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 931 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 932 } 933 } 934 935 static int 936 isert_init_tx_hdrs(struct isert_conn *isert_conn, 937 struct iser_tx_desc *tx_desc) 938 { 939 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 940 u64 dma_addr; 941 942 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 943 ISER_HEADERS_LEN, DMA_TO_DEVICE); 944 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 945 pr_err("ib_dma_mapping_error() failed\n"); 946 return -ENOMEM; 947 } 948 949 tx_desc->dma_addr = dma_addr; 950 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 951 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 952 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 953 954 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 955 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 956 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 957 958 return 0; 959 } 960 961 static void 962 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 963 struct ib_send_wr *send_wr, bool coalesce) 964 { 965 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 966 967 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 968 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 969 send_wr->opcode = IB_WR_SEND; 970 send_wr->sg_list = &tx_desc->tx_sg[0]; 971 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 972 /* 973 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 974 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 975 */ 976 mutex_lock(&isert_conn->conn_mutex); 977 if (coalesce && isert_conn->state == ISER_CONN_UP && 978 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 979 tx_desc->llnode_active = true; 980 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 981 mutex_unlock(&isert_conn->conn_mutex); 982 return; 983 } 984 isert_conn->conn_comp_batch = 0; 985 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 986 mutex_unlock(&isert_conn->conn_mutex); 987 988 send_wr->send_flags = IB_SEND_SIGNALED; 989 } 990 991 static int 992 isert_rdma_post_recvl(struct isert_conn *isert_conn) 993 { 994 struct ib_recv_wr rx_wr, *rx_wr_fail; 995 struct ib_sge sge; 996 int ret; 997 998 memset(&sge, 0, sizeof(struct ib_sge)); 999 sge.addr = isert_conn->login_req_dma; 1000 sge.length = ISER_RX_LOGIN_SIZE; 1001 sge.lkey = isert_conn->conn_mr->lkey; 1002 1003 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 1004 sge.addr, sge.length, sge.lkey); 1005 1006 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 1007 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 1008 rx_wr.sg_list = &sge; 1009 rx_wr.num_sge = 1; 1010 1011 isert_conn->post_recv_buf_count++; 1012 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 1013 if (ret) { 1014 pr_err("ib_post_recv() failed: %d\n", ret); 1015 isert_conn->post_recv_buf_count--; 1016 } 1017 1018 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 1019 return ret; 1020 } 1021 1022 static int 1023 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 1024 u32 length) 1025 { 1026 struct isert_conn *isert_conn = conn->context; 1027 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1028 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 1029 int ret; 1030 1031 isert_create_send_desc(isert_conn, NULL, tx_desc); 1032 1033 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 1034 sizeof(struct iscsi_hdr)); 1035 1036 isert_init_tx_hdrs(isert_conn, tx_desc); 1037 1038 if (length > 0) { 1039 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1040 1041 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1042 length, DMA_TO_DEVICE); 1043 1044 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1045 1046 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1047 length, DMA_TO_DEVICE); 1048 1049 tx_dsg->addr = isert_conn->login_rsp_dma; 1050 tx_dsg->length = length; 1051 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1052 tx_desc->num_sge = 2; 1053 } 1054 if (!login->login_failed) { 1055 if (login->login_complete) { 1056 if (!conn->sess->sess_ops->SessionType && 1057 isert_conn->conn_device->use_fastreg) { 1058 /* Normal Session and fastreg is used */ 1059 u8 pi_support = login->np->tpg_np->tpg->tpg_attrib.t10_pi; 1060 1061 ret = isert_conn_create_fastreg_pool(isert_conn, 1062 pi_support); 1063 if (ret) { 1064 pr_err("Conn: %p failed to create" 1065 " fastreg pool\n", isert_conn); 1066 return ret; 1067 } 1068 } 1069 1070 ret = isert_alloc_rx_descriptors(isert_conn); 1071 if (ret) 1072 return ret; 1073 1074 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 1075 if (ret) 1076 return ret; 1077 1078 isert_conn->state = ISER_CONN_UP; 1079 goto post_send; 1080 } 1081 1082 ret = isert_rdma_post_recvl(isert_conn); 1083 if (ret) 1084 return ret; 1085 } 1086 post_send: 1087 ret = isert_post_send(isert_conn, tx_desc); 1088 if (ret) 1089 return ret; 1090 1091 return 0; 1092 } 1093 1094 static void 1095 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 1096 struct isert_conn *isert_conn) 1097 { 1098 struct iscsi_conn *conn = isert_conn->conn; 1099 struct iscsi_login *login = conn->conn_login; 1100 int size; 1101 1102 if (!login) { 1103 pr_err("conn->conn_login is NULL\n"); 1104 dump_stack(); 1105 return; 1106 } 1107 1108 if (login->first_request) { 1109 struct iscsi_login_req *login_req = 1110 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1111 /* 1112 * Setup the initial iscsi_login values from the leading 1113 * login request PDU. 1114 */ 1115 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1116 login->current_stage = 1117 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1118 >> 2; 1119 login->version_min = login_req->min_version; 1120 login->version_max = login_req->max_version; 1121 memcpy(login->isid, login_req->isid, 6); 1122 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1123 login->init_task_tag = login_req->itt; 1124 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1125 login->cid = be16_to_cpu(login_req->cid); 1126 login->tsih = be16_to_cpu(login_req->tsih); 1127 } 1128 1129 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1130 1131 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1132 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1133 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1134 memcpy(login->req_buf, &rx_desc->data[0], size); 1135 1136 if (login->first_request) { 1137 complete(&isert_conn->conn_login_comp); 1138 return; 1139 } 1140 schedule_delayed_work(&conn->login_work, 0); 1141 } 1142 1143 static struct iscsi_cmd 1144 *isert_allocate_cmd(struct iscsi_conn *conn) 1145 { 1146 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1147 struct isert_cmd *isert_cmd; 1148 struct iscsi_cmd *cmd; 1149 1150 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1151 if (!cmd) { 1152 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1153 return NULL; 1154 } 1155 isert_cmd = iscsit_priv_cmd(cmd); 1156 isert_cmd->conn = isert_conn; 1157 isert_cmd->iscsi_cmd = cmd; 1158 1159 return cmd; 1160 } 1161 1162 static int 1163 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1164 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1165 struct iser_rx_desc *rx_desc, unsigned char *buf) 1166 { 1167 struct iscsi_conn *conn = isert_conn->conn; 1168 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1169 struct scatterlist *sg; 1170 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1171 bool dump_payload = false; 1172 1173 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1174 if (rc < 0) 1175 return rc; 1176 1177 imm_data = cmd->immediate_data; 1178 imm_data_len = cmd->first_burst_len; 1179 unsol_data = cmd->unsolicited_data; 1180 1181 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1182 if (rc < 0) { 1183 return 0; 1184 } else if (rc > 0) { 1185 dump_payload = true; 1186 goto sequence_cmd; 1187 } 1188 1189 if (!imm_data) 1190 return 0; 1191 1192 sg = &cmd->se_cmd.t_data_sg[0]; 1193 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1194 1195 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1196 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1197 1198 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1199 1200 cmd->write_data_done += imm_data_len; 1201 1202 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1203 spin_lock_bh(&cmd->istate_lock); 1204 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1205 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1206 spin_unlock_bh(&cmd->istate_lock); 1207 } 1208 1209 sequence_cmd: 1210 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1211 1212 if (!rc && dump_payload == false && unsol_data) 1213 iscsit_set_unsoliticed_dataout(cmd); 1214 else if (dump_payload && imm_data) 1215 target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); 1216 1217 return 0; 1218 } 1219 1220 static int 1221 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1222 struct iser_rx_desc *rx_desc, unsigned char *buf) 1223 { 1224 struct scatterlist *sg_start; 1225 struct iscsi_conn *conn = isert_conn->conn; 1226 struct iscsi_cmd *cmd = NULL; 1227 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1228 u32 unsol_data_len = ntoh24(hdr->dlength); 1229 int rc, sg_nents, sg_off, page_off; 1230 1231 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1232 if (rc < 0) 1233 return rc; 1234 else if (!cmd) 1235 return 0; 1236 /* 1237 * FIXME: Unexpected unsolicited_data out 1238 */ 1239 if (!cmd->unsolicited_data) { 1240 pr_err("Received unexpected solicited data payload\n"); 1241 dump_stack(); 1242 return -1; 1243 } 1244 1245 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1246 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1247 1248 sg_off = cmd->write_data_done / PAGE_SIZE; 1249 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1250 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1251 page_off = cmd->write_data_done % PAGE_SIZE; 1252 /* 1253 * FIXME: Non page-aligned unsolicited_data out 1254 */ 1255 if (page_off) { 1256 pr_err("Received unexpected non-page aligned data payload\n"); 1257 dump_stack(); 1258 return -1; 1259 } 1260 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1261 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1262 1263 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1264 unsol_data_len); 1265 1266 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1267 if (rc < 0) 1268 return rc; 1269 1270 return 0; 1271 } 1272 1273 static int 1274 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1275 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1276 unsigned char *buf) 1277 { 1278 struct iscsi_conn *conn = isert_conn->conn; 1279 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1280 int rc; 1281 1282 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1283 if (rc < 0) 1284 return rc; 1285 /* 1286 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1287 */ 1288 1289 return iscsit_process_nop_out(conn, cmd, hdr); 1290 } 1291 1292 static int 1293 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1294 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1295 struct iscsi_text *hdr) 1296 { 1297 struct iscsi_conn *conn = isert_conn->conn; 1298 u32 payload_length = ntoh24(hdr->dlength); 1299 int rc; 1300 unsigned char *text_in; 1301 1302 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1303 if (rc < 0) 1304 return rc; 1305 1306 text_in = kzalloc(payload_length, GFP_KERNEL); 1307 if (!text_in) { 1308 pr_err("Unable to allocate text_in of payload_length: %u\n", 1309 payload_length); 1310 return -ENOMEM; 1311 } 1312 cmd->text_in_ptr = text_in; 1313 1314 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1315 1316 return iscsit_process_text_cmd(conn, cmd, hdr); 1317 } 1318 1319 static int 1320 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1321 uint32_t read_stag, uint64_t read_va, 1322 uint32_t write_stag, uint64_t write_va) 1323 { 1324 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1325 struct iscsi_conn *conn = isert_conn->conn; 1326 struct iscsi_session *sess = conn->sess; 1327 struct iscsi_cmd *cmd; 1328 struct isert_cmd *isert_cmd; 1329 int ret = -EINVAL; 1330 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1331 1332 if (sess->sess_ops->SessionType && 1333 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1334 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1335 " ignoring\n", opcode); 1336 return 0; 1337 } 1338 1339 switch (opcode) { 1340 case ISCSI_OP_SCSI_CMD: 1341 cmd = isert_allocate_cmd(conn); 1342 if (!cmd) 1343 break; 1344 1345 isert_cmd = iscsit_priv_cmd(cmd); 1346 isert_cmd->read_stag = read_stag; 1347 isert_cmd->read_va = read_va; 1348 isert_cmd->write_stag = write_stag; 1349 isert_cmd->write_va = write_va; 1350 1351 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1352 rx_desc, (unsigned char *)hdr); 1353 break; 1354 case ISCSI_OP_NOOP_OUT: 1355 cmd = isert_allocate_cmd(conn); 1356 if (!cmd) 1357 break; 1358 1359 isert_cmd = iscsit_priv_cmd(cmd); 1360 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1361 rx_desc, (unsigned char *)hdr); 1362 break; 1363 case ISCSI_OP_SCSI_DATA_OUT: 1364 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1365 (unsigned char *)hdr); 1366 break; 1367 case ISCSI_OP_SCSI_TMFUNC: 1368 cmd = isert_allocate_cmd(conn); 1369 if (!cmd) 1370 break; 1371 1372 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1373 (unsigned char *)hdr); 1374 break; 1375 case ISCSI_OP_LOGOUT: 1376 cmd = isert_allocate_cmd(conn); 1377 if (!cmd) 1378 break; 1379 1380 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1381 if (ret > 0) 1382 wait_for_completion_timeout(&conn->conn_logout_comp, 1383 SECONDS_FOR_LOGOUT_COMP * 1384 HZ); 1385 break; 1386 case ISCSI_OP_TEXT: 1387 cmd = isert_allocate_cmd(conn); 1388 if (!cmd) 1389 break; 1390 1391 isert_cmd = iscsit_priv_cmd(cmd); 1392 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1393 rx_desc, (struct iscsi_text *)hdr); 1394 break; 1395 default: 1396 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1397 dump_stack(); 1398 break; 1399 } 1400 1401 return ret; 1402 } 1403 1404 static void 1405 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1406 { 1407 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1408 uint64_t read_va = 0, write_va = 0; 1409 uint32_t read_stag = 0, write_stag = 0; 1410 int rc; 1411 1412 switch (iser_hdr->flags & 0xF0) { 1413 case ISCSI_CTRL: 1414 if (iser_hdr->flags & ISER_RSV) { 1415 read_stag = be32_to_cpu(iser_hdr->read_stag); 1416 read_va = be64_to_cpu(iser_hdr->read_va); 1417 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1418 read_stag, (unsigned long long)read_va); 1419 } 1420 if (iser_hdr->flags & ISER_WSV) { 1421 write_stag = be32_to_cpu(iser_hdr->write_stag); 1422 write_va = be64_to_cpu(iser_hdr->write_va); 1423 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1424 write_stag, (unsigned long long)write_va); 1425 } 1426 1427 pr_debug("ISER ISCSI_CTRL PDU\n"); 1428 break; 1429 case ISER_HELLO: 1430 pr_err("iSER Hello message\n"); 1431 break; 1432 default: 1433 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1434 break; 1435 } 1436 1437 rc = isert_rx_opcode(isert_conn, rx_desc, 1438 read_stag, read_va, write_stag, write_va); 1439 } 1440 1441 static void 1442 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1443 unsigned long xfer_len) 1444 { 1445 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1446 struct iscsi_hdr *hdr; 1447 u64 rx_dma; 1448 int rx_buflen, outstanding; 1449 1450 if ((char *)desc == isert_conn->login_req_buf) { 1451 rx_dma = isert_conn->login_req_dma; 1452 rx_buflen = ISER_RX_LOGIN_SIZE; 1453 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1454 rx_dma, rx_buflen); 1455 } else { 1456 rx_dma = desc->dma_addr; 1457 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1458 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1459 rx_dma, rx_buflen); 1460 } 1461 1462 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1463 1464 hdr = &desc->iscsi_header; 1465 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1466 hdr->opcode, hdr->itt, hdr->flags, 1467 (int)(xfer_len - ISER_HEADERS_LEN)); 1468 1469 if ((char *)desc == isert_conn->login_req_buf) 1470 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1471 isert_conn); 1472 else 1473 isert_rx_do_work(desc, isert_conn); 1474 1475 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1476 DMA_FROM_DEVICE); 1477 1478 isert_conn->post_recv_buf_count--; 1479 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1480 isert_conn->post_recv_buf_count); 1481 1482 if ((char *)desc == isert_conn->login_req_buf) 1483 return; 1484 1485 outstanding = isert_conn->post_recv_buf_count; 1486 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1487 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1488 ISERT_MIN_POSTED_RX); 1489 err = isert_post_recv(isert_conn, count); 1490 if (err) { 1491 pr_err("isert_post_recv() count: %d failed, %d\n", 1492 count, err); 1493 } 1494 } 1495 } 1496 1497 static int 1498 isert_map_data_buf(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1499 struct scatterlist *sg, u32 nents, u32 length, u32 offset, 1500 enum iser_ib_op_code op, struct isert_data_buf *data) 1501 { 1502 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1503 1504 data->dma_dir = op == ISER_IB_RDMA_WRITE ? 1505 DMA_TO_DEVICE : DMA_FROM_DEVICE; 1506 1507 data->len = length - offset; 1508 data->offset = offset; 1509 data->sg_off = data->offset / PAGE_SIZE; 1510 1511 data->sg = &sg[data->sg_off]; 1512 data->nents = min_t(unsigned int, nents - data->sg_off, 1513 ISCSI_ISER_SG_TABLESIZE); 1514 data->len = min_t(unsigned int, data->len, ISCSI_ISER_SG_TABLESIZE * 1515 PAGE_SIZE); 1516 1517 data->dma_nents = ib_dma_map_sg(ib_dev, data->sg, data->nents, 1518 data->dma_dir); 1519 if (unlikely(!data->dma_nents)) { 1520 pr_err("Cmd: unable to dma map SGs %p\n", sg); 1521 return -EINVAL; 1522 } 1523 1524 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 1525 isert_cmd, data->dma_nents, data->sg, data->nents, data->len); 1526 1527 return 0; 1528 } 1529 1530 static void 1531 isert_unmap_data_buf(struct isert_conn *isert_conn, struct isert_data_buf *data) 1532 { 1533 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1534 1535 ib_dma_unmap_sg(ib_dev, data->sg, data->nents, data->dma_dir); 1536 memset(data, 0, sizeof(*data)); 1537 } 1538 1539 1540 1541 static void 1542 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1543 { 1544 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1545 1546 pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1547 1548 if (wr->data.sg) { 1549 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1550 isert_unmap_data_buf(isert_conn, &wr->data); 1551 } 1552 1553 if (wr->send_wr) { 1554 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1555 kfree(wr->send_wr); 1556 wr->send_wr = NULL; 1557 } 1558 1559 if (wr->ib_sge) { 1560 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 1561 kfree(wr->ib_sge); 1562 wr->ib_sge = NULL; 1563 } 1564 } 1565 1566 static void 1567 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1568 { 1569 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1570 LIST_HEAD(unmap_list); 1571 1572 pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd); 1573 1574 if (wr->fr_desc) { 1575 pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n", 1576 isert_cmd, wr->fr_desc); 1577 if (wr->fr_desc->ind & ISERT_PROTECTED) { 1578 isert_unmap_data_buf(isert_conn, &wr->prot); 1579 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1580 } 1581 spin_lock_bh(&isert_conn->conn_lock); 1582 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 1583 spin_unlock_bh(&isert_conn->conn_lock); 1584 wr->fr_desc = NULL; 1585 } 1586 1587 if (wr->data.sg) { 1588 pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1589 isert_unmap_data_buf(isert_conn, &wr->data); 1590 } 1591 1592 wr->ib_sge = NULL; 1593 wr->send_wr = NULL; 1594 } 1595 1596 static void 1597 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1598 { 1599 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1600 struct isert_conn *isert_conn = isert_cmd->conn; 1601 struct iscsi_conn *conn = isert_conn->conn; 1602 struct isert_device *device = isert_conn->conn_device; 1603 1604 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1605 1606 switch (cmd->iscsi_opcode) { 1607 case ISCSI_OP_SCSI_CMD: 1608 spin_lock_bh(&conn->cmd_lock); 1609 if (!list_empty(&cmd->i_conn_node)) 1610 list_del_init(&cmd->i_conn_node); 1611 spin_unlock_bh(&conn->cmd_lock); 1612 1613 if (cmd->data_direction == DMA_TO_DEVICE) { 1614 iscsit_stop_dataout_timer(cmd); 1615 /* 1616 * Check for special case during comp_err where 1617 * WRITE_PENDING has been handed off from core, 1618 * but requires an extra target_put_sess_cmd() 1619 * before transport_generic_free_cmd() below. 1620 */ 1621 if (comp_err && 1622 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1623 struct se_cmd *se_cmd = &cmd->se_cmd; 1624 1625 target_put_sess_cmd(se_cmd->se_sess, se_cmd); 1626 } 1627 } 1628 1629 device->unreg_rdma_mem(isert_cmd, isert_conn); 1630 transport_generic_free_cmd(&cmd->se_cmd, 0); 1631 break; 1632 case ISCSI_OP_SCSI_TMFUNC: 1633 spin_lock_bh(&conn->cmd_lock); 1634 if (!list_empty(&cmd->i_conn_node)) 1635 list_del_init(&cmd->i_conn_node); 1636 spin_unlock_bh(&conn->cmd_lock); 1637 1638 transport_generic_free_cmd(&cmd->se_cmd, 0); 1639 break; 1640 case ISCSI_OP_REJECT: 1641 case ISCSI_OP_NOOP_OUT: 1642 case ISCSI_OP_TEXT: 1643 spin_lock_bh(&conn->cmd_lock); 1644 if (!list_empty(&cmd->i_conn_node)) 1645 list_del_init(&cmd->i_conn_node); 1646 spin_unlock_bh(&conn->cmd_lock); 1647 1648 /* 1649 * Handle special case for REJECT when iscsi_add_reject*() has 1650 * overwritten the original iscsi_opcode assignment, and the 1651 * associated cmd->se_cmd needs to be released. 1652 */ 1653 if (cmd->se_cmd.se_tfo != NULL) { 1654 pr_debug("Calling transport_generic_free_cmd from" 1655 " isert_put_cmd for 0x%02x\n", 1656 cmd->iscsi_opcode); 1657 transport_generic_free_cmd(&cmd->se_cmd, 0); 1658 break; 1659 } 1660 /* 1661 * Fall-through 1662 */ 1663 default: 1664 iscsit_release_cmd(cmd); 1665 break; 1666 } 1667 } 1668 1669 static void 1670 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1671 { 1672 if (tx_desc->dma_addr != 0) { 1673 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1674 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1675 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1676 tx_desc->dma_addr = 0; 1677 } 1678 } 1679 1680 static void 1681 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1682 struct ib_device *ib_dev, bool comp_err) 1683 { 1684 if (isert_cmd->pdu_buf_dma != 0) { 1685 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1686 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1687 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1688 isert_cmd->pdu_buf_dma = 0; 1689 } 1690 1691 isert_unmap_tx_desc(tx_desc, ib_dev); 1692 isert_put_cmd(isert_cmd, comp_err); 1693 } 1694 1695 static int 1696 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1697 { 1698 struct ib_mr_status mr_status; 1699 int ret; 1700 1701 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1702 if (ret) { 1703 pr_err("ib_check_mr_status failed, ret %d\n", ret); 1704 goto fail_mr_status; 1705 } 1706 1707 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1708 u64 sec_offset_err; 1709 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1710 1711 switch (mr_status.sig_err.err_type) { 1712 case IB_SIG_BAD_GUARD: 1713 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1714 break; 1715 case IB_SIG_BAD_REFTAG: 1716 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1717 break; 1718 case IB_SIG_BAD_APPTAG: 1719 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1720 break; 1721 } 1722 sec_offset_err = mr_status.sig_err.sig_err_offset; 1723 do_div(sec_offset_err, block_size); 1724 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1725 1726 pr_err("isert: PI error found type %d at sector 0x%llx " 1727 "expected 0x%x vs actual 0x%x\n", 1728 mr_status.sig_err.err_type, 1729 (unsigned long long)se_cmd->bad_sector, 1730 mr_status.sig_err.expected, 1731 mr_status.sig_err.actual); 1732 ret = 1; 1733 } 1734 1735 fail_mr_status: 1736 return ret; 1737 } 1738 1739 static void 1740 isert_completion_rdma_write(struct iser_tx_desc *tx_desc, 1741 struct isert_cmd *isert_cmd) 1742 { 1743 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1744 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1745 struct se_cmd *se_cmd = &cmd->se_cmd; 1746 struct isert_conn *isert_conn = isert_cmd->conn; 1747 struct isert_device *device = isert_conn->conn_device; 1748 int ret = 0; 1749 1750 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1751 ret = isert_check_pi_status(se_cmd, 1752 wr->fr_desc->pi_ctx->sig_mr); 1753 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1754 } 1755 1756 device->unreg_rdma_mem(isert_cmd, isert_conn); 1757 wr->send_wr_num = 0; 1758 if (ret) 1759 transport_send_check_condition_and_sense(se_cmd, 1760 se_cmd->pi_err, 0); 1761 else 1762 isert_put_response(isert_conn->conn, cmd); 1763 } 1764 1765 static void 1766 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1767 struct isert_cmd *isert_cmd) 1768 { 1769 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1770 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1771 struct se_cmd *se_cmd = &cmd->se_cmd; 1772 struct isert_conn *isert_conn = isert_cmd->conn; 1773 struct isert_device *device = isert_conn->conn_device; 1774 int ret = 0; 1775 1776 if (wr->fr_desc && wr->fr_desc->ind & ISERT_PROTECTED) { 1777 ret = isert_check_pi_status(se_cmd, 1778 wr->fr_desc->pi_ctx->sig_mr); 1779 wr->fr_desc->ind &= ~ISERT_PROTECTED; 1780 } 1781 1782 iscsit_stop_dataout_timer(cmd); 1783 device->unreg_rdma_mem(isert_cmd, isert_conn); 1784 cmd->write_data_done = wr->data.len; 1785 wr->send_wr_num = 0; 1786 1787 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1788 spin_lock_bh(&cmd->istate_lock); 1789 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1790 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1791 spin_unlock_bh(&cmd->istate_lock); 1792 1793 if (ret) 1794 transport_send_check_condition_and_sense(se_cmd, 1795 se_cmd->pi_err, 0); 1796 else 1797 target_execute_cmd(se_cmd); 1798 } 1799 1800 static void 1801 isert_do_control_comp(struct work_struct *work) 1802 { 1803 struct isert_cmd *isert_cmd = container_of(work, 1804 struct isert_cmd, comp_work); 1805 struct isert_conn *isert_conn = isert_cmd->conn; 1806 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1807 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1808 1809 switch (cmd->i_state) { 1810 case ISTATE_SEND_TASKMGTRSP: 1811 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1812 1813 atomic_dec(&isert_conn->post_send_buf_count); 1814 iscsit_tmr_post_handler(cmd, cmd->conn); 1815 1816 cmd->i_state = ISTATE_SENT_STATUS; 1817 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1818 break; 1819 case ISTATE_SEND_REJECT: 1820 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1821 atomic_dec(&isert_conn->post_send_buf_count); 1822 1823 cmd->i_state = ISTATE_SENT_STATUS; 1824 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1825 break; 1826 case ISTATE_SEND_LOGOUTRSP: 1827 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1828 1829 atomic_dec(&isert_conn->post_send_buf_count); 1830 iscsit_logout_post_handler(cmd, cmd->conn); 1831 break; 1832 case ISTATE_SEND_TEXTRSP: 1833 atomic_dec(&isert_conn->post_send_buf_count); 1834 cmd->i_state = ISTATE_SENT_STATUS; 1835 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev, false); 1836 break; 1837 default: 1838 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1839 dump_stack(); 1840 break; 1841 } 1842 } 1843 1844 static void 1845 isert_response_completion(struct iser_tx_desc *tx_desc, 1846 struct isert_cmd *isert_cmd, 1847 struct isert_conn *isert_conn, 1848 struct ib_device *ib_dev) 1849 { 1850 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1851 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1852 1853 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1854 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1855 cmd->i_state == ISTATE_SEND_REJECT || 1856 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1857 isert_unmap_tx_desc(tx_desc, ib_dev); 1858 1859 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1860 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1861 return; 1862 } 1863 1864 /** 1865 * If send_wr_num is 0 this means that we got 1866 * RDMA completion and we cleared it and we should 1867 * simply decrement the response post. else the 1868 * response is incorporated in send_wr_num, just 1869 * sub it. 1870 **/ 1871 if (wr->send_wr_num) 1872 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1873 else 1874 atomic_dec(&isert_conn->post_send_buf_count); 1875 1876 cmd->i_state = ISTATE_SENT_STATUS; 1877 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1878 } 1879 1880 static void 1881 __isert_send_completion(struct iser_tx_desc *tx_desc, 1882 struct isert_conn *isert_conn) 1883 { 1884 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1885 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1886 struct isert_rdma_wr *wr; 1887 1888 if (!isert_cmd) { 1889 atomic_dec(&isert_conn->post_send_buf_count); 1890 isert_unmap_tx_desc(tx_desc, ib_dev); 1891 return; 1892 } 1893 wr = &isert_cmd->rdma_wr; 1894 1895 switch (wr->iser_ib_op) { 1896 case ISER_IB_RECV: 1897 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1898 dump_stack(); 1899 break; 1900 case ISER_IB_SEND: 1901 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1902 isert_response_completion(tx_desc, isert_cmd, 1903 isert_conn, ib_dev); 1904 break; 1905 case ISER_IB_RDMA_WRITE: 1906 pr_debug("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1907 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1908 isert_completion_rdma_write(tx_desc, isert_cmd); 1909 break; 1910 case ISER_IB_RDMA_READ: 1911 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1912 1913 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1914 isert_completion_rdma_read(tx_desc, isert_cmd); 1915 break; 1916 default: 1917 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1918 dump_stack(); 1919 break; 1920 } 1921 } 1922 1923 static void 1924 isert_send_completion(struct iser_tx_desc *tx_desc, 1925 struct isert_conn *isert_conn) 1926 { 1927 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1928 struct iser_tx_desc *t; 1929 /* 1930 * Drain coalesced completion llist starting from comp_llnode_batch 1931 * setup in isert_init_send_wr(), and then complete trailing tx_desc. 1932 */ 1933 while (llnode) { 1934 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1935 llnode = llist_next(llnode); 1936 __isert_send_completion(t, isert_conn); 1937 } 1938 __isert_send_completion(tx_desc, isert_conn); 1939 } 1940 1941 static void 1942 isert_cq_drain_comp_llist(struct isert_conn *isert_conn, struct ib_device *ib_dev) 1943 { 1944 struct llist_node *llnode; 1945 struct isert_rdma_wr *wr; 1946 struct iser_tx_desc *t; 1947 1948 mutex_lock(&isert_conn->conn_mutex); 1949 llnode = llist_del_all(&isert_conn->conn_comp_llist); 1950 isert_conn->conn_comp_batch = 0; 1951 mutex_unlock(&isert_conn->conn_mutex); 1952 1953 while (llnode) { 1954 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1955 llnode = llist_next(llnode); 1956 wr = &t->isert_cmd->rdma_wr; 1957 1958 /** 1959 * If send_wr_num is 0 this means that we got 1960 * RDMA completion and we cleared it and we should 1961 * simply decrement the response post. else the 1962 * response is incorporated in send_wr_num, just 1963 * sub it. 1964 **/ 1965 if (wr->send_wr_num) 1966 atomic_sub(wr->send_wr_num, 1967 &isert_conn->post_send_buf_count); 1968 else 1969 atomic_dec(&isert_conn->post_send_buf_count); 1970 1971 isert_completion_put(t, t->isert_cmd, ib_dev, true); 1972 } 1973 } 1974 1975 static void 1976 isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1977 { 1978 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1979 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1980 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1981 struct isert_rdma_wr *wr; 1982 struct iser_tx_desc *t; 1983 1984 while (llnode) { 1985 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1986 llnode = llist_next(llnode); 1987 wr = &t->isert_cmd->rdma_wr; 1988 1989 /** 1990 * If send_wr_num is 0 this means that we got 1991 * RDMA completion and we cleared it and we should 1992 * simply decrement the response post. else the 1993 * response is incorporated in send_wr_num, just 1994 * sub it. 1995 **/ 1996 if (wr->send_wr_num) 1997 atomic_sub(wr->send_wr_num, 1998 &isert_conn->post_send_buf_count); 1999 else 2000 atomic_dec(&isert_conn->post_send_buf_count); 2001 2002 isert_completion_put(t, t->isert_cmd, ib_dev, true); 2003 } 2004 tx_desc->comp_llnode_batch = NULL; 2005 2006 if (!isert_cmd) 2007 isert_unmap_tx_desc(tx_desc, ib_dev); 2008 else 2009 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 2010 } 2011 2012 static void 2013 isert_cq_rx_comp_err(struct isert_conn *isert_conn) 2014 { 2015 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2016 struct iscsi_conn *conn = isert_conn->conn; 2017 2018 if (isert_conn->post_recv_buf_count) 2019 return; 2020 2021 isert_cq_drain_comp_llist(isert_conn, ib_dev); 2022 2023 if (conn->sess) { 2024 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2025 target_wait_for_sess_cmds(conn->sess->se_sess); 2026 } 2027 2028 while (atomic_read(&isert_conn->post_send_buf_count)) 2029 msleep(3000); 2030 2031 mutex_lock(&isert_conn->conn_mutex); 2032 isert_conn->state = ISER_CONN_DOWN; 2033 mutex_unlock(&isert_conn->conn_mutex); 2034 2035 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 2036 2037 complete(&isert_conn->conn_wait_comp_err); 2038 } 2039 2040 static void 2041 isert_cq_tx_work(struct work_struct *work) 2042 { 2043 struct isert_cq_desc *cq_desc = container_of(work, 2044 struct isert_cq_desc, cq_tx_work); 2045 struct isert_device *device = cq_desc->device; 2046 int cq_index = cq_desc->cq_index; 2047 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 2048 struct isert_conn *isert_conn; 2049 struct iser_tx_desc *tx_desc; 2050 struct ib_wc wc; 2051 2052 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 2053 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 2054 isert_conn = wc.qp->qp_context; 2055 2056 if (wc.status == IB_WC_SUCCESS) { 2057 isert_send_completion(tx_desc, isert_conn); 2058 } else { 2059 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2060 pr_debug("TX wc.status: 0x%08x\n", wc.status); 2061 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 2062 2063 if (wc.wr_id != ISER_FASTREG_LI_WRID) { 2064 if (tx_desc->llnode_active) 2065 continue; 2066 2067 atomic_dec(&isert_conn->post_send_buf_count); 2068 isert_cq_tx_comp_err(tx_desc, isert_conn); 2069 } 2070 } 2071 } 2072 2073 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 2074 } 2075 2076 static void 2077 isert_cq_tx_callback(struct ib_cq *cq, void *context) 2078 { 2079 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2080 2081 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 2082 } 2083 2084 static void 2085 isert_cq_rx_work(struct work_struct *work) 2086 { 2087 struct isert_cq_desc *cq_desc = container_of(work, 2088 struct isert_cq_desc, cq_rx_work); 2089 struct isert_device *device = cq_desc->device; 2090 int cq_index = cq_desc->cq_index; 2091 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 2092 struct isert_conn *isert_conn; 2093 struct iser_rx_desc *rx_desc; 2094 struct ib_wc wc; 2095 unsigned long xfer_len; 2096 2097 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 2098 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 2099 isert_conn = wc.qp->qp_context; 2100 2101 if (wc.status == IB_WC_SUCCESS) { 2102 xfer_len = (unsigned long)wc.byte_len; 2103 isert_rx_completion(rx_desc, isert_conn, xfer_len); 2104 } else { 2105 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 2106 if (wc.status != IB_WC_WR_FLUSH_ERR) { 2107 pr_debug("RX wc.status: 0x%08x\n", wc.status); 2108 pr_debug("RX wc.vendor_err: 0x%08x\n", 2109 wc.vendor_err); 2110 } 2111 isert_conn->post_recv_buf_count--; 2112 isert_cq_rx_comp_err(isert_conn); 2113 } 2114 } 2115 2116 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 2117 } 2118 2119 static void 2120 isert_cq_rx_callback(struct ib_cq *cq, void *context) 2121 { 2122 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 2123 2124 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 2125 } 2126 2127 static int 2128 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 2129 { 2130 struct ib_send_wr *wr_failed; 2131 int ret; 2132 2133 atomic_inc(&isert_conn->post_send_buf_count); 2134 2135 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 2136 &wr_failed); 2137 if (ret) { 2138 pr_err("ib_post_send failed with %d\n", ret); 2139 atomic_dec(&isert_conn->post_send_buf_count); 2140 return ret; 2141 } 2142 return ret; 2143 } 2144 2145 static int 2146 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2147 { 2148 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2149 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2150 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2151 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 2152 &isert_cmd->tx_desc.iscsi_header; 2153 2154 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2155 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 2156 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2157 /* 2158 * Attach SENSE DATA payload to iSCSI Response PDU 2159 */ 2160 if (cmd->se_cmd.sense_buffer && 2161 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 2162 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 2163 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2164 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2165 u32 padding, pdu_len; 2166 2167 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 2168 cmd->sense_buffer); 2169 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 2170 2171 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 2172 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 2173 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 2174 2175 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2176 (void *)cmd->sense_buffer, pdu_len, 2177 DMA_TO_DEVICE); 2178 2179 isert_cmd->pdu_buf_len = pdu_len; 2180 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2181 tx_dsg->length = pdu_len; 2182 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2183 isert_cmd->tx_desc.num_sge = 2; 2184 } 2185 2186 isert_init_send_wr(isert_conn, isert_cmd, send_wr, true); 2187 2188 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2189 2190 return isert_post_response(isert_conn, isert_cmd); 2191 } 2192 2193 static void 2194 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2195 { 2196 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2197 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2198 struct isert_device *device = isert_conn->conn_device; 2199 2200 spin_lock_bh(&conn->cmd_lock); 2201 if (!list_empty(&cmd->i_conn_node)) 2202 list_del_init(&cmd->i_conn_node); 2203 spin_unlock_bh(&conn->cmd_lock); 2204 2205 if (cmd->data_direction == DMA_TO_DEVICE) 2206 iscsit_stop_dataout_timer(cmd); 2207 2208 device->unreg_rdma_mem(isert_cmd, isert_conn); 2209 } 2210 2211 static enum target_prot_op 2212 isert_get_sup_prot_ops(struct iscsi_conn *conn) 2213 { 2214 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2215 struct isert_device *device = isert_conn->conn_device; 2216 2217 if (device->pi_capable) 2218 return TARGET_PROT_ALL; 2219 2220 return TARGET_PROT_NORMAL; 2221 } 2222 2223 static int 2224 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 2225 bool nopout_response) 2226 { 2227 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2228 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2229 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2230 2231 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2232 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 2233 &isert_cmd->tx_desc.iscsi_header, 2234 nopout_response); 2235 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2236 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2237 2238 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2239 2240 return isert_post_response(isert_conn, isert_cmd); 2241 } 2242 2243 static int 2244 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2245 { 2246 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2247 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2248 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2249 2250 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2251 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 2252 &isert_cmd->tx_desc.iscsi_header); 2253 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2254 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2255 2256 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2257 2258 return isert_post_response(isert_conn, isert_cmd); 2259 } 2260 2261 static int 2262 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2263 { 2264 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2265 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2266 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2267 2268 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2269 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 2270 &isert_cmd->tx_desc.iscsi_header); 2271 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2272 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2273 2274 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2275 2276 return isert_post_response(isert_conn, isert_cmd); 2277 } 2278 2279 static int 2280 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2281 { 2282 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2283 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2284 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2285 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2286 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2287 struct iscsi_reject *hdr = 2288 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 2289 2290 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2291 iscsit_build_reject(cmd, conn, hdr); 2292 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2293 2294 hton24(hdr->dlength, ISCSI_HDR_LEN); 2295 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2296 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2297 DMA_TO_DEVICE); 2298 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2299 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2300 tx_dsg->length = ISCSI_HDR_LEN; 2301 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2302 isert_cmd->tx_desc.num_sge = 2; 2303 2304 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2305 2306 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2307 2308 return isert_post_response(isert_conn, isert_cmd); 2309 } 2310 2311 static int 2312 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2313 { 2314 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2315 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2316 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2317 struct iscsi_text_rsp *hdr = 2318 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2319 u32 txt_rsp_len; 2320 int rc; 2321 2322 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2323 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2324 if (rc < 0) 2325 return rc; 2326 2327 txt_rsp_len = rc; 2328 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2329 2330 if (txt_rsp_len) { 2331 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2332 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2333 void *txt_rsp_buf = cmd->buf_ptr; 2334 2335 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2336 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2337 2338 isert_cmd->pdu_buf_len = txt_rsp_len; 2339 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2340 tx_dsg->length = txt_rsp_len; 2341 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2342 isert_cmd->tx_desc.num_sge = 2; 2343 } 2344 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2345 2346 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2347 2348 return isert_post_response(isert_conn, isert_cmd); 2349 } 2350 2351 static int 2352 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2353 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2354 u32 data_left, u32 offset) 2355 { 2356 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2357 struct scatterlist *sg_start, *tmp_sg; 2358 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2359 u32 sg_off, page_off; 2360 int i = 0, sg_nents; 2361 2362 sg_off = offset / PAGE_SIZE; 2363 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2364 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2365 page_off = offset % PAGE_SIZE; 2366 2367 send_wr->sg_list = ib_sge; 2368 send_wr->num_sge = sg_nents; 2369 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2370 /* 2371 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2372 */ 2373 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2374 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2375 (unsigned long long)tmp_sg->dma_address, 2376 tmp_sg->length, page_off); 2377 2378 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2379 ib_sge->length = min_t(u32, data_left, 2380 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2381 ib_sge->lkey = isert_conn->conn_mr->lkey; 2382 2383 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2384 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2385 page_off = 0; 2386 data_left -= ib_sge->length; 2387 ib_sge++; 2388 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2389 } 2390 2391 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2392 send_wr->sg_list, send_wr->num_sge); 2393 2394 return sg_nents; 2395 } 2396 2397 static int 2398 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2399 struct isert_rdma_wr *wr) 2400 { 2401 struct se_cmd *se_cmd = &cmd->se_cmd; 2402 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2403 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2404 struct isert_data_buf *data = &wr->data; 2405 struct ib_send_wr *send_wr; 2406 struct ib_sge *ib_sge; 2407 u32 offset, data_len, data_left, rdma_write_max, va_offset = 0; 2408 int ret = 0, i, ib_sge_cnt; 2409 2410 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2411 2412 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2413 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2414 se_cmd->t_data_nents, se_cmd->data_length, 2415 offset, wr->iser_ib_op, &wr->data); 2416 if (ret) 2417 return ret; 2418 2419 data_left = data->len; 2420 offset = data->offset; 2421 2422 ib_sge = kzalloc(sizeof(struct ib_sge) * data->nents, GFP_KERNEL); 2423 if (!ib_sge) { 2424 pr_warn("Unable to allocate ib_sge\n"); 2425 ret = -ENOMEM; 2426 goto unmap_cmd; 2427 } 2428 wr->ib_sge = ib_sge; 2429 2430 wr->send_wr_num = DIV_ROUND_UP(data->nents, isert_conn->max_sge); 2431 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2432 GFP_KERNEL); 2433 if (!wr->send_wr) { 2434 pr_debug("Unable to allocate wr->send_wr\n"); 2435 ret = -ENOMEM; 2436 goto unmap_cmd; 2437 } 2438 2439 wr->isert_cmd = isert_cmd; 2440 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2441 2442 for (i = 0; i < wr->send_wr_num; i++) { 2443 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2444 data_len = min(data_left, rdma_write_max); 2445 2446 send_wr->send_flags = 0; 2447 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2448 send_wr->opcode = IB_WR_RDMA_WRITE; 2449 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2450 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2451 if (i + 1 == wr->send_wr_num) 2452 send_wr->next = &isert_cmd->tx_desc.send_wr; 2453 else 2454 send_wr->next = &wr->send_wr[i + 1]; 2455 } else { 2456 send_wr->opcode = IB_WR_RDMA_READ; 2457 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2458 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2459 if (i + 1 == wr->send_wr_num) 2460 send_wr->send_flags = IB_SEND_SIGNALED; 2461 else 2462 send_wr->next = &wr->send_wr[i + 1]; 2463 } 2464 2465 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2466 send_wr, data_len, offset); 2467 ib_sge += ib_sge_cnt; 2468 2469 offset += data_len; 2470 va_offset += data_len; 2471 data_left -= data_len; 2472 } 2473 2474 return 0; 2475 unmap_cmd: 2476 isert_unmap_data_buf(isert_conn, data); 2477 2478 return ret; 2479 } 2480 2481 static int 2482 isert_map_fr_pagelist(struct ib_device *ib_dev, 2483 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2484 { 2485 u64 start_addr, end_addr, page, chunk_start = 0; 2486 struct scatterlist *tmp_sg; 2487 int i = 0, new_chunk, last_ent, n_pages; 2488 2489 n_pages = 0; 2490 new_chunk = 1; 2491 last_ent = sg_nents - 1; 2492 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2493 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2494 if (new_chunk) 2495 chunk_start = start_addr; 2496 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2497 2498 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 2499 i, (unsigned long long)tmp_sg->dma_address, 2500 tmp_sg->length); 2501 2502 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2503 new_chunk = 0; 2504 continue; 2505 } 2506 new_chunk = 1; 2507 2508 page = chunk_start & PAGE_MASK; 2509 do { 2510 fr_pl[n_pages++] = page; 2511 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 2512 n_pages - 1, page); 2513 page += PAGE_SIZE; 2514 } while (page < end_addr); 2515 } 2516 2517 return n_pages; 2518 } 2519 2520 static int 2521 isert_fast_reg_mr(struct isert_conn *isert_conn, 2522 struct fast_reg_descriptor *fr_desc, 2523 struct isert_data_buf *mem, 2524 enum isert_indicator ind, 2525 struct ib_sge *sge) 2526 { 2527 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2528 struct ib_mr *mr; 2529 struct ib_fast_reg_page_list *frpl; 2530 struct ib_send_wr fr_wr, inv_wr; 2531 struct ib_send_wr *bad_wr, *wr = NULL; 2532 int ret, pagelist_len; 2533 u32 page_off; 2534 u8 key; 2535 2536 if (mem->dma_nents == 1) { 2537 sge->lkey = isert_conn->conn_mr->lkey; 2538 sge->addr = ib_sg_dma_address(ib_dev, &mem->sg[0]); 2539 sge->length = ib_sg_dma_len(ib_dev, &mem->sg[0]); 2540 pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 2541 __func__, __LINE__, sge->addr, sge->length, 2542 sge->lkey); 2543 return 0; 2544 } 2545 2546 if (ind == ISERT_DATA_KEY_VALID) { 2547 /* Registering data buffer */ 2548 mr = fr_desc->data_mr; 2549 frpl = fr_desc->data_frpl; 2550 } else { 2551 /* Registering protection buffer */ 2552 mr = fr_desc->pi_ctx->prot_mr; 2553 frpl = fr_desc->pi_ctx->prot_frpl; 2554 } 2555 2556 page_off = mem->offset % PAGE_SIZE; 2557 2558 pr_debug("Use fr_desc %p sg_nents %d offset %u\n", 2559 fr_desc, mem->nents, mem->offset); 2560 2561 pagelist_len = isert_map_fr_pagelist(ib_dev, mem->sg, mem->nents, 2562 &frpl->page_list[0]); 2563 2564 if (!(fr_desc->ind & ISERT_DATA_KEY_VALID)) { 2565 memset(&inv_wr, 0, sizeof(inv_wr)); 2566 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 2567 inv_wr.opcode = IB_WR_LOCAL_INV; 2568 inv_wr.ex.invalidate_rkey = mr->rkey; 2569 wr = &inv_wr; 2570 /* Bump the key */ 2571 key = (u8)(mr->rkey & 0x000000FF); 2572 ib_update_fast_reg_key(mr, ++key); 2573 } 2574 2575 /* Prepare FASTREG WR */ 2576 memset(&fr_wr, 0, sizeof(fr_wr)); 2577 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2578 fr_wr.opcode = IB_WR_FAST_REG_MR; 2579 fr_wr.wr.fast_reg.iova_start = frpl->page_list[0] + page_off; 2580 fr_wr.wr.fast_reg.page_list = frpl; 2581 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2582 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2583 fr_wr.wr.fast_reg.length = mem->len; 2584 fr_wr.wr.fast_reg.rkey = mr->rkey; 2585 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2586 2587 if (!wr) 2588 wr = &fr_wr; 2589 else 2590 wr->next = &fr_wr; 2591 2592 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2593 if (ret) { 2594 pr_err("fast registration failed, ret:%d\n", ret); 2595 return ret; 2596 } 2597 fr_desc->ind &= ~ind; 2598 2599 sge->lkey = mr->lkey; 2600 sge->addr = frpl->page_list[0] + page_off; 2601 sge->length = mem->len; 2602 2603 pr_debug("%s:%d sge: addr: 0x%llx length: %u lkey: %x\n", 2604 __func__, __LINE__, sge->addr, sge->length, 2605 sge->lkey); 2606 2607 return ret; 2608 } 2609 2610 static inline enum ib_t10_dif_type 2611 se2ib_prot_type(enum target_prot_type prot_type) 2612 { 2613 switch (prot_type) { 2614 case TARGET_DIF_TYPE0_PROT: 2615 return IB_T10DIF_NONE; 2616 case TARGET_DIF_TYPE1_PROT: 2617 return IB_T10DIF_TYPE1; 2618 case TARGET_DIF_TYPE2_PROT: 2619 return IB_T10DIF_TYPE2; 2620 case TARGET_DIF_TYPE3_PROT: 2621 return IB_T10DIF_TYPE3; 2622 default: 2623 return IB_T10DIF_NONE; 2624 } 2625 } 2626 2627 static int 2628 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2629 { 2630 enum ib_t10_dif_type ib_prot_type = se2ib_prot_type(se_cmd->prot_type); 2631 2632 sig_attrs->mem.sig_type = IB_SIG_TYPE_T10_DIF; 2633 sig_attrs->wire.sig_type = IB_SIG_TYPE_T10_DIF; 2634 sig_attrs->mem.sig.dif.pi_interval = 2635 se_cmd->se_dev->dev_attrib.block_size; 2636 sig_attrs->wire.sig.dif.pi_interval = 2637 se_cmd->se_dev->dev_attrib.block_size; 2638 2639 switch (se_cmd->prot_op) { 2640 case TARGET_PROT_DIN_INSERT: 2641 case TARGET_PROT_DOUT_STRIP: 2642 sig_attrs->mem.sig.dif.type = IB_T10DIF_NONE; 2643 sig_attrs->wire.sig.dif.type = ib_prot_type; 2644 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 2645 sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed; 2646 break; 2647 case TARGET_PROT_DOUT_INSERT: 2648 case TARGET_PROT_DIN_STRIP: 2649 sig_attrs->mem.sig.dif.type = ib_prot_type; 2650 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 2651 sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed; 2652 sig_attrs->wire.sig.dif.type = IB_T10DIF_NONE; 2653 break; 2654 case TARGET_PROT_DIN_PASS: 2655 case TARGET_PROT_DOUT_PASS: 2656 sig_attrs->mem.sig.dif.type = ib_prot_type; 2657 sig_attrs->mem.sig.dif.bg_type = IB_T10DIF_CRC; 2658 sig_attrs->mem.sig.dif.ref_tag = se_cmd->reftag_seed; 2659 sig_attrs->wire.sig.dif.type = ib_prot_type; 2660 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 2661 sig_attrs->wire.sig.dif.ref_tag = se_cmd->reftag_seed; 2662 break; 2663 default: 2664 pr_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2665 return -EINVAL; 2666 } 2667 2668 return 0; 2669 } 2670 2671 static inline u8 2672 isert_set_prot_checks(u8 prot_checks) 2673 { 2674 return (prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2675 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2676 (prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2677 } 2678 2679 static int 2680 isert_reg_sig_mr(struct isert_conn *isert_conn, struct se_cmd *se_cmd, 2681 struct fast_reg_descriptor *fr_desc, 2682 struct ib_sge *data_sge, struct ib_sge *prot_sge, 2683 struct ib_sge *sig_sge) 2684 { 2685 struct ib_send_wr sig_wr, inv_wr; 2686 struct ib_send_wr *bad_wr, *wr = NULL; 2687 struct pi_context *pi_ctx = fr_desc->pi_ctx; 2688 struct ib_sig_attrs sig_attrs; 2689 int ret; 2690 u32 key; 2691 2692 memset(&sig_attrs, 0, sizeof(sig_attrs)); 2693 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2694 if (ret) 2695 goto err; 2696 2697 sig_attrs.check_mask = isert_set_prot_checks(se_cmd->prot_checks); 2698 2699 if (!(fr_desc->ind & ISERT_SIG_KEY_VALID)) { 2700 memset(&inv_wr, 0, sizeof(inv_wr)); 2701 inv_wr.opcode = IB_WR_LOCAL_INV; 2702 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 2703 inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; 2704 wr = &inv_wr; 2705 /* Bump the key */ 2706 key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); 2707 ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); 2708 } 2709 2710 memset(&sig_wr, 0, sizeof(sig_wr)); 2711 sig_wr.opcode = IB_WR_REG_SIG_MR; 2712 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 2713 sig_wr.sg_list = data_sge; 2714 sig_wr.num_sge = 1; 2715 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE; 2716 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 2717 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 2718 if (se_cmd->t_prot_sg) 2719 sig_wr.wr.sig_handover.prot = prot_sge; 2720 2721 if (!wr) 2722 wr = &sig_wr; 2723 else 2724 wr->next = &sig_wr; 2725 2726 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2727 if (ret) { 2728 pr_err("fast registration failed, ret:%d\n", ret); 2729 goto err; 2730 } 2731 fr_desc->ind &= ~ISERT_SIG_KEY_VALID; 2732 2733 sig_sge->lkey = pi_ctx->sig_mr->lkey; 2734 sig_sge->addr = 0; 2735 sig_sge->length = se_cmd->data_length; 2736 if (se_cmd->prot_op != TARGET_PROT_DIN_STRIP && 2737 se_cmd->prot_op != TARGET_PROT_DOUT_INSERT) 2738 /* 2739 * We have protection guards on the wire 2740 * so we need to set a larget transfer 2741 */ 2742 sig_sge->length += se_cmd->prot_length; 2743 2744 pr_debug("sig_sge: addr: 0x%llx length: %u lkey: %x\n", 2745 sig_sge->addr, sig_sge->length, 2746 sig_sge->lkey); 2747 err: 2748 return ret; 2749 } 2750 2751 static int 2752 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2753 struct isert_rdma_wr *wr) 2754 { 2755 struct se_cmd *se_cmd = &cmd->se_cmd; 2756 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2757 struct isert_conn *isert_conn = conn->context; 2758 struct ib_sge data_sge; 2759 struct ib_send_wr *send_wr; 2760 struct fast_reg_descriptor *fr_desc = NULL; 2761 u32 offset; 2762 int ret = 0; 2763 unsigned long flags; 2764 2765 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2766 2767 offset = wr->iser_ib_op == ISER_IB_RDMA_READ ? cmd->write_data_done : 0; 2768 ret = isert_map_data_buf(isert_conn, isert_cmd, se_cmd->t_data_sg, 2769 se_cmd->t_data_nents, se_cmd->data_length, 2770 offset, wr->iser_ib_op, &wr->data); 2771 if (ret) 2772 return ret; 2773 2774 if (wr->data.dma_nents != 1 || 2775 se_cmd->prot_op != TARGET_PROT_NORMAL) { 2776 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2777 fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2778 struct fast_reg_descriptor, list); 2779 list_del(&fr_desc->list); 2780 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2781 wr->fr_desc = fr_desc; 2782 } 2783 2784 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->data, 2785 ISERT_DATA_KEY_VALID, &data_sge); 2786 if (ret) 2787 goto unmap_cmd; 2788 2789 if (se_cmd->prot_op != TARGET_PROT_NORMAL) { 2790 struct ib_sge prot_sge, sig_sge; 2791 2792 if (se_cmd->t_prot_sg) { 2793 ret = isert_map_data_buf(isert_conn, isert_cmd, 2794 se_cmd->t_prot_sg, 2795 se_cmd->t_prot_nents, 2796 se_cmd->prot_length, 2797 0, wr->iser_ib_op, &wr->prot); 2798 if (ret) 2799 goto unmap_cmd; 2800 2801 ret = isert_fast_reg_mr(isert_conn, fr_desc, &wr->prot, 2802 ISERT_PROT_KEY_VALID, &prot_sge); 2803 if (ret) 2804 goto unmap_prot_cmd; 2805 } 2806 2807 ret = isert_reg_sig_mr(isert_conn, se_cmd, fr_desc, 2808 &data_sge, &prot_sge, &sig_sge); 2809 if (ret) 2810 goto unmap_prot_cmd; 2811 2812 fr_desc->ind |= ISERT_PROTECTED; 2813 memcpy(&wr->s_ib_sge, &sig_sge, sizeof(sig_sge)); 2814 } else 2815 memcpy(&wr->s_ib_sge, &data_sge, sizeof(data_sge)); 2816 2817 wr->ib_sge = &wr->s_ib_sge; 2818 wr->send_wr_num = 1; 2819 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2820 wr->send_wr = &wr->s_send_wr; 2821 wr->isert_cmd = isert_cmd; 2822 2823 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2824 send_wr->sg_list = &wr->s_ib_sge; 2825 send_wr->num_sge = 1; 2826 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2827 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2828 send_wr->opcode = IB_WR_RDMA_WRITE; 2829 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2830 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2831 send_wr->send_flags = se_cmd->prot_op == TARGET_PROT_NORMAL ? 2832 0 : IB_SEND_SIGNALED; 2833 } else { 2834 send_wr->opcode = IB_WR_RDMA_READ; 2835 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2836 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2837 send_wr->send_flags = IB_SEND_SIGNALED; 2838 } 2839 2840 return 0; 2841 unmap_prot_cmd: 2842 if (se_cmd->t_prot_sg) 2843 isert_unmap_data_buf(isert_conn, &wr->prot); 2844 unmap_cmd: 2845 if (fr_desc) { 2846 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2847 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2848 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2849 } 2850 isert_unmap_data_buf(isert_conn, &wr->data); 2851 2852 return ret; 2853 } 2854 2855 static int 2856 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2857 { 2858 struct se_cmd *se_cmd = &cmd->se_cmd; 2859 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2860 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2861 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2862 struct isert_device *device = isert_conn->conn_device; 2863 struct ib_send_wr *wr_failed; 2864 int rc; 2865 2866 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 2867 isert_cmd, se_cmd->data_length); 2868 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2869 rc = device->reg_rdma_mem(conn, cmd, wr); 2870 if (rc) { 2871 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2872 return rc; 2873 } 2874 2875 if (se_cmd->prot_op == TARGET_PROT_NORMAL) { 2876 /* 2877 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2878 */ 2879 isert_create_send_desc(isert_conn, isert_cmd, 2880 &isert_cmd->tx_desc); 2881 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2882 &isert_cmd->tx_desc.iscsi_header); 2883 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2884 isert_init_send_wr(isert_conn, isert_cmd, 2885 &isert_cmd->tx_desc.send_wr, true); 2886 isert_cmd->rdma_wr.s_send_wr.next = &isert_cmd->tx_desc.send_wr; 2887 wr->send_wr_num += 1; 2888 } 2889 2890 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2891 2892 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2893 if (rc) { 2894 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2895 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2896 } 2897 2898 if (se_cmd->prot_op == TARGET_PROT_NORMAL) 2899 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data " 2900 "READ\n", isert_cmd); 2901 else 2902 pr_debug("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", 2903 isert_cmd); 2904 2905 return 1; 2906 } 2907 2908 static int 2909 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2910 { 2911 struct se_cmd *se_cmd = &cmd->se_cmd; 2912 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2913 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2914 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2915 struct isert_device *device = isert_conn->conn_device; 2916 struct ib_send_wr *wr_failed; 2917 int rc; 2918 2919 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2920 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2921 wr->iser_ib_op = ISER_IB_RDMA_READ; 2922 rc = device->reg_rdma_mem(conn, cmd, wr); 2923 if (rc) { 2924 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2925 return rc; 2926 } 2927 2928 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2929 2930 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2931 if (rc) { 2932 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2933 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2934 } 2935 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2936 isert_cmd); 2937 2938 return 0; 2939 } 2940 2941 static int 2942 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2943 { 2944 int ret; 2945 2946 switch (state) { 2947 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2948 ret = isert_put_nopin(cmd, conn, false); 2949 break; 2950 default: 2951 pr_err("Unknown immediate state: 0x%02x\n", state); 2952 ret = -EINVAL; 2953 break; 2954 } 2955 2956 return ret; 2957 } 2958 2959 static int 2960 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2961 { 2962 int ret; 2963 2964 switch (state) { 2965 case ISTATE_SEND_LOGOUTRSP: 2966 ret = isert_put_logout_rsp(cmd, conn); 2967 if (!ret) { 2968 pr_debug("Returning iSER Logout -EAGAIN\n"); 2969 ret = -EAGAIN; 2970 } 2971 break; 2972 case ISTATE_SEND_NOPIN: 2973 ret = isert_put_nopin(cmd, conn, true); 2974 break; 2975 case ISTATE_SEND_TASKMGTRSP: 2976 ret = isert_put_tm_rsp(cmd, conn); 2977 break; 2978 case ISTATE_SEND_REJECT: 2979 ret = isert_put_reject(cmd, conn); 2980 break; 2981 case ISTATE_SEND_TEXTRSP: 2982 ret = isert_put_text_rsp(cmd, conn); 2983 break; 2984 case ISTATE_SEND_STATUS: 2985 /* 2986 * Special case for sending non GOOD SCSI status from TX thread 2987 * context during pre se_cmd excecution failure. 2988 */ 2989 ret = isert_put_response(conn, cmd); 2990 break; 2991 default: 2992 pr_err("Unknown response state: 0x%02x\n", state); 2993 ret = -EINVAL; 2994 break; 2995 } 2996 2997 return ret; 2998 } 2999 3000 static int 3001 isert_setup_np(struct iscsi_np *np, 3002 struct __kernel_sockaddr_storage *ksockaddr) 3003 { 3004 struct isert_np *isert_np; 3005 struct rdma_cm_id *isert_lid; 3006 struct sockaddr *sa; 3007 int ret; 3008 3009 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 3010 if (!isert_np) { 3011 pr_err("Unable to allocate struct isert_np\n"); 3012 return -ENOMEM; 3013 } 3014 sema_init(&isert_np->np_sem, 0); 3015 mutex_init(&isert_np->np_accept_mutex); 3016 INIT_LIST_HEAD(&isert_np->np_accept_list); 3017 init_completion(&isert_np->np_login_comp); 3018 3019 sa = (struct sockaddr *)ksockaddr; 3020 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 3021 /* 3022 * Setup the np->np_sockaddr from the passed sockaddr setup 3023 * in iscsi_target_configfs.c code.. 3024 */ 3025 memcpy(&np->np_sockaddr, ksockaddr, 3026 sizeof(struct __kernel_sockaddr_storage)); 3027 3028 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 3029 IB_QPT_RC); 3030 if (IS_ERR(isert_lid)) { 3031 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 3032 PTR_ERR(isert_lid)); 3033 ret = PTR_ERR(isert_lid); 3034 goto out; 3035 } 3036 3037 ret = rdma_bind_addr(isert_lid, sa); 3038 if (ret) { 3039 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 3040 goto out_lid; 3041 } 3042 3043 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 3044 if (ret) { 3045 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 3046 goto out_lid; 3047 } 3048 3049 isert_np->np_cm_id = isert_lid; 3050 np->np_context = isert_np; 3051 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 3052 3053 return 0; 3054 3055 out_lid: 3056 rdma_destroy_id(isert_lid); 3057 out: 3058 kfree(isert_np); 3059 return ret; 3060 } 3061 3062 static int 3063 isert_rdma_accept(struct isert_conn *isert_conn) 3064 { 3065 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3066 struct rdma_conn_param cp; 3067 int ret; 3068 3069 memset(&cp, 0, sizeof(struct rdma_conn_param)); 3070 cp.responder_resources = isert_conn->responder_resources; 3071 cp.initiator_depth = isert_conn->initiator_depth; 3072 cp.retry_count = 7; 3073 cp.rnr_retry_count = 7; 3074 3075 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 3076 3077 ret = rdma_accept(cm_id, &cp); 3078 if (ret) { 3079 pr_err("rdma_accept() failed with: %d\n", ret); 3080 return ret; 3081 } 3082 3083 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 3084 3085 return 0; 3086 } 3087 3088 static int 3089 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 3090 { 3091 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 3092 int ret; 3093 3094 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 3095 /* 3096 * For login requests after the first PDU, isert_rx_login_req() will 3097 * kick schedule_delayed_work(&conn->login_work) as the packet is 3098 * received, which turns this callback from iscsi_target_do_login_rx() 3099 * into a NOP. 3100 */ 3101 if (!login->first_request) 3102 return 0; 3103 3104 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 3105 if (ret) 3106 return ret; 3107 3108 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 3109 return 0; 3110 } 3111 3112 static void 3113 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 3114 struct isert_conn *isert_conn) 3115 { 3116 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 3117 struct rdma_route *cm_route = &cm_id->route; 3118 struct sockaddr_in *sock_in; 3119 struct sockaddr_in6 *sock_in6; 3120 3121 conn->login_family = np->np_sockaddr.ss_family; 3122 3123 if (np->np_sockaddr.ss_family == AF_INET6) { 3124 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 3125 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 3126 &sock_in6->sin6_addr.in6_u); 3127 conn->login_port = ntohs(sock_in6->sin6_port); 3128 3129 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 3130 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 3131 &sock_in6->sin6_addr.in6_u); 3132 conn->local_port = ntohs(sock_in6->sin6_port); 3133 } else { 3134 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 3135 sprintf(conn->login_ip, "%pI4", 3136 &sock_in->sin_addr.s_addr); 3137 conn->login_port = ntohs(sock_in->sin_port); 3138 3139 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 3140 sprintf(conn->local_ip, "%pI4", 3141 &sock_in->sin_addr.s_addr); 3142 conn->local_port = ntohs(sock_in->sin_port); 3143 } 3144 } 3145 3146 static int 3147 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 3148 { 3149 struct isert_np *isert_np = (struct isert_np *)np->np_context; 3150 struct isert_conn *isert_conn; 3151 int max_accept = 0, ret; 3152 3153 accept_wait: 3154 ret = down_interruptible(&isert_np->np_sem); 3155 if (max_accept > 5) 3156 return -ENODEV; 3157 3158 spin_lock_bh(&np->np_thread_lock); 3159 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 3160 spin_unlock_bh(&np->np_thread_lock); 3161 pr_debug("np_thread_state %d for isert_accept_np\n", 3162 np->np_thread_state); 3163 /** 3164 * No point in stalling here when np_thread 3165 * is in state RESET/SHUTDOWN/EXIT - bail 3166 **/ 3167 return -ENODEV; 3168 } 3169 spin_unlock_bh(&np->np_thread_lock); 3170 3171 mutex_lock(&isert_np->np_accept_mutex); 3172 if (list_empty(&isert_np->np_accept_list)) { 3173 mutex_unlock(&isert_np->np_accept_mutex); 3174 max_accept++; 3175 goto accept_wait; 3176 } 3177 isert_conn = list_first_entry(&isert_np->np_accept_list, 3178 struct isert_conn, conn_accept_node); 3179 list_del_init(&isert_conn->conn_accept_node); 3180 mutex_unlock(&isert_np->np_accept_mutex); 3181 3182 conn->context = isert_conn; 3183 isert_conn->conn = conn; 3184 max_accept = 0; 3185 3186 ret = isert_rdma_post_recvl(isert_conn); 3187 if (ret) 3188 return ret; 3189 3190 ret = isert_rdma_accept(isert_conn); 3191 if (ret) 3192 return ret; 3193 3194 isert_set_conn_info(np, conn, isert_conn); 3195 3196 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 3197 return 0; 3198 } 3199 3200 static void 3201 isert_free_np(struct iscsi_np *np) 3202 { 3203 struct isert_np *isert_np = (struct isert_np *)np->np_context; 3204 3205 rdma_destroy_id(isert_np->np_cm_id); 3206 3207 np->np_context = NULL; 3208 kfree(isert_np); 3209 } 3210 3211 static void isert_wait_conn(struct iscsi_conn *conn) 3212 { 3213 struct isert_conn *isert_conn = conn->context; 3214 3215 pr_debug("isert_wait_conn: Starting \n"); 3216 3217 mutex_lock(&isert_conn->conn_mutex); 3218 if (isert_conn->conn_cm_id) { 3219 pr_debug("Calling rdma_disconnect from isert_wait_conn\n"); 3220 rdma_disconnect(isert_conn->conn_cm_id); 3221 } 3222 /* 3223 * Only wait for conn_wait_comp_err if the isert_conn made it 3224 * into full feature phase.. 3225 */ 3226 if (isert_conn->state == ISER_CONN_INIT) { 3227 mutex_unlock(&isert_conn->conn_mutex); 3228 return; 3229 } 3230 if (isert_conn->state == ISER_CONN_UP) 3231 isert_conn->state = ISER_CONN_TERMINATING; 3232 mutex_unlock(&isert_conn->conn_mutex); 3233 3234 wait_for_completion(&isert_conn->conn_wait_comp_err); 3235 3236 wait_for_completion(&isert_conn->conn_wait); 3237 } 3238 3239 static void isert_free_conn(struct iscsi_conn *conn) 3240 { 3241 struct isert_conn *isert_conn = conn->context; 3242 3243 isert_put_conn(isert_conn); 3244 } 3245 3246 static struct iscsit_transport iser_target_transport = { 3247 .name = "IB/iSER", 3248 .transport_type = ISCSI_INFINIBAND, 3249 .priv_size = sizeof(struct isert_cmd), 3250 .owner = THIS_MODULE, 3251 .iscsit_setup_np = isert_setup_np, 3252 .iscsit_accept_np = isert_accept_np, 3253 .iscsit_free_np = isert_free_np, 3254 .iscsit_wait_conn = isert_wait_conn, 3255 .iscsit_free_conn = isert_free_conn, 3256 .iscsit_get_login_rx = isert_get_login_rx, 3257 .iscsit_put_login_tx = isert_put_login_tx, 3258 .iscsit_immediate_queue = isert_immediate_queue, 3259 .iscsit_response_queue = isert_response_queue, 3260 .iscsit_get_dataout = isert_get_dataout, 3261 .iscsit_queue_data_in = isert_put_datain, 3262 .iscsit_queue_status = isert_put_response, 3263 .iscsit_aborted_task = isert_aborted_task, 3264 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 3265 }; 3266 3267 static int __init isert_init(void) 3268 { 3269 int ret; 3270 3271 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 3272 if (!isert_rx_wq) { 3273 pr_err("Unable to allocate isert_rx_wq\n"); 3274 return -ENOMEM; 3275 } 3276 3277 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 3278 if (!isert_comp_wq) { 3279 pr_err("Unable to allocate isert_comp_wq\n"); 3280 ret = -ENOMEM; 3281 goto destroy_rx_wq; 3282 } 3283 3284 iscsit_register_transport(&iser_target_transport); 3285 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 3286 return 0; 3287 3288 destroy_rx_wq: 3289 destroy_workqueue(isert_rx_wq); 3290 return ret; 3291 } 3292 3293 static void __exit isert_exit(void) 3294 { 3295 flush_scheduled_work(); 3296 destroy_workqueue(isert_comp_wq); 3297 destroy_workqueue(isert_rx_wq); 3298 iscsit_unregister_transport(&iser_target_transport); 3299 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 3300 } 3301 3302 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 3303 MODULE_VERSION("0.1"); 3304 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 3305 MODULE_LICENSE("GPL"); 3306 3307 module_init(isert_init); 3308 module_exit(isert_exit); 3309