1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <linux/llist.h> 26 #include <rdma/ib_verbs.h> 27 #include <rdma/rdma_cm.h> 28 #include <target/target_core_base.h> 29 #include <target/target_core_fabric.h> 30 #include <target/iscsi/iscsi_transport.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 39 static DEFINE_MUTEX(device_list_mutex); 40 static LIST_HEAD(device_list); 41 static struct workqueue_struct *isert_rx_wq; 42 static struct workqueue_struct *isert_comp_wq; 43 44 static void 45 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 46 static int 47 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 48 struct isert_rdma_wr *wr); 49 static void 50 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51 static int 52 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 53 struct isert_rdma_wr *wr); 54 55 static void 56 isert_qp_event_callback(struct ib_event *e, void *context) 57 { 58 struct isert_conn *isert_conn = (struct isert_conn *)context; 59 60 pr_err("isert_qp_event_callback event: %d\n", e->event); 61 switch (e->event) { 62 case IB_EVENT_COMM_EST: 63 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 64 break; 65 case IB_EVENT_QP_LAST_WQE_REACHED: 66 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 67 break; 68 default: 69 break; 70 } 71 } 72 73 static int 74 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 75 { 76 int ret; 77 78 ret = ib_query_device(ib_dev, devattr); 79 if (ret) { 80 pr_err("ib_query_device() failed: %d\n", ret); 81 return ret; 82 } 83 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 84 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 85 86 return 0; 87 } 88 89 static int 90 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 91 { 92 struct isert_device *device = isert_conn->conn_device; 93 struct ib_qp_init_attr attr; 94 int ret, index, min_index = 0; 95 96 mutex_lock(&device_list_mutex); 97 for (index = 0; index < device->cqs_used; index++) 98 if (device->cq_active_qps[index] < 99 device->cq_active_qps[min_index]) 100 min_index = index; 101 device->cq_active_qps[min_index]++; 102 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 103 mutex_unlock(&device_list_mutex); 104 105 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 106 attr.event_handler = isert_qp_event_callback; 107 attr.qp_context = isert_conn; 108 attr.send_cq = device->dev_tx_cq[min_index]; 109 attr.recv_cq = device->dev_rx_cq[min_index]; 110 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 111 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 112 /* 113 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 114 * work-around for RDMA_READ.. 115 */ 116 attr.cap.max_send_sge = device->dev_attr.max_sge - 2; 117 isert_conn->max_sge = attr.cap.max_send_sge; 118 119 attr.cap.max_recv_sge = 1; 120 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 121 attr.qp_type = IB_QPT_RC; 122 123 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 124 cma_id->device); 125 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 126 isert_conn->conn_pd->device); 127 128 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 129 if (ret) { 130 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 131 return ret; 132 } 133 isert_conn->conn_qp = cma_id->qp; 134 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 135 136 return 0; 137 } 138 139 static void 140 isert_cq_event_callback(struct ib_event *e, void *context) 141 { 142 pr_debug("isert_cq_event_callback event: %d\n", e->event); 143 } 144 145 static int 146 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 147 { 148 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 149 struct iser_rx_desc *rx_desc; 150 struct ib_sge *rx_sg; 151 u64 dma_addr; 152 int i, j; 153 154 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 155 sizeof(struct iser_rx_desc), GFP_KERNEL); 156 if (!isert_conn->conn_rx_descs) 157 goto fail; 158 159 rx_desc = isert_conn->conn_rx_descs; 160 161 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 162 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 163 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 164 if (ib_dma_mapping_error(ib_dev, dma_addr)) 165 goto dma_map_fail; 166 167 rx_desc->dma_addr = dma_addr; 168 169 rx_sg = &rx_desc->rx_sg; 170 rx_sg->addr = rx_desc->dma_addr; 171 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 172 rx_sg->lkey = isert_conn->conn_mr->lkey; 173 } 174 175 isert_conn->conn_rx_desc_head = 0; 176 return 0; 177 178 dma_map_fail: 179 rx_desc = isert_conn->conn_rx_descs; 180 for (j = 0; j < i; j++, rx_desc++) { 181 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 182 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 183 } 184 kfree(isert_conn->conn_rx_descs); 185 isert_conn->conn_rx_descs = NULL; 186 fail: 187 return -ENOMEM; 188 } 189 190 static void 191 isert_free_rx_descriptors(struct isert_conn *isert_conn) 192 { 193 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 194 struct iser_rx_desc *rx_desc; 195 int i; 196 197 if (!isert_conn->conn_rx_descs) 198 return; 199 200 rx_desc = isert_conn->conn_rx_descs; 201 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 202 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 203 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 204 } 205 206 kfree(isert_conn->conn_rx_descs); 207 isert_conn->conn_rx_descs = NULL; 208 } 209 210 static void isert_cq_tx_work(struct work_struct *); 211 static void isert_cq_tx_callback(struct ib_cq *, void *); 212 static void isert_cq_rx_work(struct work_struct *); 213 static void isert_cq_rx_callback(struct ib_cq *, void *); 214 215 static int 216 isert_create_device_ib_res(struct isert_device *device) 217 { 218 struct ib_device *ib_dev = device->ib_device; 219 struct isert_cq_desc *cq_desc; 220 struct ib_device_attr *dev_attr; 221 int ret = 0, i, j; 222 223 dev_attr = &device->dev_attr; 224 ret = isert_query_device(ib_dev, dev_attr); 225 if (ret) 226 return ret; 227 228 /* asign function handlers */ 229 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) { 230 device->use_fastreg = 1; 231 device->reg_rdma_mem = isert_reg_rdma; 232 device->unreg_rdma_mem = isert_unreg_rdma; 233 } else { 234 device->use_fastreg = 0; 235 device->reg_rdma_mem = isert_map_rdma; 236 device->unreg_rdma_mem = isert_unmap_cmd; 237 } 238 239 device->cqs_used = min_t(int, num_online_cpus(), 240 device->ib_device->num_comp_vectors); 241 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 242 pr_debug("Using %d CQs, device %s supports %d vectors support " 243 "Fast registration %d\n", 244 device->cqs_used, device->ib_device->name, 245 device->ib_device->num_comp_vectors, device->use_fastreg); 246 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 247 device->cqs_used, GFP_KERNEL); 248 if (!device->cq_desc) { 249 pr_err("Unable to allocate device->cq_desc\n"); 250 return -ENOMEM; 251 } 252 cq_desc = device->cq_desc; 253 254 for (i = 0; i < device->cqs_used; i++) { 255 cq_desc[i].device = device; 256 cq_desc[i].cq_index = i; 257 258 INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 259 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 260 isert_cq_rx_callback, 261 isert_cq_event_callback, 262 (void *)&cq_desc[i], 263 ISER_MAX_RX_CQ_LEN, i); 264 if (IS_ERR(device->dev_rx_cq[i])) { 265 ret = PTR_ERR(device->dev_rx_cq[i]); 266 device->dev_rx_cq[i] = NULL; 267 goto out_cq; 268 } 269 270 INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 271 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 272 isert_cq_tx_callback, 273 isert_cq_event_callback, 274 (void *)&cq_desc[i], 275 ISER_MAX_TX_CQ_LEN, i); 276 if (IS_ERR(device->dev_tx_cq[i])) { 277 ret = PTR_ERR(device->dev_tx_cq[i]); 278 device->dev_tx_cq[i] = NULL; 279 goto out_cq; 280 } 281 282 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 283 if (ret) 284 goto out_cq; 285 286 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 287 if (ret) 288 goto out_cq; 289 } 290 291 return 0; 292 293 out_cq: 294 for (j = 0; j < i; j++) { 295 cq_desc = &device->cq_desc[j]; 296 297 if (device->dev_rx_cq[j]) { 298 cancel_work_sync(&cq_desc->cq_rx_work); 299 ib_destroy_cq(device->dev_rx_cq[j]); 300 } 301 if (device->dev_tx_cq[j]) { 302 cancel_work_sync(&cq_desc->cq_tx_work); 303 ib_destroy_cq(device->dev_tx_cq[j]); 304 } 305 } 306 kfree(device->cq_desc); 307 308 return ret; 309 } 310 311 static void 312 isert_free_device_ib_res(struct isert_device *device) 313 { 314 struct isert_cq_desc *cq_desc; 315 int i; 316 317 for (i = 0; i < device->cqs_used; i++) { 318 cq_desc = &device->cq_desc[i]; 319 320 cancel_work_sync(&cq_desc->cq_rx_work); 321 cancel_work_sync(&cq_desc->cq_tx_work); 322 ib_destroy_cq(device->dev_rx_cq[i]); 323 ib_destroy_cq(device->dev_tx_cq[i]); 324 device->dev_rx_cq[i] = NULL; 325 device->dev_tx_cq[i] = NULL; 326 } 327 328 kfree(device->cq_desc); 329 } 330 331 static void 332 isert_device_try_release(struct isert_device *device) 333 { 334 mutex_lock(&device_list_mutex); 335 device->refcount--; 336 if (!device->refcount) { 337 isert_free_device_ib_res(device); 338 list_del(&device->dev_node); 339 kfree(device); 340 } 341 mutex_unlock(&device_list_mutex); 342 } 343 344 static struct isert_device * 345 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 346 { 347 struct isert_device *device; 348 int ret; 349 350 mutex_lock(&device_list_mutex); 351 list_for_each_entry(device, &device_list, dev_node) { 352 if (device->ib_device->node_guid == cma_id->device->node_guid) { 353 device->refcount++; 354 mutex_unlock(&device_list_mutex); 355 return device; 356 } 357 } 358 359 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 360 if (!device) { 361 mutex_unlock(&device_list_mutex); 362 return ERR_PTR(-ENOMEM); 363 } 364 365 INIT_LIST_HEAD(&device->dev_node); 366 367 device->ib_device = cma_id->device; 368 ret = isert_create_device_ib_res(device); 369 if (ret) { 370 kfree(device); 371 mutex_unlock(&device_list_mutex); 372 return ERR_PTR(ret); 373 } 374 375 device->refcount++; 376 list_add_tail(&device->dev_node, &device_list); 377 mutex_unlock(&device_list_mutex); 378 379 return device; 380 } 381 382 static void 383 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 384 { 385 struct fast_reg_descriptor *fr_desc, *tmp; 386 int i = 0; 387 388 if (list_empty(&isert_conn->conn_fr_pool)) 389 return; 390 391 pr_debug("Freeing conn %p fastreg pool", isert_conn); 392 393 list_for_each_entry_safe(fr_desc, tmp, 394 &isert_conn->conn_fr_pool, list) { 395 list_del(&fr_desc->list); 396 ib_free_fast_reg_page_list(fr_desc->data_frpl); 397 ib_dereg_mr(fr_desc->data_mr); 398 kfree(fr_desc); 399 ++i; 400 } 401 402 if (i < isert_conn->conn_fr_pool_size) 403 pr_warn("Pool still has %d regions registered\n", 404 isert_conn->conn_fr_pool_size - i); 405 } 406 407 static int 408 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 409 struct fast_reg_descriptor *fr_desc) 410 { 411 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 412 ISCSI_ISER_SG_TABLESIZE); 413 if (IS_ERR(fr_desc->data_frpl)) { 414 pr_err("Failed to allocate data frpl err=%ld\n", 415 PTR_ERR(fr_desc->data_frpl)); 416 return PTR_ERR(fr_desc->data_frpl); 417 } 418 419 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 420 if (IS_ERR(fr_desc->data_mr)) { 421 pr_err("Failed to allocate data frmr err=%ld\n", 422 PTR_ERR(fr_desc->data_mr)); 423 ib_free_fast_reg_page_list(fr_desc->data_frpl); 424 return PTR_ERR(fr_desc->data_mr); 425 } 426 pr_debug("Create fr_desc %p page_list %p\n", 427 fr_desc, fr_desc->data_frpl->page_list); 428 429 fr_desc->valid = true; 430 431 return 0; 432 } 433 434 static int 435 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 436 { 437 struct fast_reg_descriptor *fr_desc; 438 struct isert_device *device = isert_conn->conn_device; 439 int i, ret; 440 441 INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 442 isert_conn->conn_fr_pool_size = 0; 443 for (i = 0; i < ISCSI_DEF_XMIT_CMDS_MAX; i++) { 444 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 445 if (!fr_desc) { 446 pr_err("Failed to allocate fast_reg descriptor\n"); 447 ret = -ENOMEM; 448 goto err; 449 } 450 451 ret = isert_create_fr_desc(device->ib_device, 452 isert_conn->conn_pd, fr_desc); 453 if (ret) { 454 pr_err("Failed to create fastreg descriptor err=%d\n", 455 ret); 456 kfree(fr_desc); 457 goto err; 458 } 459 460 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 461 isert_conn->conn_fr_pool_size++; 462 } 463 464 pr_debug("Creating conn %p fastreg pool size=%d", 465 isert_conn, isert_conn->conn_fr_pool_size); 466 467 return 0; 468 469 err: 470 isert_conn_free_fastreg_pool(isert_conn); 471 return ret; 472 } 473 474 static int 475 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 476 { 477 struct iscsi_np *np = cma_id->context; 478 struct isert_np *isert_np = np->np_context; 479 struct isert_conn *isert_conn; 480 struct isert_device *device; 481 struct ib_device *ib_dev = cma_id->device; 482 int ret = 0; 483 484 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 485 cma_id, cma_id->context); 486 487 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 488 if (!isert_conn) { 489 pr_err("Unable to allocate isert_conn\n"); 490 return -ENOMEM; 491 } 492 isert_conn->state = ISER_CONN_INIT; 493 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 494 init_completion(&isert_conn->conn_login_comp); 495 init_completion(&isert_conn->conn_wait); 496 init_completion(&isert_conn->conn_wait_comp_err); 497 kref_init(&isert_conn->conn_kref); 498 kref_get(&isert_conn->conn_kref); 499 mutex_init(&isert_conn->conn_mutex); 500 spin_lock_init(&isert_conn->conn_lock); 501 502 cma_id->context = isert_conn; 503 isert_conn->conn_cm_id = cma_id; 504 isert_conn->responder_resources = event->param.conn.responder_resources; 505 isert_conn->initiator_depth = event->param.conn.initiator_depth; 506 pr_debug("Using responder_resources: %u initiator_depth: %u\n", 507 isert_conn->responder_resources, isert_conn->initiator_depth); 508 509 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 510 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 511 if (!isert_conn->login_buf) { 512 pr_err("Unable to allocate isert_conn->login_buf\n"); 513 ret = -ENOMEM; 514 goto out; 515 } 516 517 isert_conn->login_req_buf = isert_conn->login_buf; 518 isert_conn->login_rsp_buf = isert_conn->login_buf + 519 ISCSI_DEF_MAX_RECV_SEG_LEN; 520 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 521 isert_conn->login_buf, isert_conn->login_req_buf, 522 isert_conn->login_rsp_buf); 523 524 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 525 (void *)isert_conn->login_req_buf, 526 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 527 528 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 529 if (ret) { 530 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 531 ret); 532 isert_conn->login_req_dma = 0; 533 goto out_login_buf; 534 } 535 536 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 537 (void *)isert_conn->login_rsp_buf, 538 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 539 540 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 541 if (ret) { 542 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 543 ret); 544 isert_conn->login_rsp_dma = 0; 545 goto out_req_dma_map; 546 } 547 548 device = isert_device_find_by_ib_dev(cma_id); 549 if (IS_ERR(device)) { 550 ret = PTR_ERR(device); 551 goto out_rsp_dma_map; 552 } 553 554 isert_conn->conn_device = device; 555 isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 556 if (IS_ERR(isert_conn->conn_pd)) { 557 ret = PTR_ERR(isert_conn->conn_pd); 558 pr_err("ib_alloc_pd failed for conn %p: ret=%d\n", 559 isert_conn, ret); 560 goto out_pd; 561 } 562 563 isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 564 IB_ACCESS_LOCAL_WRITE); 565 if (IS_ERR(isert_conn->conn_mr)) { 566 ret = PTR_ERR(isert_conn->conn_mr); 567 pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 568 isert_conn, ret); 569 goto out_mr; 570 } 571 572 if (device->use_fastreg) { 573 ret = isert_conn_create_fastreg_pool(isert_conn); 574 if (ret) { 575 pr_err("Conn: %p failed to create fastreg pool\n", 576 isert_conn); 577 goto out_fastreg; 578 } 579 } 580 581 ret = isert_conn_setup_qp(isert_conn, cma_id); 582 if (ret) 583 goto out_conn_dev; 584 585 mutex_lock(&isert_np->np_accept_mutex); 586 list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 587 mutex_unlock(&isert_np->np_accept_mutex); 588 589 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 590 wake_up(&isert_np->np_accept_wq); 591 return 0; 592 593 out_conn_dev: 594 if (device->use_fastreg) 595 isert_conn_free_fastreg_pool(isert_conn); 596 out_fastreg: 597 ib_dereg_mr(isert_conn->conn_mr); 598 out_mr: 599 ib_dealloc_pd(isert_conn->conn_pd); 600 out_pd: 601 isert_device_try_release(device); 602 out_rsp_dma_map: 603 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 604 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 605 out_req_dma_map: 606 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 607 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 608 out_login_buf: 609 kfree(isert_conn->login_buf); 610 out: 611 kfree(isert_conn); 612 return ret; 613 } 614 615 static void 616 isert_connect_release(struct isert_conn *isert_conn) 617 { 618 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 619 struct isert_device *device = isert_conn->conn_device; 620 int cq_index; 621 622 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 623 624 if (device && device->use_fastreg) 625 isert_conn_free_fastreg_pool(isert_conn); 626 627 if (isert_conn->conn_qp) { 628 cq_index = ((struct isert_cq_desc *) 629 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 630 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 631 isert_conn->conn_device->cq_active_qps[cq_index]--; 632 633 rdma_destroy_qp(isert_conn->conn_cm_id); 634 } 635 636 isert_free_rx_descriptors(isert_conn); 637 rdma_destroy_id(isert_conn->conn_cm_id); 638 639 ib_dereg_mr(isert_conn->conn_mr); 640 ib_dealloc_pd(isert_conn->conn_pd); 641 642 if (isert_conn->login_buf) { 643 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 644 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 645 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 646 ISCSI_DEF_MAX_RECV_SEG_LEN, 647 DMA_FROM_DEVICE); 648 kfree(isert_conn->login_buf); 649 } 650 kfree(isert_conn); 651 652 if (device) 653 isert_device_try_release(device); 654 655 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 656 } 657 658 static void 659 isert_connected_handler(struct rdma_cm_id *cma_id) 660 { 661 return; 662 } 663 664 static void 665 isert_release_conn_kref(struct kref *kref) 666 { 667 struct isert_conn *isert_conn = container_of(kref, 668 struct isert_conn, conn_kref); 669 670 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 671 current->comm, current->pid); 672 673 isert_connect_release(isert_conn); 674 } 675 676 static void 677 isert_put_conn(struct isert_conn *isert_conn) 678 { 679 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 680 } 681 682 static void 683 isert_disconnect_work(struct work_struct *work) 684 { 685 struct isert_conn *isert_conn = container_of(work, 686 struct isert_conn, conn_logout_work); 687 688 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 689 mutex_lock(&isert_conn->conn_mutex); 690 if (isert_conn->state == ISER_CONN_UP) 691 isert_conn->state = ISER_CONN_TERMINATING; 692 693 if (isert_conn->post_recv_buf_count == 0 && 694 atomic_read(&isert_conn->post_send_buf_count) == 0) { 695 mutex_unlock(&isert_conn->conn_mutex); 696 goto wake_up; 697 } 698 if (!isert_conn->conn_cm_id) { 699 mutex_unlock(&isert_conn->conn_mutex); 700 isert_put_conn(isert_conn); 701 return; 702 } 703 if (!isert_conn->logout_posted) { 704 pr_debug("Calling rdma_disconnect for !logout_posted from" 705 " isert_disconnect_work\n"); 706 rdma_disconnect(isert_conn->conn_cm_id); 707 mutex_unlock(&isert_conn->conn_mutex); 708 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 709 goto wake_up; 710 } 711 mutex_unlock(&isert_conn->conn_mutex); 712 713 wake_up: 714 complete(&isert_conn->conn_wait); 715 isert_put_conn(isert_conn); 716 } 717 718 static void 719 isert_disconnected_handler(struct rdma_cm_id *cma_id) 720 { 721 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 722 723 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 724 schedule_work(&isert_conn->conn_logout_work); 725 } 726 727 static int 728 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 729 { 730 int ret = 0; 731 732 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 733 event->event, event->status, cma_id->context, cma_id); 734 735 switch (event->event) { 736 case RDMA_CM_EVENT_CONNECT_REQUEST: 737 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 738 ret = isert_connect_request(cma_id, event); 739 break; 740 case RDMA_CM_EVENT_ESTABLISHED: 741 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 742 isert_connected_handler(cma_id); 743 break; 744 case RDMA_CM_EVENT_DISCONNECTED: 745 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 746 isert_disconnected_handler(cma_id); 747 break; 748 case RDMA_CM_EVENT_DEVICE_REMOVAL: 749 case RDMA_CM_EVENT_ADDR_CHANGE: 750 break; 751 case RDMA_CM_EVENT_CONNECT_ERROR: 752 default: 753 pr_err("Unknown RDMA CMA event: %d\n", event->event); 754 break; 755 } 756 757 if (ret != 0) { 758 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 759 event->event, ret); 760 dump_stack(); 761 } 762 763 return ret; 764 } 765 766 static int 767 isert_post_recv(struct isert_conn *isert_conn, u32 count) 768 { 769 struct ib_recv_wr *rx_wr, *rx_wr_failed; 770 int i, ret; 771 unsigned int rx_head = isert_conn->conn_rx_desc_head; 772 struct iser_rx_desc *rx_desc; 773 774 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 775 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 776 rx_wr->wr_id = (unsigned long)rx_desc; 777 rx_wr->sg_list = &rx_desc->rx_sg; 778 rx_wr->num_sge = 1; 779 rx_wr->next = rx_wr + 1; 780 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 781 } 782 783 rx_wr--; 784 rx_wr->next = NULL; /* mark end of work requests list */ 785 786 isert_conn->post_recv_buf_count += count; 787 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 788 &rx_wr_failed); 789 if (ret) { 790 pr_err("ib_post_recv() failed with ret: %d\n", ret); 791 isert_conn->post_recv_buf_count -= count; 792 } else { 793 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 794 isert_conn->conn_rx_desc_head = rx_head; 795 } 796 return ret; 797 } 798 799 static int 800 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 801 { 802 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 803 struct ib_send_wr send_wr, *send_wr_failed; 804 int ret; 805 806 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 807 ISER_HEADERS_LEN, DMA_TO_DEVICE); 808 809 send_wr.next = NULL; 810 send_wr.wr_id = (unsigned long)tx_desc; 811 send_wr.sg_list = tx_desc->tx_sg; 812 send_wr.num_sge = tx_desc->num_sge; 813 send_wr.opcode = IB_WR_SEND; 814 send_wr.send_flags = IB_SEND_SIGNALED; 815 816 atomic_inc(&isert_conn->post_send_buf_count); 817 818 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 819 if (ret) { 820 pr_err("ib_post_send() failed, ret: %d\n", ret); 821 atomic_dec(&isert_conn->post_send_buf_count); 822 } 823 824 return ret; 825 } 826 827 static void 828 isert_create_send_desc(struct isert_conn *isert_conn, 829 struct isert_cmd *isert_cmd, 830 struct iser_tx_desc *tx_desc) 831 { 832 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 833 834 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 835 ISER_HEADERS_LEN, DMA_TO_DEVICE); 836 837 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 838 tx_desc->iser_header.flags = ISER_VER; 839 840 tx_desc->num_sge = 1; 841 tx_desc->isert_cmd = isert_cmd; 842 843 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 844 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 845 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 846 } 847 } 848 849 static int 850 isert_init_tx_hdrs(struct isert_conn *isert_conn, 851 struct iser_tx_desc *tx_desc) 852 { 853 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 854 u64 dma_addr; 855 856 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 857 ISER_HEADERS_LEN, DMA_TO_DEVICE); 858 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 859 pr_err("ib_dma_mapping_error() failed\n"); 860 return -ENOMEM; 861 } 862 863 tx_desc->dma_addr = dma_addr; 864 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 865 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 866 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 867 868 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 869 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 870 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 871 872 return 0; 873 } 874 875 static void 876 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 877 struct ib_send_wr *send_wr, bool coalesce) 878 { 879 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 880 881 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 882 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 883 send_wr->opcode = IB_WR_SEND; 884 send_wr->sg_list = &tx_desc->tx_sg[0]; 885 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 886 /* 887 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 888 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 889 */ 890 mutex_lock(&isert_conn->conn_mutex); 891 if (coalesce && isert_conn->state == ISER_CONN_UP && 892 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 893 tx_desc->llnode_active = true; 894 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 895 mutex_unlock(&isert_conn->conn_mutex); 896 return; 897 } 898 isert_conn->conn_comp_batch = 0; 899 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 900 mutex_unlock(&isert_conn->conn_mutex); 901 902 send_wr->send_flags = IB_SEND_SIGNALED; 903 } 904 905 static int 906 isert_rdma_post_recvl(struct isert_conn *isert_conn) 907 { 908 struct ib_recv_wr rx_wr, *rx_wr_fail; 909 struct ib_sge sge; 910 int ret; 911 912 memset(&sge, 0, sizeof(struct ib_sge)); 913 sge.addr = isert_conn->login_req_dma; 914 sge.length = ISER_RX_LOGIN_SIZE; 915 sge.lkey = isert_conn->conn_mr->lkey; 916 917 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 918 sge.addr, sge.length, sge.lkey); 919 920 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 921 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 922 rx_wr.sg_list = &sge; 923 rx_wr.num_sge = 1; 924 925 isert_conn->post_recv_buf_count++; 926 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 927 if (ret) { 928 pr_err("ib_post_recv() failed: %d\n", ret); 929 isert_conn->post_recv_buf_count--; 930 } 931 932 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 933 return ret; 934 } 935 936 static int 937 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 938 u32 length) 939 { 940 struct isert_conn *isert_conn = conn->context; 941 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 942 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 943 int ret; 944 945 isert_create_send_desc(isert_conn, NULL, tx_desc); 946 947 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 948 sizeof(struct iscsi_hdr)); 949 950 isert_init_tx_hdrs(isert_conn, tx_desc); 951 952 if (length > 0) { 953 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 954 955 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 956 length, DMA_TO_DEVICE); 957 958 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 959 960 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 961 length, DMA_TO_DEVICE); 962 963 tx_dsg->addr = isert_conn->login_rsp_dma; 964 tx_dsg->length = length; 965 tx_dsg->lkey = isert_conn->conn_mr->lkey; 966 tx_desc->num_sge = 2; 967 } 968 if (!login->login_failed) { 969 if (login->login_complete) { 970 ret = isert_alloc_rx_descriptors(isert_conn); 971 if (ret) 972 return ret; 973 974 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 975 if (ret) 976 return ret; 977 978 isert_conn->state = ISER_CONN_UP; 979 goto post_send; 980 } 981 982 ret = isert_rdma_post_recvl(isert_conn); 983 if (ret) 984 return ret; 985 } 986 post_send: 987 ret = isert_post_send(isert_conn, tx_desc); 988 if (ret) 989 return ret; 990 991 return 0; 992 } 993 994 static void 995 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 996 struct isert_conn *isert_conn) 997 { 998 struct iscsi_conn *conn = isert_conn->conn; 999 struct iscsi_login *login = conn->conn_login; 1000 int size; 1001 1002 if (!login) { 1003 pr_err("conn->conn_login is NULL\n"); 1004 dump_stack(); 1005 return; 1006 } 1007 1008 if (login->first_request) { 1009 struct iscsi_login_req *login_req = 1010 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1011 /* 1012 * Setup the initial iscsi_login values from the leading 1013 * login request PDU. 1014 */ 1015 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1016 login->current_stage = 1017 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1018 >> 2; 1019 login->version_min = login_req->min_version; 1020 login->version_max = login_req->max_version; 1021 memcpy(login->isid, login_req->isid, 6); 1022 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1023 login->init_task_tag = login_req->itt; 1024 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1025 login->cid = be16_to_cpu(login_req->cid); 1026 login->tsih = be16_to_cpu(login_req->tsih); 1027 } 1028 1029 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1030 1031 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1032 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1033 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1034 memcpy(login->req_buf, &rx_desc->data[0], size); 1035 1036 if (login->first_request) { 1037 complete(&isert_conn->conn_login_comp); 1038 return; 1039 } 1040 schedule_delayed_work(&conn->login_work, 0); 1041 } 1042 1043 static struct iscsi_cmd 1044 *isert_allocate_cmd(struct iscsi_conn *conn) 1045 { 1046 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1047 struct isert_cmd *isert_cmd; 1048 struct iscsi_cmd *cmd; 1049 1050 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1051 if (!cmd) { 1052 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1053 return NULL; 1054 } 1055 isert_cmd = iscsit_priv_cmd(cmd); 1056 isert_cmd->conn = isert_conn; 1057 isert_cmd->iscsi_cmd = cmd; 1058 1059 return cmd; 1060 } 1061 1062 static int 1063 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1064 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1065 struct iser_rx_desc *rx_desc, unsigned char *buf) 1066 { 1067 struct iscsi_conn *conn = isert_conn->conn; 1068 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1069 struct scatterlist *sg; 1070 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1071 bool dump_payload = false; 1072 1073 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1074 if (rc < 0) 1075 return rc; 1076 1077 imm_data = cmd->immediate_data; 1078 imm_data_len = cmd->first_burst_len; 1079 unsol_data = cmd->unsolicited_data; 1080 1081 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1082 if (rc < 0) { 1083 return 0; 1084 } else if (rc > 0) { 1085 dump_payload = true; 1086 goto sequence_cmd; 1087 } 1088 1089 if (!imm_data) 1090 return 0; 1091 1092 sg = &cmd->se_cmd.t_data_sg[0]; 1093 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1094 1095 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1096 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1097 1098 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1099 1100 cmd->write_data_done += imm_data_len; 1101 1102 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1103 spin_lock_bh(&cmd->istate_lock); 1104 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1105 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1106 spin_unlock_bh(&cmd->istate_lock); 1107 } 1108 1109 sequence_cmd: 1110 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1111 1112 if (!rc && dump_payload == false && unsol_data) 1113 iscsit_set_unsoliticed_dataout(cmd); 1114 1115 return 0; 1116 } 1117 1118 static int 1119 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1120 struct iser_rx_desc *rx_desc, unsigned char *buf) 1121 { 1122 struct scatterlist *sg_start; 1123 struct iscsi_conn *conn = isert_conn->conn; 1124 struct iscsi_cmd *cmd = NULL; 1125 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1126 u32 unsol_data_len = ntoh24(hdr->dlength); 1127 int rc, sg_nents, sg_off, page_off; 1128 1129 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1130 if (rc < 0) 1131 return rc; 1132 else if (!cmd) 1133 return 0; 1134 /* 1135 * FIXME: Unexpected unsolicited_data out 1136 */ 1137 if (!cmd->unsolicited_data) { 1138 pr_err("Received unexpected solicited data payload\n"); 1139 dump_stack(); 1140 return -1; 1141 } 1142 1143 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1144 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1145 1146 sg_off = cmd->write_data_done / PAGE_SIZE; 1147 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1148 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1149 page_off = cmd->write_data_done % PAGE_SIZE; 1150 /* 1151 * FIXME: Non page-aligned unsolicited_data out 1152 */ 1153 if (page_off) { 1154 pr_err("Received unexpected non-page aligned data payload\n"); 1155 dump_stack(); 1156 return -1; 1157 } 1158 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1159 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1160 1161 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1162 unsol_data_len); 1163 1164 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1165 if (rc < 0) 1166 return rc; 1167 1168 return 0; 1169 } 1170 1171 static int 1172 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1173 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1174 unsigned char *buf) 1175 { 1176 struct iscsi_conn *conn = isert_conn->conn; 1177 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1178 int rc; 1179 1180 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1181 if (rc < 0) 1182 return rc; 1183 /* 1184 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1185 */ 1186 1187 return iscsit_process_nop_out(conn, cmd, hdr); 1188 } 1189 1190 static int 1191 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1192 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1193 struct iscsi_text *hdr) 1194 { 1195 struct iscsi_conn *conn = isert_conn->conn; 1196 u32 payload_length = ntoh24(hdr->dlength); 1197 int rc; 1198 unsigned char *text_in; 1199 1200 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1201 if (rc < 0) 1202 return rc; 1203 1204 text_in = kzalloc(payload_length, GFP_KERNEL); 1205 if (!text_in) { 1206 pr_err("Unable to allocate text_in of payload_length: %u\n", 1207 payload_length); 1208 return -ENOMEM; 1209 } 1210 cmd->text_in_ptr = text_in; 1211 1212 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1213 1214 return iscsit_process_text_cmd(conn, cmd, hdr); 1215 } 1216 1217 static int 1218 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1219 uint32_t read_stag, uint64_t read_va, 1220 uint32_t write_stag, uint64_t write_va) 1221 { 1222 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1223 struct iscsi_conn *conn = isert_conn->conn; 1224 struct iscsi_session *sess = conn->sess; 1225 struct iscsi_cmd *cmd; 1226 struct isert_cmd *isert_cmd; 1227 int ret = -EINVAL; 1228 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1229 1230 if (sess->sess_ops->SessionType && 1231 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1232 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1233 " ignoring\n", opcode); 1234 return 0; 1235 } 1236 1237 switch (opcode) { 1238 case ISCSI_OP_SCSI_CMD: 1239 cmd = isert_allocate_cmd(conn); 1240 if (!cmd) 1241 break; 1242 1243 isert_cmd = iscsit_priv_cmd(cmd); 1244 isert_cmd->read_stag = read_stag; 1245 isert_cmd->read_va = read_va; 1246 isert_cmd->write_stag = write_stag; 1247 isert_cmd->write_va = write_va; 1248 1249 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1250 rx_desc, (unsigned char *)hdr); 1251 break; 1252 case ISCSI_OP_NOOP_OUT: 1253 cmd = isert_allocate_cmd(conn); 1254 if (!cmd) 1255 break; 1256 1257 isert_cmd = iscsit_priv_cmd(cmd); 1258 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1259 rx_desc, (unsigned char *)hdr); 1260 break; 1261 case ISCSI_OP_SCSI_DATA_OUT: 1262 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1263 (unsigned char *)hdr); 1264 break; 1265 case ISCSI_OP_SCSI_TMFUNC: 1266 cmd = isert_allocate_cmd(conn); 1267 if (!cmd) 1268 break; 1269 1270 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1271 (unsigned char *)hdr); 1272 break; 1273 case ISCSI_OP_LOGOUT: 1274 cmd = isert_allocate_cmd(conn); 1275 if (!cmd) 1276 break; 1277 1278 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1279 if (ret > 0) 1280 wait_for_completion_timeout(&conn->conn_logout_comp, 1281 SECONDS_FOR_LOGOUT_COMP * 1282 HZ); 1283 break; 1284 case ISCSI_OP_TEXT: 1285 cmd = isert_allocate_cmd(conn); 1286 if (!cmd) 1287 break; 1288 1289 isert_cmd = iscsit_priv_cmd(cmd); 1290 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1291 rx_desc, (struct iscsi_text *)hdr); 1292 break; 1293 default: 1294 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1295 dump_stack(); 1296 break; 1297 } 1298 1299 return ret; 1300 } 1301 1302 static void 1303 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1304 { 1305 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1306 uint64_t read_va = 0, write_va = 0; 1307 uint32_t read_stag = 0, write_stag = 0; 1308 int rc; 1309 1310 switch (iser_hdr->flags & 0xF0) { 1311 case ISCSI_CTRL: 1312 if (iser_hdr->flags & ISER_RSV) { 1313 read_stag = be32_to_cpu(iser_hdr->read_stag); 1314 read_va = be64_to_cpu(iser_hdr->read_va); 1315 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1316 read_stag, (unsigned long long)read_va); 1317 } 1318 if (iser_hdr->flags & ISER_WSV) { 1319 write_stag = be32_to_cpu(iser_hdr->write_stag); 1320 write_va = be64_to_cpu(iser_hdr->write_va); 1321 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1322 write_stag, (unsigned long long)write_va); 1323 } 1324 1325 pr_debug("ISER ISCSI_CTRL PDU\n"); 1326 break; 1327 case ISER_HELLO: 1328 pr_err("iSER Hello message\n"); 1329 break; 1330 default: 1331 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1332 break; 1333 } 1334 1335 rc = isert_rx_opcode(isert_conn, rx_desc, 1336 read_stag, read_va, write_stag, write_va); 1337 } 1338 1339 static void 1340 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1341 unsigned long xfer_len) 1342 { 1343 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1344 struct iscsi_hdr *hdr; 1345 u64 rx_dma; 1346 int rx_buflen, outstanding; 1347 1348 if ((char *)desc == isert_conn->login_req_buf) { 1349 rx_dma = isert_conn->login_req_dma; 1350 rx_buflen = ISER_RX_LOGIN_SIZE; 1351 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1352 rx_dma, rx_buflen); 1353 } else { 1354 rx_dma = desc->dma_addr; 1355 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1356 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1357 rx_dma, rx_buflen); 1358 } 1359 1360 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1361 1362 hdr = &desc->iscsi_header; 1363 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1364 hdr->opcode, hdr->itt, hdr->flags, 1365 (int)(xfer_len - ISER_HEADERS_LEN)); 1366 1367 if ((char *)desc == isert_conn->login_req_buf) 1368 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1369 isert_conn); 1370 else 1371 isert_rx_do_work(desc, isert_conn); 1372 1373 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1374 DMA_FROM_DEVICE); 1375 1376 isert_conn->post_recv_buf_count--; 1377 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1378 isert_conn->post_recv_buf_count); 1379 1380 if ((char *)desc == isert_conn->login_req_buf) 1381 return; 1382 1383 outstanding = isert_conn->post_recv_buf_count; 1384 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1385 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1386 ISERT_MIN_POSTED_RX); 1387 err = isert_post_recv(isert_conn, count); 1388 if (err) { 1389 pr_err("isert_post_recv() count: %d failed, %d\n", 1390 count, err); 1391 } 1392 } 1393 } 1394 1395 static void 1396 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1397 { 1398 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1399 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1400 1401 pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1402 if (wr->sge) { 1403 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1404 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1405 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1406 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1407 wr->sge = NULL; 1408 } 1409 1410 if (wr->send_wr) { 1411 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1412 kfree(wr->send_wr); 1413 wr->send_wr = NULL; 1414 } 1415 1416 if (wr->ib_sge) { 1417 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 1418 kfree(wr->ib_sge); 1419 wr->ib_sge = NULL; 1420 } 1421 } 1422 1423 static void 1424 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1425 { 1426 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1427 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1428 LIST_HEAD(unmap_list); 1429 1430 pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd); 1431 1432 if (wr->fr_desc) { 1433 pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n", 1434 isert_cmd, wr->fr_desc); 1435 spin_lock_bh(&isert_conn->conn_lock); 1436 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 1437 spin_unlock_bh(&isert_conn->conn_lock); 1438 wr->fr_desc = NULL; 1439 } 1440 1441 if (wr->sge) { 1442 pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1443 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1444 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1445 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1446 wr->sge = NULL; 1447 } 1448 1449 wr->ib_sge = NULL; 1450 wr->send_wr = NULL; 1451 } 1452 1453 static void 1454 isert_put_cmd(struct isert_cmd *isert_cmd) 1455 { 1456 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1457 struct isert_conn *isert_conn = isert_cmd->conn; 1458 struct iscsi_conn *conn = isert_conn->conn; 1459 struct isert_device *device = isert_conn->conn_device; 1460 1461 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1462 1463 switch (cmd->iscsi_opcode) { 1464 case ISCSI_OP_SCSI_CMD: 1465 spin_lock_bh(&conn->cmd_lock); 1466 if (!list_empty(&cmd->i_conn_node)) 1467 list_del_init(&cmd->i_conn_node); 1468 spin_unlock_bh(&conn->cmd_lock); 1469 1470 if (cmd->data_direction == DMA_TO_DEVICE) 1471 iscsit_stop_dataout_timer(cmd); 1472 1473 device->unreg_rdma_mem(isert_cmd, isert_conn); 1474 transport_generic_free_cmd(&cmd->se_cmd, 0); 1475 break; 1476 case ISCSI_OP_SCSI_TMFUNC: 1477 spin_lock_bh(&conn->cmd_lock); 1478 if (!list_empty(&cmd->i_conn_node)) 1479 list_del_init(&cmd->i_conn_node); 1480 spin_unlock_bh(&conn->cmd_lock); 1481 1482 transport_generic_free_cmd(&cmd->se_cmd, 0); 1483 break; 1484 case ISCSI_OP_REJECT: 1485 case ISCSI_OP_NOOP_OUT: 1486 case ISCSI_OP_TEXT: 1487 spin_lock_bh(&conn->cmd_lock); 1488 if (!list_empty(&cmd->i_conn_node)) 1489 list_del_init(&cmd->i_conn_node); 1490 spin_unlock_bh(&conn->cmd_lock); 1491 1492 /* 1493 * Handle special case for REJECT when iscsi_add_reject*() has 1494 * overwritten the original iscsi_opcode assignment, and the 1495 * associated cmd->se_cmd needs to be released. 1496 */ 1497 if (cmd->se_cmd.se_tfo != NULL) { 1498 pr_debug("Calling transport_generic_free_cmd from" 1499 " isert_put_cmd for 0x%02x\n", 1500 cmd->iscsi_opcode); 1501 transport_generic_free_cmd(&cmd->se_cmd, 0); 1502 break; 1503 } 1504 /* 1505 * Fall-through 1506 */ 1507 default: 1508 iscsit_release_cmd(cmd); 1509 break; 1510 } 1511 } 1512 1513 static void 1514 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1515 { 1516 if (tx_desc->dma_addr != 0) { 1517 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1518 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1519 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1520 tx_desc->dma_addr = 0; 1521 } 1522 } 1523 1524 static void 1525 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1526 struct ib_device *ib_dev) 1527 { 1528 if (isert_cmd->pdu_buf_dma != 0) { 1529 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1530 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1531 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1532 isert_cmd->pdu_buf_dma = 0; 1533 } 1534 1535 isert_unmap_tx_desc(tx_desc, ib_dev); 1536 isert_put_cmd(isert_cmd); 1537 } 1538 1539 static void 1540 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1541 struct isert_cmd *isert_cmd) 1542 { 1543 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1544 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1545 struct se_cmd *se_cmd = &cmd->se_cmd; 1546 struct isert_conn *isert_conn = isert_cmd->conn; 1547 struct isert_device *device = isert_conn->conn_device; 1548 1549 iscsit_stop_dataout_timer(cmd); 1550 device->unreg_rdma_mem(isert_cmd, isert_conn); 1551 cmd->write_data_done = wr->cur_rdma_length; 1552 wr->send_wr_num = 0; 1553 1554 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1555 spin_lock_bh(&cmd->istate_lock); 1556 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1557 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1558 spin_unlock_bh(&cmd->istate_lock); 1559 1560 target_execute_cmd(se_cmd); 1561 } 1562 1563 static void 1564 isert_do_control_comp(struct work_struct *work) 1565 { 1566 struct isert_cmd *isert_cmd = container_of(work, 1567 struct isert_cmd, comp_work); 1568 struct isert_conn *isert_conn = isert_cmd->conn; 1569 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1570 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1571 1572 switch (cmd->i_state) { 1573 case ISTATE_SEND_TASKMGTRSP: 1574 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1575 1576 atomic_dec(&isert_conn->post_send_buf_count); 1577 iscsit_tmr_post_handler(cmd, cmd->conn); 1578 1579 cmd->i_state = ISTATE_SENT_STATUS; 1580 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1581 break; 1582 case ISTATE_SEND_REJECT: 1583 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1584 atomic_dec(&isert_conn->post_send_buf_count); 1585 1586 cmd->i_state = ISTATE_SENT_STATUS; 1587 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1588 break; 1589 case ISTATE_SEND_LOGOUTRSP: 1590 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1591 /* 1592 * Call atomic_dec(&isert_conn->post_send_buf_count) 1593 * from isert_wait_conn() 1594 */ 1595 isert_conn->logout_posted = true; 1596 iscsit_logout_post_handler(cmd, cmd->conn); 1597 break; 1598 case ISTATE_SEND_TEXTRSP: 1599 atomic_dec(&isert_conn->post_send_buf_count); 1600 cmd->i_state = ISTATE_SENT_STATUS; 1601 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1602 break; 1603 default: 1604 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1605 dump_stack(); 1606 break; 1607 } 1608 } 1609 1610 static void 1611 isert_response_completion(struct iser_tx_desc *tx_desc, 1612 struct isert_cmd *isert_cmd, 1613 struct isert_conn *isert_conn, 1614 struct ib_device *ib_dev) 1615 { 1616 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1617 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1618 1619 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1620 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1621 cmd->i_state == ISTATE_SEND_REJECT || 1622 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1623 isert_unmap_tx_desc(tx_desc, ib_dev); 1624 1625 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1626 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1627 return; 1628 } 1629 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count); 1630 1631 cmd->i_state = ISTATE_SENT_STATUS; 1632 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1633 } 1634 1635 static void 1636 __isert_send_completion(struct iser_tx_desc *tx_desc, 1637 struct isert_conn *isert_conn) 1638 { 1639 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1640 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1641 struct isert_rdma_wr *wr; 1642 1643 if (!isert_cmd) { 1644 atomic_dec(&isert_conn->post_send_buf_count); 1645 isert_unmap_tx_desc(tx_desc, ib_dev); 1646 return; 1647 } 1648 wr = &isert_cmd->rdma_wr; 1649 1650 switch (wr->iser_ib_op) { 1651 case ISER_IB_RECV: 1652 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1653 dump_stack(); 1654 break; 1655 case ISER_IB_SEND: 1656 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1657 isert_response_completion(tx_desc, isert_cmd, 1658 isert_conn, ib_dev); 1659 break; 1660 case ISER_IB_RDMA_WRITE: 1661 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1662 dump_stack(); 1663 break; 1664 case ISER_IB_RDMA_READ: 1665 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1666 1667 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 1668 isert_completion_rdma_read(tx_desc, isert_cmd); 1669 break; 1670 default: 1671 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1672 dump_stack(); 1673 break; 1674 } 1675 } 1676 1677 static void 1678 isert_send_completion(struct iser_tx_desc *tx_desc, 1679 struct isert_conn *isert_conn) 1680 { 1681 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1682 struct iser_tx_desc *t; 1683 /* 1684 * Drain coalesced completion llist starting from comp_llnode_batch 1685 * setup in isert_init_send_wr(), and then complete trailing tx_desc. 1686 */ 1687 while (llnode) { 1688 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1689 llnode = llist_next(llnode); 1690 __isert_send_completion(t, isert_conn); 1691 } 1692 __isert_send_completion(tx_desc, isert_conn); 1693 } 1694 1695 static void 1696 isert_cq_drain_comp_llist(struct isert_conn *isert_conn, struct ib_device *ib_dev) 1697 { 1698 struct llist_node *llnode; 1699 struct isert_rdma_wr *wr; 1700 struct iser_tx_desc *t; 1701 1702 mutex_lock(&isert_conn->conn_mutex); 1703 llnode = llist_del_all(&isert_conn->conn_comp_llist); 1704 isert_conn->conn_comp_batch = 0; 1705 mutex_unlock(&isert_conn->conn_mutex); 1706 1707 while (llnode) { 1708 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1709 llnode = llist_next(llnode); 1710 wr = &t->isert_cmd->rdma_wr; 1711 1712 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count); 1713 isert_completion_put(t, t->isert_cmd, ib_dev); 1714 } 1715 } 1716 1717 static void 1718 isert_cq_tx_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1719 { 1720 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1721 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1722 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1723 struct isert_rdma_wr *wr; 1724 struct iser_tx_desc *t; 1725 1726 while (llnode) { 1727 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1728 llnode = llist_next(llnode); 1729 wr = &t->isert_cmd->rdma_wr; 1730 1731 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count); 1732 isert_completion_put(t, t->isert_cmd, ib_dev); 1733 } 1734 tx_desc->comp_llnode_batch = NULL; 1735 1736 if (!isert_cmd) 1737 isert_unmap_tx_desc(tx_desc, ib_dev); 1738 else 1739 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1740 } 1741 1742 static void 1743 isert_cq_rx_comp_err(struct isert_conn *isert_conn) 1744 { 1745 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1746 struct iscsi_conn *conn = isert_conn->conn; 1747 1748 if (isert_conn->post_recv_buf_count) 1749 return; 1750 1751 isert_cq_drain_comp_llist(isert_conn, ib_dev); 1752 1753 if (conn->sess) { 1754 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 1755 target_wait_for_sess_cmds(conn->sess->se_sess); 1756 } 1757 1758 while (atomic_read(&isert_conn->post_send_buf_count)) 1759 msleep(3000); 1760 1761 mutex_lock(&isert_conn->conn_mutex); 1762 isert_conn->state = ISER_CONN_DOWN; 1763 mutex_unlock(&isert_conn->conn_mutex); 1764 1765 complete(&isert_conn->conn_wait_comp_err); 1766 } 1767 1768 static void 1769 isert_cq_tx_work(struct work_struct *work) 1770 { 1771 struct isert_cq_desc *cq_desc = container_of(work, 1772 struct isert_cq_desc, cq_tx_work); 1773 struct isert_device *device = cq_desc->device; 1774 int cq_index = cq_desc->cq_index; 1775 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 1776 struct isert_conn *isert_conn; 1777 struct iser_tx_desc *tx_desc; 1778 struct ib_wc wc; 1779 1780 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 1781 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 1782 isert_conn = wc.qp->qp_context; 1783 1784 if (wc.status == IB_WC_SUCCESS) { 1785 isert_send_completion(tx_desc, isert_conn); 1786 } else { 1787 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1788 pr_debug("TX wc.status: 0x%08x\n", wc.status); 1789 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 1790 1791 if (wc.wr_id != ISER_FASTREG_LI_WRID) { 1792 if (tx_desc->llnode_active) 1793 continue; 1794 1795 atomic_dec(&isert_conn->post_send_buf_count); 1796 isert_cq_tx_comp_err(tx_desc, isert_conn); 1797 } 1798 } 1799 } 1800 1801 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 1802 } 1803 1804 static void 1805 isert_cq_tx_callback(struct ib_cq *cq, void *context) 1806 { 1807 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1808 1809 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 1810 } 1811 1812 static void 1813 isert_cq_rx_work(struct work_struct *work) 1814 { 1815 struct isert_cq_desc *cq_desc = container_of(work, 1816 struct isert_cq_desc, cq_rx_work); 1817 struct isert_device *device = cq_desc->device; 1818 int cq_index = cq_desc->cq_index; 1819 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 1820 struct isert_conn *isert_conn; 1821 struct iser_rx_desc *rx_desc; 1822 struct ib_wc wc; 1823 unsigned long xfer_len; 1824 1825 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 1826 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 1827 isert_conn = wc.qp->qp_context; 1828 1829 if (wc.status == IB_WC_SUCCESS) { 1830 xfer_len = (unsigned long)wc.byte_len; 1831 isert_rx_completion(rx_desc, isert_conn, xfer_len); 1832 } else { 1833 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1834 if (wc.status != IB_WC_WR_FLUSH_ERR) { 1835 pr_debug("RX wc.status: 0x%08x\n", wc.status); 1836 pr_debug("RX wc.vendor_err: 0x%08x\n", 1837 wc.vendor_err); 1838 } 1839 isert_conn->post_recv_buf_count--; 1840 isert_cq_rx_comp_err(isert_conn); 1841 } 1842 } 1843 1844 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 1845 } 1846 1847 static void 1848 isert_cq_rx_callback(struct ib_cq *cq, void *context) 1849 { 1850 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1851 1852 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 1853 } 1854 1855 static int 1856 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1857 { 1858 struct ib_send_wr *wr_failed; 1859 int ret; 1860 1861 atomic_inc(&isert_conn->post_send_buf_count); 1862 1863 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 1864 &wr_failed); 1865 if (ret) { 1866 pr_err("ib_post_send failed with %d\n", ret); 1867 atomic_dec(&isert_conn->post_send_buf_count); 1868 return ret; 1869 } 1870 return ret; 1871 } 1872 1873 static int 1874 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1875 { 1876 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1877 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1878 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1879 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1880 &isert_cmd->tx_desc.iscsi_header; 1881 1882 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1883 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1884 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1885 /* 1886 * Attach SENSE DATA payload to iSCSI Response PDU 1887 */ 1888 if (cmd->se_cmd.sense_buffer && 1889 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1890 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1891 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1892 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1893 u32 padding, pdu_len; 1894 1895 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1896 cmd->sense_buffer); 1897 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1898 1899 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1900 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1901 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1902 1903 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1904 (void *)cmd->sense_buffer, pdu_len, 1905 DMA_TO_DEVICE); 1906 1907 isert_cmd->pdu_buf_len = pdu_len; 1908 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1909 tx_dsg->length = pdu_len; 1910 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1911 isert_cmd->tx_desc.num_sge = 2; 1912 } 1913 1914 isert_init_send_wr(isert_conn, isert_cmd, send_wr, true); 1915 1916 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1917 1918 return isert_post_response(isert_conn, isert_cmd); 1919 } 1920 1921 static int 1922 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1923 bool nopout_response) 1924 { 1925 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1926 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1927 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1928 1929 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1930 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1931 &isert_cmd->tx_desc.iscsi_header, 1932 nopout_response); 1933 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1934 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1935 1936 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1937 1938 return isert_post_response(isert_conn, isert_cmd); 1939 } 1940 1941 static int 1942 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1943 { 1944 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1945 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1946 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1947 1948 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1949 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1950 &isert_cmd->tx_desc.iscsi_header); 1951 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1952 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1953 1954 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1955 1956 return isert_post_response(isert_conn, isert_cmd); 1957 } 1958 1959 static int 1960 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1961 { 1962 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1963 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1964 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1965 1966 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1967 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1968 &isert_cmd->tx_desc.iscsi_header); 1969 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1970 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1971 1972 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1973 1974 return isert_post_response(isert_conn, isert_cmd); 1975 } 1976 1977 static int 1978 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1979 { 1980 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1981 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1982 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1983 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1984 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1985 struct iscsi_reject *hdr = 1986 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1987 1988 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1989 iscsit_build_reject(cmd, conn, hdr); 1990 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1991 1992 hton24(hdr->dlength, ISCSI_HDR_LEN); 1993 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1994 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1995 DMA_TO_DEVICE); 1996 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1997 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1998 tx_dsg->length = ISCSI_HDR_LEN; 1999 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2000 isert_cmd->tx_desc.num_sge = 2; 2001 2002 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2003 2004 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2005 2006 return isert_post_response(isert_conn, isert_cmd); 2007 } 2008 2009 static int 2010 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2011 { 2012 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2013 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2014 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2015 struct iscsi_text_rsp *hdr = 2016 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2017 u32 txt_rsp_len; 2018 int rc; 2019 2020 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2021 rc = iscsit_build_text_rsp(cmd, conn, hdr); 2022 if (rc < 0) 2023 return rc; 2024 2025 txt_rsp_len = rc; 2026 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2027 2028 if (txt_rsp_len) { 2029 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2030 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2031 void *txt_rsp_buf = cmd->buf_ptr; 2032 2033 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2034 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2035 2036 isert_cmd->pdu_buf_len = txt_rsp_len; 2037 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2038 tx_dsg->length = txt_rsp_len; 2039 tx_dsg->lkey = isert_conn->conn_mr->lkey; 2040 isert_cmd->tx_desc.num_sge = 2; 2041 } 2042 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 2043 2044 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 2045 2046 return isert_post_response(isert_conn, isert_cmd); 2047 } 2048 2049 static int 2050 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 2051 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 2052 u32 data_left, u32 offset) 2053 { 2054 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2055 struct scatterlist *sg_start, *tmp_sg; 2056 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2057 u32 sg_off, page_off; 2058 int i = 0, sg_nents; 2059 2060 sg_off = offset / PAGE_SIZE; 2061 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2062 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2063 page_off = offset % PAGE_SIZE; 2064 2065 send_wr->sg_list = ib_sge; 2066 send_wr->num_sge = sg_nents; 2067 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2068 /* 2069 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2070 */ 2071 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2072 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2073 (unsigned long long)tmp_sg->dma_address, 2074 tmp_sg->length, page_off); 2075 2076 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2077 ib_sge->length = min_t(u32, data_left, 2078 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2079 ib_sge->lkey = isert_conn->conn_mr->lkey; 2080 2081 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2082 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2083 page_off = 0; 2084 data_left -= ib_sge->length; 2085 ib_sge++; 2086 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2087 } 2088 2089 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2090 send_wr->sg_list, send_wr->num_sge); 2091 2092 return sg_nents; 2093 } 2094 2095 static int 2096 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2097 struct isert_rdma_wr *wr) 2098 { 2099 struct se_cmd *se_cmd = &cmd->se_cmd; 2100 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2101 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2102 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2103 struct ib_send_wr *send_wr; 2104 struct ib_sge *ib_sge; 2105 struct scatterlist *sg_start; 2106 u32 sg_off = 0, sg_nents; 2107 u32 offset = 0, data_len, data_left, rdma_write_max, va_offset = 0; 2108 int ret = 0, count, i, ib_sge_cnt; 2109 2110 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2111 data_left = se_cmd->data_length; 2112 } else { 2113 sg_off = cmd->write_data_done / PAGE_SIZE; 2114 data_left = se_cmd->data_length - cmd->write_data_done; 2115 offset = cmd->write_data_done; 2116 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2117 } 2118 2119 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2120 sg_nents = se_cmd->t_data_nents - sg_off; 2121 2122 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2123 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2124 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2125 if (unlikely(!count)) { 2126 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2127 return -EINVAL; 2128 } 2129 wr->sge = sg_start; 2130 wr->num_sge = sg_nents; 2131 wr->cur_rdma_length = data_left; 2132 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2133 isert_cmd, count, sg_start, sg_nents, data_left); 2134 2135 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 2136 if (!ib_sge) { 2137 pr_warn("Unable to allocate ib_sge\n"); 2138 ret = -ENOMEM; 2139 goto unmap_sg; 2140 } 2141 wr->ib_sge = ib_sge; 2142 2143 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 2144 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2145 GFP_KERNEL); 2146 if (!wr->send_wr) { 2147 pr_debug("Unable to allocate wr->send_wr\n"); 2148 ret = -ENOMEM; 2149 goto unmap_sg; 2150 } 2151 2152 wr->isert_cmd = isert_cmd; 2153 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2154 2155 for (i = 0; i < wr->send_wr_num; i++) { 2156 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2157 data_len = min(data_left, rdma_write_max); 2158 2159 send_wr->send_flags = 0; 2160 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2161 send_wr->opcode = IB_WR_RDMA_WRITE; 2162 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2163 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2164 if (i + 1 == wr->send_wr_num) 2165 send_wr->next = &isert_cmd->tx_desc.send_wr; 2166 else 2167 send_wr->next = &wr->send_wr[i + 1]; 2168 } else { 2169 send_wr->opcode = IB_WR_RDMA_READ; 2170 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2171 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2172 if (i + 1 == wr->send_wr_num) 2173 send_wr->send_flags = IB_SEND_SIGNALED; 2174 else 2175 send_wr->next = &wr->send_wr[i + 1]; 2176 } 2177 2178 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2179 send_wr, data_len, offset); 2180 ib_sge += ib_sge_cnt; 2181 2182 offset += data_len; 2183 va_offset += data_len; 2184 data_left -= data_len; 2185 } 2186 2187 return 0; 2188 unmap_sg: 2189 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2190 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2191 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2192 return ret; 2193 } 2194 2195 static int 2196 isert_map_fr_pagelist(struct ib_device *ib_dev, 2197 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2198 { 2199 u64 start_addr, end_addr, page, chunk_start = 0; 2200 struct scatterlist *tmp_sg; 2201 int i = 0, new_chunk, last_ent, n_pages; 2202 2203 n_pages = 0; 2204 new_chunk = 1; 2205 last_ent = sg_nents - 1; 2206 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2207 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2208 if (new_chunk) 2209 chunk_start = start_addr; 2210 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2211 2212 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 2213 i, (unsigned long long)tmp_sg->dma_address, 2214 tmp_sg->length); 2215 2216 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2217 new_chunk = 0; 2218 continue; 2219 } 2220 new_chunk = 1; 2221 2222 page = chunk_start & PAGE_MASK; 2223 do { 2224 fr_pl[n_pages++] = page; 2225 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 2226 n_pages - 1, page); 2227 page += PAGE_SIZE; 2228 } while (page < end_addr); 2229 } 2230 2231 return n_pages; 2232 } 2233 2234 static int 2235 isert_fast_reg_mr(struct fast_reg_descriptor *fr_desc, 2236 struct isert_conn *isert_conn, struct scatterlist *sg_start, 2237 struct ib_sge *ib_sge, u32 sg_nents, u32 offset, 2238 unsigned int data_len) 2239 { 2240 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2241 struct ib_send_wr fr_wr, inv_wr; 2242 struct ib_send_wr *bad_wr, *wr = NULL; 2243 int ret, pagelist_len; 2244 u32 page_off; 2245 u8 key; 2246 2247 sg_nents = min_t(unsigned int, sg_nents, ISCSI_ISER_SG_TABLESIZE); 2248 page_off = offset % PAGE_SIZE; 2249 2250 pr_debug("Use fr_desc %p sg_nents %d offset %u\n", 2251 fr_desc, sg_nents, offset); 2252 2253 pagelist_len = isert_map_fr_pagelist(ib_dev, sg_start, sg_nents, 2254 &fr_desc->data_frpl->page_list[0]); 2255 2256 if (!fr_desc->valid) { 2257 memset(&inv_wr, 0, sizeof(inv_wr)); 2258 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 2259 inv_wr.opcode = IB_WR_LOCAL_INV; 2260 inv_wr.ex.invalidate_rkey = fr_desc->data_mr->rkey; 2261 wr = &inv_wr; 2262 /* Bump the key */ 2263 key = (u8)(fr_desc->data_mr->rkey & 0x000000FF); 2264 ib_update_fast_reg_key(fr_desc->data_mr, ++key); 2265 } 2266 2267 /* Prepare FASTREG WR */ 2268 memset(&fr_wr, 0, sizeof(fr_wr)); 2269 fr_wr.wr_id = ISER_FASTREG_LI_WRID; 2270 fr_wr.opcode = IB_WR_FAST_REG_MR; 2271 fr_wr.wr.fast_reg.iova_start = 2272 fr_desc->data_frpl->page_list[0] + page_off; 2273 fr_wr.wr.fast_reg.page_list = fr_desc->data_frpl; 2274 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2275 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2276 fr_wr.wr.fast_reg.length = data_len; 2277 fr_wr.wr.fast_reg.rkey = fr_desc->data_mr->rkey; 2278 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2279 2280 if (!wr) 2281 wr = &fr_wr; 2282 else 2283 wr->next = &fr_wr; 2284 2285 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2286 if (ret) { 2287 pr_err("fast registration failed, ret:%d\n", ret); 2288 return ret; 2289 } 2290 fr_desc->valid = false; 2291 2292 ib_sge->lkey = fr_desc->data_mr->lkey; 2293 ib_sge->addr = fr_desc->data_frpl->page_list[0] + page_off; 2294 ib_sge->length = data_len; 2295 2296 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2297 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2298 2299 return ret; 2300 } 2301 2302 static int 2303 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2304 struct isert_rdma_wr *wr) 2305 { 2306 struct se_cmd *se_cmd = &cmd->se_cmd; 2307 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2308 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2309 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2310 struct ib_send_wr *send_wr; 2311 struct ib_sge *ib_sge; 2312 struct scatterlist *sg_start; 2313 struct fast_reg_descriptor *fr_desc; 2314 u32 sg_off = 0, sg_nents; 2315 u32 offset = 0, data_len, data_left, rdma_write_max; 2316 int ret = 0, count; 2317 unsigned long flags; 2318 2319 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2320 data_left = se_cmd->data_length; 2321 } else { 2322 offset = cmd->write_data_done; 2323 sg_off = offset / PAGE_SIZE; 2324 data_left = se_cmd->data_length - cmd->write_data_done; 2325 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2326 } 2327 2328 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2329 sg_nents = se_cmd->t_data_nents - sg_off; 2330 2331 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2332 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2333 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2334 if (unlikely(!count)) { 2335 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2336 return -EINVAL; 2337 } 2338 wr->sge = sg_start; 2339 wr->num_sge = sg_nents; 2340 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2341 isert_cmd, count, sg_start, sg_nents, data_left); 2342 2343 memset(&wr->s_ib_sge, 0, sizeof(*ib_sge)); 2344 ib_sge = &wr->s_ib_sge; 2345 wr->ib_sge = ib_sge; 2346 2347 wr->send_wr_num = 1; 2348 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2349 wr->send_wr = &wr->s_send_wr; 2350 2351 wr->isert_cmd = isert_cmd; 2352 rdma_write_max = ISCSI_ISER_SG_TABLESIZE * PAGE_SIZE; 2353 2354 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2355 send_wr->sg_list = ib_sge; 2356 send_wr->num_sge = 1; 2357 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2358 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2359 send_wr->opcode = IB_WR_RDMA_WRITE; 2360 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2361 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2362 send_wr->send_flags = 0; 2363 send_wr->next = &isert_cmd->tx_desc.send_wr; 2364 } else { 2365 send_wr->opcode = IB_WR_RDMA_READ; 2366 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2367 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2368 send_wr->send_flags = IB_SEND_SIGNALED; 2369 } 2370 2371 data_len = min(data_left, rdma_write_max); 2372 wr->cur_rdma_length = data_len; 2373 2374 /* if there is a single dma entry, dma mr is sufficient */ 2375 if (count == 1) { 2376 ib_sge->addr = ib_sg_dma_address(ib_dev, &sg_start[0]); 2377 ib_sge->length = ib_sg_dma_len(ib_dev, &sg_start[0]); 2378 ib_sge->lkey = isert_conn->conn_mr->lkey; 2379 wr->fr_desc = NULL; 2380 } else { 2381 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2382 fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2383 struct fast_reg_descriptor, list); 2384 list_del(&fr_desc->list); 2385 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2386 wr->fr_desc = fr_desc; 2387 2388 ret = isert_fast_reg_mr(fr_desc, isert_conn, sg_start, 2389 ib_sge, sg_nents, offset, data_len); 2390 if (ret) { 2391 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2392 goto unmap_sg; 2393 } 2394 } 2395 2396 return 0; 2397 2398 unmap_sg: 2399 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2400 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2401 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2402 return ret; 2403 } 2404 2405 static int 2406 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2407 { 2408 struct se_cmd *se_cmd = &cmd->se_cmd; 2409 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2410 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2411 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2412 struct isert_device *device = isert_conn->conn_device; 2413 struct ib_send_wr *wr_failed; 2414 int rc; 2415 2416 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 2417 isert_cmd, se_cmd->data_length); 2418 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2419 rc = device->reg_rdma_mem(conn, cmd, wr); 2420 if (rc) { 2421 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2422 return rc; 2423 } 2424 2425 /* 2426 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2427 */ 2428 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2429 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2430 &isert_cmd->tx_desc.iscsi_header); 2431 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2432 isert_init_send_wr(isert_conn, isert_cmd, 2433 &isert_cmd->tx_desc.send_wr, true); 2434 2435 atomic_add(wr->send_wr_num + 1, &isert_conn->post_send_buf_count); 2436 2437 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2438 if (rc) { 2439 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2440 atomic_sub(wr->send_wr_num + 1, &isert_conn->post_send_buf_count); 2441 } 2442 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data READ\n", 2443 isert_cmd); 2444 2445 return 1; 2446 } 2447 2448 static int 2449 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2450 { 2451 struct se_cmd *se_cmd = &cmd->se_cmd; 2452 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2453 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2454 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2455 struct isert_device *device = isert_conn->conn_device; 2456 struct ib_send_wr *wr_failed; 2457 int rc; 2458 2459 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2460 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2461 wr->iser_ib_op = ISER_IB_RDMA_READ; 2462 rc = device->reg_rdma_mem(conn, cmd, wr); 2463 if (rc) { 2464 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2465 return rc; 2466 } 2467 2468 atomic_add(wr->send_wr_num, &isert_conn->post_send_buf_count); 2469 2470 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2471 if (rc) { 2472 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2473 atomic_sub(wr->send_wr_num, &isert_conn->post_send_buf_count); 2474 } 2475 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2476 isert_cmd); 2477 2478 return 0; 2479 } 2480 2481 static int 2482 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2483 { 2484 int ret; 2485 2486 switch (state) { 2487 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2488 ret = isert_put_nopin(cmd, conn, false); 2489 break; 2490 default: 2491 pr_err("Unknown immediate state: 0x%02x\n", state); 2492 ret = -EINVAL; 2493 break; 2494 } 2495 2496 return ret; 2497 } 2498 2499 static int 2500 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2501 { 2502 int ret; 2503 2504 switch (state) { 2505 case ISTATE_SEND_LOGOUTRSP: 2506 ret = isert_put_logout_rsp(cmd, conn); 2507 if (!ret) { 2508 pr_debug("Returning iSER Logout -EAGAIN\n"); 2509 ret = -EAGAIN; 2510 } 2511 break; 2512 case ISTATE_SEND_NOPIN: 2513 ret = isert_put_nopin(cmd, conn, true); 2514 break; 2515 case ISTATE_SEND_TASKMGTRSP: 2516 ret = isert_put_tm_rsp(cmd, conn); 2517 break; 2518 case ISTATE_SEND_REJECT: 2519 ret = isert_put_reject(cmd, conn); 2520 break; 2521 case ISTATE_SEND_TEXTRSP: 2522 ret = isert_put_text_rsp(cmd, conn); 2523 break; 2524 case ISTATE_SEND_STATUS: 2525 /* 2526 * Special case for sending non GOOD SCSI status from TX thread 2527 * context during pre se_cmd excecution failure. 2528 */ 2529 ret = isert_put_response(conn, cmd); 2530 break; 2531 default: 2532 pr_err("Unknown response state: 0x%02x\n", state); 2533 ret = -EINVAL; 2534 break; 2535 } 2536 2537 return ret; 2538 } 2539 2540 static int 2541 isert_setup_np(struct iscsi_np *np, 2542 struct __kernel_sockaddr_storage *ksockaddr) 2543 { 2544 struct isert_np *isert_np; 2545 struct rdma_cm_id *isert_lid; 2546 struct sockaddr *sa; 2547 int ret; 2548 2549 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2550 if (!isert_np) { 2551 pr_err("Unable to allocate struct isert_np\n"); 2552 return -ENOMEM; 2553 } 2554 init_waitqueue_head(&isert_np->np_accept_wq); 2555 mutex_init(&isert_np->np_accept_mutex); 2556 INIT_LIST_HEAD(&isert_np->np_accept_list); 2557 init_completion(&isert_np->np_login_comp); 2558 2559 sa = (struct sockaddr *)ksockaddr; 2560 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 2561 /* 2562 * Setup the np->np_sockaddr from the passed sockaddr setup 2563 * in iscsi_target_configfs.c code.. 2564 */ 2565 memcpy(&np->np_sockaddr, ksockaddr, 2566 sizeof(struct __kernel_sockaddr_storage)); 2567 2568 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 2569 IB_QPT_RC); 2570 if (IS_ERR(isert_lid)) { 2571 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 2572 PTR_ERR(isert_lid)); 2573 ret = PTR_ERR(isert_lid); 2574 goto out; 2575 } 2576 2577 ret = rdma_bind_addr(isert_lid, sa); 2578 if (ret) { 2579 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 2580 goto out_lid; 2581 } 2582 2583 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 2584 if (ret) { 2585 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 2586 goto out_lid; 2587 } 2588 2589 isert_np->np_cm_id = isert_lid; 2590 np->np_context = isert_np; 2591 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 2592 2593 return 0; 2594 2595 out_lid: 2596 rdma_destroy_id(isert_lid); 2597 out: 2598 kfree(isert_np); 2599 return ret; 2600 } 2601 2602 static int 2603 isert_check_accept_queue(struct isert_np *isert_np) 2604 { 2605 int empty; 2606 2607 mutex_lock(&isert_np->np_accept_mutex); 2608 empty = list_empty(&isert_np->np_accept_list); 2609 mutex_unlock(&isert_np->np_accept_mutex); 2610 2611 return empty; 2612 } 2613 2614 static int 2615 isert_rdma_accept(struct isert_conn *isert_conn) 2616 { 2617 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2618 struct rdma_conn_param cp; 2619 int ret; 2620 2621 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2622 cp.responder_resources = isert_conn->responder_resources; 2623 cp.initiator_depth = isert_conn->initiator_depth; 2624 cp.retry_count = 7; 2625 cp.rnr_retry_count = 7; 2626 2627 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 2628 2629 ret = rdma_accept(cm_id, &cp); 2630 if (ret) { 2631 pr_err("rdma_accept() failed with: %d\n", ret); 2632 return ret; 2633 } 2634 2635 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 2636 2637 return 0; 2638 } 2639 2640 static int 2641 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2642 { 2643 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2644 int ret; 2645 2646 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 2647 /* 2648 * For login requests after the first PDU, isert_rx_login_req() will 2649 * kick schedule_delayed_work(&conn->login_work) as the packet is 2650 * received, which turns this callback from iscsi_target_do_login_rx() 2651 * into a NOP. 2652 */ 2653 if (!login->first_request) 2654 return 0; 2655 2656 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 2657 if (ret) 2658 return ret; 2659 2660 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 2661 return 0; 2662 } 2663 2664 static void 2665 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2666 struct isert_conn *isert_conn) 2667 { 2668 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2669 struct rdma_route *cm_route = &cm_id->route; 2670 struct sockaddr_in *sock_in; 2671 struct sockaddr_in6 *sock_in6; 2672 2673 conn->login_family = np->np_sockaddr.ss_family; 2674 2675 if (np->np_sockaddr.ss_family == AF_INET6) { 2676 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 2677 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 2678 &sock_in6->sin6_addr.in6_u); 2679 conn->login_port = ntohs(sock_in6->sin6_port); 2680 2681 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 2682 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 2683 &sock_in6->sin6_addr.in6_u); 2684 conn->local_port = ntohs(sock_in6->sin6_port); 2685 } else { 2686 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 2687 sprintf(conn->login_ip, "%pI4", 2688 &sock_in->sin_addr.s_addr); 2689 conn->login_port = ntohs(sock_in->sin_port); 2690 2691 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 2692 sprintf(conn->local_ip, "%pI4", 2693 &sock_in->sin_addr.s_addr); 2694 conn->local_port = ntohs(sock_in->sin_port); 2695 } 2696 } 2697 2698 static int 2699 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2700 { 2701 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2702 struct isert_conn *isert_conn; 2703 int max_accept = 0, ret; 2704 2705 accept_wait: 2706 ret = wait_event_interruptible(isert_np->np_accept_wq, 2707 !isert_check_accept_queue(isert_np) || 2708 np->np_thread_state == ISCSI_NP_THREAD_RESET); 2709 if (max_accept > 5) 2710 return -ENODEV; 2711 2712 spin_lock_bh(&np->np_thread_lock); 2713 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 2714 spin_unlock_bh(&np->np_thread_lock); 2715 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 2716 return -ENODEV; 2717 } 2718 spin_unlock_bh(&np->np_thread_lock); 2719 2720 mutex_lock(&isert_np->np_accept_mutex); 2721 if (list_empty(&isert_np->np_accept_list)) { 2722 mutex_unlock(&isert_np->np_accept_mutex); 2723 max_accept++; 2724 goto accept_wait; 2725 } 2726 isert_conn = list_first_entry(&isert_np->np_accept_list, 2727 struct isert_conn, conn_accept_node); 2728 list_del_init(&isert_conn->conn_accept_node); 2729 mutex_unlock(&isert_np->np_accept_mutex); 2730 2731 conn->context = isert_conn; 2732 isert_conn->conn = conn; 2733 max_accept = 0; 2734 2735 ret = isert_rdma_post_recvl(isert_conn); 2736 if (ret) 2737 return ret; 2738 2739 ret = isert_rdma_accept(isert_conn); 2740 if (ret) 2741 return ret; 2742 2743 isert_set_conn_info(np, conn, isert_conn); 2744 2745 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 2746 return 0; 2747 } 2748 2749 static void 2750 isert_free_np(struct iscsi_np *np) 2751 { 2752 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2753 2754 rdma_destroy_id(isert_np->np_cm_id); 2755 2756 np->np_context = NULL; 2757 kfree(isert_np); 2758 } 2759 2760 static void isert_wait_conn(struct iscsi_conn *conn) 2761 { 2762 struct isert_conn *isert_conn = conn->context; 2763 2764 pr_debug("isert_wait_conn: Starting \n"); 2765 /* 2766 * Decrement post_send_buf_count for special case when called 2767 * from isert_do_control_comp() -> iscsit_logout_post_handler() 2768 */ 2769 mutex_lock(&isert_conn->conn_mutex); 2770 if (isert_conn->logout_posted) 2771 atomic_dec(&isert_conn->post_send_buf_count); 2772 2773 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 2774 pr_debug("Calling rdma_disconnect from isert_wait_conn\n"); 2775 rdma_disconnect(isert_conn->conn_cm_id); 2776 } 2777 /* 2778 * Only wait for conn_wait_comp_err if the isert_conn made it 2779 * into full feature phase.. 2780 */ 2781 if (isert_conn->state == ISER_CONN_INIT) { 2782 mutex_unlock(&isert_conn->conn_mutex); 2783 return; 2784 } 2785 if (isert_conn->state == ISER_CONN_UP) 2786 isert_conn->state = ISER_CONN_TERMINATING; 2787 mutex_unlock(&isert_conn->conn_mutex); 2788 2789 wait_for_completion(&isert_conn->conn_wait_comp_err); 2790 2791 wait_for_completion(&isert_conn->conn_wait); 2792 } 2793 2794 static void isert_free_conn(struct iscsi_conn *conn) 2795 { 2796 struct isert_conn *isert_conn = conn->context; 2797 2798 isert_put_conn(isert_conn); 2799 } 2800 2801 static struct iscsit_transport iser_target_transport = { 2802 .name = "IB/iSER", 2803 .transport_type = ISCSI_INFINIBAND, 2804 .priv_size = sizeof(struct isert_cmd), 2805 .owner = THIS_MODULE, 2806 .iscsit_setup_np = isert_setup_np, 2807 .iscsit_accept_np = isert_accept_np, 2808 .iscsit_free_np = isert_free_np, 2809 .iscsit_wait_conn = isert_wait_conn, 2810 .iscsit_free_conn = isert_free_conn, 2811 .iscsit_get_login_rx = isert_get_login_rx, 2812 .iscsit_put_login_tx = isert_put_login_tx, 2813 .iscsit_immediate_queue = isert_immediate_queue, 2814 .iscsit_response_queue = isert_response_queue, 2815 .iscsit_get_dataout = isert_get_dataout, 2816 .iscsit_queue_data_in = isert_put_datain, 2817 .iscsit_queue_status = isert_put_response, 2818 }; 2819 2820 static int __init isert_init(void) 2821 { 2822 int ret; 2823 2824 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 2825 if (!isert_rx_wq) { 2826 pr_err("Unable to allocate isert_rx_wq\n"); 2827 return -ENOMEM; 2828 } 2829 2830 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 2831 if (!isert_comp_wq) { 2832 pr_err("Unable to allocate isert_comp_wq\n"); 2833 ret = -ENOMEM; 2834 goto destroy_rx_wq; 2835 } 2836 2837 iscsit_register_transport(&iser_target_transport); 2838 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2839 return 0; 2840 2841 destroy_rx_wq: 2842 destroy_workqueue(isert_rx_wq); 2843 return ret; 2844 } 2845 2846 static void __exit isert_exit(void) 2847 { 2848 destroy_workqueue(isert_comp_wq); 2849 destroy_workqueue(isert_rx_wq); 2850 iscsit_unregister_transport(&iser_target_transport); 2851 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 2852 } 2853 2854 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2855 MODULE_VERSION("0.1"); 2856 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2857 MODULE_LICENSE("GPL"); 2858 2859 module_init(isert_init); 2860 module_exit(isert_exit); 2861