1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 RisingTide Systems LLC. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 31 #include "isert_proto.h" 32 #include "ib_isert.h" 33 34 #define ISERT_MAX_CONN 8 35 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 37 38 static DEFINE_MUTEX(device_list_mutex); 39 static LIST_HEAD(device_list); 40 static struct workqueue_struct *isert_rx_wq; 41 static struct workqueue_struct *isert_comp_wq; 42 static struct kmem_cache *isert_cmd_cache; 43 44 static void 45 isert_qp_event_callback(struct ib_event *e, void *context) 46 { 47 struct isert_conn *isert_conn = (struct isert_conn *)context; 48 49 pr_err("isert_qp_event_callback event: %d\n", e->event); 50 switch (e->event) { 51 case IB_EVENT_COMM_EST: 52 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 53 break; 54 case IB_EVENT_QP_LAST_WQE_REACHED: 55 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 56 break; 57 default: 58 break; 59 } 60 } 61 62 static int 63 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 64 { 65 int ret; 66 67 ret = ib_query_device(ib_dev, devattr); 68 if (ret) { 69 pr_err("ib_query_device() failed: %d\n", ret); 70 return ret; 71 } 72 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 73 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 74 75 return 0; 76 } 77 78 static int 79 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 80 { 81 struct isert_device *device = isert_conn->conn_device; 82 struct ib_qp_init_attr attr; 83 struct ib_device_attr devattr; 84 int ret, index, min_index = 0; 85 86 memset(&devattr, 0, sizeof(struct ib_device_attr)); 87 ret = isert_query_device(cma_id->device, &devattr); 88 if (ret) 89 return ret; 90 91 mutex_lock(&device_list_mutex); 92 for (index = 0; index < device->cqs_used; index++) 93 if (device->cq_active_qps[index] < 94 device->cq_active_qps[min_index]) 95 min_index = index; 96 device->cq_active_qps[min_index]++; 97 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 98 mutex_unlock(&device_list_mutex); 99 100 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 101 attr.event_handler = isert_qp_event_callback; 102 attr.qp_context = isert_conn; 103 attr.send_cq = device->dev_tx_cq[min_index]; 104 attr.recv_cq = device->dev_rx_cq[min_index]; 105 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 106 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 107 /* 108 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 109 * work-around for RDMA_READ.. 110 */ 111 attr.cap.max_send_sge = devattr.max_sge - 2; 112 isert_conn->max_sge = attr.cap.max_send_sge; 113 114 attr.cap.max_recv_sge = 1; 115 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 116 attr.qp_type = IB_QPT_RC; 117 118 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 119 cma_id->device); 120 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 121 isert_conn->conn_pd->device); 122 123 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 124 if (ret) { 125 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 126 return ret; 127 } 128 isert_conn->conn_qp = cma_id->qp; 129 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 130 131 return 0; 132 } 133 134 static void 135 isert_cq_event_callback(struct ib_event *e, void *context) 136 { 137 pr_debug("isert_cq_event_callback event: %d\n", e->event); 138 } 139 140 static int 141 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 142 { 143 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 144 struct iser_rx_desc *rx_desc; 145 struct ib_sge *rx_sg; 146 u64 dma_addr; 147 int i, j; 148 149 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 150 sizeof(struct iser_rx_desc), GFP_KERNEL); 151 if (!isert_conn->conn_rx_descs) 152 goto fail; 153 154 rx_desc = isert_conn->conn_rx_descs; 155 156 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 157 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 158 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 159 if (ib_dma_mapping_error(ib_dev, dma_addr)) 160 goto dma_map_fail; 161 162 rx_desc->dma_addr = dma_addr; 163 164 rx_sg = &rx_desc->rx_sg; 165 rx_sg->addr = rx_desc->dma_addr; 166 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 167 rx_sg->lkey = isert_conn->conn_mr->lkey; 168 } 169 170 isert_conn->conn_rx_desc_head = 0; 171 return 0; 172 173 dma_map_fail: 174 rx_desc = isert_conn->conn_rx_descs; 175 for (j = 0; j < i; j++, rx_desc++) { 176 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 177 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 178 } 179 kfree(isert_conn->conn_rx_descs); 180 isert_conn->conn_rx_descs = NULL; 181 fail: 182 return -ENOMEM; 183 } 184 185 static void 186 isert_free_rx_descriptors(struct isert_conn *isert_conn) 187 { 188 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 189 struct iser_rx_desc *rx_desc; 190 int i; 191 192 if (!isert_conn->conn_rx_descs) 193 return; 194 195 rx_desc = isert_conn->conn_rx_descs; 196 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 197 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 198 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 199 } 200 201 kfree(isert_conn->conn_rx_descs); 202 isert_conn->conn_rx_descs = NULL; 203 } 204 205 static void isert_cq_tx_callback(struct ib_cq *, void *); 206 static void isert_cq_rx_callback(struct ib_cq *, void *); 207 208 static int 209 isert_create_device_ib_res(struct isert_device *device) 210 { 211 struct ib_device *ib_dev = device->ib_device; 212 struct isert_cq_desc *cq_desc; 213 int ret = 0, i, j; 214 215 device->cqs_used = min_t(int, num_online_cpus(), 216 device->ib_device->num_comp_vectors); 217 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 218 pr_debug("Using %d CQs, device %s supports %d vectors\n", 219 device->cqs_used, device->ib_device->name, 220 device->ib_device->num_comp_vectors); 221 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 222 device->cqs_used, GFP_KERNEL); 223 if (!device->cq_desc) { 224 pr_err("Unable to allocate device->cq_desc\n"); 225 return -ENOMEM; 226 } 227 cq_desc = device->cq_desc; 228 229 device->dev_pd = ib_alloc_pd(ib_dev); 230 if (IS_ERR(device->dev_pd)) { 231 ret = PTR_ERR(device->dev_pd); 232 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret); 233 goto out_cq_desc; 234 } 235 236 for (i = 0; i < device->cqs_used; i++) { 237 cq_desc[i].device = device; 238 cq_desc[i].cq_index = i; 239 240 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 241 isert_cq_rx_callback, 242 isert_cq_event_callback, 243 (void *)&cq_desc[i], 244 ISER_MAX_RX_CQ_LEN, i); 245 if (IS_ERR(device->dev_rx_cq[i])) 246 goto out_cq; 247 248 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 249 isert_cq_tx_callback, 250 isert_cq_event_callback, 251 (void *)&cq_desc[i], 252 ISER_MAX_TX_CQ_LEN, i); 253 if (IS_ERR(device->dev_tx_cq[i])) 254 goto out_cq; 255 256 if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP)) 257 goto out_cq; 258 259 if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP)) 260 goto out_cq; 261 } 262 263 device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE); 264 if (IS_ERR(device->dev_mr)) { 265 ret = PTR_ERR(device->dev_mr); 266 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret); 267 goto out_cq; 268 } 269 270 return 0; 271 272 out_cq: 273 for (j = 0; j < i; j++) { 274 cq_desc = &device->cq_desc[j]; 275 276 if (device->dev_rx_cq[j]) { 277 cancel_work_sync(&cq_desc->cq_rx_work); 278 ib_destroy_cq(device->dev_rx_cq[j]); 279 } 280 if (device->dev_tx_cq[j]) { 281 cancel_work_sync(&cq_desc->cq_tx_work); 282 ib_destroy_cq(device->dev_tx_cq[j]); 283 } 284 } 285 ib_dealloc_pd(device->dev_pd); 286 287 out_cq_desc: 288 kfree(device->cq_desc); 289 290 return ret; 291 } 292 293 static void 294 isert_free_device_ib_res(struct isert_device *device) 295 { 296 struct isert_cq_desc *cq_desc; 297 int i; 298 299 for (i = 0; i < device->cqs_used; i++) { 300 cq_desc = &device->cq_desc[i]; 301 302 cancel_work_sync(&cq_desc->cq_rx_work); 303 cancel_work_sync(&cq_desc->cq_tx_work); 304 ib_destroy_cq(device->dev_rx_cq[i]); 305 ib_destroy_cq(device->dev_tx_cq[i]); 306 device->dev_rx_cq[i] = NULL; 307 device->dev_tx_cq[i] = NULL; 308 } 309 310 ib_dereg_mr(device->dev_mr); 311 ib_dealloc_pd(device->dev_pd); 312 kfree(device->cq_desc); 313 } 314 315 static void 316 isert_device_try_release(struct isert_device *device) 317 { 318 mutex_lock(&device_list_mutex); 319 device->refcount--; 320 if (!device->refcount) { 321 isert_free_device_ib_res(device); 322 list_del(&device->dev_node); 323 kfree(device); 324 } 325 mutex_unlock(&device_list_mutex); 326 } 327 328 static struct isert_device * 329 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 330 { 331 struct isert_device *device; 332 int ret; 333 334 mutex_lock(&device_list_mutex); 335 list_for_each_entry(device, &device_list, dev_node) { 336 if (device->ib_device->node_guid == cma_id->device->node_guid) { 337 device->refcount++; 338 mutex_unlock(&device_list_mutex); 339 return device; 340 } 341 } 342 343 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 344 if (!device) { 345 mutex_unlock(&device_list_mutex); 346 return ERR_PTR(-ENOMEM); 347 } 348 349 INIT_LIST_HEAD(&device->dev_node); 350 351 device->ib_device = cma_id->device; 352 ret = isert_create_device_ib_res(device); 353 if (ret) { 354 kfree(device); 355 mutex_unlock(&device_list_mutex); 356 return ERR_PTR(ret); 357 } 358 359 device->refcount++; 360 list_add_tail(&device->dev_node, &device_list); 361 mutex_unlock(&device_list_mutex); 362 363 return device; 364 } 365 366 static int 367 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 368 { 369 struct iscsi_np *np = cma_id->context; 370 struct isert_np *isert_np = np->np_context; 371 struct isert_conn *isert_conn; 372 struct isert_device *device; 373 struct ib_device *ib_dev = cma_id->device; 374 int ret = 0; 375 376 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 377 cma_id, cma_id->context); 378 379 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 380 if (!isert_conn) { 381 pr_err("Unable to allocate isert_conn\n"); 382 return -ENOMEM; 383 } 384 isert_conn->state = ISER_CONN_INIT; 385 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 386 init_completion(&isert_conn->conn_login_comp); 387 init_waitqueue_head(&isert_conn->conn_wait); 388 init_waitqueue_head(&isert_conn->conn_wait_comp_err); 389 kref_init(&isert_conn->conn_kref); 390 kref_get(&isert_conn->conn_kref); 391 mutex_init(&isert_conn->conn_mutex); 392 393 cma_id->context = isert_conn; 394 isert_conn->conn_cm_id = cma_id; 395 isert_conn->responder_resources = event->param.conn.responder_resources; 396 isert_conn->initiator_depth = event->param.conn.initiator_depth; 397 pr_debug("Using responder_resources: %u initiator_depth: %u\n", 398 isert_conn->responder_resources, isert_conn->initiator_depth); 399 400 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 401 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 402 if (!isert_conn->login_buf) { 403 pr_err("Unable to allocate isert_conn->login_buf\n"); 404 ret = -ENOMEM; 405 goto out; 406 } 407 408 isert_conn->login_req_buf = isert_conn->login_buf; 409 isert_conn->login_rsp_buf = isert_conn->login_buf + 410 ISCSI_DEF_MAX_RECV_SEG_LEN; 411 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 412 isert_conn->login_buf, isert_conn->login_req_buf, 413 isert_conn->login_rsp_buf); 414 415 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 416 (void *)isert_conn->login_req_buf, 417 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 418 419 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 420 if (ret) { 421 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 422 ret); 423 isert_conn->login_req_dma = 0; 424 goto out_login_buf; 425 } 426 427 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 428 (void *)isert_conn->login_rsp_buf, 429 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 430 431 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 432 if (ret) { 433 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 434 ret); 435 isert_conn->login_rsp_dma = 0; 436 goto out_req_dma_map; 437 } 438 439 device = isert_device_find_by_ib_dev(cma_id); 440 if (IS_ERR(device)) { 441 ret = PTR_ERR(device); 442 goto out_rsp_dma_map; 443 } 444 445 isert_conn->conn_device = device; 446 isert_conn->conn_pd = device->dev_pd; 447 isert_conn->conn_mr = device->dev_mr; 448 449 ret = isert_conn_setup_qp(isert_conn, cma_id); 450 if (ret) 451 goto out_conn_dev; 452 453 mutex_lock(&isert_np->np_accept_mutex); 454 list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 455 mutex_unlock(&isert_np->np_accept_mutex); 456 457 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 458 wake_up(&isert_np->np_accept_wq); 459 return 0; 460 461 out_conn_dev: 462 isert_device_try_release(device); 463 out_rsp_dma_map: 464 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 465 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 466 out_req_dma_map: 467 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 468 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 469 out_login_buf: 470 kfree(isert_conn->login_buf); 471 out: 472 kfree(isert_conn); 473 return ret; 474 } 475 476 static void 477 isert_connect_release(struct isert_conn *isert_conn) 478 { 479 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 480 struct isert_device *device = isert_conn->conn_device; 481 int cq_index; 482 483 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 484 485 if (isert_conn->conn_qp) { 486 cq_index = ((struct isert_cq_desc *) 487 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 488 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 489 isert_conn->conn_device->cq_active_qps[cq_index]--; 490 491 rdma_destroy_qp(isert_conn->conn_cm_id); 492 } 493 494 isert_free_rx_descriptors(isert_conn); 495 rdma_destroy_id(isert_conn->conn_cm_id); 496 497 if (isert_conn->login_buf) { 498 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 499 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 500 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 501 ISCSI_DEF_MAX_RECV_SEG_LEN, 502 DMA_FROM_DEVICE); 503 kfree(isert_conn->login_buf); 504 } 505 kfree(isert_conn); 506 507 if (device) 508 isert_device_try_release(device); 509 510 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 511 } 512 513 static void 514 isert_connected_handler(struct rdma_cm_id *cma_id) 515 { 516 return; 517 } 518 519 static void 520 isert_release_conn_kref(struct kref *kref) 521 { 522 struct isert_conn *isert_conn = container_of(kref, 523 struct isert_conn, conn_kref); 524 525 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 526 current->comm, current->pid); 527 528 isert_connect_release(isert_conn); 529 } 530 531 static void 532 isert_put_conn(struct isert_conn *isert_conn) 533 { 534 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 535 } 536 537 static void 538 isert_disconnect_work(struct work_struct *work) 539 { 540 struct isert_conn *isert_conn = container_of(work, 541 struct isert_conn, conn_logout_work); 542 543 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 544 mutex_lock(&isert_conn->conn_mutex); 545 isert_conn->state = ISER_CONN_DOWN; 546 547 if (isert_conn->post_recv_buf_count == 0 && 548 atomic_read(&isert_conn->post_send_buf_count) == 0) { 549 pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); 550 mutex_unlock(&isert_conn->conn_mutex); 551 goto wake_up; 552 } 553 if (!isert_conn->conn_cm_id) { 554 mutex_unlock(&isert_conn->conn_mutex); 555 isert_put_conn(isert_conn); 556 return; 557 } 558 if (!isert_conn->logout_posted) { 559 pr_debug("Calling rdma_disconnect for !logout_posted from" 560 " isert_disconnect_work\n"); 561 rdma_disconnect(isert_conn->conn_cm_id); 562 mutex_unlock(&isert_conn->conn_mutex); 563 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 564 goto wake_up; 565 } 566 mutex_unlock(&isert_conn->conn_mutex); 567 568 wake_up: 569 wake_up(&isert_conn->conn_wait); 570 isert_put_conn(isert_conn); 571 } 572 573 static void 574 isert_disconnected_handler(struct rdma_cm_id *cma_id) 575 { 576 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 577 578 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 579 schedule_work(&isert_conn->conn_logout_work); 580 } 581 582 static int 583 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 584 { 585 int ret = 0; 586 587 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 588 event->event, event->status, cma_id->context, cma_id); 589 590 switch (event->event) { 591 case RDMA_CM_EVENT_CONNECT_REQUEST: 592 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 593 ret = isert_connect_request(cma_id, event); 594 break; 595 case RDMA_CM_EVENT_ESTABLISHED: 596 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 597 isert_connected_handler(cma_id); 598 break; 599 case RDMA_CM_EVENT_DISCONNECTED: 600 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 601 isert_disconnected_handler(cma_id); 602 break; 603 case RDMA_CM_EVENT_DEVICE_REMOVAL: 604 case RDMA_CM_EVENT_ADDR_CHANGE: 605 break; 606 case RDMA_CM_EVENT_CONNECT_ERROR: 607 default: 608 pr_err("Unknown RDMA CMA event: %d\n", event->event); 609 break; 610 } 611 612 if (ret != 0) { 613 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 614 event->event, ret); 615 dump_stack(); 616 } 617 618 return ret; 619 } 620 621 static int 622 isert_post_recv(struct isert_conn *isert_conn, u32 count) 623 { 624 struct ib_recv_wr *rx_wr, *rx_wr_failed; 625 int i, ret; 626 unsigned int rx_head = isert_conn->conn_rx_desc_head; 627 struct iser_rx_desc *rx_desc; 628 629 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 630 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 631 rx_wr->wr_id = (unsigned long)rx_desc; 632 rx_wr->sg_list = &rx_desc->rx_sg; 633 rx_wr->num_sge = 1; 634 rx_wr->next = rx_wr + 1; 635 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 636 } 637 638 rx_wr--; 639 rx_wr->next = NULL; /* mark end of work requests list */ 640 641 isert_conn->post_recv_buf_count += count; 642 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 643 &rx_wr_failed); 644 if (ret) { 645 pr_err("ib_post_recv() failed with ret: %d\n", ret); 646 isert_conn->post_recv_buf_count -= count; 647 } else { 648 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 649 isert_conn->conn_rx_desc_head = rx_head; 650 } 651 return ret; 652 } 653 654 static int 655 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 656 { 657 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 658 struct ib_send_wr send_wr, *send_wr_failed; 659 int ret; 660 661 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 662 ISER_HEADERS_LEN, DMA_TO_DEVICE); 663 664 send_wr.next = NULL; 665 send_wr.wr_id = (unsigned long)tx_desc; 666 send_wr.sg_list = tx_desc->tx_sg; 667 send_wr.num_sge = tx_desc->num_sge; 668 send_wr.opcode = IB_WR_SEND; 669 send_wr.send_flags = IB_SEND_SIGNALED; 670 671 atomic_inc(&isert_conn->post_send_buf_count); 672 673 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 674 if (ret) { 675 pr_err("ib_post_send() failed, ret: %d\n", ret); 676 atomic_dec(&isert_conn->post_send_buf_count); 677 } 678 679 return ret; 680 } 681 682 static void 683 isert_create_send_desc(struct isert_conn *isert_conn, 684 struct isert_cmd *isert_cmd, 685 struct iser_tx_desc *tx_desc) 686 { 687 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 688 689 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 690 ISER_HEADERS_LEN, DMA_TO_DEVICE); 691 692 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 693 tx_desc->iser_header.flags = ISER_VER; 694 695 tx_desc->num_sge = 1; 696 tx_desc->isert_cmd = isert_cmd; 697 698 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 699 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 700 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 701 } 702 } 703 704 static int 705 isert_init_tx_hdrs(struct isert_conn *isert_conn, 706 struct iser_tx_desc *tx_desc) 707 { 708 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 709 u64 dma_addr; 710 711 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 712 ISER_HEADERS_LEN, DMA_TO_DEVICE); 713 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 714 pr_err("ib_dma_mapping_error() failed\n"); 715 return -ENOMEM; 716 } 717 718 tx_desc->dma_addr = dma_addr; 719 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 720 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 721 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 722 723 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 724 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 725 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 726 727 return 0; 728 } 729 730 static void 731 isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr) 732 { 733 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 734 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 735 send_wr->opcode = IB_WR_SEND; 736 send_wr->send_flags = IB_SEND_SIGNALED; 737 send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0]; 738 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 739 } 740 741 static int 742 isert_rdma_post_recvl(struct isert_conn *isert_conn) 743 { 744 struct ib_recv_wr rx_wr, *rx_wr_fail; 745 struct ib_sge sge; 746 int ret; 747 748 memset(&sge, 0, sizeof(struct ib_sge)); 749 sge.addr = isert_conn->login_req_dma; 750 sge.length = ISER_RX_LOGIN_SIZE; 751 sge.lkey = isert_conn->conn_mr->lkey; 752 753 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 754 sge.addr, sge.length, sge.lkey); 755 756 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 757 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 758 rx_wr.sg_list = &sge; 759 rx_wr.num_sge = 1; 760 761 isert_conn->post_recv_buf_count++; 762 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 763 if (ret) { 764 pr_err("ib_post_recv() failed: %d\n", ret); 765 isert_conn->post_recv_buf_count--; 766 } 767 768 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 769 return ret; 770 } 771 772 static int 773 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 774 u32 length) 775 { 776 struct isert_conn *isert_conn = conn->context; 777 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 778 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 779 int ret; 780 781 isert_create_send_desc(isert_conn, NULL, tx_desc); 782 783 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 784 sizeof(struct iscsi_hdr)); 785 786 isert_init_tx_hdrs(isert_conn, tx_desc); 787 788 if (length > 0) { 789 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 790 791 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 792 length, DMA_TO_DEVICE); 793 794 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 795 796 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 797 length, DMA_TO_DEVICE); 798 799 tx_dsg->addr = isert_conn->login_rsp_dma; 800 tx_dsg->length = length; 801 tx_dsg->lkey = isert_conn->conn_mr->lkey; 802 tx_desc->num_sge = 2; 803 } 804 if (!login->login_failed) { 805 if (login->login_complete) { 806 ret = isert_alloc_rx_descriptors(isert_conn); 807 if (ret) 808 return ret; 809 810 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 811 if (ret) 812 return ret; 813 814 isert_conn->state = ISER_CONN_UP; 815 goto post_send; 816 } 817 818 ret = isert_rdma_post_recvl(isert_conn); 819 if (ret) 820 return ret; 821 } 822 post_send: 823 ret = isert_post_send(isert_conn, tx_desc); 824 if (ret) 825 return ret; 826 827 return 0; 828 } 829 830 static void 831 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 832 struct isert_conn *isert_conn) 833 { 834 struct iscsi_conn *conn = isert_conn->conn; 835 struct iscsi_login *login = conn->conn_login; 836 int size; 837 838 if (!login) { 839 pr_err("conn->conn_login is NULL\n"); 840 dump_stack(); 841 return; 842 } 843 844 if (login->first_request) { 845 struct iscsi_login_req *login_req = 846 (struct iscsi_login_req *)&rx_desc->iscsi_header; 847 /* 848 * Setup the initial iscsi_login values from the leading 849 * login request PDU. 850 */ 851 login->leading_connection = (!login_req->tsih) ? 1 : 0; 852 login->current_stage = 853 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 854 >> 2; 855 login->version_min = login_req->min_version; 856 login->version_max = login_req->max_version; 857 memcpy(login->isid, login_req->isid, 6); 858 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 859 login->init_task_tag = login_req->itt; 860 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 861 login->cid = be16_to_cpu(login_req->cid); 862 login->tsih = be16_to_cpu(login_req->tsih); 863 } 864 865 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 866 867 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 868 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 869 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 870 memcpy(login->req_buf, &rx_desc->data[0], size); 871 872 complete(&isert_conn->conn_login_comp); 873 } 874 875 static void 876 isert_release_cmd(struct iscsi_cmd *cmd) 877 { 878 struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd, 879 iscsi_cmd); 880 881 pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd); 882 883 kfree(cmd->buf_ptr); 884 kfree(cmd->tmr_req); 885 886 kmem_cache_free(isert_cmd_cache, isert_cmd); 887 } 888 889 static struct iscsi_cmd 890 *isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp) 891 { 892 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 893 struct isert_cmd *isert_cmd; 894 895 isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp); 896 if (!isert_cmd) { 897 pr_err("Unable to allocate isert_cmd\n"); 898 return NULL; 899 } 900 isert_cmd->conn = isert_conn; 901 isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd; 902 903 return &isert_cmd->iscsi_cmd; 904 } 905 906 static int 907 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 908 struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc, 909 unsigned char *buf) 910 { 911 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 912 struct iscsi_conn *conn = isert_conn->conn; 913 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 914 struct scatterlist *sg; 915 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 916 bool dump_payload = false; 917 918 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 919 if (rc < 0) 920 return rc; 921 922 imm_data = cmd->immediate_data; 923 imm_data_len = cmd->first_burst_len; 924 unsol_data = cmd->unsolicited_data; 925 926 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 927 if (rc < 0) { 928 return 0; 929 } else if (rc > 0) { 930 dump_payload = true; 931 goto sequence_cmd; 932 } 933 934 if (!imm_data) 935 return 0; 936 937 sg = &cmd->se_cmd.t_data_sg[0]; 938 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 939 940 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 941 sg, sg_nents, &rx_desc->data[0], imm_data_len); 942 943 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 944 945 cmd->write_data_done += imm_data_len; 946 947 if (cmd->write_data_done == cmd->se_cmd.data_length) { 948 spin_lock_bh(&cmd->istate_lock); 949 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 950 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 951 spin_unlock_bh(&cmd->istate_lock); 952 } 953 954 sequence_cmd: 955 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 956 957 if (!rc && dump_payload == false && unsol_data) 958 iscsit_set_unsoliticed_dataout(cmd); 959 960 return 0; 961 } 962 963 static int 964 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 965 struct iser_rx_desc *rx_desc, unsigned char *buf) 966 { 967 struct scatterlist *sg_start; 968 struct iscsi_conn *conn = isert_conn->conn; 969 struct iscsi_cmd *cmd = NULL; 970 struct iscsi_data *hdr = (struct iscsi_data *)buf; 971 u32 unsol_data_len = ntoh24(hdr->dlength); 972 int rc, sg_nents, sg_off, page_off; 973 974 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 975 if (rc < 0) 976 return rc; 977 else if (!cmd) 978 return 0; 979 /* 980 * FIXME: Unexpected unsolicited_data out 981 */ 982 if (!cmd->unsolicited_data) { 983 pr_err("Received unexpected solicited data payload\n"); 984 dump_stack(); 985 return -1; 986 } 987 988 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 989 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 990 991 sg_off = cmd->write_data_done / PAGE_SIZE; 992 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 993 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 994 page_off = cmd->write_data_done % PAGE_SIZE; 995 /* 996 * FIXME: Non page-aligned unsolicited_data out 997 */ 998 if (page_off) { 999 pr_err("Received unexpected non-page aligned data payload\n"); 1000 dump_stack(); 1001 return -1; 1002 } 1003 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1004 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1005 1006 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1007 unsol_data_len); 1008 1009 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1010 if (rc < 0) 1011 return rc; 1012 1013 return 0; 1014 } 1015 1016 static int 1017 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1018 struct iser_rx_desc *rx_desc, unsigned char *buf) 1019 { 1020 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1021 struct iscsi_conn *conn = isert_conn->conn; 1022 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1023 int rc; 1024 1025 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1026 if (rc < 0) 1027 return rc; 1028 /* 1029 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1030 */ 1031 1032 return iscsit_process_nop_out(conn, cmd, hdr); 1033 } 1034 1035 static int 1036 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1037 struct iser_rx_desc *rx_desc, struct iscsi_text *hdr) 1038 { 1039 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1040 struct iscsi_conn *conn = isert_conn->conn; 1041 u32 payload_length = ntoh24(hdr->dlength); 1042 int rc; 1043 unsigned char *text_in; 1044 1045 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1046 if (rc < 0) 1047 return rc; 1048 1049 text_in = kzalloc(payload_length, GFP_KERNEL); 1050 if (!text_in) { 1051 pr_err("Unable to allocate text_in of payload_length: %u\n", 1052 payload_length); 1053 return -ENOMEM; 1054 } 1055 cmd->text_in_ptr = text_in; 1056 1057 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1058 1059 return iscsit_process_text_cmd(conn, cmd, hdr); 1060 } 1061 1062 static int 1063 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1064 uint32_t read_stag, uint64_t read_va, 1065 uint32_t write_stag, uint64_t write_va) 1066 { 1067 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1068 struct iscsi_conn *conn = isert_conn->conn; 1069 struct iscsi_session *sess = conn->sess; 1070 struct iscsi_cmd *cmd; 1071 struct isert_cmd *isert_cmd; 1072 int ret = -EINVAL; 1073 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1074 1075 if (sess->sess_ops->SessionType && 1076 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1077 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1078 " ignoring\n", opcode); 1079 return 0; 1080 } 1081 1082 switch (opcode) { 1083 case ISCSI_OP_SCSI_CMD: 1084 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1085 if (!cmd) 1086 break; 1087 1088 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd); 1089 isert_cmd->read_stag = read_stag; 1090 isert_cmd->read_va = read_va; 1091 isert_cmd->write_stag = write_stag; 1092 isert_cmd->write_va = write_va; 1093 1094 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, 1095 rx_desc, (unsigned char *)hdr); 1096 break; 1097 case ISCSI_OP_NOOP_OUT: 1098 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1099 if (!cmd) 1100 break; 1101 1102 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd); 1103 ret = isert_handle_nop_out(isert_conn, isert_cmd, 1104 rx_desc, (unsigned char *)hdr); 1105 break; 1106 case ISCSI_OP_SCSI_DATA_OUT: 1107 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1108 (unsigned char *)hdr); 1109 break; 1110 case ISCSI_OP_SCSI_TMFUNC: 1111 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1112 if (!cmd) 1113 break; 1114 1115 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1116 (unsigned char *)hdr); 1117 break; 1118 case ISCSI_OP_LOGOUT: 1119 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1120 if (!cmd) 1121 break; 1122 1123 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1124 if (ret > 0) 1125 wait_for_completion_timeout(&conn->conn_logout_comp, 1126 SECONDS_FOR_LOGOUT_COMP * 1127 HZ); 1128 break; 1129 case ISCSI_OP_TEXT: 1130 cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); 1131 if (!cmd) 1132 break; 1133 1134 isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd); 1135 ret = isert_handle_text_cmd(isert_conn, isert_cmd, 1136 rx_desc, (struct iscsi_text *)hdr); 1137 break; 1138 default: 1139 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1140 dump_stack(); 1141 break; 1142 } 1143 1144 return ret; 1145 } 1146 1147 static void 1148 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1149 { 1150 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1151 uint64_t read_va = 0, write_va = 0; 1152 uint32_t read_stag = 0, write_stag = 0; 1153 int rc; 1154 1155 switch (iser_hdr->flags & 0xF0) { 1156 case ISCSI_CTRL: 1157 if (iser_hdr->flags & ISER_RSV) { 1158 read_stag = be32_to_cpu(iser_hdr->read_stag); 1159 read_va = be64_to_cpu(iser_hdr->read_va); 1160 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1161 read_stag, (unsigned long long)read_va); 1162 } 1163 if (iser_hdr->flags & ISER_WSV) { 1164 write_stag = be32_to_cpu(iser_hdr->write_stag); 1165 write_va = be64_to_cpu(iser_hdr->write_va); 1166 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1167 write_stag, (unsigned long long)write_va); 1168 } 1169 1170 pr_debug("ISER ISCSI_CTRL PDU\n"); 1171 break; 1172 case ISER_HELLO: 1173 pr_err("iSER Hello message\n"); 1174 break; 1175 default: 1176 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1177 break; 1178 } 1179 1180 rc = isert_rx_opcode(isert_conn, rx_desc, 1181 read_stag, read_va, write_stag, write_va); 1182 } 1183 1184 static void 1185 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1186 unsigned long xfer_len) 1187 { 1188 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1189 struct iscsi_hdr *hdr; 1190 u64 rx_dma; 1191 int rx_buflen, outstanding; 1192 1193 if ((char *)desc == isert_conn->login_req_buf) { 1194 rx_dma = isert_conn->login_req_dma; 1195 rx_buflen = ISER_RX_LOGIN_SIZE; 1196 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1197 rx_dma, rx_buflen); 1198 } else { 1199 rx_dma = desc->dma_addr; 1200 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1201 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1202 rx_dma, rx_buflen); 1203 } 1204 1205 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1206 1207 hdr = &desc->iscsi_header; 1208 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1209 hdr->opcode, hdr->itt, hdr->flags, 1210 (int)(xfer_len - ISER_HEADERS_LEN)); 1211 1212 if ((char *)desc == isert_conn->login_req_buf) 1213 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1214 isert_conn); 1215 else 1216 isert_rx_do_work(desc, isert_conn); 1217 1218 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1219 DMA_FROM_DEVICE); 1220 1221 isert_conn->post_recv_buf_count--; 1222 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1223 isert_conn->post_recv_buf_count); 1224 1225 if ((char *)desc == isert_conn->login_req_buf) 1226 return; 1227 1228 outstanding = isert_conn->post_recv_buf_count; 1229 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1230 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1231 ISERT_MIN_POSTED_RX); 1232 err = isert_post_recv(isert_conn, count); 1233 if (err) { 1234 pr_err("isert_post_recv() count: %d failed, %d\n", 1235 count, err); 1236 } 1237 } 1238 } 1239 1240 static void 1241 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1242 { 1243 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1244 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1245 1246 pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n"); 1247 1248 if (wr->sge) { 1249 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE); 1250 wr->sge = NULL; 1251 } 1252 1253 kfree(wr->send_wr); 1254 wr->send_wr = NULL; 1255 1256 kfree(isert_cmd->ib_sge); 1257 isert_cmd->ib_sge = NULL; 1258 } 1259 1260 static void 1261 isert_put_cmd(struct isert_cmd *isert_cmd) 1262 { 1263 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1264 struct isert_conn *isert_conn = isert_cmd->conn; 1265 struct iscsi_conn *conn = isert_conn->conn; 1266 1267 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1268 1269 switch (cmd->iscsi_opcode) { 1270 case ISCSI_OP_SCSI_CMD: 1271 spin_lock_bh(&conn->cmd_lock); 1272 if (!list_empty(&cmd->i_conn_node)) 1273 list_del(&cmd->i_conn_node); 1274 spin_unlock_bh(&conn->cmd_lock); 1275 1276 if (cmd->data_direction == DMA_TO_DEVICE) 1277 iscsit_stop_dataout_timer(cmd); 1278 1279 isert_unmap_cmd(isert_cmd, isert_conn); 1280 transport_generic_free_cmd(&cmd->se_cmd, 0); 1281 break; 1282 case ISCSI_OP_SCSI_TMFUNC: 1283 spin_lock_bh(&conn->cmd_lock); 1284 if (!list_empty(&cmd->i_conn_node)) 1285 list_del(&cmd->i_conn_node); 1286 spin_unlock_bh(&conn->cmd_lock); 1287 1288 transport_generic_free_cmd(&cmd->se_cmd, 0); 1289 break; 1290 case ISCSI_OP_REJECT: 1291 case ISCSI_OP_NOOP_OUT: 1292 case ISCSI_OP_TEXT: 1293 spin_lock_bh(&conn->cmd_lock); 1294 if (!list_empty(&cmd->i_conn_node)) 1295 list_del(&cmd->i_conn_node); 1296 spin_unlock_bh(&conn->cmd_lock); 1297 1298 /* 1299 * Handle special case for REJECT when iscsi_add_reject*() has 1300 * overwritten the original iscsi_opcode assignment, and the 1301 * associated cmd->se_cmd needs to be released. 1302 */ 1303 if (cmd->se_cmd.se_tfo != NULL) { 1304 pr_debug("Calling transport_generic_free_cmd from" 1305 " isert_put_cmd for 0x%02x\n", 1306 cmd->iscsi_opcode); 1307 transport_generic_free_cmd(&cmd->se_cmd, 0); 1308 break; 1309 } 1310 /* 1311 * Fall-through 1312 */ 1313 default: 1314 isert_release_cmd(cmd); 1315 break; 1316 } 1317 } 1318 1319 static void 1320 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1321 { 1322 if (tx_desc->dma_addr != 0) { 1323 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1324 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1325 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1326 tx_desc->dma_addr = 0; 1327 } 1328 } 1329 1330 static void 1331 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1332 struct ib_device *ib_dev) 1333 { 1334 if (isert_cmd->pdu_buf_dma != 0) { 1335 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1336 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1337 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1338 isert_cmd->pdu_buf_dma = 0; 1339 } 1340 1341 isert_unmap_tx_desc(tx_desc, ib_dev); 1342 isert_put_cmd(isert_cmd); 1343 } 1344 1345 static void 1346 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1347 struct isert_cmd *isert_cmd) 1348 { 1349 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1350 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1351 struct se_cmd *se_cmd = &cmd->se_cmd; 1352 struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device; 1353 1354 iscsit_stop_dataout_timer(cmd); 1355 1356 if (wr->sge) { 1357 pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n"); 1358 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE); 1359 wr->sge = NULL; 1360 } 1361 1362 if (isert_cmd->ib_sge) { 1363 pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n"); 1364 kfree(isert_cmd->ib_sge); 1365 isert_cmd->ib_sge = NULL; 1366 } 1367 1368 cmd->write_data_done = se_cmd->data_length; 1369 1370 pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n"); 1371 spin_lock_bh(&cmd->istate_lock); 1372 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1373 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1374 spin_unlock_bh(&cmd->istate_lock); 1375 1376 target_execute_cmd(se_cmd); 1377 } 1378 1379 static void 1380 isert_do_control_comp(struct work_struct *work) 1381 { 1382 struct isert_cmd *isert_cmd = container_of(work, 1383 struct isert_cmd, comp_work); 1384 struct isert_conn *isert_conn = isert_cmd->conn; 1385 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1386 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1387 1388 switch (cmd->i_state) { 1389 case ISTATE_SEND_TASKMGTRSP: 1390 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1391 1392 atomic_dec(&isert_conn->post_send_buf_count); 1393 iscsit_tmr_post_handler(cmd, cmd->conn); 1394 1395 cmd->i_state = ISTATE_SENT_STATUS; 1396 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1397 break; 1398 case ISTATE_SEND_REJECT: 1399 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1400 atomic_dec(&isert_conn->post_send_buf_count); 1401 1402 cmd->i_state = ISTATE_SENT_STATUS; 1403 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1404 break; 1405 case ISTATE_SEND_LOGOUTRSP: 1406 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1407 /* 1408 * Call atomic_dec(&isert_conn->post_send_buf_count) 1409 * from isert_free_conn() 1410 */ 1411 isert_conn->logout_posted = true; 1412 iscsit_logout_post_handler(cmd, cmd->conn); 1413 break; 1414 case ISTATE_SEND_TEXTRSP: 1415 atomic_dec(&isert_conn->post_send_buf_count); 1416 cmd->i_state = ISTATE_SENT_STATUS; 1417 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1418 break; 1419 default: 1420 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1421 dump_stack(); 1422 break; 1423 } 1424 } 1425 1426 static void 1427 isert_response_completion(struct iser_tx_desc *tx_desc, 1428 struct isert_cmd *isert_cmd, 1429 struct isert_conn *isert_conn, 1430 struct ib_device *ib_dev) 1431 { 1432 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1433 1434 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1435 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1436 cmd->i_state == ISTATE_SEND_REJECT || 1437 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1438 isert_unmap_tx_desc(tx_desc, ib_dev); 1439 1440 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1441 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1442 return; 1443 } 1444 atomic_dec(&isert_conn->post_send_buf_count); 1445 1446 cmd->i_state = ISTATE_SENT_STATUS; 1447 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1448 } 1449 1450 static void 1451 isert_send_completion(struct iser_tx_desc *tx_desc, 1452 struct isert_conn *isert_conn) 1453 { 1454 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1455 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1456 struct isert_rdma_wr *wr; 1457 1458 if (!isert_cmd) { 1459 atomic_dec(&isert_conn->post_send_buf_count); 1460 isert_unmap_tx_desc(tx_desc, ib_dev); 1461 return; 1462 } 1463 wr = &isert_cmd->rdma_wr; 1464 1465 switch (wr->iser_ib_op) { 1466 case ISER_IB_RECV: 1467 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1468 dump_stack(); 1469 break; 1470 case ISER_IB_SEND: 1471 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1472 isert_response_completion(tx_desc, isert_cmd, 1473 isert_conn, ib_dev); 1474 break; 1475 case ISER_IB_RDMA_WRITE: 1476 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1477 dump_stack(); 1478 break; 1479 case ISER_IB_RDMA_READ: 1480 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1481 1482 atomic_dec(&isert_conn->post_send_buf_count); 1483 isert_completion_rdma_read(tx_desc, isert_cmd); 1484 break; 1485 default: 1486 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1487 dump_stack(); 1488 break; 1489 } 1490 } 1491 1492 static void 1493 isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1494 { 1495 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1496 1497 if (tx_desc) { 1498 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1499 1500 if (!isert_cmd) 1501 isert_unmap_tx_desc(tx_desc, ib_dev); 1502 else 1503 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1504 } 1505 1506 if (isert_conn->post_recv_buf_count == 0 && 1507 atomic_read(&isert_conn->post_send_buf_count) == 0) { 1508 pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 1509 pr_debug("Calling wake_up from isert_cq_comp_err\n"); 1510 1511 mutex_lock(&isert_conn->conn_mutex); 1512 if (isert_conn->state != ISER_CONN_DOWN) 1513 isert_conn->state = ISER_CONN_TERMINATING; 1514 mutex_unlock(&isert_conn->conn_mutex); 1515 1516 wake_up(&isert_conn->conn_wait_comp_err); 1517 } 1518 } 1519 1520 static void 1521 isert_cq_tx_work(struct work_struct *work) 1522 { 1523 struct isert_cq_desc *cq_desc = container_of(work, 1524 struct isert_cq_desc, cq_tx_work); 1525 struct isert_device *device = cq_desc->device; 1526 int cq_index = cq_desc->cq_index; 1527 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 1528 struct isert_conn *isert_conn; 1529 struct iser_tx_desc *tx_desc; 1530 struct ib_wc wc; 1531 1532 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 1533 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 1534 isert_conn = wc.qp->qp_context; 1535 1536 if (wc.status == IB_WC_SUCCESS) { 1537 isert_send_completion(tx_desc, isert_conn); 1538 } else { 1539 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1540 pr_debug("TX wc.status: 0x%08x\n", wc.status); 1541 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 1542 atomic_dec(&isert_conn->post_send_buf_count); 1543 isert_cq_comp_err(tx_desc, isert_conn); 1544 } 1545 } 1546 1547 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 1548 } 1549 1550 static void 1551 isert_cq_tx_callback(struct ib_cq *cq, void *context) 1552 { 1553 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1554 1555 INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work); 1556 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 1557 } 1558 1559 static void 1560 isert_cq_rx_work(struct work_struct *work) 1561 { 1562 struct isert_cq_desc *cq_desc = container_of(work, 1563 struct isert_cq_desc, cq_rx_work); 1564 struct isert_device *device = cq_desc->device; 1565 int cq_index = cq_desc->cq_index; 1566 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 1567 struct isert_conn *isert_conn; 1568 struct iser_rx_desc *rx_desc; 1569 struct ib_wc wc; 1570 unsigned long xfer_len; 1571 1572 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 1573 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 1574 isert_conn = wc.qp->qp_context; 1575 1576 if (wc.status == IB_WC_SUCCESS) { 1577 xfer_len = (unsigned long)wc.byte_len; 1578 isert_rx_completion(rx_desc, isert_conn, xfer_len); 1579 } else { 1580 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1581 if (wc.status != IB_WC_WR_FLUSH_ERR) { 1582 pr_debug("RX wc.status: 0x%08x\n", wc.status); 1583 pr_debug("RX wc.vendor_err: 0x%08x\n", 1584 wc.vendor_err); 1585 } 1586 isert_conn->post_recv_buf_count--; 1587 isert_cq_comp_err(NULL, isert_conn); 1588 } 1589 } 1590 1591 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 1592 } 1593 1594 static void 1595 isert_cq_rx_callback(struct ib_cq *cq, void *context) 1596 { 1597 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1598 1599 INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work); 1600 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 1601 } 1602 1603 static int 1604 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1605 { 1606 struct ib_send_wr *wr_failed; 1607 int ret; 1608 1609 atomic_inc(&isert_conn->post_send_buf_count); 1610 1611 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 1612 &wr_failed); 1613 if (ret) { 1614 pr_err("ib_post_send failed with %d\n", ret); 1615 atomic_dec(&isert_conn->post_send_buf_count); 1616 return ret; 1617 } 1618 return ret; 1619 } 1620 1621 static int 1622 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1623 { 1624 struct isert_cmd *isert_cmd = container_of(cmd, 1625 struct isert_cmd, iscsi_cmd); 1626 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1627 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1628 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1629 &isert_cmd->tx_desc.iscsi_header; 1630 1631 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1632 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1633 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1634 /* 1635 * Attach SENSE DATA payload to iSCSI Response PDU 1636 */ 1637 if (cmd->se_cmd.sense_buffer && 1638 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1639 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1640 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1641 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1642 u32 padding, pdu_len; 1643 1644 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1645 cmd->sense_buffer); 1646 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1647 1648 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1649 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1650 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1651 1652 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1653 (void *)cmd->sense_buffer, pdu_len, 1654 DMA_TO_DEVICE); 1655 1656 isert_cmd->pdu_buf_len = pdu_len; 1657 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1658 tx_dsg->length = pdu_len; 1659 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1660 isert_cmd->tx_desc.num_sge = 2; 1661 } 1662 1663 isert_init_send_wr(isert_cmd, send_wr); 1664 1665 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1666 1667 return isert_post_response(isert_conn, isert_cmd); 1668 } 1669 1670 static int 1671 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1672 bool nopout_response) 1673 { 1674 struct isert_cmd *isert_cmd = container_of(cmd, 1675 struct isert_cmd, iscsi_cmd); 1676 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1677 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1678 1679 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1680 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1681 &isert_cmd->tx_desc.iscsi_header, 1682 nopout_response); 1683 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1684 isert_init_send_wr(isert_cmd, send_wr); 1685 1686 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1687 1688 return isert_post_response(isert_conn, isert_cmd); 1689 } 1690 1691 static int 1692 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1693 { 1694 struct isert_cmd *isert_cmd = container_of(cmd, 1695 struct isert_cmd, iscsi_cmd); 1696 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1697 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1698 1699 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1700 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1701 &isert_cmd->tx_desc.iscsi_header); 1702 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1703 isert_init_send_wr(isert_cmd, send_wr); 1704 1705 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1706 1707 return isert_post_response(isert_conn, isert_cmd); 1708 } 1709 1710 static int 1711 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1712 { 1713 struct isert_cmd *isert_cmd = container_of(cmd, 1714 struct isert_cmd, iscsi_cmd); 1715 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1716 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1717 1718 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1719 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1720 &isert_cmd->tx_desc.iscsi_header); 1721 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1722 isert_init_send_wr(isert_cmd, send_wr); 1723 1724 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1725 1726 return isert_post_response(isert_conn, isert_cmd); 1727 } 1728 1729 static int 1730 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1731 { 1732 struct isert_cmd *isert_cmd = container_of(cmd, 1733 struct isert_cmd, iscsi_cmd); 1734 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1735 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1736 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1737 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1738 struct iscsi_reject *hdr = 1739 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1740 1741 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1742 iscsit_build_reject(cmd, conn, hdr); 1743 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1744 1745 hton24(hdr->dlength, ISCSI_HDR_LEN); 1746 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1747 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1748 DMA_TO_DEVICE); 1749 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1750 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1751 tx_dsg->length = ISCSI_HDR_LEN; 1752 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1753 isert_cmd->tx_desc.num_sge = 2; 1754 1755 isert_init_send_wr(isert_cmd, send_wr); 1756 1757 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1758 1759 return isert_post_response(isert_conn, isert_cmd); 1760 } 1761 1762 static int 1763 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1764 { 1765 struct isert_cmd *isert_cmd = container_of(cmd, 1766 struct isert_cmd, iscsi_cmd); 1767 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1768 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1769 struct iscsi_text_rsp *hdr = 1770 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1771 u32 txt_rsp_len; 1772 int rc; 1773 1774 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1775 rc = iscsit_build_text_rsp(cmd, conn, hdr); 1776 if (rc < 0) 1777 return rc; 1778 1779 txt_rsp_len = rc; 1780 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1781 1782 if (txt_rsp_len) { 1783 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1784 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1785 void *txt_rsp_buf = cmd->buf_ptr; 1786 1787 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1788 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1789 1790 isert_cmd->pdu_buf_len = txt_rsp_len; 1791 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1792 tx_dsg->length = txt_rsp_len; 1793 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1794 isert_cmd->tx_desc.num_sge = 2; 1795 } 1796 isert_init_send_wr(isert_cmd, send_wr); 1797 1798 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1799 1800 return isert_post_response(isert_conn, isert_cmd); 1801 } 1802 1803 static int 1804 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1805 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 1806 u32 data_left, u32 offset) 1807 { 1808 struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; 1809 struct scatterlist *sg_start, *tmp_sg; 1810 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1811 u32 sg_off, page_off; 1812 int i = 0, sg_nents; 1813 1814 sg_off = offset / PAGE_SIZE; 1815 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1816 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 1817 page_off = offset % PAGE_SIZE; 1818 1819 send_wr->sg_list = ib_sge; 1820 send_wr->num_sge = sg_nents; 1821 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 1822 /* 1823 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 1824 */ 1825 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 1826 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 1827 (unsigned long long)tmp_sg->dma_address, 1828 tmp_sg->length, page_off); 1829 1830 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 1831 ib_sge->length = min_t(u32, data_left, 1832 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 1833 ib_sge->lkey = isert_conn->conn_mr->lkey; 1834 1835 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u\n", 1836 ib_sge->addr, ib_sge->length); 1837 page_off = 0; 1838 data_left -= ib_sge->length; 1839 ib_sge++; 1840 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 1841 } 1842 1843 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 1844 send_wr->sg_list, send_wr->num_sge); 1845 1846 return sg_nents; 1847 } 1848 1849 static int 1850 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1851 { 1852 struct se_cmd *se_cmd = &cmd->se_cmd; 1853 struct isert_cmd *isert_cmd = container_of(cmd, 1854 struct isert_cmd, iscsi_cmd); 1855 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1856 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1857 struct ib_send_wr *wr_failed, *send_wr; 1858 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1859 struct ib_sge *ib_sge; 1860 struct scatterlist *sg; 1861 u32 offset = 0, data_len, data_left, rdma_write_max; 1862 int rc, ret = 0, count, sg_nents, i, ib_sge_cnt; 1863 1864 pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length); 1865 1866 sg = &se_cmd->t_data_sg[0]; 1867 sg_nents = se_cmd->t_data_nents; 1868 1869 count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE); 1870 if (unlikely(!count)) { 1871 pr_err("Unable to map put_datain SGs\n"); 1872 return -EINVAL; 1873 } 1874 wr->sge = sg; 1875 wr->num_sge = sg_nents; 1876 pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n", 1877 count, sg, sg_nents); 1878 1879 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 1880 if (!ib_sge) { 1881 pr_warn("Unable to allocate datain ib_sge\n"); 1882 ret = -ENOMEM; 1883 goto unmap_sg; 1884 } 1885 isert_cmd->ib_sge = ib_sge; 1886 1887 pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n", 1888 ib_sge, se_cmd->t_data_nents); 1889 1890 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 1891 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 1892 GFP_KERNEL); 1893 if (!wr->send_wr) { 1894 pr_err("Unable to allocate wr->send_wr\n"); 1895 ret = -ENOMEM; 1896 goto unmap_sg; 1897 } 1898 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n", 1899 wr->send_wr, wr->send_wr_num); 1900 1901 iscsit_increment_maxcmdsn(cmd, conn->sess); 1902 cmd->stat_sn = conn->stat_sn++; 1903 1904 wr->isert_cmd = isert_cmd; 1905 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 1906 data_left = se_cmd->data_length; 1907 1908 for (i = 0; i < wr->send_wr_num; i++) { 1909 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 1910 data_len = min(data_left, rdma_write_max); 1911 1912 send_wr->opcode = IB_WR_RDMA_WRITE; 1913 send_wr->send_flags = 0; 1914 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 1915 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 1916 1917 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 1918 send_wr, data_len, offset); 1919 ib_sge += ib_sge_cnt; 1920 1921 if (i + 1 == wr->send_wr_num) 1922 send_wr->next = &isert_cmd->tx_desc.send_wr; 1923 else 1924 send_wr->next = &wr->send_wr[i + 1]; 1925 1926 offset += data_len; 1927 data_left -= data_len; 1928 } 1929 /* 1930 * Build isert_conn->tx_desc for iSCSI response PDU and attach 1931 */ 1932 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1933 iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *) 1934 &isert_cmd->tx_desc.iscsi_header); 1935 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1936 isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr); 1937 1938 atomic_inc(&isert_conn->post_send_buf_count); 1939 1940 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 1941 if (rc) { 1942 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 1943 atomic_dec(&isert_conn->post_send_buf_count); 1944 } 1945 pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n"); 1946 return 1; 1947 1948 unmap_sg: 1949 ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE); 1950 return ret; 1951 } 1952 1953 static int 1954 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 1955 { 1956 struct se_cmd *se_cmd = &cmd->se_cmd; 1957 struct isert_cmd *isert_cmd = container_of(cmd, 1958 struct isert_cmd, iscsi_cmd); 1959 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1960 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1961 struct ib_send_wr *wr_failed, *send_wr; 1962 struct ib_sge *ib_sge; 1963 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1964 struct scatterlist *sg_start; 1965 u32 sg_off, sg_nents, page_off, va_offset = 0; 1966 u32 offset = 0, data_len, data_left, rdma_write_max; 1967 int rc, ret = 0, count, i, ib_sge_cnt; 1968 1969 pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n", 1970 se_cmd->data_length, cmd->write_data_done); 1971 1972 sg_off = cmd->write_data_done / PAGE_SIZE; 1973 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1974 page_off = cmd->write_data_done % PAGE_SIZE; 1975 1976 pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n", 1977 sg_off, sg_start, page_off); 1978 1979 data_left = se_cmd->data_length - cmd->write_data_done; 1980 sg_nents = se_cmd->t_data_nents - sg_off; 1981 1982 pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n", 1983 data_left, sg_nents); 1984 1985 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE); 1986 if (unlikely(!count)) { 1987 pr_err("Unable to map get_dataout SGs\n"); 1988 return -EINVAL; 1989 } 1990 wr->sge = sg_start; 1991 wr->num_sge = sg_nents; 1992 pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n", 1993 count, sg_start, sg_nents); 1994 1995 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 1996 if (!ib_sge) { 1997 pr_warn("Unable to allocate dataout ib_sge\n"); 1998 ret = -ENOMEM; 1999 goto unmap_sg; 2000 } 2001 isert_cmd->ib_sge = ib_sge; 2002 2003 pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n", 2004 ib_sge, sg_nents); 2005 2006 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 2007 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2008 GFP_KERNEL); 2009 if (!wr->send_wr) { 2010 pr_debug("Unable to allocate wr->send_wr\n"); 2011 ret = -ENOMEM; 2012 goto unmap_sg; 2013 } 2014 pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n", 2015 wr->send_wr, wr->send_wr_num); 2016 2017 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2018 2019 wr->iser_ib_op = ISER_IB_RDMA_READ; 2020 wr->isert_cmd = isert_cmd; 2021 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2022 offset = cmd->write_data_done; 2023 2024 for (i = 0; i < wr->send_wr_num; i++) { 2025 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2026 data_len = min(data_left, rdma_write_max); 2027 2028 send_wr->opcode = IB_WR_RDMA_READ; 2029 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2030 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2031 2032 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2033 send_wr, data_len, offset); 2034 ib_sge += ib_sge_cnt; 2035 2036 if (i + 1 == wr->send_wr_num) 2037 send_wr->send_flags = IB_SEND_SIGNALED; 2038 else 2039 send_wr->next = &wr->send_wr[i + 1]; 2040 2041 offset += data_len; 2042 va_offset += data_len; 2043 data_left -= data_len; 2044 } 2045 2046 atomic_inc(&isert_conn->post_send_buf_count); 2047 2048 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2049 if (rc) { 2050 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2051 atomic_dec(&isert_conn->post_send_buf_count); 2052 } 2053 pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n"); 2054 return 0; 2055 2056 unmap_sg: 2057 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE); 2058 return ret; 2059 } 2060 2061 static int 2062 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2063 { 2064 int ret; 2065 2066 switch (state) { 2067 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2068 ret = isert_put_nopin(cmd, conn, false); 2069 break; 2070 default: 2071 pr_err("Unknown immediate state: 0x%02x\n", state); 2072 ret = -EINVAL; 2073 break; 2074 } 2075 2076 return ret; 2077 } 2078 2079 static int 2080 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2081 { 2082 int ret; 2083 2084 switch (state) { 2085 case ISTATE_SEND_LOGOUTRSP: 2086 ret = isert_put_logout_rsp(cmd, conn); 2087 if (!ret) { 2088 pr_debug("Returning iSER Logout -EAGAIN\n"); 2089 ret = -EAGAIN; 2090 } 2091 break; 2092 case ISTATE_SEND_NOPIN: 2093 ret = isert_put_nopin(cmd, conn, true); 2094 break; 2095 case ISTATE_SEND_TASKMGTRSP: 2096 ret = isert_put_tm_rsp(cmd, conn); 2097 break; 2098 case ISTATE_SEND_REJECT: 2099 ret = isert_put_reject(cmd, conn); 2100 break; 2101 case ISTATE_SEND_TEXTRSP: 2102 ret = isert_put_text_rsp(cmd, conn); 2103 break; 2104 case ISTATE_SEND_STATUS: 2105 /* 2106 * Special case for sending non GOOD SCSI status from TX thread 2107 * context during pre se_cmd excecution failure. 2108 */ 2109 ret = isert_put_response(conn, cmd); 2110 break; 2111 default: 2112 pr_err("Unknown response state: 0x%02x\n", state); 2113 ret = -EINVAL; 2114 break; 2115 } 2116 2117 return ret; 2118 } 2119 2120 static int 2121 isert_setup_np(struct iscsi_np *np, 2122 struct __kernel_sockaddr_storage *ksockaddr) 2123 { 2124 struct isert_np *isert_np; 2125 struct rdma_cm_id *isert_lid; 2126 struct sockaddr *sa; 2127 int ret; 2128 2129 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2130 if (!isert_np) { 2131 pr_err("Unable to allocate struct isert_np\n"); 2132 return -ENOMEM; 2133 } 2134 init_waitqueue_head(&isert_np->np_accept_wq); 2135 mutex_init(&isert_np->np_accept_mutex); 2136 INIT_LIST_HEAD(&isert_np->np_accept_list); 2137 init_completion(&isert_np->np_login_comp); 2138 2139 sa = (struct sockaddr *)ksockaddr; 2140 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 2141 /* 2142 * Setup the np->np_sockaddr from the passed sockaddr setup 2143 * in iscsi_target_configfs.c code.. 2144 */ 2145 memcpy(&np->np_sockaddr, ksockaddr, 2146 sizeof(struct __kernel_sockaddr_storage)); 2147 2148 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 2149 IB_QPT_RC); 2150 if (IS_ERR(isert_lid)) { 2151 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 2152 PTR_ERR(isert_lid)); 2153 ret = PTR_ERR(isert_lid); 2154 goto out; 2155 } 2156 2157 ret = rdma_bind_addr(isert_lid, sa); 2158 if (ret) { 2159 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 2160 goto out_lid; 2161 } 2162 2163 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 2164 if (ret) { 2165 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 2166 goto out_lid; 2167 } 2168 2169 isert_np->np_cm_id = isert_lid; 2170 np->np_context = isert_np; 2171 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 2172 2173 return 0; 2174 2175 out_lid: 2176 rdma_destroy_id(isert_lid); 2177 out: 2178 kfree(isert_np); 2179 return ret; 2180 } 2181 2182 static int 2183 isert_check_accept_queue(struct isert_np *isert_np) 2184 { 2185 int empty; 2186 2187 mutex_lock(&isert_np->np_accept_mutex); 2188 empty = list_empty(&isert_np->np_accept_list); 2189 mutex_unlock(&isert_np->np_accept_mutex); 2190 2191 return empty; 2192 } 2193 2194 static int 2195 isert_rdma_accept(struct isert_conn *isert_conn) 2196 { 2197 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2198 struct rdma_conn_param cp; 2199 int ret; 2200 2201 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2202 cp.responder_resources = isert_conn->responder_resources; 2203 cp.initiator_depth = isert_conn->initiator_depth; 2204 cp.retry_count = 7; 2205 cp.rnr_retry_count = 7; 2206 2207 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 2208 2209 ret = rdma_accept(cm_id, &cp); 2210 if (ret) { 2211 pr_err("rdma_accept() failed with: %d\n", ret); 2212 return ret; 2213 } 2214 2215 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 2216 2217 return 0; 2218 } 2219 2220 static int 2221 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2222 { 2223 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2224 int ret; 2225 2226 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 2227 2228 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 2229 if (ret) 2230 return ret; 2231 2232 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 2233 return 0; 2234 } 2235 2236 static void 2237 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2238 struct isert_conn *isert_conn) 2239 { 2240 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2241 struct rdma_route *cm_route = &cm_id->route; 2242 struct sockaddr_in *sock_in; 2243 struct sockaddr_in6 *sock_in6; 2244 2245 conn->login_family = np->np_sockaddr.ss_family; 2246 2247 if (np->np_sockaddr.ss_family == AF_INET6) { 2248 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 2249 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 2250 &sock_in6->sin6_addr.in6_u); 2251 conn->login_port = ntohs(sock_in6->sin6_port); 2252 2253 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 2254 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 2255 &sock_in6->sin6_addr.in6_u); 2256 conn->local_port = ntohs(sock_in6->sin6_port); 2257 } else { 2258 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 2259 sprintf(conn->login_ip, "%pI4", 2260 &sock_in->sin_addr.s_addr); 2261 conn->login_port = ntohs(sock_in->sin_port); 2262 2263 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 2264 sprintf(conn->local_ip, "%pI4", 2265 &sock_in->sin_addr.s_addr); 2266 conn->local_port = ntohs(sock_in->sin_port); 2267 } 2268 } 2269 2270 static int 2271 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2272 { 2273 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2274 struct isert_conn *isert_conn; 2275 int max_accept = 0, ret; 2276 2277 accept_wait: 2278 ret = wait_event_interruptible(isert_np->np_accept_wq, 2279 !isert_check_accept_queue(isert_np) || 2280 np->np_thread_state == ISCSI_NP_THREAD_RESET); 2281 if (max_accept > 5) 2282 return -ENODEV; 2283 2284 spin_lock_bh(&np->np_thread_lock); 2285 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 2286 spin_unlock_bh(&np->np_thread_lock); 2287 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 2288 return -ENODEV; 2289 } 2290 spin_unlock_bh(&np->np_thread_lock); 2291 2292 mutex_lock(&isert_np->np_accept_mutex); 2293 if (list_empty(&isert_np->np_accept_list)) { 2294 mutex_unlock(&isert_np->np_accept_mutex); 2295 max_accept++; 2296 goto accept_wait; 2297 } 2298 isert_conn = list_first_entry(&isert_np->np_accept_list, 2299 struct isert_conn, conn_accept_node); 2300 list_del_init(&isert_conn->conn_accept_node); 2301 mutex_unlock(&isert_np->np_accept_mutex); 2302 2303 conn->context = isert_conn; 2304 isert_conn->conn = conn; 2305 max_accept = 0; 2306 2307 ret = isert_rdma_post_recvl(isert_conn); 2308 if (ret) 2309 return ret; 2310 2311 ret = isert_rdma_accept(isert_conn); 2312 if (ret) 2313 return ret; 2314 2315 isert_set_conn_info(np, conn, isert_conn); 2316 2317 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 2318 return 0; 2319 } 2320 2321 static void 2322 isert_free_np(struct iscsi_np *np) 2323 { 2324 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2325 2326 rdma_destroy_id(isert_np->np_cm_id); 2327 2328 np->np_context = NULL; 2329 kfree(isert_np); 2330 } 2331 2332 static int isert_check_state(struct isert_conn *isert_conn, int state) 2333 { 2334 int ret; 2335 2336 mutex_lock(&isert_conn->conn_mutex); 2337 ret = (isert_conn->state == state); 2338 mutex_unlock(&isert_conn->conn_mutex); 2339 2340 return ret; 2341 } 2342 2343 static void isert_free_conn(struct iscsi_conn *conn) 2344 { 2345 struct isert_conn *isert_conn = conn->context; 2346 2347 pr_debug("isert_free_conn: Starting \n"); 2348 /* 2349 * Decrement post_send_buf_count for special case when called 2350 * from isert_do_control_comp() -> iscsit_logout_post_handler() 2351 */ 2352 mutex_lock(&isert_conn->conn_mutex); 2353 if (isert_conn->logout_posted) 2354 atomic_dec(&isert_conn->post_send_buf_count); 2355 2356 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 2357 pr_debug("Calling rdma_disconnect from isert_free_conn\n"); 2358 rdma_disconnect(isert_conn->conn_cm_id); 2359 } 2360 /* 2361 * Only wait for conn_wait_comp_err if the isert_conn made it 2362 * into full feature phase.. 2363 */ 2364 if (isert_conn->state == ISER_CONN_UP) { 2365 pr_debug("isert_free_conn: Before wait_event comp_err %d\n", 2366 isert_conn->state); 2367 mutex_unlock(&isert_conn->conn_mutex); 2368 2369 wait_event(isert_conn->conn_wait_comp_err, 2370 (isert_check_state(isert_conn, ISER_CONN_TERMINATING))); 2371 2372 wait_event(isert_conn->conn_wait, 2373 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2374 2375 isert_put_conn(isert_conn); 2376 return; 2377 } 2378 if (isert_conn->state == ISER_CONN_INIT) { 2379 mutex_unlock(&isert_conn->conn_mutex); 2380 isert_put_conn(isert_conn); 2381 return; 2382 } 2383 pr_debug("isert_free_conn: wait_event conn_wait %d\n", 2384 isert_conn->state); 2385 mutex_unlock(&isert_conn->conn_mutex); 2386 2387 wait_event(isert_conn->conn_wait, 2388 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2389 2390 isert_put_conn(isert_conn); 2391 } 2392 2393 static struct iscsit_transport iser_target_transport = { 2394 .name = "IB/iSER", 2395 .transport_type = ISCSI_INFINIBAND, 2396 .owner = THIS_MODULE, 2397 .iscsit_setup_np = isert_setup_np, 2398 .iscsit_accept_np = isert_accept_np, 2399 .iscsit_free_np = isert_free_np, 2400 .iscsit_free_conn = isert_free_conn, 2401 .iscsit_alloc_cmd = isert_alloc_cmd, 2402 .iscsit_get_login_rx = isert_get_login_rx, 2403 .iscsit_put_login_tx = isert_put_login_tx, 2404 .iscsit_immediate_queue = isert_immediate_queue, 2405 .iscsit_response_queue = isert_response_queue, 2406 .iscsit_get_dataout = isert_get_dataout, 2407 .iscsit_queue_data_in = isert_put_datain, 2408 .iscsit_queue_status = isert_put_response, 2409 }; 2410 2411 static int __init isert_init(void) 2412 { 2413 int ret; 2414 2415 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 2416 if (!isert_rx_wq) { 2417 pr_err("Unable to allocate isert_rx_wq\n"); 2418 return -ENOMEM; 2419 } 2420 2421 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 2422 if (!isert_comp_wq) { 2423 pr_err("Unable to allocate isert_comp_wq\n"); 2424 ret = -ENOMEM; 2425 goto destroy_rx_wq; 2426 } 2427 2428 isert_cmd_cache = kmem_cache_create("isert_cmd_cache", 2429 sizeof(struct isert_cmd), __alignof__(struct isert_cmd), 2430 0, NULL); 2431 if (!isert_cmd_cache) { 2432 pr_err("Unable to create isert_cmd_cache\n"); 2433 ret = -ENOMEM; 2434 goto destroy_tx_cq; 2435 } 2436 2437 iscsit_register_transport(&iser_target_transport); 2438 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2439 return 0; 2440 2441 destroy_tx_cq: 2442 destroy_workqueue(isert_comp_wq); 2443 destroy_rx_wq: 2444 destroy_workqueue(isert_rx_wq); 2445 return ret; 2446 } 2447 2448 static void __exit isert_exit(void) 2449 { 2450 kmem_cache_destroy(isert_cmd_cache); 2451 destroy_workqueue(isert_comp_wq); 2452 destroy_workqueue(isert_rx_wq); 2453 iscsit_unregister_transport(&iser_target_transport); 2454 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 2455 } 2456 2457 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2458 MODULE_VERSION("0.1"); 2459 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2460 MODULE_LICENSE("GPL"); 2461 2462 module_init(isert_init); 2463 module_exit(isert_exit); 2464