1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "ib_isert.h" 33 34 #define ISERT_MAX_CONN 8 35 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36 #define ISER_MAX_TX_CQ_LEN \ 37 ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 static int isert_debug_level; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static int 51 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 52 static int 53 isert_login_post_recv(struct isert_conn *isert_conn); 54 static int 55 isert_rdma_accept(struct isert_conn *isert_conn); 56 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 57 58 static void isert_release_work(struct work_struct *work); 59 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 61 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 62 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 63 64 static inline bool 65 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 66 { 67 return (conn->pi_support && 68 cmd->prot_op != TARGET_PROT_NORMAL); 69 } 70 71 72 static void 73 isert_qp_event_callback(struct ib_event *e, void *context) 74 { 75 struct isert_conn *isert_conn = context; 76 77 isert_err("%s (%d): conn %p\n", 78 ib_event_msg(e->event), e->event, isert_conn); 79 80 switch (e->event) { 81 case IB_EVENT_COMM_EST: 82 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 83 break; 84 case IB_EVENT_QP_LAST_WQE_REACHED: 85 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 86 break; 87 default: 88 break; 89 } 90 } 91 92 static struct isert_comp * 93 isert_comp_get(struct isert_conn *isert_conn) 94 { 95 struct isert_device *device = isert_conn->device; 96 struct isert_comp *comp; 97 int i, min = 0; 98 99 mutex_lock(&device_list_mutex); 100 for (i = 0; i < device->comps_used; i++) 101 if (device->comps[i].active_qps < 102 device->comps[min].active_qps) 103 min = i; 104 comp = &device->comps[min]; 105 comp->active_qps++; 106 mutex_unlock(&device_list_mutex); 107 108 isert_info("conn %p, using comp %p min_index: %d\n", 109 isert_conn, comp, min); 110 111 return comp; 112 } 113 114 static void 115 isert_comp_put(struct isert_comp *comp) 116 { 117 mutex_lock(&device_list_mutex); 118 comp->active_qps--; 119 mutex_unlock(&device_list_mutex); 120 } 121 122 static struct ib_qp * 123 isert_create_qp(struct isert_conn *isert_conn, 124 struct isert_comp *comp, 125 struct rdma_cm_id *cma_id) 126 { 127 struct isert_device *device = isert_conn->device; 128 struct ib_qp_init_attr attr; 129 int ret; 130 131 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 132 attr.event_handler = isert_qp_event_callback; 133 attr.qp_context = isert_conn; 134 attr.send_cq = comp->cq; 135 attr.recv_cq = comp->cq; 136 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 137 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 138 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX; 139 attr.cap.max_send_sge = device->ib_device->attrs.max_sge; 140 isert_conn->max_sge = min(device->ib_device->attrs.max_sge, 141 device->ib_device->attrs.max_sge_rd); 142 attr.cap.max_recv_sge = 1; 143 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 144 attr.qp_type = IB_QPT_RC; 145 if (device->pi_capable) 146 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 147 148 ret = rdma_create_qp(cma_id, device->pd, &attr); 149 if (ret) { 150 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 151 return ERR_PTR(ret); 152 } 153 154 return cma_id->qp; 155 } 156 157 static int 158 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 159 { 160 struct isert_comp *comp; 161 int ret; 162 163 comp = isert_comp_get(isert_conn); 164 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 165 if (IS_ERR(isert_conn->qp)) { 166 ret = PTR_ERR(isert_conn->qp); 167 goto err; 168 } 169 170 return 0; 171 err: 172 isert_comp_put(comp); 173 return ret; 174 } 175 176 static int 177 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 178 { 179 struct isert_device *device = isert_conn->device; 180 struct ib_device *ib_dev = device->ib_device; 181 struct iser_rx_desc *rx_desc; 182 struct ib_sge *rx_sg; 183 u64 dma_addr; 184 int i, j; 185 186 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 187 sizeof(struct iser_rx_desc), GFP_KERNEL); 188 if (!isert_conn->rx_descs) 189 goto fail; 190 191 rx_desc = isert_conn->rx_descs; 192 193 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 194 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 195 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 196 if (ib_dma_mapping_error(ib_dev, dma_addr)) 197 goto dma_map_fail; 198 199 rx_desc->dma_addr = dma_addr; 200 201 rx_sg = &rx_desc->rx_sg; 202 rx_sg->addr = rx_desc->dma_addr; 203 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 204 rx_sg->lkey = device->pd->local_dma_lkey; 205 rx_desc->rx_cqe.done = isert_recv_done; 206 } 207 208 return 0; 209 210 dma_map_fail: 211 rx_desc = isert_conn->rx_descs; 212 for (j = 0; j < i; j++, rx_desc++) { 213 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 214 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 215 } 216 kfree(isert_conn->rx_descs); 217 isert_conn->rx_descs = NULL; 218 fail: 219 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 220 221 return -ENOMEM; 222 } 223 224 static void 225 isert_free_rx_descriptors(struct isert_conn *isert_conn) 226 { 227 struct ib_device *ib_dev = isert_conn->device->ib_device; 228 struct iser_rx_desc *rx_desc; 229 int i; 230 231 if (!isert_conn->rx_descs) 232 return; 233 234 rx_desc = isert_conn->rx_descs; 235 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 236 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 237 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 238 } 239 240 kfree(isert_conn->rx_descs); 241 isert_conn->rx_descs = NULL; 242 } 243 244 static void 245 isert_free_comps(struct isert_device *device) 246 { 247 int i; 248 249 for (i = 0; i < device->comps_used; i++) { 250 struct isert_comp *comp = &device->comps[i]; 251 252 if (comp->cq) 253 ib_free_cq(comp->cq); 254 } 255 kfree(device->comps); 256 } 257 258 static int 259 isert_alloc_comps(struct isert_device *device) 260 { 261 int i, max_cqe, ret = 0; 262 263 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 264 device->ib_device->num_comp_vectors)); 265 266 isert_info("Using %d CQs, %s supports %d vectors support " 267 "pi_capable %d\n", 268 device->comps_used, device->ib_device->name, 269 device->ib_device->num_comp_vectors, 270 device->pi_capable); 271 272 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 273 GFP_KERNEL); 274 if (!device->comps) { 275 isert_err("Unable to allocate completion contexts\n"); 276 return -ENOMEM; 277 } 278 279 max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 280 281 for (i = 0; i < device->comps_used; i++) { 282 struct isert_comp *comp = &device->comps[i]; 283 284 comp->device = device; 285 comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 286 IB_POLL_WORKQUEUE); 287 if (IS_ERR(comp->cq)) { 288 isert_err("Unable to allocate cq\n"); 289 ret = PTR_ERR(comp->cq); 290 comp->cq = NULL; 291 goto out_cq; 292 } 293 } 294 295 return 0; 296 out_cq: 297 isert_free_comps(device); 298 return ret; 299 } 300 301 static int 302 isert_create_device_ib_res(struct isert_device *device) 303 { 304 struct ib_device *ib_dev = device->ib_device; 305 int ret; 306 307 isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge); 308 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 309 310 ret = isert_alloc_comps(device); 311 if (ret) 312 goto out; 313 314 device->pd = ib_alloc_pd(ib_dev); 315 if (IS_ERR(device->pd)) { 316 ret = PTR_ERR(device->pd); 317 isert_err("failed to allocate pd, device %p, ret=%d\n", 318 device, ret); 319 goto out_cq; 320 } 321 322 /* Check signature cap */ 323 device->pi_capable = ib_dev->attrs.device_cap_flags & 324 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 325 326 return 0; 327 328 out_cq: 329 isert_free_comps(device); 330 out: 331 if (ret > 0) 332 ret = -EINVAL; 333 return ret; 334 } 335 336 static void 337 isert_free_device_ib_res(struct isert_device *device) 338 { 339 isert_info("device %p\n", device); 340 341 ib_dealloc_pd(device->pd); 342 isert_free_comps(device); 343 } 344 345 static void 346 isert_device_put(struct isert_device *device) 347 { 348 mutex_lock(&device_list_mutex); 349 device->refcount--; 350 isert_info("device %p refcount %d\n", device, device->refcount); 351 if (!device->refcount) { 352 isert_free_device_ib_res(device); 353 list_del(&device->dev_node); 354 kfree(device); 355 } 356 mutex_unlock(&device_list_mutex); 357 } 358 359 static struct isert_device * 360 isert_device_get(struct rdma_cm_id *cma_id) 361 { 362 struct isert_device *device; 363 int ret; 364 365 mutex_lock(&device_list_mutex); 366 list_for_each_entry(device, &device_list, dev_node) { 367 if (device->ib_device->node_guid == cma_id->device->node_guid) { 368 device->refcount++; 369 isert_info("Found iser device %p refcount %d\n", 370 device, device->refcount); 371 mutex_unlock(&device_list_mutex); 372 return device; 373 } 374 } 375 376 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 377 if (!device) { 378 mutex_unlock(&device_list_mutex); 379 return ERR_PTR(-ENOMEM); 380 } 381 382 INIT_LIST_HEAD(&device->dev_node); 383 384 device->ib_device = cma_id->device; 385 ret = isert_create_device_ib_res(device); 386 if (ret) { 387 kfree(device); 388 mutex_unlock(&device_list_mutex); 389 return ERR_PTR(ret); 390 } 391 392 device->refcount++; 393 list_add_tail(&device->dev_node, &device_list); 394 isert_info("Created a new iser device %p refcount %d\n", 395 device, device->refcount); 396 mutex_unlock(&device_list_mutex); 397 398 return device; 399 } 400 401 static void 402 isert_init_conn(struct isert_conn *isert_conn) 403 { 404 isert_conn->state = ISER_CONN_INIT; 405 INIT_LIST_HEAD(&isert_conn->node); 406 init_completion(&isert_conn->login_comp); 407 init_completion(&isert_conn->login_req_comp); 408 kref_init(&isert_conn->kref); 409 mutex_init(&isert_conn->mutex); 410 INIT_WORK(&isert_conn->release_work, isert_release_work); 411 } 412 413 static void 414 isert_free_login_buf(struct isert_conn *isert_conn) 415 { 416 struct ib_device *ib_dev = isert_conn->device->ib_device; 417 418 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 419 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 420 kfree(isert_conn->login_rsp_buf); 421 422 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 423 ISER_RX_PAYLOAD_SIZE, 424 DMA_FROM_DEVICE); 425 kfree(isert_conn->login_req_buf); 426 } 427 428 static int 429 isert_alloc_login_buf(struct isert_conn *isert_conn, 430 struct ib_device *ib_dev) 431 { 432 int ret; 433 434 isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 435 GFP_KERNEL); 436 if (!isert_conn->login_req_buf) { 437 isert_err("Unable to allocate isert_conn->login_buf\n"); 438 return -ENOMEM; 439 } 440 441 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 442 isert_conn->login_req_buf, 443 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 444 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 445 if (ret) { 446 isert_err("login_req_dma mapping error: %d\n", ret); 447 isert_conn->login_req_dma = 0; 448 goto out_free_login_req_buf; 449 } 450 451 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 452 if (!isert_conn->login_rsp_buf) { 453 isert_err("Unable to allocate isert_conn->login_rspbuf\n"); 454 goto out_unmap_login_req_buf; 455 } 456 457 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 458 isert_conn->login_rsp_buf, 459 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 460 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 461 if (ret) { 462 isert_err("login_rsp_dma mapping error: %d\n", ret); 463 isert_conn->login_rsp_dma = 0; 464 goto out_free_login_rsp_buf; 465 } 466 467 return 0; 468 469 out_free_login_rsp_buf: 470 kfree(isert_conn->login_rsp_buf); 471 out_unmap_login_req_buf: 472 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 473 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 474 out_free_login_req_buf: 475 kfree(isert_conn->login_req_buf); 476 return ret; 477 } 478 479 static void 480 isert_set_nego_params(struct isert_conn *isert_conn, 481 struct rdma_conn_param *param) 482 { 483 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 484 485 /* Set max inflight RDMA READ requests */ 486 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 487 attr->max_qp_init_rd_atom); 488 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 489 490 if (param->private_data) { 491 u8 flags = *(u8 *)param->private_data; 492 493 /* 494 * use remote invalidation if the both initiator 495 * and the HCA support it 496 */ 497 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 498 (attr->device_cap_flags & 499 IB_DEVICE_MEM_MGT_EXTENSIONS); 500 if (isert_conn->snd_w_inv) 501 isert_info("Using remote invalidation\n"); 502 } 503 } 504 505 static int 506 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 507 { 508 struct isert_np *isert_np = cma_id->context; 509 struct iscsi_np *np = isert_np->np; 510 struct isert_conn *isert_conn; 511 struct isert_device *device; 512 int ret = 0; 513 514 spin_lock_bh(&np->np_thread_lock); 515 if (!np->enabled) { 516 spin_unlock_bh(&np->np_thread_lock); 517 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 518 return rdma_reject(cma_id, NULL, 0); 519 } 520 spin_unlock_bh(&np->np_thread_lock); 521 522 isert_dbg("cma_id: %p, portal: %p\n", 523 cma_id, cma_id->context); 524 525 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 526 if (!isert_conn) 527 return -ENOMEM; 528 529 isert_init_conn(isert_conn); 530 isert_conn->cm_id = cma_id; 531 532 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 533 if (ret) 534 goto out; 535 536 device = isert_device_get(cma_id); 537 if (IS_ERR(device)) { 538 ret = PTR_ERR(device); 539 goto out_rsp_dma_map; 540 } 541 isert_conn->device = device; 542 543 isert_set_nego_params(isert_conn, &event->param.conn); 544 545 ret = isert_conn_setup_qp(isert_conn, cma_id); 546 if (ret) 547 goto out_conn_dev; 548 549 ret = isert_login_post_recv(isert_conn); 550 if (ret) 551 goto out_conn_dev; 552 553 ret = isert_rdma_accept(isert_conn); 554 if (ret) 555 goto out_conn_dev; 556 557 mutex_lock(&isert_np->mutex); 558 list_add_tail(&isert_conn->node, &isert_np->accepted); 559 mutex_unlock(&isert_np->mutex); 560 561 return 0; 562 563 out_conn_dev: 564 isert_device_put(device); 565 out_rsp_dma_map: 566 isert_free_login_buf(isert_conn); 567 out: 568 kfree(isert_conn); 569 rdma_reject(cma_id, NULL, 0); 570 return ret; 571 } 572 573 static void 574 isert_connect_release(struct isert_conn *isert_conn) 575 { 576 struct isert_device *device = isert_conn->device; 577 578 isert_dbg("conn %p\n", isert_conn); 579 580 BUG_ON(!device); 581 582 isert_free_rx_descriptors(isert_conn); 583 if (isert_conn->cm_id) 584 rdma_destroy_id(isert_conn->cm_id); 585 586 if (isert_conn->qp) { 587 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 588 589 isert_comp_put(comp); 590 ib_destroy_qp(isert_conn->qp); 591 } 592 593 if (isert_conn->login_req_buf) 594 isert_free_login_buf(isert_conn); 595 596 isert_device_put(device); 597 598 kfree(isert_conn); 599 } 600 601 static void 602 isert_connected_handler(struct rdma_cm_id *cma_id) 603 { 604 struct isert_conn *isert_conn = cma_id->qp->qp_context; 605 struct isert_np *isert_np = cma_id->context; 606 607 isert_info("conn %p\n", isert_conn); 608 609 mutex_lock(&isert_conn->mutex); 610 isert_conn->state = ISER_CONN_UP; 611 kref_get(&isert_conn->kref); 612 mutex_unlock(&isert_conn->mutex); 613 614 mutex_lock(&isert_np->mutex); 615 list_move_tail(&isert_conn->node, &isert_np->pending); 616 mutex_unlock(&isert_np->mutex); 617 618 isert_info("np %p: Allow accept_np to continue\n", isert_np); 619 up(&isert_np->sem); 620 } 621 622 static void 623 isert_release_kref(struct kref *kref) 624 { 625 struct isert_conn *isert_conn = container_of(kref, 626 struct isert_conn, kref); 627 628 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 629 current->pid); 630 631 isert_connect_release(isert_conn); 632 } 633 634 static void 635 isert_put_conn(struct isert_conn *isert_conn) 636 { 637 kref_put(&isert_conn->kref, isert_release_kref); 638 } 639 640 static void 641 isert_handle_unbound_conn(struct isert_conn *isert_conn) 642 { 643 struct isert_np *isert_np = isert_conn->cm_id->context; 644 645 mutex_lock(&isert_np->mutex); 646 if (!list_empty(&isert_conn->node)) { 647 /* 648 * This means iscsi doesn't know this connection 649 * so schedule a cleanup ourselves 650 */ 651 list_del_init(&isert_conn->node); 652 isert_put_conn(isert_conn); 653 queue_work(isert_release_wq, &isert_conn->release_work); 654 } 655 mutex_unlock(&isert_np->mutex); 656 } 657 658 /** 659 * isert_conn_terminate() - Initiate connection termination 660 * @isert_conn: isert connection struct 661 * 662 * Notes: 663 * In case the connection state is BOUND, move state 664 * to TEMINATING and start teardown sequence (rdma_disconnect). 665 * In case the connection state is UP, complete flush as well. 666 * 667 * This routine must be called with mutex held. Thus it is 668 * safe to call multiple times. 669 */ 670 static void 671 isert_conn_terminate(struct isert_conn *isert_conn) 672 { 673 int err; 674 675 if (isert_conn->state >= ISER_CONN_TERMINATING) 676 return; 677 678 isert_info("Terminating conn %p state %d\n", 679 isert_conn, isert_conn->state); 680 isert_conn->state = ISER_CONN_TERMINATING; 681 err = rdma_disconnect(isert_conn->cm_id); 682 if (err) 683 isert_warn("Failed rdma_disconnect isert_conn %p\n", 684 isert_conn); 685 } 686 687 static int 688 isert_np_cma_handler(struct isert_np *isert_np, 689 enum rdma_cm_event_type event) 690 { 691 isert_dbg("%s (%d): isert np %p\n", 692 rdma_event_msg(event), event, isert_np); 693 694 switch (event) { 695 case RDMA_CM_EVENT_DEVICE_REMOVAL: 696 isert_np->cm_id = NULL; 697 break; 698 case RDMA_CM_EVENT_ADDR_CHANGE: 699 isert_np->cm_id = isert_setup_id(isert_np); 700 if (IS_ERR(isert_np->cm_id)) { 701 isert_err("isert np %p setup id failed: %ld\n", 702 isert_np, PTR_ERR(isert_np->cm_id)); 703 isert_np->cm_id = NULL; 704 } 705 break; 706 default: 707 isert_err("isert np %p Unexpected event %d\n", 708 isert_np, event); 709 } 710 711 return -1; 712 } 713 714 static int 715 isert_disconnected_handler(struct rdma_cm_id *cma_id, 716 enum rdma_cm_event_type event) 717 { 718 struct isert_conn *isert_conn = cma_id->qp->qp_context; 719 720 mutex_lock(&isert_conn->mutex); 721 switch (isert_conn->state) { 722 case ISER_CONN_TERMINATING: 723 break; 724 case ISER_CONN_UP: 725 isert_conn_terminate(isert_conn); 726 ib_drain_qp(isert_conn->qp); 727 isert_handle_unbound_conn(isert_conn); 728 break; 729 case ISER_CONN_BOUND: 730 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 731 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 732 break; 733 default: 734 isert_warn("conn %p teminating in state %d\n", 735 isert_conn, isert_conn->state); 736 } 737 mutex_unlock(&isert_conn->mutex); 738 739 return 0; 740 } 741 742 static int 743 isert_connect_error(struct rdma_cm_id *cma_id) 744 { 745 struct isert_conn *isert_conn = cma_id->qp->qp_context; 746 747 list_del_init(&isert_conn->node); 748 isert_conn->cm_id = NULL; 749 isert_put_conn(isert_conn); 750 751 return -1; 752 } 753 754 static int 755 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 756 { 757 struct isert_np *isert_np = cma_id->context; 758 int ret = 0; 759 760 isert_info("%s (%d): status %d id %p np %p\n", 761 rdma_event_msg(event->event), event->event, 762 event->status, cma_id, cma_id->context); 763 764 if (isert_np->cm_id == cma_id) 765 return isert_np_cma_handler(cma_id->context, event->event); 766 767 switch (event->event) { 768 case RDMA_CM_EVENT_CONNECT_REQUEST: 769 ret = isert_connect_request(cma_id, event); 770 if (ret) 771 isert_err("failed handle connect request %d\n", ret); 772 break; 773 case RDMA_CM_EVENT_ESTABLISHED: 774 isert_connected_handler(cma_id); 775 break; 776 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 777 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 778 case RDMA_CM_EVENT_DEVICE_REMOVAL: /* FALLTHRU */ 779 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 780 ret = isert_disconnected_handler(cma_id, event->event); 781 break; 782 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 783 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 784 case RDMA_CM_EVENT_CONNECT_ERROR: 785 ret = isert_connect_error(cma_id); 786 break; 787 default: 788 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 789 break; 790 } 791 792 return ret; 793 } 794 795 static int 796 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 797 { 798 struct ib_recv_wr *rx_wr, *rx_wr_failed; 799 int i, ret; 800 struct iser_rx_desc *rx_desc; 801 802 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 803 rx_desc = &isert_conn->rx_descs[i]; 804 805 rx_wr->wr_cqe = &rx_desc->rx_cqe; 806 rx_wr->sg_list = &rx_desc->rx_sg; 807 rx_wr->num_sge = 1; 808 rx_wr->next = rx_wr + 1; 809 } 810 rx_wr--; 811 rx_wr->next = NULL; /* mark end of work requests list */ 812 813 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 814 &rx_wr_failed); 815 if (ret) 816 isert_err("ib_post_recv() failed with ret: %d\n", ret); 817 818 return ret; 819 } 820 821 static int 822 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 823 { 824 struct ib_recv_wr *rx_wr_failed, rx_wr; 825 int ret; 826 827 rx_wr.wr_cqe = &rx_desc->rx_cqe; 828 rx_wr.sg_list = &rx_desc->rx_sg; 829 rx_wr.num_sge = 1; 830 rx_wr.next = NULL; 831 832 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 833 if (ret) 834 isert_err("ib_post_recv() failed with ret: %d\n", ret); 835 836 return ret; 837 } 838 839 static int 840 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 841 { 842 struct ib_device *ib_dev = isert_conn->cm_id->device; 843 struct ib_send_wr send_wr, *send_wr_failed; 844 int ret; 845 846 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 847 ISER_HEADERS_LEN, DMA_TO_DEVICE); 848 849 tx_desc->tx_cqe.done = isert_login_send_done; 850 851 send_wr.next = NULL; 852 send_wr.wr_cqe = &tx_desc->tx_cqe; 853 send_wr.sg_list = tx_desc->tx_sg; 854 send_wr.num_sge = tx_desc->num_sge; 855 send_wr.opcode = IB_WR_SEND; 856 send_wr.send_flags = IB_SEND_SIGNALED; 857 858 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 859 if (ret) 860 isert_err("ib_post_send() failed, ret: %d\n", ret); 861 862 return ret; 863 } 864 865 static void 866 isert_create_send_desc(struct isert_conn *isert_conn, 867 struct isert_cmd *isert_cmd, 868 struct iser_tx_desc *tx_desc) 869 { 870 struct isert_device *device = isert_conn->device; 871 struct ib_device *ib_dev = device->ib_device; 872 873 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 874 ISER_HEADERS_LEN, DMA_TO_DEVICE); 875 876 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 877 tx_desc->iser_header.flags = ISCSI_CTRL; 878 879 tx_desc->num_sge = 1; 880 881 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 882 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 883 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 884 } 885 } 886 887 static int 888 isert_init_tx_hdrs(struct isert_conn *isert_conn, 889 struct iser_tx_desc *tx_desc) 890 { 891 struct isert_device *device = isert_conn->device; 892 struct ib_device *ib_dev = device->ib_device; 893 u64 dma_addr; 894 895 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 896 ISER_HEADERS_LEN, DMA_TO_DEVICE); 897 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 898 isert_err("ib_dma_mapping_error() failed\n"); 899 return -ENOMEM; 900 } 901 902 tx_desc->dma_addr = dma_addr; 903 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 904 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 905 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 906 907 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 908 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 909 tx_desc->tx_sg[0].lkey); 910 911 return 0; 912 } 913 914 static void 915 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 916 struct ib_send_wr *send_wr) 917 { 918 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 919 920 tx_desc->tx_cqe.done = isert_send_done; 921 send_wr->wr_cqe = &tx_desc->tx_cqe; 922 923 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 924 send_wr->opcode = IB_WR_SEND_WITH_INV; 925 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 926 } else { 927 send_wr->opcode = IB_WR_SEND; 928 } 929 930 send_wr->sg_list = &tx_desc->tx_sg[0]; 931 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 932 send_wr->send_flags = IB_SEND_SIGNALED; 933 } 934 935 static int 936 isert_login_post_recv(struct isert_conn *isert_conn) 937 { 938 struct ib_recv_wr rx_wr, *rx_wr_fail; 939 struct ib_sge sge; 940 int ret; 941 942 memset(&sge, 0, sizeof(struct ib_sge)); 943 sge.addr = isert_conn->login_req_dma; 944 sge.length = ISER_RX_PAYLOAD_SIZE; 945 sge.lkey = isert_conn->device->pd->local_dma_lkey; 946 947 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 948 sge.addr, sge.length, sge.lkey); 949 950 isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 951 952 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 953 rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 954 rx_wr.sg_list = &sge; 955 rx_wr.num_sge = 1; 956 957 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 958 if (ret) 959 isert_err("ib_post_recv() failed: %d\n", ret); 960 961 return ret; 962 } 963 964 static int 965 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 966 u32 length) 967 { 968 struct isert_conn *isert_conn = conn->context; 969 struct isert_device *device = isert_conn->device; 970 struct ib_device *ib_dev = device->ib_device; 971 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 972 int ret; 973 974 isert_create_send_desc(isert_conn, NULL, tx_desc); 975 976 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 977 sizeof(struct iscsi_hdr)); 978 979 isert_init_tx_hdrs(isert_conn, tx_desc); 980 981 if (length > 0) { 982 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 983 984 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 985 length, DMA_TO_DEVICE); 986 987 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 988 989 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 990 length, DMA_TO_DEVICE); 991 992 tx_dsg->addr = isert_conn->login_rsp_dma; 993 tx_dsg->length = length; 994 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 995 tx_desc->num_sge = 2; 996 } 997 if (!login->login_failed) { 998 if (login->login_complete) { 999 ret = isert_alloc_rx_descriptors(isert_conn); 1000 if (ret) 1001 return ret; 1002 1003 ret = isert_post_recvm(isert_conn, 1004 ISERT_QP_MAX_RECV_DTOS); 1005 if (ret) 1006 return ret; 1007 1008 /* Now we are in FULL_FEATURE phase */ 1009 mutex_lock(&isert_conn->mutex); 1010 isert_conn->state = ISER_CONN_FULL_FEATURE; 1011 mutex_unlock(&isert_conn->mutex); 1012 goto post_send; 1013 } 1014 1015 ret = isert_login_post_recv(isert_conn); 1016 if (ret) 1017 return ret; 1018 } 1019 post_send: 1020 ret = isert_login_post_send(isert_conn, tx_desc); 1021 if (ret) 1022 return ret; 1023 1024 return 0; 1025 } 1026 1027 static void 1028 isert_rx_login_req(struct isert_conn *isert_conn) 1029 { 1030 struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 1031 int rx_buflen = isert_conn->login_req_len; 1032 struct iscsi_conn *conn = isert_conn->conn; 1033 struct iscsi_login *login = conn->conn_login; 1034 int size; 1035 1036 isert_info("conn %p\n", isert_conn); 1037 1038 WARN_ON_ONCE(!login); 1039 1040 if (login->first_request) { 1041 struct iscsi_login_req *login_req = 1042 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1043 /* 1044 * Setup the initial iscsi_login values from the leading 1045 * login request PDU. 1046 */ 1047 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1048 login->current_stage = 1049 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1050 >> 2; 1051 login->version_min = login_req->min_version; 1052 login->version_max = login_req->max_version; 1053 memcpy(login->isid, login_req->isid, 6); 1054 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1055 login->init_task_tag = login_req->itt; 1056 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1057 login->cid = be16_to_cpu(login_req->cid); 1058 login->tsih = be16_to_cpu(login_req->tsih); 1059 } 1060 1061 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1062 1063 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1064 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1065 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1066 MAX_KEY_VALUE_PAIRS); 1067 memcpy(login->req_buf, &rx_desc->data[0], size); 1068 1069 if (login->first_request) { 1070 complete(&isert_conn->login_comp); 1071 return; 1072 } 1073 schedule_delayed_work(&conn->login_work, 0); 1074 } 1075 1076 static struct iscsi_cmd 1077 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1078 { 1079 struct isert_conn *isert_conn = conn->context; 1080 struct isert_cmd *isert_cmd; 1081 struct iscsi_cmd *cmd; 1082 1083 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1084 if (!cmd) { 1085 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1086 return NULL; 1087 } 1088 isert_cmd = iscsit_priv_cmd(cmd); 1089 isert_cmd->conn = isert_conn; 1090 isert_cmd->iscsi_cmd = cmd; 1091 isert_cmd->rx_desc = rx_desc; 1092 1093 return cmd; 1094 } 1095 1096 static int 1097 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1098 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1099 struct iser_rx_desc *rx_desc, unsigned char *buf) 1100 { 1101 struct iscsi_conn *conn = isert_conn->conn; 1102 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1103 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1104 bool dump_payload = false; 1105 unsigned int data_len; 1106 1107 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1108 if (rc < 0) 1109 return rc; 1110 1111 imm_data = cmd->immediate_data; 1112 imm_data_len = cmd->first_burst_len; 1113 unsol_data = cmd->unsolicited_data; 1114 data_len = cmd->se_cmd.data_length; 1115 1116 if (imm_data && imm_data_len == data_len) 1117 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1118 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1119 if (rc < 0) { 1120 return 0; 1121 } else if (rc > 0) { 1122 dump_payload = true; 1123 goto sequence_cmd; 1124 } 1125 1126 if (!imm_data) 1127 return 0; 1128 1129 if (imm_data_len != data_len) { 1130 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1131 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1132 &rx_desc->data[0], imm_data_len); 1133 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1134 sg_nents, imm_data_len); 1135 } else { 1136 sg_init_table(&isert_cmd->sg, 1); 1137 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1138 cmd->se_cmd.t_data_nents = 1; 1139 sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 1140 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1141 imm_data_len); 1142 } 1143 1144 cmd->write_data_done += imm_data_len; 1145 1146 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1147 spin_lock_bh(&cmd->istate_lock); 1148 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1149 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1150 spin_unlock_bh(&cmd->istate_lock); 1151 } 1152 1153 sequence_cmd: 1154 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1155 1156 if (!rc && dump_payload == false && unsol_data) 1157 iscsit_set_unsoliticed_dataout(cmd); 1158 else if (dump_payload && imm_data) 1159 target_put_sess_cmd(&cmd->se_cmd); 1160 1161 return 0; 1162 } 1163 1164 static int 1165 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1166 struct iser_rx_desc *rx_desc, unsigned char *buf) 1167 { 1168 struct scatterlist *sg_start; 1169 struct iscsi_conn *conn = isert_conn->conn; 1170 struct iscsi_cmd *cmd = NULL; 1171 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1172 u32 unsol_data_len = ntoh24(hdr->dlength); 1173 int rc, sg_nents, sg_off, page_off; 1174 1175 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1176 if (rc < 0) 1177 return rc; 1178 else if (!cmd) 1179 return 0; 1180 /* 1181 * FIXME: Unexpected unsolicited_data out 1182 */ 1183 if (!cmd->unsolicited_data) { 1184 isert_err("Received unexpected solicited data payload\n"); 1185 dump_stack(); 1186 return -1; 1187 } 1188 1189 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1190 "write_data_done: %u, data_length: %u\n", 1191 unsol_data_len, cmd->write_data_done, 1192 cmd->se_cmd.data_length); 1193 1194 sg_off = cmd->write_data_done / PAGE_SIZE; 1195 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1196 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1197 page_off = cmd->write_data_done % PAGE_SIZE; 1198 /* 1199 * FIXME: Non page-aligned unsolicited_data out 1200 */ 1201 if (page_off) { 1202 isert_err("unexpected non-page aligned data payload\n"); 1203 dump_stack(); 1204 return -1; 1205 } 1206 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1207 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1208 sg_nents, &rx_desc->data[0], unsol_data_len); 1209 1210 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1211 unsol_data_len); 1212 1213 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1214 if (rc < 0) 1215 return rc; 1216 1217 /* 1218 * multiple data-outs on the same command can arrive - 1219 * so post the buffer before hand 1220 */ 1221 rc = isert_post_recv(isert_conn, rx_desc); 1222 if (rc) { 1223 isert_err("ib_post_recv failed with %d\n", rc); 1224 return rc; 1225 } 1226 return 0; 1227 } 1228 1229 static int 1230 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1231 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1232 unsigned char *buf) 1233 { 1234 struct iscsi_conn *conn = isert_conn->conn; 1235 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1236 int rc; 1237 1238 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1239 if (rc < 0) 1240 return rc; 1241 /* 1242 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1243 */ 1244 1245 return iscsit_process_nop_out(conn, cmd, hdr); 1246 } 1247 1248 static int 1249 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1250 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1251 struct iscsi_text *hdr) 1252 { 1253 struct iscsi_conn *conn = isert_conn->conn; 1254 u32 payload_length = ntoh24(hdr->dlength); 1255 int rc; 1256 unsigned char *text_in = NULL; 1257 1258 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1259 if (rc < 0) 1260 return rc; 1261 1262 if (payload_length) { 1263 text_in = kzalloc(payload_length, GFP_KERNEL); 1264 if (!text_in) { 1265 isert_err("Unable to allocate text_in of payload_length: %u\n", 1266 payload_length); 1267 return -ENOMEM; 1268 } 1269 } 1270 cmd->text_in_ptr = text_in; 1271 1272 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1273 1274 return iscsit_process_text_cmd(conn, cmd, hdr); 1275 } 1276 1277 static int 1278 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1279 uint32_t read_stag, uint64_t read_va, 1280 uint32_t write_stag, uint64_t write_va) 1281 { 1282 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1283 struct iscsi_conn *conn = isert_conn->conn; 1284 struct iscsi_cmd *cmd; 1285 struct isert_cmd *isert_cmd; 1286 int ret = -EINVAL; 1287 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1288 1289 if (conn->sess->sess_ops->SessionType && 1290 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1291 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1292 " ignoring\n", opcode); 1293 return 0; 1294 } 1295 1296 switch (opcode) { 1297 case ISCSI_OP_SCSI_CMD: 1298 cmd = isert_allocate_cmd(conn, rx_desc); 1299 if (!cmd) 1300 break; 1301 1302 isert_cmd = iscsit_priv_cmd(cmd); 1303 isert_cmd->read_stag = read_stag; 1304 isert_cmd->read_va = read_va; 1305 isert_cmd->write_stag = write_stag; 1306 isert_cmd->write_va = write_va; 1307 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1308 1309 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1310 rx_desc, (unsigned char *)hdr); 1311 break; 1312 case ISCSI_OP_NOOP_OUT: 1313 cmd = isert_allocate_cmd(conn, rx_desc); 1314 if (!cmd) 1315 break; 1316 1317 isert_cmd = iscsit_priv_cmd(cmd); 1318 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1319 rx_desc, (unsigned char *)hdr); 1320 break; 1321 case ISCSI_OP_SCSI_DATA_OUT: 1322 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1323 (unsigned char *)hdr); 1324 break; 1325 case ISCSI_OP_SCSI_TMFUNC: 1326 cmd = isert_allocate_cmd(conn, rx_desc); 1327 if (!cmd) 1328 break; 1329 1330 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1331 (unsigned char *)hdr); 1332 break; 1333 case ISCSI_OP_LOGOUT: 1334 cmd = isert_allocate_cmd(conn, rx_desc); 1335 if (!cmd) 1336 break; 1337 1338 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1339 break; 1340 case ISCSI_OP_TEXT: 1341 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1342 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1343 else 1344 cmd = isert_allocate_cmd(conn, rx_desc); 1345 1346 if (!cmd) 1347 break; 1348 1349 isert_cmd = iscsit_priv_cmd(cmd); 1350 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1351 rx_desc, (struct iscsi_text *)hdr); 1352 break; 1353 default: 1354 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1355 dump_stack(); 1356 break; 1357 } 1358 1359 return ret; 1360 } 1361 1362 static void 1363 isert_print_wc(struct ib_wc *wc, const char *type) 1364 { 1365 if (wc->status != IB_WC_WR_FLUSH_ERR) 1366 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1367 ib_wc_status_msg(wc->status), wc->status, 1368 wc->vendor_err); 1369 else 1370 isert_dbg("%s failure: %s (%d)\n", type, 1371 ib_wc_status_msg(wc->status), wc->status); 1372 } 1373 1374 static void 1375 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1376 { 1377 struct isert_conn *isert_conn = wc->qp->qp_context; 1378 struct ib_device *ib_dev = isert_conn->cm_id->device; 1379 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1380 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1381 struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1382 uint64_t read_va = 0, write_va = 0; 1383 uint32_t read_stag = 0, write_stag = 0; 1384 1385 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1386 isert_print_wc(wc, "recv"); 1387 if (wc->status != IB_WC_WR_FLUSH_ERR) 1388 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1389 return; 1390 } 1391 1392 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1393 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1394 1395 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1396 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1397 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1398 1399 switch (iser_ctrl->flags & 0xF0) { 1400 case ISCSI_CTRL: 1401 if (iser_ctrl->flags & ISER_RSV) { 1402 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1403 read_va = be64_to_cpu(iser_ctrl->read_va); 1404 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1405 read_stag, (unsigned long long)read_va); 1406 } 1407 if (iser_ctrl->flags & ISER_WSV) { 1408 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1409 write_va = be64_to_cpu(iser_ctrl->write_va); 1410 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1411 write_stag, (unsigned long long)write_va); 1412 } 1413 1414 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1415 break; 1416 case ISER_HELLO: 1417 isert_err("iSER Hello message\n"); 1418 break; 1419 default: 1420 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1421 break; 1422 } 1423 1424 isert_rx_opcode(isert_conn, rx_desc, 1425 read_stag, read_va, write_stag, write_va); 1426 1427 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1428 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1429 } 1430 1431 static void 1432 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1433 { 1434 struct isert_conn *isert_conn = wc->qp->qp_context; 1435 struct ib_device *ib_dev = isert_conn->cm_id->device; 1436 1437 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1438 isert_print_wc(wc, "login recv"); 1439 return; 1440 } 1441 1442 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 1443 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1444 1445 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1446 1447 if (isert_conn->conn) { 1448 struct iscsi_login *login = isert_conn->conn->conn_login; 1449 1450 if (login && !login->first_request) 1451 isert_rx_login_req(isert_conn); 1452 } 1453 1454 mutex_lock(&isert_conn->mutex); 1455 complete(&isert_conn->login_req_comp); 1456 mutex_unlock(&isert_conn->mutex); 1457 1458 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 1459 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1460 } 1461 1462 static void 1463 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1464 { 1465 struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 1466 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1467 1468 if (!cmd->rw.nr_ops) 1469 return; 1470 1471 if (isert_prot_cmd(conn, se_cmd)) { 1472 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1473 conn->cm_id->port_num, se_cmd->t_data_sg, 1474 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1475 se_cmd->t_prot_nents, dir); 1476 } else { 1477 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1478 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1479 } 1480 1481 cmd->rw.nr_ops = 0; 1482 } 1483 1484 static void 1485 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1486 { 1487 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1488 struct isert_conn *isert_conn = isert_cmd->conn; 1489 struct iscsi_conn *conn = isert_conn->conn; 1490 struct iscsi_text_rsp *hdr; 1491 1492 isert_dbg("Cmd %p\n", isert_cmd); 1493 1494 switch (cmd->iscsi_opcode) { 1495 case ISCSI_OP_SCSI_CMD: 1496 spin_lock_bh(&conn->cmd_lock); 1497 if (!list_empty(&cmd->i_conn_node)) 1498 list_del_init(&cmd->i_conn_node); 1499 spin_unlock_bh(&conn->cmd_lock); 1500 1501 if (cmd->data_direction == DMA_TO_DEVICE) { 1502 iscsit_stop_dataout_timer(cmd); 1503 /* 1504 * Check for special case during comp_err where 1505 * WRITE_PENDING has been handed off from core, 1506 * but requires an extra target_put_sess_cmd() 1507 * before transport_generic_free_cmd() below. 1508 */ 1509 if (comp_err && 1510 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1511 struct se_cmd *se_cmd = &cmd->se_cmd; 1512 1513 target_put_sess_cmd(se_cmd); 1514 } 1515 } 1516 1517 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1518 transport_generic_free_cmd(&cmd->se_cmd, 0); 1519 break; 1520 case ISCSI_OP_SCSI_TMFUNC: 1521 spin_lock_bh(&conn->cmd_lock); 1522 if (!list_empty(&cmd->i_conn_node)) 1523 list_del_init(&cmd->i_conn_node); 1524 spin_unlock_bh(&conn->cmd_lock); 1525 1526 transport_generic_free_cmd(&cmd->se_cmd, 0); 1527 break; 1528 case ISCSI_OP_REJECT: 1529 case ISCSI_OP_NOOP_OUT: 1530 case ISCSI_OP_TEXT: 1531 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1532 /* If the continue bit is on, keep the command alive */ 1533 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1534 break; 1535 1536 spin_lock_bh(&conn->cmd_lock); 1537 if (!list_empty(&cmd->i_conn_node)) 1538 list_del_init(&cmd->i_conn_node); 1539 spin_unlock_bh(&conn->cmd_lock); 1540 1541 /* 1542 * Handle special case for REJECT when iscsi_add_reject*() has 1543 * overwritten the original iscsi_opcode assignment, and the 1544 * associated cmd->se_cmd needs to be released. 1545 */ 1546 if (cmd->se_cmd.se_tfo != NULL) { 1547 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1548 cmd->iscsi_opcode); 1549 transport_generic_free_cmd(&cmd->se_cmd, 0); 1550 break; 1551 } 1552 /* 1553 * Fall-through 1554 */ 1555 default: 1556 iscsit_release_cmd(cmd); 1557 break; 1558 } 1559 } 1560 1561 static void 1562 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1563 { 1564 if (tx_desc->dma_addr != 0) { 1565 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1566 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1567 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1568 tx_desc->dma_addr = 0; 1569 } 1570 } 1571 1572 static void 1573 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1574 struct ib_device *ib_dev, bool comp_err) 1575 { 1576 if (isert_cmd->pdu_buf_dma != 0) { 1577 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1578 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1579 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1580 isert_cmd->pdu_buf_dma = 0; 1581 } 1582 1583 isert_unmap_tx_desc(tx_desc, ib_dev); 1584 isert_put_cmd(isert_cmd, comp_err); 1585 } 1586 1587 static int 1588 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1589 { 1590 struct ib_mr_status mr_status; 1591 int ret; 1592 1593 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1594 if (ret) { 1595 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1596 goto fail_mr_status; 1597 } 1598 1599 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1600 u64 sec_offset_err; 1601 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1602 1603 switch (mr_status.sig_err.err_type) { 1604 case IB_SIG_BAD_GUARD: 1605 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1606 break; 1607 case IB_SIG_BAD_REFTAG: 1608 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1609 break; 1610 case IB_SIG_BAD_APPTAG: 1611 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1612 break; 1613 } 1614 sec_offset_err = mr_status.sig_err.sig_err_offset; 1615 do_div(sec_offset_err, block_size); 1616 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1617 1618 isert_err("PI error found type %d at sector 0x%llx " 1619 "expected 0x%x vs actual 0x%x\n", 1620 mr_status.sig_err.err_type, 1621 (unsigned long long)se_cmd->bad_sector, 1622 mr_status.sig_err.expected, 1623 mr_status.sig_err.actual); 1624 ret = 1; 1625 } 1626 1627 fail_mr_status: 1628 return ret; 1629 } 1630 1631 static void 1632 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1633 { 1634 struct isert_conn *isert_conn = wc->qp->qp_context; 1635 struct isert_device *device = isert_conn->device; 1636 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1637 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1638 struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 1639 int ret = 0; 1640 1641 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1642 isert_print_wc(wc, "rdma write"); 1643 if (wc->status != IB_WC_WR_FLUSH_ERR) 1644 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1645 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1646 return; 1647 } 1648 1649 isert_dbg("Cmd %p\n", isert_cmd); 1650 1651 ret = isert_check_pi_status(cmd, isert_cmd->rw.sig->sig_mr); 1652 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1653 1654 if (ret) 1655 transport_send_check_condition_and_sense(cmd, cmd->pi_err, 0); 1656 else 1657 isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 1658 } 1659 1660 static void 1661 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1662 { 1663 struct isert_conn *isert_conn = wc->qp->qp_context; 1664 struct isert_device *device = isert_conn->device; 1665 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1666 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1667 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1668 struct se_cmd *se_cmd = &cmd->se_cmd; 1669 int ret = 0; 1670 1671 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1672 isert_print_wc(wc, "rdma read"); 1673 if (wc->status != IB_WC_WR_FLUSH_ERR) 1674 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1675 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1676 return; 1677 } 1678 1679 isert_dbg("Cmd %p\n", isert_cmd); 1680 1681 iscsit_stop_dataout_timer(cmd); 1682 1683 if (isert_prot_cmd(isert_conn, se_cmd)) 1684 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.sig->sig_mr); 1685 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1686 cmd->write_data_done = 0; 1687 1688 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1689 spin_lock_bh(&cmd->istate_lock); 1690 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1691 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1692 spin_unlock_bh(&cmd->istate_lock); 1693 1694 if (ret) { 1695 target_put_sess_cmd(se_cmd); 1696 transport_send_check_condition_and_sense(se_cmd, 1697 se_cmd->pi_err, 0); 1698 } else { 1699 target_execute_cmd(se_cmd); 1700 } 1701 } 1702 1703 static void 1704 isert_do_control_comp(struct work_struct *work) 1705 { 1706 struct isert_cmd *isert_cmd = container_of(work, 1707 struct isert_cmd, comp_work); 1708 struct isert_conn *isert_conn = isert_cmd->conn; 1709 struct ib_device *ib_dev = isert_conn->cm_id->device; 1710 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1711 1712 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1713 1714 switch (cmd->i_state) { 1715 case ISTATE_SEND_TASKMGTRSP: 1716 iscsit_tmr_post_handler(cmd, cmd->conn); 1717 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1718 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1719 cmd->i_state = ISTATE_SENT_STATUS; 1720 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1721 ib_dev, false); 1722 break; 1723 case ISTATE_SEND_LOGOUTRSP: 1724 iscsit_logout_post_handler(cmd, cmd->conn); 1725 break; 1726 default: 1727 isert_err("Unknown i_state %d\n", cmd->i_state); 1728 dump_stack(); 1729 break; 1730 } 1731 } 1732 1733 static void 1734 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1735 { 1736 struct isert_conn *isert_conn = wc->qp->qp_context; 1737 struct ib_device *ib_dev = isert_conn->cm_id->device; 1738 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1739 1740 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1741 isert_print_wc(wc, "login send"); 1742 if (wc->status != IB_WC_WR_FLUSH_ERR) 1743 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1744 } 1745 1746 isert_unmap_tx_desc(tx_desc, ib_dev); 1747 } 1748 1749 static void 1750 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1751 { 1752 struct isert_conn *isert_conn = wc->qp->qp_context; 1753 struct ib_device *ib_dev = isert_conn->cm_id->device; 1754 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1755 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1756 1757 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1758 isert_print_wc(wc, "send"); 1759 if (wc->status != IB_WC_WR_FLUSH_ERR) 1760 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1761 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1762 return; 1763 } 1764 1765 isert_dbg("Cmd %p\n", isert_cmd); 1766 1767 switch (isert_cmd->iscsi_cmd->i_state) { 1768 case ISTATE_SEND_TASKMGTRSP: 1769 case ISTATE_SEND_LOGOUTRSP: 1770 case ISTATE_SEND_REJECT: 1771 case ISTATE_SEND_TEXTRSP: 1772 isert_unmap_tx_desc(tx_desc, ib_dev); 1773 1774 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1775 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1776 return; 1777 default: 1778 isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 1779 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1780 break; 1781 } 1782 } 1783 1784 static int 1785 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1786 { 1787 struct ib_send_wr *wr_failed; 1788 int ret; 1789 1790 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1791 if (ret) { 1792 isert_err("ib_post_recv failed with %d\n", ret); 1793 return ret; 1794 } 1795 1796 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 1797 &wr_failed); 1798 if (ret) { 1799 isert_err("ib_post_send failed with %d\n", ret); 1800 return ret; 1801 } 1802 return ret; 1803 } 1804 1805 static int 1806 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1807 { 1808 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1809 struct isert_conn *isert_conn = conn->context; 1810 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1811 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1812 &isert_cmd->tx_desc.iscsi_header; 1813 1814 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1815 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1816 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1817 /* 1818 * Attach SENSE DATA payload to iSCSI Response PDU 1819 */ 1820 if (cmd->se_cmd.sense_buffer && 1821 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1822 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1823 struct isert_device *device = isert_conn->device; 1824 struct ib_device *ib_dev = device->ib_device; 1825 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1826 u32 padding, pdu_len; 1827 1828 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1829 cmd->sense_buffer); 1830 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1831 1832 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1833 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1834 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1835 1836 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1837 (void *)cmd->sense_buffer, pdu_len, 1838 DMA_TO_DEVICE); 1839 1840 isert_cmd->pdu_buf_len = pdu_len; 1841 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1842 tx_dsg->length = pdu_len; 1843 tx_dsg->lkey = device->pd->local_dma_lkey; 1844 isert_cmd->tx_desc.num_sge = 2; 1845 } 1846 1847 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1848 1849 isert_dbg("Posting SCSI Response\n"); 1850 1851 return isert_post_response(isert_conn, isert_cmd); 1852 } 1853 1854 static void 1855 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1856 { 1857 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1858 struct isert_conn *isert_conn = conn->context; 1859 1860 spin_lock_bh(&conn->cmd_lock); 1861 if (!list_empty(&cmd->i_conn_node)) 1862 list_del_init(&cmd->i_conn_node); 1863 spin_unlock_bh(&conn->cmd_lock); 1864 1865 if (cmd->data_direction == DMA_TO_DEVICE) 1866 iscsit_stop_dataout_timer(cmd); 1867 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1868 } 1869 1870 static enum target_prot_op 1871 isert_get_sup_prot_ops(struct iscsi_conn *conn) 1872 { 1873 struct isert_conn *isert_conn = conn->context; 1874 struct isert_device *device = isert_conn->device; 1875 1876 if (conn->tpg->tpg_attrib.t10_pi) { 1877 if (device->pi_capable) { 1878 isert_info("conn %p PI offload enabled\n", isert_conn); 1879 isert_conn->pi_support = true; 1880 return TARGET_PROT_ALL; 1881 } 1882 } 1883 1884 isert_info("conn %p PI offload disabled\n", isert_conn); 1885 isert_conn->pi_support = false; 1886 1887 return TARGET_PROT_NORMAL; 1888 } 1889 1890 static int 1891 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1892 bool nopout_response) 1893 { 1894 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1895 struct isert_conn *isert_conn = conn->context; 1896 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1897 1898 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1899 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1900 &isert_cmd->tx_desc.iscsi_header, 1901 nopout_response); 1902 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1903 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1904 1905 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1906 1907 return isert_post_response(isert_conn, isert_cmd); 1908 } 1909 1910 static int 1911 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1912 { 1913 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1914 struct isert_conn *isert_conn = conn->context; 1915 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1916 1917 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1918 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1919 &isert_cmd->tx_desc.iscsi_header); 1920 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1921 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1922 1923 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1924 1925 return isert_post_response(isert_conn, isert_cmd); 1926 } 1927 1928 static int 1929 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1930 { 1931 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1932 struct isert_conn *isert_conn = conn->context; 1933 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1934 1935 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1936 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1937 &isert_cmd->tx_desc.iscsi_header); 1938 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1939 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1940 1941 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1942 1943 return isert_post_response(isert_conn, isert_cmd); 1944 } 1945 1946 static int 1947 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1948 { 1949 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1950 struct isert_conn *isert_conn = conn->context; 1951 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1952 struct isert_device *device = isert_conn->device; 1953 struct ib_device *ib_dev = device->ib_device; 1954 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1955 struct iscsi_reject *hdr = 1956 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1957 1958 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1959 iscsit_build_reject(cmd, conn, hdr); 1960 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1961 1962 hton24(hdr->dlength, ISCSI_HDR_LEN); 1963 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1964 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1965 DMA_TO_DEVICE); 1966 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1967 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1968 tx_dsg->length = ISCSI_HDR_LEN; 1969 tx_dsg->lkey = device->pd->local_dma_lkey; 1970 isert_cmd->tx_desc.num_sge = 2; 1971 1972 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1973 1974 isert_dbg("conn %p Posting Reject\n", isert_conn); 1975 1976 return isert_post_response(isert_conn, isert_cmd); 1977 } 1978 1979 static int 1980 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1981 { 1982 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1983 struct isert_conn *isert_conn = conn->context; 1984 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1985 struct iscsi_text_rsp *hdr = 1986 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1987 u32 txt_rsp_len; 1988 int rc; 1989 1990 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1991 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1992 if (rc < 0) 1993 return rc; 1994 1995 txt_rsp_len = rc; 1996 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1997 1998 if (txt_rsp_len) { 1999 struct isert_device *device = isert_conn->device; 2000 struct ib_device *ib_dev = device->ib_device; 2001 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2002 void *txt_rsp_buf = cmd->buf_ptr; 2003 2004 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2005 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2006 2007 isert_cmd->pdu_buf_len = txt_rsp_len; 2008 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2009 tx_dsg->length = txt_rsp_len; 2010 tx_dsg->lkey = device->pd->local_dma_lkey; 2011 isert_cmd->tx_desc.num_sge = 2; 2012 } 2013 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2014 2015 isert_dbg("conn %p Text Response\n", isert_conn); 2016 2017 return isert_post_response(isert_conn, isert_cmd); 2018 } 2019 2020 static inline void 2021 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2022 struct ib_sig_domain *domain) 2023 { 2024 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2025 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2026 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2027 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2028 /* 2029 * At the moment we hard code those, but if in the future 2030 * the target core would like to use it, we will take it 2031 * from se_cmd. 2032 */ 2033 domain->sig.dif.apptag_check_mask = 0xffff; 2034 domain->sig.dif.app_escape = true; 2035 domain->sig.dif.ref_escape = true; 2036 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2037 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2038 domain->sig.dif.ref_remap = true; 2039 }; 2040 2041 static int 2042 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2043 { 2044 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2045 2046 switch (se_cmd->prot_op) { 2047 case TARGET_PROT_DIN_INSERT: 2048 case TARGET_PROT_DOUT_STRIP: 2049 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2050 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2051 break; 2052 case TARGET_PROT_DOUT_INSERT: 2053 case TARGET_PROT_DIN_STRIP: 2054 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2055 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2056 break; 2057 case TARGET_PROT_DIN_PASS: 2058 case TARGET_PROT_DOUT_PASS: 2059 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2060 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2061 break; 2062 default: 2063 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2064 return -EINVAL; 2065 } 2066 2067 sig_attrs->check_mask = 2068 (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2069 (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2070 (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2071 return 0; 2072 } 2073 2074 static int 2075 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2076 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2077 { 2078 struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 2079 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2080 u8 port_num = conn->cm_id->port_num; 2081 u64 addr; 2082 u32 rkey, offset; 2083 int ret; 2084 2085 if (dir == DMA_FROM_DEVICE) { 2086 addr = cmd->write_va; 2087 rkey = cmd->write_stag; 2088 offset = cmd->iscsi_cmd->write_data_done; 2089 } else { 2090 addr = cmd->read_va; 2091 rkey = cmd->read_stag; 2092 offset = 0; 2093 } 2094 2095 if (isert_prot_cmd(conn, se_cmd)) { 2096 struct ib_sig_attrs sig_attrs; 2097 2098 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2099 if (ret) 2100 return ret; 2101 2102 WARN_ON_ONCE(offset); 2103 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2104 se_cmd->t_data_sg, se_cmd->t_data_nents, 2105 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2106 &sig_attrs, addr, rkey, dir); 2107 } else { 2108 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2109 se_cmd->t_data_sg, se_cmd->t_data_nents, 2110 offset, addr, rkey, dir); 2111 } 2112 if (ret < 0) { 2113 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2114 return ret; 2115 } 2116 2117 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2118 if (ret < 0) 2119 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2120 return ret; 2121 } 2122 2123 static int 2124 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2125 { 2126 struct se_cmd *se_cmd = &cmd->se_cmd; 2127 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2128 struct isert_conn *isert_conn = conn->context; 2129 struct ib_cqe *cqe = NULL; 2130 struct ib_send_wr *chain_wr = NULL; 2131 int rc; 2132 2133 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2134 isert_cmd, se_cmd->data_length); 2135 2136 if (isert_prot_cmd(isert_conn, se_cmd)) { 2137 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2138 cqe = &isert_cmd->tx_desc.tx_cqe; 2139 } else { 2140 /* 2141 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2142 */ 2143 isert_create_send_desc(isert_conn, isert_cmd, 2144 &isert_cmd->tx_desc); 2145 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2146 &isert_cmd->tx_desc.iscsi_header); 2147 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2148 isert_init_send_wr(isert_conn, isert_cmd, 2149 &isert_cmd->tx_desc.send_wr); 2150 2151 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2152 if (rc) { 2153 isert_err("ib_post_recv failed with %d\n", rc); 2154 return rc; 2155 } 2156 2157 chain_wr = &isert_cmd->tx_desc.send_wr; 2158 } 2159 2160 isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2161 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ\n", isert_cmd); 2162 return 1; 2163 } 2164 2165 static int 2166 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2167 { 2168 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2169 2170 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2171 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2172 2173 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2174 isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2175 &isert_cmd->tx_desc.tx_cqe, NULL); 2176 2177 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2178 isert_cmd); 2179 return 0; 2180 } 2181 2182 static int 2183 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2184 { 2185 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2186 int ret = 0; 2187 2188 switch (state) { 2189 case ISTATE_REMOVE: 2190 spin_lock_bh(&conn->cmd_lock); 2191 list_del_init(&cmd->i_conn_node); 2192 spin_unlock_bh(&conn->cmd_lock); 2193 isert_put_cmd(isert_cmd, true); 2194 break; 2195 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2196 ret = isert_put_nopin(cmd, conn, false); 2197 break; 2198 default: 2199 isert_err("Unknown immediate state: 0x%02x\n", state); 2200 ret = -EINVAL; 2201 break; 2202 } 2203 2204 return ret; 2205 } 2206 2207 static int 2208 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2209 { 2210 struct isert_conn *isert_conn = conn->context; 2211 int ret; 2212 2213 switch (state) { 2214 case ISTATE_SEND_LOGOUTRSP: 2215 ret = isert_put_logout_rsp(cmd, conn); 2216 if (!ret) 2217 isert_conn->logout_posted = true; 2218 break; 2219 case ISTATE_SEND_NOPIN: 2220 ret = isert_put_nopin(cmd, conn, true); 2221 break; 2222 case ISTATE_SEND_TASKMGTRSP: 2223 ret = isert_put_tm_rsp(cmd, conn); 2224 break; 2225 case ISTATE_SEND_REJECT: 2226 ret = isert_put_reject(cmd, conn); 2227 break; 2228 case ISTATE_SEND_TEXTRSP: 2229 ret = isert_put_text_rsp(cmd, conn); 2230 break; 2231 case ISTATE_SEND_STATUS: 2232 /* 2233 * Special case for sending non GOOD SCSI status from TX thread 2234 * context during pre se_cmd excecution failure. 2235 */ 2236 ret = isert_put_response(conn, cmd); 2237 break; 2238 default: 2239 isert_err("Unknown response state: 0x%02x\n", state); 2240 ret = -EINVAL; 2241 break; 2242 } 2243 2244 return ret; 2245 } 2246 2247 struct rdma_cm_id * 2248 isert_setup_id(struct isert_np *isert_np) 2249 { 2250 struct iscsi_np *np = isert_np->np; 2251 struct rdma_cm_id *id; 2252 struct sockaddr *sa; 2253 int ret; 2254 2255 sa = (struct sockaddr *)&np->np_sockaddr; 2256 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2257 2258 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2259 RDMA_PS_TCP, IB_QPT_RC); 2260 if (IS_ERR(id)) { 2261 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2262 ret = PTR_ERR(id); 2263 goto out; 2264 } 2265 isert_dbg("id %p context %p\n", id, id->context); 2266 2267 ret = rdma_bind_addr(id, sa); 2268 if (ret) { 2269 isert_err("rdma_bind_addr() failed: %d\n", ret); 2270 goto out_id; 2271 } 2272 2273 ret = rdma_listen(id, 0); 2274 if (ret) { 2275 isert_err("rdma_listen() failed: %d\n", ret); 2276 goto out_id; 2277 } 2278 2279 return id; 2280 out_id: 2281 rdma_destroy_id(id); 2282 out: 2283 return ERR_PTR(ret); 2284 } 2285 2286 static int 2287 isert_setup_np(struct iscsi_np *np, 2288 struct sockaddr_storage *ksockaddr) 2289 { 2290 struct isert_np *isert_np; 2291 struct rdma_cm_id *isert_lid; 2292 int ret; 2293 2294 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2295 if (!isert_np) { 2296 isert_err("Unable to allocate struct isert_np\n"); 2297 return -ENOMEM; 2298 } 2299 sema_init(&isert_np->sem, 0); 2300 mutex_init(&isert_np->mutex); 2301 INIT_LIST_HEAD(&isert_np->accepted); 2302 INIT_LIST_HEAD(&isert_np->pending); 2303 isert_np->np = np; 2304 2305 /* 2306 * Setup the np->np_sockaddr from the passed sockaddr setup 2307 * in iscsi_target_configfs.c code.. 2308 */ 2309 memcpy(&np->np_sockaddr, ksockaddr, 2310 sizeof(struct sockaddr_storage)); 2311 2312 isert_lid = isert_setup_id(isert_np); 2313 if (IS_ERR(isert_lid)) { 2314 ret = PTR_ERR(isert_lid); 2315 goto out; 2316 } 2317 2318 isert_np->cm_id = isert_lid; 2319 np->np_context = isert_np; 2320 2321 return 0; 2322 2323 out: 2324 kfree(isert_np); 2325 2326 return ret; 2327 } 2328 2329 static int 2330 isert_rdma_accept(struct isert_conn *isert_conn) 2331 { 2332 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2333 struct rdma_conn_param cp; 2334 int ret; 2335 struct iser_cm_hdr rsp_hdr; 2336 2337 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2338 cp.initiator_depth = isert_conn->initiator_depth; 2339 cp.retry_count = 7; 2340 cp.rnr_retry_count = 7; 2341 2342 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2343 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2344 if (!isert_conn->snd_w_inv) 2345 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2346 cp.private_data = (void *)&rsp_hdr; 2347 cp.private_data_len = sizeof(rsp_hdr); 2348 2349 ret = rdma_accept(cm_id, &cp); 2350 if (ret) { 2351 isert_err("rdma_accept() failed with: %d\n", ret); 2352 return ret; 2353 } 2354 2355 return 0; 2356 } 2357 2358 static int 2359 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2360 { 2361 struct isert_conn *isert_conn = conn->context; 2362 int ret; 2363 2364 isert_info("before login_req comp conn: %p\n", isert_conn); 2365 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2366 if (ret) { 2367 isert_err("isert_conn %p interrupted before got login req\n", 2368 isert_conn); 2369 return ret; 2370 } 2371 reinit_completion(&isert_conn->login_req_comp); 2372 2373 /* 2374 * For login requests after the first PDU, isert_rx_login_req() will 2375 * kick schedule_delayed_work(&conn->login_work) as the packet is 2376 * received, which turns this callback from iscsi_target_do_login_rx() 2377 * into a NOP. 2378 */ 2379 if (!login->first_request) 2380 return 0; 2381 2382 isert_rx_login_req(isert_conn); 2383 2384 isert_info("before login_comp conn: %p\n", conn); 2385 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2386 if (ret) 2387 return ret; 2388 2389 isert_info("processing login->req: %p\n", login->req); 2390 2391 return 0; 2392 } 2393 2394 static void 2395 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2396 struct isert_conn *isert_conn) 2397 { 2398 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2399 struct rdma_route *cm_route = &cm_id->route; 2400 2401 conn->login_family = np->np_sockaddr.ss_family; 2402 2403 conn->login_sockaddr = cm_route->addr.dst_addr; 2404 conn->local_sockaddr = cm_route->addr.src_addr; 2405 } 2406 2407 static int 2408 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2409 { 2410 struct isert_np *isert_np = np->np_context; 2411 struct isert_conn *isert_conn; 2412 int ret; 2413 2414 accept_wait: 2415 ret = down_interruptible(&isert_np->sem); 2416 if (ret) 2417 return -ENODEV; 2418 2419 spin_lock_bh(&np->np_thread_lock); 2420 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2421 spin_unlock_bh(&np->np_thread_lock); 2422 isert_dbg("np_thread_state %d\n", 2423 np->np_thread_state); 2424 /** 2425 * No point in stalling here when np_thread 2426 * is in state RESET/SHUTDOWN/EXIT - bail 2427 **/ 2428 return -ENODEV; 2429 } 2430 spin_unlock_bh(&np->np_thread_lock); 2431 2432 mutex_lock(&isert_np->mutex); 2433 if (list_empty(&isert_np->pending)) { 2434 mutex_unlock(&isert_np->mutex); 2435 goto accept_wait; 2436 } 2437 isert_conn = list_first_entry(&isert_np->pending, 2438 struct isert_conn, node); 2439 list_del_init(&isert_conn->node); 2440 mutex_unlock(&isert_np->mutex); 2441 2442 conn->context = isert_conn; 2443 isert_conn->conn = conn; 2444 isert_conn->state = ISER_CONN_BOUND; 2445 2446 isert_set_conn_info(np, conn, isert_conn); 2447 2448 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2449 2450 return 0; 2451 } 2452 2453 static void 2454 isert_free_np(struct iscsi_np *np) 2455 { 2456 struct isert_np *isert_np = np->np_context; 2457 struct isert_conn *isert_conn, *n; 2458 2459 if (isert_np->cm_id) 2460 rdma_destroy_id(isert_np->cm_id); 2461 2462 /* 2463 * FIXME: At this point we don't have a good way to insure 2464 * that at this point we don't have hanging connections that 2465 * completed RDMA establishment but didn't start iscsi login 2466 * process. So work-around this by cleaning up what ever piled 2467 * up in accepted and pending lists. 2468 */ 2469 mutex_lock(&isert_np->mutex); 2470 if (!list_empty(&isert_np->pending)) { 2471 isert_info("Still have isert pending connections\n"); 2472 list_for_each_entry_safe(isert_conn, n, 2473 &isert_np->pending, 2474 node) { 2475 isert_info("cleaning isert_conn %p state (%d)\n", 2476 isert_conn, isert_conn->state); 2477 isert_connect_release(isert_conn); 2478 } 2479 } 2480 2481 if (!list_empty(&isert_np->accepted)) { 2482 isert_info("Still have isert accepted connections\n"); 2483 list_for_each_entry_safe(isert_conn, n, 2484 &isert_np->accepted, 2485 node) { 2486 isert_info("cleaning isert_conn %p state (%d)\n", 2487 isert_conn, isert_conn->state); 2488 isert_connect_release(isert_conn); 2489 } 2490 } 2491 mutex_unlock(&isert_np->mutex); 2492 2493 np->np_context = NULL; 2494 kfree(isert_np); 2495 } 2496 2497 static void isert_release_work(struct work_struct *work) 2498 { 2499 struct isert_conn *isert_conn = container_of(work, 2500 struct isert_conn, 2501 release_work); 2502 2503 isert_info("Starting release conn %p\n", isert_conn); 2504 2505 mutex_lock(&isert_conn->mutex); 2506 isert_conn->state = ISER_CONN_DOWN; 2507 mutex_unlock(&isert_conn->mutex); 2508 2509 isert_info("Destroying conn %p\n", isert_conn); 2510 isert_put_conn(isert_conn); 2511 } 2512 2513 static void 2514 isert_wait4logout(struct isert_conn *isert_conn) 2515 { 2516 struct iscsi_conn *conn = isert_conn->conn; 2517 2518 isert_info("conn %p\n", isert_conn); 2519 2520 if (isert_conn->logout_posted) { 2521 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2522 wait_for_completion_timeout(&conn->conn_logout_comp, 2523 SECONDS_FOR_LOGOUT_COMP * HZ); 2524 } 2525 } 2526 2527 static void 2528 isert_wait4cmds(struct iscsi_conn *conn) 2529 { 2530 isert_info("iscsi_conn %p\n", conn); 2531 2532 if (conn->sess) { 2533 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2534 target_wait_for_sess_cmds(conn->sess->se_sess); 2535 } 2536 } 2537 2538 /** 2539 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2540 * unsolicitate dataout 2541 * @conn: iscsi connection 2542 * 2543 * We might still have commands that are waiting for unsolicited 2544 * dataouts messages. We must put the extra reference on those 2545 * before blocking on the target_wait_for_session_cmds 2546 */ 2547 static void 2548 isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 2549 { 2550 struct iscsi_cmd *cmd, *tmp; 2551 static LIST_HEAD(drop_cmd_list); 2552 2553 spin_lock_bh(&conn->cmd_lock); 2554 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2555 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2556 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2557 (cmd->write_data_done < cmd->se_cmd.data_length)) 2558 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2559 } 2560 spin_unlock_bh(&conn->cmd_lock); 2561 2562 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2563 list_del_init(&cmd->i_conn_node); 2564 if (cmd->i_state != ISTATE_REMOVE) { 2565 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2566 2567 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2568 isert_put_cmd(isert_cmd, true); 2569 } 2570 } 2571 } 2572 2573 static void isert_wait_conn(struct iscsi_conn *conn) 2574 { 2575 struct isert_conn *isert_conn = conn->context; 2576 2577 isert_info("Starting conn %p\n", isert_conn); 2578 2579 mutex_lock(&isert_conn->mutex); 2580 isert_conn_terminate(isert_conn); 2581 mutex_unlock(&isert_conn->mutex); 2582 2583 ib_drain_qp(isert_conn->qp); 2584 isert_put_unsol_pending_cmds(conn); 2585 isert_wait4cmds(conn); 2586 isert_wait4logout(isert_conn); 2587 2588 queue_work(isert_release_wq, &isert_conn->release_work); 2589 } 2590 2591 static void isert_free_conn(struct iscsi_conn *conn) 2592 { 2593 struct isert_conn *isert_conn = conn->context; 2594 2595 ib_drain_qp(isert_conn->qp); 2596 isert_put_conn(isert_conn); 2597 } 2598 2599 static void isert_get_rx_pdu(struct iscsi_conn *conn) 2600 { 2601 struct completion comp; 2602 2603 init_completion(&comp); 2604 2605 wait_for_completion_interruptible(&comp); 2606 } 2607 2608 static struct iscsit_transport iser_target_transport = { 2609 .name = "IB/iSER", 2610 .transport_type = ISCSI_INFINIBAND, 2611 .rdma_shutdown = true, 2612 .priv_size = sizeof(struct isert_cmd), 2613 .owner = THIS_MODULE, 2614 .iscsit_setup_np = isert_setup_np, 2615 .iscsit_accept_np = isert_accept_np, 2616 .iscsit_free_np = isert_free_np, 2617 .iscsit_wait_conn = isert_wait_conn, 2618 .iscsit_free_conn = isert_free_conn, 2619 .iscsit_get_login_rx = isert_get_login_rx, 2620 .iscsit_put_login_tx = isert_put_login_tx, 2621 .iscsit_immediate_queue = isert_immediate_queue, 2622 .iscsit_response_queue = isert_response_queue, 2623 .iscsit_get_dataout = isert_get_dataout, 2624 .iscsit_queue_data_in = isert_put_datain, 2625 .iscsit_queue_status = isert_put_response, 2626 .iscsit_aborted_task = isert_aborted_task, 2627 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2628 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2629 }; 2630 2631 static int __init isert_init(void) 2632 { 2633 int ret; 2634 2635 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2636 WQ_UNBOUND | WQ_HIGHPRI, 0); 2637 if (!isert_comp_wq) { 2638 isert_err("Unable to allocate isert_comp_wq\n"); 2639 ret = -ENOMEM; 2640 return -ENOMEM; 2641 } 2642 2643 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2644 WQ_UNBOUND_MAX_ACTIVE); 2645 if (!isert_release_wq) { 2646 isert_err("Unable to allocate isert_release_wq\n"); 2647 ret = -ENOMEM; 2648 goto destroy_comp_wq; 2649 } 2650 2651 iscsit_register_transport(&iser_target_transport); 2652 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2653 2654 return 0; 2655 2656 destroy_comp_wq: 2657 destroy_workqueue(isert_comp_wq); 2658 2659 return ret; 2660 } 2661 2662 static void __exit isert_exit(void) 2663 { 2664 flush_scheduled_work(); 2665 destroy_workqueue(isert_release_wq); 2666 destroy_workqueue(isert_comp_wq); 2667 iscsit_unregister_transport(&iser_target_transport); 2668 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2669 } 2670 2671 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2672 MODULE_VERSION("1.0"); 2673 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2674 MODULE_LICENSE("GPL"); 2675 2676 module_init(isert_init); 2677 module_exit(isert_exit); 2678