1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <rdma/ib_verbs.h> 26 #include <rdma/rdma_cm.h> 27 #include <target/target_core_base.h> 28 #include <target/target_core_fabric.h> 29 #include <target/iscsi/iscsi_transport.h> 30 #include <linux/semaphore.h> 31 32 #include "ib_isert.h" 33 34 #define ISERT_MAX_CONN 8 35 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 36 #define ISER_MAX_TX_CQ_LEN \ 37 ((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN) 38 #define ISER_MAX_CQ_LEN (ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \ 39 ISERT_MAX_CONN) 40 41 static int isert_debug_level; 42 module_param_named(debug_level, isert_debug_level, int, 0644); 43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 44 45 static DEFINE_MUTEX(device_list_mutex); 46 static LIST_HEAD(device_list); 47 static struct workqueue_struct *isert_comp_wq; 48 static struct workqueue_struct *isert_release_wq; 49 50 static int 51 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 52 static int 53 isert_login_post_recv(struct isert_conn *isert_conn); 54 static int 55 isert_rdma_accept(struct isert_conn *isert_conn); 56 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 57 58 static void isert_release_work(struct work_struct *work); 59 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 61 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 62 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 63 64 static inline bool 65 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 66 { 67 return (conn->pi_support && 68 cmd->prot_op != TARGET_PROT_NORMAL); 69 } 70 71 72 static void 73 isert_qp_event_callback(struct ib_event *e, void *context) 74 { 75 struct isert_conn *isert_conn = context; 76 77 isert_err("%s (%d): conn %p\n", 78 ib_event_msg(e->event), e->event, isert_conn); 79 80 switch (e->event) { 81 case IB_EVENT_COMM_EST: 82 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 83 break; 84 case IB_EVENT_QP_LAST_WQE_REACHED: 85 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 86 break; 87 default: 88 break; 89 } 90 } 91 92 static struct isert_comp * 93 isert_comp_get(struct isert_conn *isert_conn) 94 { 95 struct isert_device *device = isert_conn->device; 96 struct isert_comp *comp; 97 int i, min = 0; 98 99 mutex_lock(&device_list_mutex); 100 for (i = 0; i < device->comps_used; i++) 101 if (device->comps[i].active_qps < 102 device->comps[min].active_qps) 103 min = i; 104 comp = &device->comps[min]; 105 comp->active_qps++; 106 mutex_unlock(&device_list_mutex); 107 108 isert_info("conn %p, using comp %p min_index: %d\n", 109 isert_conn, comp, min); 110 111 return comp; 112 } 113 114 static void 115 isert_comp_put(struct isert_comp *comp) 116 { 117 mutex_lock(&device_list_mutex); 118 comp->active_qps--; 119 mutex_unlock(&device_list_mutex); 120 } 121 122 static struct ib_qp * 123 isert_create_qp(struct isert_conn *isert_conn, 124 struct isert_comp *comp, 125 struct rdma_cm_id *cma_id) 126 { 127 struct isert_device *device = isert_conn->device; 128 struct ib_qp_init_attr attr; 129 int ret; 130 131 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 132 attr.event_handler = isert_qp_event_callback; 133 attr.qp_context = isert_conn; 134 attr.send_cq = comp->cq; 135 attr.recv_cq = comp->cq; 136 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 137 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 138 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX; 139 attr.cap.max_send_sge = device->ib_device->attrs.max_sge; 140 attr.cap.max_recv_sge = 1; 141 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 142 attr.qp_type = IB_QPT_RC; 143 if (device->pi_capable) 144 attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN; 145 146 ret = rdma_create_qp(cma_id, device->pd, &attr); 147 if (ret) { 148 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 149 return ERR_PTR(ret); 150 } 151 152 return cma_id->qp; 153 } 154 155 static int 156 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 157 { 158 struct isert_comp *comp; 159 int ret; 160 161 comp = isert_comp_get(isert_conn); 162 isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id); 163 if (IS_ERR(isert_conn->qp)) { 164 ret = PTR_ERR(isert_conn->qp); 165 goto err; 166 } 167 168 return 0; 169 err: 170 isert_comp_put(comp); 171 return ret; 172 } 173 174 static int 175 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 176 { 177 struct isert_device *device = isert_conn->device; 178 struct ib_device *ib_dev = device->ib_device; 179 struct iser_rx_desc *rx_desc; 180 struct ib_sge *rx_sg; 181 u64 dma_addr; 182 int i, j; 183 184 isert_conn->rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 185 sizeof(struct iser_rx_desc), GFP_KERNEL); 186 if (!isert_conn->rx_descs) 187 return -ENOMEM; 188 189 rx_desc = isert_conn->rx_descs; 190 191 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 192 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 193 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 194 if (ib_dma_mapping_error(ib_dev, dma_addr)) 195 goto dma_map_fail; 196 197 rx_desc->dma_addr = dma_addr; 198 199 rx_sg = &rx_desc->rx_sg; 200 rx_sg->addr = rx_desc->dma_addr; 201 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 202 rx_sg->lkey = device->pd->local_dma_lkey; 203 rx_desc->rx_cqe.done = isert_recv_done; 204 } 205 206 return 0; 207 208 dma_map_fail: 209 rx_desc = isert_conn->rx_descs; 210 for (j = 0; j < i; j++, rx_desc++) { 211 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 212 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 213 } 214 kfree(isert_conn->rx_descs); 215 isert_conn->rx_descs = NULL; 216 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 217 return -ENOMEM; 218 } 219 220 static void 221 isert_free_rx_descriptors(struct isert_conn *isert_conn) 222 { 223 struct ib_device *ib_dev = isert_conn->device->ib_device; 224 struct iser_rx_desc *rx_desc; 225 int i; 226 227 if (!isert_conn->rx_descs) 228 return; 229 230 rx_desc = isert_conn->rx_descs; 231 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 232 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 233 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 234 } 235 236 kfree(isert_conn->rx_descs); 237 isert_conn->rx_descs = NULL; 238 } 239 240 static void 241 isert_free_comps(struct isert_device *device) 242 { 243 int i; 244 245 for (i = 0; i < device->comps_used; i++) { 246 struct isert_comp *comp = &device->comps[i]; 247 248 if (comp->cq) 249 ib_free_cq(comp->cq); 250 } 251 kfree(device->comps); 252 } 253 254 static int 255 isert_alloc_comps(struct isert_device *device) 256 { 257 int i, max_cqe, ret = 0; 258 259 device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(), 260 device->ib_device->num_comp_vectors)); 261 262 isert_info("Using %d CQs, %s supports %d vectors support " 263 "pi_capable %d\n", 264 device->comps_used, device->ib_device->name, 265 device->ib_device->num_comp_vectors, 266 device->pi_capable); 267 268 device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp), 269 GFP_KERNEL); 270 if (!device->comps) 271 return -ENOMEM; 272 273 max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe); 274 275 for (i = 0; i < device->comps_used; i++) { 276 struct isert_comp *comp = &device->comps[i]; 277 278 comp->device = device; 279 comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i, 280 IB_POLL_WORKQUEUE); 281 if (IS_ERR(comp->cq)) { 282 isert_err("Unable to allocate cq\n"); 283 ret = PTR_ERR(comp->cq); 284 comp->cq = NULL; 285 goto out_cq; 286 } 287 } 288 289 return 0; 290 out_cq: 291 isert_free_comps(device); 292 return ret; 293 } 294 295 static int 296 isert_create_device_ib_res(struct isert_device *device) 297 { 298 struct ib_device *ib_dev = device->ib_device; 299 int ret; 300 301 isert_dbg("devattr->max_sge: %d\n", ib_dev->attrs.max_sge); 302 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 303 304 ret = isert_alloc_comps(device); 305 if (ret) 306 goto out; 307 308 device->pd = ib_alloc_pd(ib_dev, 0); 309 if (IS_ERR(device->pd)) { 310 ret = PTR_ERR(device->pd); 311 isert_err("failed to allocate pd, device %p, ret=%d\n", 312 device, ret); 313 goto out_cq; 314 } 315 316 /* Check signature cap */ 317 device->pi_capable = ib_dev->attrs.device_cap_flags & 318 IB_DEVICE_SIGNATURE_HANDOVER ? true : false; 319 320 return 0; 321 322 out_cq: 323 isert_free_comps(device); 324 out: 325 if (ret > 0) 326 ret = -EINVAL; 327 return ret; 328 } 329 330 static void 331 isert_free_device_ib_res(struct isert_device *device) 332 { 333 isert_info("device %p\n", device); 334 335 ib_dealloc_pd(device->pd); 336 isert_free_comps(device); 337 } 338 339 static void 340 isert_device_put(struct isert_device *device) 341 { 342 mutex_lock(&device_list_mutex); 343 device->refcount--; 344 isert_info("device %p refcount %d\n", device, device->refcount); 345 if (!device->refcount) { 346 isert_free_device_ib_res(device); 347 list_del(&device->dev_node); 348 kfree(device); 349 } 350 mutex_unlock(&device_list_mutex); 351 } 352 353 static struct isert_device * 354 isert_device_get(struct rdma_cm_id *cma_id) 355 { 356 struct isert_device *device; 357 int ret; 358 359 mutex_lock(&device_list_mutex); 360 list_for_each_entry(device, &device_list, dev_node) { 361 if (device->ib_device->node_guid == cma_id->device->node_guid) { 362 device->refcount++; 363 isert_info("Found iser device %p refcount %d\n", 364 device, device->refcount); 365 mutex_unlock(&device_list_mutex); 366 return device; 367 } 368 } 369 370 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 371 if (!device) { 372 mutex_unlock(&device_list_mutex); 373 return ERR_PTR(-ENOMEM); 374 } 375 376 INIT_LIST_HEAD(&device->dev_node); 377 378 device->ib_device = cma_id->device; 379 ret = isert_create_device_ib_res(device); 380 if (ret) { 381 kfree(device); 382 mutex_unlock(&device_list_mutex); 383 return ERR_PTR(ret); 384 } 385 386 device->refcount++; 387 list_add_tail(&device->dev_node, &device_list); 388 isert_info("Created a new iser device %p refcount %d\n", 389 device, device->refcount); 390 mutex_unlock(&device_list_mutex); 391 392 return device; 393 } 394 395 static void 396 isert_init_conn(struct isert_conn *isert_conn) 397 { 398 isert_conn->state = ISER_CONN_INIT; 399 INIT_LIST_HEAD(&isert_conn->node); 400 init_completion(&isert_conn->login_comp); 401 init_completion(&isert_conn->login_req_comp); 402 init_waitqueue_head(&isert_conn->rem_wait); 403 kref_init(&isert_conn->kref); 404 mutex_init(&isert_conn->mutex); 405 INIT_WORK(&isert_conn->release_work, isert_release_work); 406 } 407 408 static void 409 isert_free_login_buf(struct isert_conn *isert_conn) 410 { 411 struct ib_device *ib_dev = isert_conn->device->ib_device; 412 413 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 414 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 415 kfree(isert_conn->login_rsp_buf); 416 417 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 418 ISER_RX_PAYLOAD_SIZE, 419 DMA_FROM_DEVICE); 420 kfree(isert_conn->login_req_buf); 421 } 422 423 static int 424 isert_alloc_login_buf(struct isert_conn *isert_conn, 425 struct ib_device *ib_dev) 426 { 427 int ret; 428 429 isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf), 430 GFP_KERNEL); 431 if (!isert_conn->login_req_buf) 432 return -ENOMEM; 433 434 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 435 isert_conn->login_req_buf, 436 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 437 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 438 if (ret) { 439 isert_err("login_req_dma mapping error: %d\n", ret); 440 isert_conn->login_req_dma = 0; 441 goto out_free_login_req_buf; 442 } 443 444 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 445 if (!isert_conn->login_rsp_buf) { 446 ret = -ENOMEM; 447 goto out_unmap_login_req_buf; 448 } 449 450 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 451 isert_conn->login_rsp_buf, 452 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 453 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 454 if (ret) { 455 isert_err("login_rsp_dma mapping error: %d\n", ret); 456 isert_conn->login_rsp_dma = 0; 457 goto out_free_login_rsp_buf; 458 } 459 460 return 0; 461 462 out_free_login_rsp_buf: 463 kfree(isert_conn->login_rsp_buf); 464 out_unmap_login_req_buf: 465 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 466 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 467 out_free_login_req_buf: 468 kfree(isert_conn->login_req_buf); 469 return ret; 470 } 471 472 static void 473 isert_set_nego_params(struct isert_conn *isert_conn, 474 struct rdma_conn_param *param) 475 { 476 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 477 478 /* Set max inflight RDMA READ requests */ 479 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 480 attr->max_qp_init_rd_atom); 481 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 482 483 if (param->private_data) { 484 u8 flags = *(u8 *)param->private_data; 485 486 /* 487 * use remote invalidation if the both initiator 488 * and the HCA support it 489 */ 490 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 491 (attr->device_cap_flags & 492 IB_DEVICE_MEM_MGT_EXTENSIONS); 493 if (isert_conn->snd_w_inv) 494 isert_info("Using remote invalidation\n"); 495 } 496 } 497 498 static int 499 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 500 { 501 struct isert_np *isert_np = cma_id->context; 502 struct iscsi_np *np = isert_np->np; 503 struct isert_conn *isert_conn; 504 struct isert_device *device; 505 int ret = 0; 506 507 spin_lock_bh(&np->np_thread_lock); 508 if (!np->enabled) { 509 spin_unlock_bh(&np->np_thread_lock); 510 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 511 return rdma_reject(cma_id, NULL, 0); 512 } 513 spin_unlock_bh(&np->np_thread_lock); 514 515 isert_dbg("cma_id: %p, portal: %p\n", 516 cma_id, cma_id->context); 517 518 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 519 if (!isert_conn) 520 return -ENOMEM; 521 522 isert_init_conn(isert_conn); 523 isert_conn->cm_id = cma_id; 524 525 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 526 if (ret) 527 goto out; 528 529 device = isert_device_get(cma_id); 530 if (IS_ERR(device)) { 531 ret = PTR_ERR(device); 532 goto out_rsp_dma_map; 533 } 534 isert_conn->device = device; 535 536 isert_set_nego_params(isert_conn, &event->param.conn); 537 538 ret = isert_conn_setup_qp(isert_conn, cma_id); 539 if (ret) 540 goto out_conn_dev; 541 542 ret = isert_login_post_recv(isert_conn); 543 if (ret) 544 goto out_conn_dev; 545 546 ret = isert_rdma_accept(isert_conn); 547 if (ret) 548 goto out_conn_dev; 549 550 mutex_lock(&isert_np->mutex); 551 list_add_tail(&isert_conn->node, &isert_np->accepted); 552 mutex_unlock(&isert_np->mutex); 553 554 return 0; 555 556 out_conn_dev: 557 isert_device_put(device); 558 out_rsp_dma_map: 559 isert_free_login_buf(isert_conn); 560 out: 561 kfree(isert_conn); 562 rdma_reject(cma_id, NULL, 0); 563 return ret; 564 } 565 566 static void 567 isert_connect_release(struct isert_conn *isert_conn) 568 { 569 struct isert_device *device = isert_conn->device; 570 571 isert_dbg("conn %p\n", isert_conn); 572 573 BUG_ON(!device); 574 575 isert_free_rx_descriptors(isert_conn); 576 if (isert_conn->cm_id && 577 !isert_conn->dev_removed) 578 rdma_destroy_id(isert_conn->cm_id); 579 580 if (isert_conn->qp) { 581 struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context; 582 583 isert_comp_put(comp); 584 ib_destroy_qp(isert_conn->qp); 585 } 586 587 if (isert_conn->login_req_buf) 588 isert_free_login_buf(isert_conn); 589 590 isert_device_put(device); 591 592 if (isert_conn->dev_removed) 593 wake_up_interruptible(&isert_conn->rem_wait); 594 else 595 kfree(isert_conn); 596 } 597 598 static void 599 isert_connected_handler(struct rdma_cm_id *cma_id) 600 { 601 struct isert_conn *isert_conn = cma_id->qp->qp_context; 602 struct isert_np *isert_np = cma_id->context; 603 604 isert_info("conn %p\n", isert_conn); 605 606 mutex_lock(&isert_conn->mutex); 607 isert_conn->state = ISER_CONN_UP; 608 kref_get(&isert_conn->kref); 609 mutex_unlock(&isert_conn->mutex); 610 611 mutex_lock(&isert_np->mutex); 612 list_move_tail(&isert_conn->node, &isert_np->pending); 613 mutex_unlock(&isert_np->mutex); 614 615 isert_info("np %p: Allow accept_np to continue\n", isert_np); 616 up(&isert_np->sem); 617 } 618 619 static void 620 isert_release_kref(struct kref *kref) 621 { 622 struct isert_conn *isert_conn = container_of(kref, 623 struct isert_conn, kref); 624 625 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 626 current->pid); 627 628 isert_connect_release(isert_conn); 629 } 630 631 static void 632 isert_put_conn(struct isert_conn *isert_conn) 633 { 634 kref_put(&isert_conn->kref, isert_release_kref); 635 } 636 637 static void 638 isert_handle_unbound_conn(struct isert_conn *isert_conn) 639 { 640 struct isert_np *isert_np = isert_conn->cm_id->context; 641 642 mutex_lock(&isert_np->mutex); 643 if (!list_empty(&isert_conn->node)) { 644 /* 645 * This means iscsi doesn't know this connection 646 * so schedule a cleanup ourselves 647 */ 648 list_del_init(&isert_conn->node); 649 isert_put_conn(isert_conn); 650 queue_work(isert_release_wq, &isert_conn->release_work); 651 } 652 mutex_unlock(&isert_np->mutex); 653 } 654 655 /** 656 * isert_conn_terminate() - Initiate connection termination 657 * @isert_conn: isert connection struct 658 * 659 * Notes: 660 * In case the connection state is BOUND, move state 661 * to TEMINATING and start teardown sequence (rdma_disconnect). 662 * In case the connection state is UP, complete flush as well. 663 * 664 * This routine must be called with mutex held. Thus it is 665 * safe to call multiple times. 666 */ 667 static void 668 isert_conn_terminate(struct isert_conn *isert_conn) 669 { 670 int err; 671 672 if (isert_conn->state >= ISER_CONN_TERMINATING) 673 return; 674 675 isert_info("Terminating conn %p state %d\n", 676 isert_conn, isert_conn->state); 677 isert_conn->state = ISER_CONN_TERMINATING; 678 err = rdma_disconnect(isert_conn->cm_id); 679 if (err) 680 isert_warn("Failed rdma_disconnect isert_conn %p\n", 681 isert_conn); 682 } 683 684 static int 685 isert_np_cma_handler(struct isert_np *isert_np, 686 enum rdma_cm_event_type event) 687 { 688 isert_dbg("%s (%d): isert np %p\n", 689 rdma_event_msg(event), event, isert_np); 690 691 switch (event) { 692 case RDMA_CM_EVENT_DEVICE_REMOVAL: 693 isert_np->cm_id = NULL; 694 break; 695 case RDMA_CM_EVENT_ADDR_CHANGE: 696 isert_np->cm_id = isert_setup_id(isert_np); 697 if (IS_ERR(isert_np->cm_id)) { 698 isert_err("isert np %p setup id failed: %ld\n", 699 isert_np, PTR_ERR(isert_np->cm_id)); 700 isert_np->cm_id = NULL; 701 } 702 break; 703 default: 704 isert_err("isert np %p Unexpected event %d\n", 705 isert_np, event); 706 } 707 708 return -1; 709 } 710 711 static int 712 isert_disconnected_handler(struct rdma_cm_id *cma_id, 713 enum rdma_cm_event_type event) 714 { 715 struct isert_conn *isert_conn = cma_id->qp->qp_context; 716 717 mutex_lock(&isert_conn->mutex); 718 switch (isert_conn->state) { 719 case ISER_CONN_TERMINATING: 720 break; 721 case ISER_CONN_UP: 722 isert_conn_terminate(isert_conn); 723 ib_drain_qp(isert_conn->qp); 724 isert_handle_unbound_conn(isert_conn); 725 break; 726 case ISER_CONN_BOUND: 727 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 728 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 729 break; 730 default: 731 isert_warn("conn %p terminating in state %d\n", 732 isert_conn, isert_conn->state); 733 } 734 mutex_unlock(&isert_conn->mutex); 735 736 return 0; 737 } 738 739 static int 740 isert_connect_error(struct rdma_cm_id *cma_id) 741 { 742 struct isert_conn *isert_conn = cma_id->qp->qp_context; 743 744 list_del_init(&isert_conn->node); 745 isert_conn->cm_id = NULL; 746 isert_put_conn(isert_conn); 747 748 return -1; 749 } 750 751 static int 752 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 753 { 754 struct isert_np *isert_np = cma_id->context; 755 struct isert_conn *isert_conn; 756 int ret = 0; 757 758 isert_info("%s (%d): status %d id %p np %p\n", 759 rdma_event_msg(event->event), event->event, 760 event->status, cma_id, cma_id->context); 761 762 if (isert_np->cm_id == cma_id) 763 return isert_np_cma_handler(cma_id->context, event->event); 764 765 switch (event->event) { 766 case RDMA_CM_EVENT_CONNECT_REQUEST: 767 ret = isert_connect_request(cma_id, event); 768 if (ret) 769 isert_err("failed handle connect request %d\n", ret); 770 break; 771 case RDMA_CM_EVENT_ESTABLISHED: 772 isert_connected_handler(cma_id); 773 break; 774 case RDMA_CM_EVENT_ADDR_CHANGE: /* FALLTHRU */ 775 case RDMA_CM_EVENT_DISCONNECTED: /* FALLTHRU */ 776 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 777 ret = isert_disconnected_handler(cma_id, event->event); 778 break; 779 case RDMA_CM_EVENT_DEVICE_REMOVAL: 780 isert_conn = cma_id->qp->qp_context; 781 isert_conn->dev_removed = true; 782 isert_disconnected_handler(cma_id, event->event); 783 wait_event_interruptible(isert_conn->rem_wait, 784 isert_conn->state == ISER_CONN_DOWN); 785 kfree(isert_conn); 786 /* 787 * return non-zero from the callback to destroy 788 * the rdma cm id 789 */ 790 return 1; 791 case RDMA_CM_EVENT_REJECTED: /* FALLTHRU */ 792 isert_info("Connection rejected: %s\n", 793 rdma_reject_msg(cma_id, event->status)); 794 case RDMA_CM_EVENT_UNREACHABLE: /* FALLTHRU */ 795 case RDMA_CM_EVENT_CONNECT_ERROR: 796 ret = isert_connect_error(cma_id); 797 break; 798 default: 799 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 800 break; 801 } 802 803 return ret; 804 } 805 806 static int 807 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 808 { 809 struct ib_recv_wr *rx_wr, *rx_wr_failed; 810 int i, ret; 811 struct iser_rx_desc *rx_desc; 812 813 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 814 rx_desc = &isert_conn->rx_descs[i]; 815 816 rx_wr->wr_cqe = &rx_desc->rx_cqe; 817 rx_wr->sg_list = &rx_desc->rx_sg; 818 rx_wr->num_sge = 1; 819 rx_wr->next = rx_wr + 1; 820 rx_desc->in_use = false; 821 } 822 rx_wr--; 823 rx_wr->next = NULL; /* mark end of work requests list */ 824 825 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, 826 &rx_wr_failed); 827 if (ret) 828 isert_err("ib_post_recv() failed with ret: %d\n", ret); 829 830 return ret; 831 } 832 833 static int 834 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 835 { 836 struct ib_recv_wr *rx_wr_failed, rx_wr; 837 int ret; 838 839 if (!rx_desc->in_use) { 840 /* 841 * if the descriptor is not in-use we already reposted it 842 * for recv, so just silently return 843 */ 844 return 0; 845 } 846 847 rx_desc->in_use = false; 848 rx_wr.wr_cqe = &rx_desc->rx_cqe; 849 rx_wr.sg_list = &rx_desc->rx_sg; 850 rx_wr.num_sge = 1; 851 rx_wr.next = NULL; 852 853 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_failed); 854 if (ret) 855 isert_err("ib_post_recv() failed with ret: %d\n", ret); 856 857 return ret; 858 } 859 860 static int 861 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 862 { 863 struct ib_device *ib_dev = isert_conn->cm_id->device; 864 struct ib_send_wr send_wr, *send_wr_failed; 865 int ret; 866 867 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 868 ISER_HEADERS_LEN, DMA_TO_DEVICE); 869 870 tx_desc->tx_cqe.done = isert_login_send_done; 871 872 send_wr.next = NULL; 873 send_wr.wr_cqe = &tx_desc->tx_cqe; 874 send_wr.sg_list = tx_desc->tx_sg; 875 send_wr.num_sge = tx_desc->num_sge; 876 send_wr.opcode = IB_WR_SEND; 877 send_wr.send_flags = IB_SEND_SIGNALED; 878 879 ret = ib_post_send(isert_conn->qp, &send_wr, &send_wr_failed); 880 if (ret) 881 isert_err("ib_post_send() failed, ret: %d\n", ret); 882 883 return ret; 884 } 885 886 static void 887 isert_create_send_desc(struct isert_conn *isert_conn, 888 struct isert_cmd *isert_cmd, 889 struct iser_tx_desc *tx_desc) 890 { 891 struct isert_device *device = isert_conn->device; 892 struct ib_device *ib_dev = device->ib_device; 893 894 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 895 ISER_HEADERS_LEN, DMA_TO_DEVICE); 896 897 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 898 tx_desc->iser_header.flags = ISCSI_CTRL; 899 900 tx_desc->num_sge = 1; 901 902 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 903 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 904 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 905 } 906 } 907 908 static int 909 isert_init_tx_hdrs(struct isert_conn *isert_conn, 910 struct iser_tx_desc *tx_desc) 911 { 912 struct isert_device *device = isert_conn->device; 913 struct ib_device *ib_dev = device->ib_device; 914 u64 dma_addr; 915 916 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 917 ISER_HEADERS_LEN, DMA_TO_DEVICE); 918 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 919 isert_err("ib_dma_mapping_error() failed\n"); 920 return -ENOMEM; 921 } 922 923 tx_desc->dma_addr = dma_addr; 924 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 925 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 926 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 927 928 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 929 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 930 tx_desc->tx_sg[0].lkey); 931 932 return 0; 933 } 934 935 static void 936 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 937 struct ib_send_wr *send_wr) 938 { 939 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 940 941 tx_desc->tx_cqe.done = isert_send_done; 942 send_wr->wr_cqe = &tx_desc->tx_cqe; 943 944 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 945 send_wr->opcode = IB_WR_SEND_WITH_INV; 946 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 947 } else { 948 send_wr->opcode = IB_WR_SEND; 949 } 950 951 send_wr->sg_list = &tx_desc->tx_sg[0]; 952 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 953 send_wr->send_flags = IB_SEND_SIGNALED; 954 } 955 956 static int 957 isert_login_post_recv(struct isert_conn *isert_conn) 958 { 959 struct ib_recv_wr rx_wr, *rx_wr_fail; 960 struct ib_sge sge; 961 int ret; 962 963 memset(&sge, 0, sizeof(struct ib_sge)); 964 sge.addr = isert_conn->login_req_dma; 965 sge.length = ISER_RX_PAYLOAD_SIZE; 966 sge.lkey = isert_conn->device->pd->local_dma_lkey; 967 968 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 969 sge.addr, sge.length, sge.lkey); 970 971 isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done; 972 973 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 974 rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe; 975 rx_wr.sg_list = &sge; 976 rx_wr.num_sge = 1; 977 978 ret = ib_post_recv(isert_conn->qp, &rx_wr, &rx_wr_fail); 979 if (ret) 980 isert_err("ib_post_recv() failed: %d\n", ret); 981 982 return ret; 983 } 984 985 static int 986 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 987 u32 length) 988 { 989 struct isert_conn *isert_conn = conn->context; 990 struct isert_device *device = isert_conn->device; 991 struct ib_device *ib_dev = device->ib_device; 992 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 993 int ret; 994 995 isert_create_send_desc(isert_conn, NULL, tx_desc); 996 997 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 998 sizeof(struct iscsi_hdr)); 999 1000 isert_init_tx_hdrs(isert_conn, tx_desc); 1001 1002 if (length > 0) { 1003 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 1004 1005 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 1006 length, DMA_TO_DEVICE); 1007 1008 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 1009 1010 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 1011 length, DMA_TO_DEVICE); 1012 1013 tx_dsg->addr = isert_conn->login_rsp_dma; 1014 tx_dsg->length = length; 1015 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 1016 tx_desc->num_sge = 2; 1017 } 1018 if (!login->login_failed) { 1019 if (login->login_complete) { 1020 ret = isert_alloc_rx_descriptors(isert_conn); 1021 if (ret) 1022 return ret; 1023 1024 ret = isert_post_recvm(isert_conn, 1025 ISERT_QP_MAX_RECV_DTOS); 1026 if (ret) 1027 return ret; 1028 1029 /* Now we are in FULL_FEATURE phase */ 1030 mutex_lock(&isert_conn->mutex); 1031 isert_conn->state = ISER_CONN_FULL_FEATURE; 1032 mutex_unlock(&isert_conn->mutex); 1033 goto post_send; 1034 } 1035 1036 ret = isert_login_post_recv(isert_conn); 1037 if (ret) 1038 return ret; 1039 } 1040 post_send: 1041 ret = isert_login_post_send(isert_conn, tx_desc); 1042 if (ret) 1043 return ret; 1044 1045 return 0; 1046 } 1047 1048 static void 1049 isert_rx_login_req(struct isert_conn *isert_conn) 1050 { 1051 struct iser_rx_desc *rx_desc = isert_conn->login_req_buf; 1052 int rx_buflen = isert_conn->login_req_len; 1053 struct iscsi_conn *conn = isert_conn->conn; 1054 struct iscsi_login *login = conn->conn_login; 1055 int size; 1056 1057 isert_info("conn %p\n", isert_conn); 1058 1059 WARN_ON_ONCE(!login); 1060 1061 if (login->first_request) { 1062 struct iscsi_login_req *login_req = 1063 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1064 /* 1065 * Setup the initial iscsi_login values from the leading 1066 * login request PDU. 1067 */ 1068 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1069 login->current_stage = 1070 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1071 >> 2; 1072 login->version_min = login_req->min_version; 1073 login->version_max = login_req->max_version; 1074 memcpy(login->isid, login_req->isid, 6); 1075 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1076 login->init_task_tag = login_req->itt; 1077 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1078 login->cid = be16_to_cpu(login_req->cid); 1079 login->tsih = be16_to_cpu(login_req->tsih); 1080 } 1081 1082 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1083 1084 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1085 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1086 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1087 MAX_KEY_VALUE_PAIRS); 1088 memcpy(login->req_buf, &rx_desc->data[0], size); 1089 1090 if (login->first_request) { 1091 complete(&isert_conn->login_comp); 1092 return; 1093 } 1094 schedule_delayed_work(&conn->login_work, 0); 1095 } 1096 1097 static struct iscsi_cmd 1098 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1099 { 1100 struct isert_conn *isert_conn = conn->context; 1101 struct isert_cmd *isert_cmd; 1102 struct iscsi_cmd *cmd; 1103 1104 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1105 if (!cmd) { 1106 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1107 return NULL; 1108 } 1109 isert_cmd = iscsit_priv_cmd(cmd); 1110 isert_cmd->conn = isert_conn; 1111 isert_cmd->iscsi_cmd = cmd; 1112 isert_cmd->rx_desc = rx_desc; 1113 1114 return cmd; 1115 } 1116 1117 static int 1118 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1119 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1120 struct iser_rx_desc *rx_desc, unsigned char *buf) 1121 { 1122 struct iscsi_conn *conn = isert_conn->conn; 1123 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1124 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1125 bool dump_payload = false; 1126 unsigned int data_len; 1127 1128 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1129 if (rc < 0) 1130 return rc; 1131 1132 imm_data = cmd->immediate_data; 1133 imm_data_len = cmd->first_burst_len; 1134 unsol_data = cmd->unsolicited_data; 1135 data_len = cmd->se_cmd.data_length; 1136 1137 if (imm_data && imm_data_len == data_len) 1138 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1139 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1140 if (rc < 0) { 1141 return 0; 1142 } else if (rc > 0) { 1143 dump_payload = true; 1144 goto sequence_cmd; 1145 } 1146 1147 if (!imm_data) 1148 return 0; 1149 1150 if (imm_data_len != data_len) { 1151 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1152 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1153 &rx_desc->data[0], imm_data_len); 1154 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1155 sg_nents, imm_data_len); 1156 } else { 1157 sg_init_table(&isert_cmd->sg, 1); 1158 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1159 cmd->se_cmd.t_data_nents = 1; 1160 sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len); 1161 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1162 imm_data_len); 1163 } 1164 1165 cmd->write_data_done += imm_data_len; 1166 1167 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1168 spin_lock_bh(&cmd->istate_lock); 1169 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1170 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1171 spin_unlock_bh(&cmd->istate_lock); 1172 } 1173 1174 sequence_cmd: 1175 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1176 1177 if (!rc && dump_payload == false && unsol_data) 1178 iscsit_set_unsoliticed_dataout(cmd); 1179 else if (dump_payload && imm_data) 1180 target_put_sess_cmd(&cmd->se_cmd); 1181 1182 return 0; 1183 } 1184 1185 static int 1186 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1187 struct iser_rx_desc *rx_desc, unsigned char *buf) 1188 { 1189 struct scatterlist *sg_start; 1190 struct iscsi_conn *conn = isert_conn->conn; 1191 struct iscsi_cmd *cmd = NULL; 1192 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1193 u32 unsol_data_len = ntoh24(hdr->dlength); 1194 int rc, sg_nents, sg_off, page_off; 1195 1196 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1197 if (rc < 0) 1198 return rc; 1199 else if (!cmd) 1200 return 0; 1201 /* 1202 * FIXME: Unexpected unsolicited_data out 1203 */ 1204 if (!cmd->unsolicited_data) { 1205 isert_err("Received unexpected solicited data payload\n"); 1206 dump_stack(); 1207 return -1; 1208 } 1209 1210 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1211 "write_data_done: %u, data_length: %u\n", 1212 unsol_data_len, cmd->write_data_done, 1213 cmd->se_cmd.data_length); 1214 1215 sg_off = cmd->write_data_done / PAGE_SIZE; 1216 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1217 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1218 page_off = cmd->write_data_done % PAGE_SIZE; 1219 /* 1220 * FIXME: Non page-aligned unsolicited_data out 1221 */ 1222 if (page_off) { 1223 isert_err("unexpected non-page aligned data payload\n"); 1224 dump_stack(); 1225 return -1; 1226 } 1227 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1228 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1229 sg_nents, &rx_desc->data[0], unsol_data_len); 1230 1231 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1232 unsol_data_len); 1233 1234 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1235 if (rc < 0) 1236 return rc; 1237 1238 /* 1239 * multiple data-outs on the same command can arrive - 1240 * so post the buffer before hand 1241 */ 1242 rc = isert_post_recv(isert_conn, rx_desc); 1243 if (rc) { 1244 isert_err("ib_post_recv failed with %d\n", rc); 1245 return rc; 1246 } 1247 return 0; 1248 } 1249 1250 static int 1251 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1252 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1253 unsigned char *buf) 1254 { 1255 struct iscsi_conn *conn = isert_conn->conn; 1256 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1257 int rc; 1258 1259 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1260 if (rc < 0) 1261 return rc; 1262 /* 1263 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1264 */ 1265 1266 return iscsit_process_nop_out(conn, cmd, hdr); 1267 } 1268 1269 static int 1270 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1271 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1272 struct iscsi_text *hdr) 1273 { 1274 struct iscsi_conn *conn = isert_conn->conn; 1275 u32 payload_length = ntoh24(hdr->dlength); 1276 int rc; 1277 unsigned char *text_in = NULL; 1278 1279 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1280 if (rc < 0) 1281 return rc; 1282 1283 if (payload_length) { 1284 text_in = kzalloc(payload_length, GFP_KERNEL); 1285 if (!text_in) 1286 return -ENOMEM; 1287 } 1288 cmd->text_in_ptr = text_in; 1289 1290 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1291 1292 return iscsit_process_text_cmd(conn, cmd, hdr); 1293 } 1294 1295 static int 1296 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1297 uint32_t read_stag, uint64_t read_va, 1298 uint32_t write_stag, uint64_t write_va) 1299 { 1300 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1301 struct iscsi_conn *conn = isert_conn->conn; 1302 struct iscsi_cmd *cmd; 1303 struct isert_cmd *isert_cmd; 1304 int ret = -EINVAL; 1305 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1306 1307 if (conn->sess->sess_ops->SessionType && 1308 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1309 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1310 " ignoring\n", opcode); 1311 return 0; 1312 } 1313 1314 switch (opcode) { 1315 case ISCSI_OP_SCSI_CMD: 1316 cmd = isert_allocate_cmd(conn, rx_desc); 1317 if (!cmd) 1318 break; 1319 1320 isert_cmd = iscsit_priv_cmd(cmd); 1321 isert_cmd->read_stag = read_stag; 1322 isert_cmd->read_va = read_va; 1323 isert_cmd->write_stag = write_stag; 1324 isert_cmd->write_va = write_va; 1325 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1326 1327 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1328 rx_desc, (unsigned char *)hdr); 1329 break; 1330 case ISCSI_OP_NOOP_OUT: 1331 cmd = isert_allocate_cmd(conn, rx_desc); 1332 if (!cmd) 1333 break; 1334 1335 isert_cmd = iscsit_priv_cmd(cmd); 1336 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1337 rx_desc, (unsigned char *)hdr); 1338 break; 1339 case ISCSI_OP_SCSI_DATA_OUT: 1340 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1341 (unsigned char *)hdr); 1342 break; 1343 case ISCSI_OP_SCSI_TMFUNC: 1344 cmd = isert_allocate_cmd(conn, rx_desc); 1345 if (!cmd) 1346 break; 1347 1348 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1349 (unsigned char *)hdr); 1350 break; 1351 case ISCSI_OP_LOGOUT: 1352 cmd = isert_allocate_cmd(conn, rx_desc); 1353 if (!cmd) 1354 break; 1355 1356 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1357 break; 1358 case ISCSI_OP_TEXT: 1359 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1360 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1361 else 1362 cmd = isert_allocate_cmd(conn, rx_desc); 1363 1364 if (!cmd) 1365 break; 1366 1367 isert_cmd = iscsit_priv_cmd(cmd); 1368 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1369 rx_desc, (struct iscsi_text *)hdr); 1370 break; 1371 default: 1372 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1373 dump_stack(); 1374 break; 1375 } 1376 1377 return ret; 1378 } 1379 1380 static void 1381 isert_print_wc(struct ib_wc *wc, const char *type) 1382 { 1383 if (wc->status != IB_WC_WR_FLUSH_ERR) 1384 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1385 ib_wc_status_msg(wc->status), wc->status, 1386 wc->vendor_err); 1387 else 1388 isert_dbg("%s failure: %s (%d)\n", type, 1389 ib_wc_status_msg(wc->status), wc->status); 1390 } 1391 1392 static void 1393 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1394 { 1395 struct isert_conn *isert_conn = wc->qp->qp_context; 1396 struct ib_device *ib_dev = isert_conn->cm_id->device; 1397 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1398 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1399 struct iser_ctrl *iser_ctrl = &rx_desc->iser_header; 1400 uint64_t read_va = 0, write_va = 0; 1401 uint32_t read_stag = 0, write_stag = 0; 1402 1403 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1404 isert_print_wc(wc, "recv"); 1405 if (wc->status != IB_WC_WR_FLUSH_ERR) 1406 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1407 return; 1408 } 1409 1410 rx_desc->in_use = true; 1411 1412 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1413 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1414 1415 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1416 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1417 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1418 1419 switch (iser_ctrl->flags & 0xF0) { 1420 case ISCSI_CTRL: 1421 if (iser_ctrl->flags & ISER_RSV) { 1422 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1423 read_va = be64_to_cpu(iser_ctrl->read_va); 1424 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1425 read_stag, (unsigned long long)read_va); 1426 } 1427 if (iser_ctrl->flags & ISER_WSV) { 1428 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1429 write_va = be64_to_cpu(iser_ctrl->write_va); 1430 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1431 write_stag, (unsigned long long)write_va); 1432 } 1433 1434 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1435 break; 1436 case ISER_HELLO: 1437 isert_err("iSER Hello message\n"); 1438 break; 1439 default: 1440 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1441 break; 1442 } 1443 1444 isert_rx_opcode(isert_conn, rx_desc, 1445 read_stag, read_va, write_stag, write_va); 1446 1447 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1448 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1449 } 1450 1451 static void 1452 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1453 { 1454 struct isert_conn *isert_conn = wc->qp->qp_context; 1455 struct ib_device *ib_dev = isert_conn->device->ib_device; 1456 1457 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1458 isert_print_wc(wc, "login recv"); 1459 return; 1460 } 1461 1462 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma, 1463 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1464 1465 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1466 1467 if (isert_conn->conn) { 1468 struct iscsi_login *login = isert_conn->conn->conn_login; 1469 1470 if (login && !login->first_request) 1471 isert_rx_login_req(isert_conn); 1472 } 1473 1474 mutex_lock(&isert_conn->mutex); 1475 complete(&isert_conn->login_req_comp); 1476 mutex_unlock(&isert_conn->mutex); 1477 1478 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma, 1479 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 1480 } 1481 1482 static void 1483 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1484 { 1485 struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 1486 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1487 1488 if (!cmd->rw.nr_ops) 1489 return; 1490 1491 if (isert_prot_cmd(conn, se_cmd)) { 1492 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1493 conn->cm_id->port_num, se_cmd->t_data_sg, 1494 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1495 se_cmd->t_prot_nents, dir); 1496 } else { 1497 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1498 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1499 } 1500 1501 cmd->rw.nr_ops = 0; 1502 } 1503 1504 static void 1505 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1506 { 1507 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1508 struct isert_conn *isert_conn = isert_cmd->conn; 1509 struct iscsi_conn *conn = isert_conn->conn; 1510 struct iscsi_text_rsp *hdr; 1511 1512 isert_dbg("Cmd %p\n", isert_cmd); 1513 1514 switch (cmd->iscsi_opcode) { 1515 case ISCSI_OP_SCSI_CMD: 1516 spin_lock_bh(&conn->cmd_lock); 1517 if (!list_empty(&cmd->i_conn_node)) 1518 list_del_init(&cmd->i_conn_node); 1519 spin_unlock_bh(&conn->cmd_lock); 1520 1521 if (cmd->data_direction == DMA_TO_DEVICE) { 1522 iscsit_stop_dataout_timer(cmd); 1523 /* 1524 * Check for special case during comp_err where 1525 * WRITE_PENDING has been handed off from core, 1526 * but requires an extra target_put_sess_cmd() 1527 * before transport_generic_free_cmd() below. 1528 */ 1529 if (comp_err && 1530 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1531 struct se_cmd *se_cmd = &cmd->se_cmd; 1532 1533 target_put_sess_cmd(se_cmd); 1534 } 1535 } 1536 1537 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1538 transport_generic_free_cmd(&cmd->se_cmd, 0); 1539 break; 1540 case ISCSI_OP_SCSI_TMFUNC: 1541 spin_lock_bh(&conn->cmd_lock); 1542 if (!list_empty(&cmd->i_conn_node)) 1543 list_del_init(&cmd->i_conn_node); 1544 spin_unlock_bh(&conn->cmd_lock); 1545 1546 transport_generic_free_cmd(&cmd->se_cmd, 0); 1547 break; 1548 case ISCSI_OP_REJECT: 1549 case ISCSI_OP_NOOP_OUT: 1550 case ISCSI_OP_TEXT: 1551 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1552 /* If the continue bit is on, keep the command alive */ 1553 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1554 break; 1555 1556 spin_lock_bh(&conn->cmd_lock); 1557 if (!list_empty(&cmd->i_conn_node)) 1558 list_del_init(&cmd->i_conn_node); 1559 spin_unlock_bh(&conn->cmd_lock); 1560 1561 /* 1562 * Handle special case for REJECT when iscsi_add_reject*() has 1563 * overwritten the original iscsi_opcode assignment, and the 1564 * associated cmd->se_cmd needs to be released. 1565 */ 1566 if (cmd->se_cmd.se_tfo != NULL) { 1567 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1568 cmd->iscsi_opcode); 1569 transport_generic_free_cmd(&cmd->se_cmd, 0); 1570 break; 1571 } 1572 /* 1573 * Fall-through 1574 */ 1575 default: 1576 iscsit_release_cmd(cmd); 1577 break; 1578 } 1579 } 1580 1581 static void 1582 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1583 { 1584 if (tx_desc->dma_addr != 0) { 1585 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1586 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1587 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1588 tx_desc->dma_addr = 0; 1589 } 1590 } 1591 1592 static void 1593 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1594 struct ib_device *ib_dev, bool comp_err) 1595 { 1596 if (isert_cmd->pdu_buf_dma != 0) { 1597 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1598 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1599 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1600 isert_cmd->pdu_buf_dma = 0; 1601 } 1602 1603 isert_unmap_tx_desc(tx_desc, ib_dev); 1604 isert_put_cmd(isert_cmd, comp_err); 1605 } 1606 1607 static int 1608 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1609 { 1610 struct ib_mr_status mr_status; 1611 int ret; 1612 1613 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1614 if (ret) { 1615 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1616 goto fail_mr_status; 1617 } 1618 1619 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1620 u64 sec_offset_err; 1621 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1622 1623 switch (mr_status.sig_err.err_type) { 1624 case IB_SIG_BAD_GUARD: 1625 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1626 break; 1627 case IB_SIG_BAD_REFTAG: 1628 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1629 break; 1630 case IB_SIG_BAD_APPTAG: 1631 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1632 break; 1633 } 1634 sec_offset_err = mr_status.sig_err.sig_err_offset; 1635 do_div(sec_offset_err, block_size); 1636 se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba; 1637 1638 isert_err("PI error found type %d at sector 0x%llx " 1639 "expected 0x%x vs actual 0x%x\n", 1640 mr_status.sig_err.err_type, 1641 (unsigned long long)se_cmd->bad_sector, 1642 mr_status.sig_err.expected, 1643 mr_status.sig_err.actual); 1644 ret = 1; 1645 } 1646 1647 fail_mr_status: 1648 return ret; 1649 } 1650 1651 static void 1652 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1653 { 1654 struct isert_conn *isert_conn = wc->qp->qp_context; 1655 struct isert_device *device = isert_conn->device; 1656 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1657 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1658 struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 1659 int ret = 0; 1660 1661 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1662 isert_print_wc(wc, "rdma write"); 1663 if (wc->status != IB_WC_WR_FLUSH_ERR) 1664 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1665 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1666 return; 1667 } 1668 1669 isert_dbg("Cmd %p\n", isert_cmd); 1670 1671 ret = isert_check_pi_status(cmd, isert_cmd->rw.sig->sig_mr); 1672 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1673 1674 if (ret) { 1675 /* 1676 * transport_generic_request_failure() expects to have 1677 * plus two references to handle queue-full, so re-add 1678 * one here as target-core will have already dropped 1679 * it after the first isert_put_datain() callback. 1680 */ 1681 kref_get(&cmd->cmd_kref); 1682 transport_generic_request_failure(cmd, cmd->pi_err); 1683 } else { 1684 /* 1685 * XXX: isert_put_response() failure is not retried. 1686 */ 1687 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 1688 if (ret) 1689 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1690 } 1691 } 1692 1693 static void 1694 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1695 { 1696 struct isert_conn *isert_conn = wc->qp->qp_context; 1697 struct isert_device *device = isert_conn->device; 1698 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1699 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1700 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1701 struct se_cmd *se_cmd = &cmd->se_cmd; 1702 int ret = 0; 1703 1704 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1705 isert_print_wc(wc, "rdma read"); 1706 if (wc->status != IB_WC_WR_FLUSH_ERR) 1707 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1708 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1709 return; 1710 } 1711 1712 isert_dbg("Cmd %p\n", isert_cmd); 1713 1714 iscsit_stop_dataout_timer(cmd); 1715 1716 if (isert_prot_cmd(isert_conn, se_cmd)) 1717 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.sig->sig_mr); 1718 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1719 cmd->write_data_done = 0; 1720 1721 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1722 spin_lock_bh(&cmd->istate_lock); 1723 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1724 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1725 spin_unlock_bh(&cmd->istate_lock); 1726 1727 /* 1728 * transport_generic_request_failure() will drop the extra 1729 * se_cmd->cmd_kref reference after T10-PI error, and handle 1730 * any non-zero ->queue_status() callback error retries. 1731 */ 1732 if (ret) 1733 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1734 else 1735 target_execute_cmd(se_cmd); 1736 } 1737 1738 static void 1739 isert_do_control_comp(struct work_struct *work) 1740 { 1741 struct isert_cmd *isert_cmd = container_of(work, 1742 struct isert_cmd, comp_work); 1743 struct isert_conn *isert_conn = isert_cmd->conn; 1744 struct ib_device *ib_dev = isert_conn->cm_id->device; 1745 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1746 1747 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1748 1749 switch (cmd->i_state) { 1750 case ISTATE_SEND_TASKMGTRSP: 1751 iscsit_tmr_post_handler(cmd, cmd->conn); 1752 case ISTATE_SEND_REJECT: /* FALLTHRU */ 1753 case ISTATE_SEND_TEXTRSP: /* FALLTHRU */ 1754 cmd->i_state = ISTATE_SENT_STATUS; 1755 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1756 ib_dev, false); 1757 break; 1758 case ISTATE_SEND_LOGOUTRSP: 1759 iscsit_logout_post_handler(cmd, cmd->conn); 1760 break; 1761 default: 1762 isert_err("Unknown i_state %d\n", cmd->i_state); 1763 dump_stack(); 1764 break; 1765 } 1766 } 1767 1768 static void 1769 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1770 { 1771 struct isert_conn *isert_conn = wc->qp->qp_context; 1772 struct ib_device *ib_dev = isert_conn->cm_id->device; 1773 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1774 1775 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1776 isert_print_wc(wc, "login send"); 1777 if (wc->status != IB_WC_WR_FLUSH_ERR) 1778 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1779 } 1780 1781 isert_unmap_tx_desc(tx_desc, ib_dev); 1782 } 1783 1784 static void 1785 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1786 { 1787 struct isert_conn *isert_conn = wc->qp->qp_context; 1788 struct ib_device *ib_dev = isert_conn->cm_id->device; 1789 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1790 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1791 1792 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1793 isert_print_wc(wc, "send"); 1794 if (wc->status != IB_WC_WR_FLUSH_ERR) 1795 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1796 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1797 return; 1798 } 1799 1800 isert_dbg("Cmd %p\n", isert_cmd); 1801 1802 switch (isert_cmd->iscsi_cmd->i_state) { 1803 case ISTATE_SEND_TASKMGTRSP: 1804 case ISTATE_SEND_LOGOUTRSP: 1805 case ISTATE_SEND_REJECT: 1806 case ISTATE_SEND_TEXTRSP: 1807 isert_unmap_tx_desc(tx_desc, ib_dev); 1808 1809 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1810 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1811 return; 1812 default: 1813 isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 1814 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1815 break; 1816 } 1817 } 1818 1819 static int 1820 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1821 { 1822 struct ib_send_wr *wr_failed; 1823 int ret; 1824 1825 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1826 if (ret) { 1827 isert_err("ib_post_recv failed with %d\n", ret); 1828 return ret; 1829 } 1830 1831 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, 1832 &wr_failed); 1833 if (ret) { 1834 isert_err("ib_post_send failed with %d\n", ret); 1835 return ret; 1836 } 1837 return ret; 1838 } 1839 1840 static int 1841 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1842 { 1843 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1844 struct isert_conn *isert_conn = conn->context; 1845 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1846 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1847 &isert_cmd->tx_desc.iscsi_header; 1848 1849 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1850 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1851 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1852 /* 1853 * Attach SENSE DATA payload to iSCSI Response PDU 1854 */ 1855 if (cmd->se_cmd.sense_buffer && 1856 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1857 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1858 struct isert_device *device = isert_conn->device; 1859 struct ib_device *ib_dev = device->ib_device; 1860 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1861 u32 padding, pdu_len; 1862 1863 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1864 cmd->sense_buffer); 1865 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1866 1867 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1868 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1869 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1870 1871 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1872 (void *)cmd->sense_buffer, pdu_len, 1873 DMA_TO_DEVICE); 1874 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1875 return -ENOMEM; 1876 1877 isert_cmd->pdu_buf_len = pdu_len; 1878 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1879 tx_dsg->length = pdu_len; 1880 tx_dsg->lkey = device->pd->local_dma_lkey; 1881 isert_cmd->tx_desc.num_sge = 2; 1882 } 1883 1884 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1885 1886 isert_dbg("Posting SCSI Response\n"); 1887 1888 return isert_post_response(isert_conn, isert_cmd); 1889 } 1890 1891 static void 1892 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1893 { 1894 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1895 struct isert_conn *isert_conn = conn->context; 1896 1897 spin_lock_bh(&conn->cmd_lock); 1898 if (!list_empty(&cmd->i_conn_node)) 1899 list_del_init(&cmd->i_conn_node); 1900 spin_unlock_bh(&conn->cmd_lock); 1901 1902 if (cmd->data_direction == DMA_TO_DEVICE) 1903 iscsit_stop_dataout_timer(cmd); 1904 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1905 } 1906 1907 static enum target_prot_op 1908 isert_get_sup_prot_ops(struct iscsi_conn *conn) 1909 { 1910 struct isert_conn *isert_conn = conn->context; 1911 struct isert_device *device = isert_conn->device; 1912 1913 if (conn->tpg->tpg_attrib.t10_pi) { 1914 if (device->pi_capable) { 1915 isert_info("conn %p PI offload enabled\n", isert_conn); 1916 isert_conn->pi_support = true; 1917 return TARGET_PROT_ALL; 1918 } 1919 } 1920 1921 isert_info("conn %p PI offload disabled\n", isert_conn); 1922 isert_conn->pi_support = false; 1923 1924 return TARGET_PROT_NORMAL; 1925 } 1926 1927 static int 1928 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1929 bool nopout_response) 1930 { 1931 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1932 struct isert_conn *isert_conn = conn->context; 1933 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1934 1935 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1936 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1937 &isert_cmd->tx_desc.iscsi_header, 1938 nopout_response); 1939 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1940 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1941 1942 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1943 1944 return isert_post_response(isert_conn, isert_cmd); 1945 } 1946 1947 static int 1948 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1949 { 1950 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1951 struct isert_conn *isert_conn = conn->context; 1952 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1953 1954 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1955 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1956 &isert_cmd->tx_desc.iscsi_header); 1957 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1958 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1959 1960 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1961 1962 return isert_post_response(isert_conn, isert_cmd); 1963 } 1964 1965 static int 1966 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1967 { 1968 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1969 struct isert_conn *isert_conn = conn->context; 1970 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1971 1972 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1973 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1974 &isert_cmd->tx_desc.iscsi_header); 1975 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1976 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1977 1978 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1979 1980 return isert_post_response(isert_conn, isert_cmd); 1981 } 1982 1983 static int 1984 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1985 { 1986 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1987 struct isert_conn *isert_conn = conn->context; 1988 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1989 struct isert_device *device = isert_conn->device; 1990 struct ib_device *ib_dev = device->ib_device; 1991 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1992 struct iscsi_reject *hdr = 1993 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1994 1995 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1996 iscsit_build_reject(cmd, conn, hdr); 1997 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1998 1999 hton24(hdr->dlength, ISCSI_HDR_LEN); 2000 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2001 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 2002 DMA_TO_DEVICE); 2003 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2004 return -ENOMEM; 2005 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 2006 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2007 tx_dsg->length = ISCSI_HDR_LEN; 2008 tx_dsg->lkey = device->pd->local_dma_lkey; 2009 isert_cmd->tx_desc.num_sge = 2; 2010 2011 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2012 2013 isert_dbg("conn %p Posting Reject\n", isert_conn); 2014 2015 return isert_post_response(isert_conn, isert_cmd); 2016 } 2017 2018 static int 2019 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 2020 { 2021 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2022 struct isert_conn *isert_conn = conn->context; 2023 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 2024 struct iscsi_text_rsp *hdr = 2025 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 2026 u32 txt_rsp_len; 2027 int rc; 2028 2029 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2030 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 2031 if (rc < 0) 2032 return rc; 2033 2034 txt_rsp_len = rc; 2035 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2036 2037 if (txt_rsp_len) { 2038 struct isert_device *device = isert_conn->device; 2039 struct ib_device *ib_dev = device->ib_device; 2040 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 2041 void *txt_rsp_buf = cmd->buf_ptr; 2042 2043 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 2044 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 2045 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 2046 return -ENOMEM; 2047 2048 isert_cmd->pdu_buf_len = txt_rsp_len; 2049 tx_dsg->addr = isert_cmd->pdu_buf_dma; 2050 tx_dsg->length = txt_rsp_len; 2051 tx_dsg->lkey = device->pd->local_dma_lkey; 2052 isert_cmd->tx_desc.num_sge = 2; 2053 } 2054 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 2055 2056 isert_dbg("conn %p Text Response\n", isert_conn); 2057 2058 return isert_post_response(isert_conn, isert_cmd); 2059 } 2060 2061 static inline void 2062 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs, 2063 struct ib_sig_domain *domain) 2064 { 2065 domain->sig_type = IB_SIG_TYPE_T10_DIF; 2066 domain->sig.dif.bg_type = IB_T10DIF_CRC; 2067 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 2068 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 2069 /* 2070 * At the moment we hard code those, but if in the future 2071 * the target core would like to use it, we will take it 2072 * from se_cmd. 2073 */ 2074 domain->sig.dif.apptag_check_mask = 0xffff; 2075 domain->sig.dif.app_escape = true; 2076 domain->sig.dif.ref_escape = true; 2077 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 2078 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 2079 domain->sig.dif.ref_remap = true; 2080 }; 2081 2082 static int 2083 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2084 { 2085 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2086 2087 switch (se_cmd->prot_op) { 2088 case TARGET_PROT_DIN_INSERT: 2089 case TARGET_PROT_DOUT_STRIP: 2090 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2091 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2092 break; 2093 case TARGET_PROT_DOUT_INSERT: 2094 case TARGET_PROT_DIN_STRIP: 2095 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2096 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2097 break; 2098 case TARGET_PROT_DIN_PASS: 2099 case TARGET_PROT_DOUT_PASS: 2100 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire); 2101 isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem); 2102 break; 2103 default: 2104 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2105 return -EINVAL; 2106 } 2107 2108 sig_attrs->check_mask = 2109 (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD ? 0xc0 : 0) | 2110 (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x30 : 0) | 2111 (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG ? 0x0f : 0); 2112 return 0; 2113 } 2114 2115 static int 2116 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2117 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2118 { 2119 struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 2120 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2121 u8 port_num = conn->cm_id->port_num; 2122 u64 addr; 2123 u32 rkey, offset; 2124 int ret; 2125 2126 if (dir == DMA_FROM_DEVICE) { 2127 addr = cmd->write_va; 2128 rkey = cmd->write_stag; 2129 offset = cmd->iscsi_cmd->write_data_done; 2130 } else { 2131 addr = cmd->read_va; 2132 rkey = cmd->read_stag; 2133 offset = 0; 2134 } 2135 2136 if (isert_prot_cmd(conn, se_cmd)) { 2137 struct ib_sig_attrs sig_attrs; 2138 2139 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2140 if (ret) 2141 return ret; 2142 2143 WARN_ON_ONCE(offset); 2144 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2145 se_cmd->t_data_sg, se_cmd->t_data_nents, 2146 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2147 &sig_attrs, addr, rkey, dir); 2148 } else { 2149 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2150 se_cmd->t_data_sg, se_cmd->t_data_nents, 2151 offset, addr, rkey, dir); 2152 } 2153 if (ret < 0) { 2154 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2155 return ret; 2156 } 2157 2158 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2159 if (ret < 0) 2160 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2161 return ret; 2162 } 2163 2164 static int 2165 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2166 { 2167 struct se_cmd *se_cmd = &cmd->se_cmd; 2168 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2169 struct isert_conn *isert_conn = conn->context; 2170 struct ib_cqe *cqe = NULL; 2171 struct ib_send_wr *chain_wr = NULL; 2172 int rc; 2173 2174 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2175 isert_cmd, se_cmd->data_length); 2176 2177 if (isert_prot_cmd(isert_conn, se_cmd)) { 2178 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2179 cqe = &isert_cmd->tx_desc.tx_cqe; 2180 } else { 2181 /* 2182 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2183 */ 2184 isert_create_send_desc(isert_conn, isert_cmd, 2185 &isert_cmd->tx_desc); 2186 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2187 &isert_cmd->tx_desc.iscsi_header); 2188 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2189 isert_init_send_wr(isert_conn, isert_cmd, 2190 &isert_cmd->tx_desc.send_wr); 2191 2192 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2193 if (rc) { 2194 isert_err("ib_post_recv failed with %d\n", rc); 2195 return rc; 2196 } 2197 2198 chain_wr = &isert_cmd->tx_desc.send_wr; 2199 } 2200 2201 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2202 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2203 isert_cmd, rc); 2204 return rc; 2205 } 2206 2207 static int 2208 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2209 { 2210 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2211 int ret; 2212 2213 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2214 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2215 2216 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2217 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2218 &isert_cmd->tx_desc.tx_cqe, NULL); 2219 2220 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2221 isert_cmd, ret); 2222 return ret; 2223 } 2224 2225 static int 2226 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2227 { 2228 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2229 int ret = 0; 2230 2231 switch (state) { 2232 case ISTATE_REMOVE: 2233 spin_lock_bh(&conn->cmd_lock); 2234 list_del_init(&cmd->i_conn_node); 2235 spin_unlock_bh(&conn->cmd_lock); 2236 isert_put_cmd(isert_cmd, true); 2237 break; 2238 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2239 ret = isert_put_nopin(cmd, conn, false); 2240 break; 2241 default: 2242 isert_err("Unknown immediate state: 0x%02x\n", state); 2243 ret = -EINVAL; 2244 break; 2245 } 2246 2247 return ret; 2248 } 2249 2250 static int 2251 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2252 { 2253 struct isert_conn *isert_conn = conn->context; 2254 int ret; 2255 2256 switch (state) { 2257 case ISTATE_SEND_LOGOUTRSP: 2258 ret = isert_put_logout_rsp(cmd, conn); 2259 if (!ret) 2260 isert_conn->logout_posted = true; 2261 break; 2262 case ISTATE_SEND_NOPIN: 2263 ret = isert_put_nopin(cmd, conn, true); 2264 break; 2265 case ISTATE_SEND_TASKMGTRSP: 2266 ret = isert_put_tm_rsp(cmd, conn); 2267 break; 2268 case ISTATE_SEND_REJECT: 2269 ret = isert_put_reject(cmd, conn); 2270 break; 2271 case ISTATE_SEND_TEXTRSP: 2272 ret = isert_put_text_rsp(cmd, conn); 2273 break; 2274 case ISTATE_SEND_STATUS: 2275 /* 2276 * Special case for sending non GOOD SCSI status from TX thread 2277 * context during pre se_cmd excecution failure. 2278 */ 2279 ret = isert_put_response(conn, cmd); 2280 break; 2281 default: 2282 isert_err("Unknown response state: 0x%02x\n", state); 2283 ret = -EINVAL; 2284 break; 2285 } 2286 2287 return ret; 2288 } 2289 2290 struct rdma_cm_id * 2291 isert_setup_id(struct isert_np *isert_np) 2292 { 2293 struct iscsi_np *np = isert_np->np; 2294 struct rdma_cm_id *id; 2295 struct sockaddr *sa; 2296 int ret; 2297 2298 sa = (struct sockaddr *)&np->np_sockaddr; 2299 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2300 2301 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2302 RDMA_PS_TCP, IB_QPT_RC); 2303 if (IS_ERR(id)) { 2304 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2305 ret = PTR_ERR(id); 2306 goto out; 2307 } 2308 isert_dbg("id %p context %p\n", id, id->context); 2309 2310 ret = rdma_bind_addr(id, sa); 2311 if (ret) { 2312 isert_err("rdma_bind_addr() failed: %d\n", ret); 2313 goto out_id; 2314 } 2315 2316 ret = rdma_listen(id, 0); 2317 if (ret) { 2318 isert_err("rdma_listen() failed: %d\n", ret); 2319 goto out_id; 2320 } 2321 2322 return id; 2323 out_id: 2324 rdma_destroy_id(id); 2325 out: 2326 return ERR_PTR(ret); 2327 } 2328 2329 static int 2330 isert_setup_np(struct iscsi_np *np, 2331 struct sockaddr_storage *ksockaddr) 2332 { 2333 struct isert_np *isert_np; 2334 struct rdma_cm_id *isert_lid; 2335 int ret; 2336 2337 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2338 if (!isert_np) 2339 return -ENOMEM; 2340 2341 sema_init(&isert_np->sem, 0); 2342 mutex_init(&isert_np->mutex); 2343 INIT_LIST_HEAD(&isert_np->accepted); 2344 INIT_LIST_HEAD(&isert_np->pending); 2345 isert_np->np = np; 2346 2347 /* 2348 * Setup the np->np_sockaddr from the passed sockaddr setup 2349 * in iscsi_target_configfs.c code.. 2350 */ 2351 memcpy(&np->np_sockaddr, ksockaddr, 2352 sizeof(struct sockaddr_storage)); 2353 2354 isert_lid = isert_setup_id(isert_np); 2355 if (IS_ERR(isert_lid)) { 2356 ret = PTR_ERR(isert_lid); 2357 goto out; 2358 } 2359 2360 isert_np->cm_id = isert_lid; 2361 np->np_context = isert_np; 2362 2363 return 0; 2364 2365 out: 2366 kfree(isert_np); 2367 2368 return ret; 2369 } 2370 2371 static int 2372 isert_rdma_accept(struct isert_conn *isert_conn) 2373 { 2374 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2375 struct rdma_conn_param cp; 2376 int ret; 2377 struct iser_cm_hdr rsp_hdr; 2378 2379 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2380 cp.initiator_depth = isert_conn->initiator_depth; 2381 cp.retry_count = 7; 2382 cp.rnr_retry_count = 7; 2383 2384 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2385 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2386 if (!isert_conn->snd_w_inv) 2387 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2388 cp.private_data = (void *)&rsp_hdr; 2389 cp.private_data_len = sizeof(rsp_hdr); 2390 2391 ret = rdma_accept(cm_id, &cp); 2392 if (ret) { 2393 isert_err("rdma_accept() failed with: %d\n", ret); 2394 return ret; 2395 } 2396 2397 return 0; 2398 } 2399 2400 static int 2401 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2402 { 2403 struct isert_conn *isert_conn = conn->context; 2404 int ret; 2405 2406 isert_info("before login_req comp conn: %p\n", isert_conn); 2407 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2408 if (ret) { 2409 isert_err("isert_conn %p interrupted before got login req\n", 2410 isert_conn); 2411 return ret; 2412 } 2413 reinit_completion(&isert_conn->login_req_comp); 2414 2415 /* 2416 * For login requests after the first PDU, isert_rx_login_req() will 2417 * kick schedule_delayed_work(&conn->login_work) as the packet is 2418 * received, which turns this callback from iscsi_target_do_login_rx() 2419 * into a NOP. 2420 */ 2421 if (!login->first_request) 2422 return 0; 2423 2424 isert_rx_login_req(isert_conn); 2425 2426 isert_info("before login_comp conn: %p\n", conn); 2427 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2428 if (ret) 2429 return ret; 2430 2431 isert_info("processing login->req: %p\n", login->req); 2432 2433 return 0; 2434 } 2435 2436 static void 2437 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2438 struct isert_conn *isert_conn) 2439 { 2440 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2441 struct rdma_route *cm_route = &cm_id->route; 2442 2443 conn->login_family = np->np_sockaddr.ss_family; 2444 2445 conn->login_sockaddr = cm_route->addr.dst_addr; 2446 conn->local_sockaddr = cm_route->addr.src_addr; 2447 } 2448 2449 static int 2450 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2451 { 2452 struct isert_np *isert_np = np->np_context; 2453 struct isert_conn *isert_conn; 2454 int ret; 2455 2456 accept_wait: 2457 ret = down_interruptible(&isert_np->sem); 2458 if (ret) 2459 return -ENODEV; 2460 2461 spin_lock_bh(&np->np_thread_lock); 2462 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2463 spin_unlock_bh(&np->np_thread_lock); 2464 isert_dbg("np_thread_state %d\n", 2465 np->np_thread_state); 2466 /** 2467 * No point in stalling here when np_thread 2468 * is in state RESET/SHUTDOWN/EXIT - bail 2469 **/ 2470 return -ENODEV; 2471 } 2472 spin_unlock_bh(&np->np_thread_lock); 2473 2474 mutex_lock(&isert_np->mutex); 2475 if (list_empty(&isert_np->pending)) { 2476 mutex_unlock(&isert_np->mutex); 2477 goto accept_wait; 2478 } 2479 isert_conn = list_first_entry(&isert_np->pending, 2480 struct isert_conn, node); 2481 list_del_init(&isert_conn->node); 2482 mutex_unlock(&isert_np->mutex); 2483 2484 conn->context = isert_conn; 2485 isert_conn->conn = conn; 2486 isert_conn->state = ISER_CONN_BOUND; 2487 2488 isert_set_conn_info(np, conn, isert_conn); 2489 2490 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2491 2492 return 0; 2493 } 2494 2495 static void 2496 isert_free_np(struct iscsi_np *np) 2497 { 2498 struct isert_np *isert_np = np->np_context; 2499 struct isert_conn *isert_conn, *n; 2500 2501 if (isert_np->cm_id) 2502 rdma_destroy_id(isert_np->cm_id); 2503 2504 /* 2505 * FIXME: At this point we don't have a good way to insure 2506 * that at this point we don't have hanging connections that 2507 * completed RDMA establishment but didn't start iscsi login 2508 * process. So work-around this by cleaning up what ever piled 2509 * up in accepted and pending lists. 2510 */ 2511 mutex_lock(&isert_np->mutex); 2512 if (!list_empty(&isert_np->pending)) { 2513 isert_info("Still have isert pending connections\n"); 2514 list_for_each_entry_safe(isert_conn, n, 2515 &isert_np->pending, 2516 node) { 2517 isert_info("cleaning isert_conn %p state (%d)\n", 2518 isert_conn, isert_conn->state); 2519 isert_connect_release(isert_conn); 2520 } 2521 } 2522 2523 if (!list_empty(&isert_np->accepted)) { 2524 isert_info("Still have isert accepted connections\n"); 2525 list_for_each_entry_safe(isert_conn, n, 2526 &isert_np->accepted, 2527 node) { 2528 isert_info("cleaning isert_conn %p state (%d)\n", 2529 isert_conn, isert_conn->state); 2530 isert_connect_release(isert_conn); 2531 } 2532 } 2533 mutex_unlock(&isert_np->mutex); 2534 2535 np->np_context = NULL; 2536 kfree(isert_np); 2537 } 2538 2539 static void isert_release_work(struct work_struct *work) 2540 { 2541 struct isert_conn *isert_conn = container_of(work, 2542 struct isert_conn, 2543 release_work); 2544 2545 isert_info("Starting release conn %p\n", isert_conn); 2546 2547 mutex_lock(&isert_conn->mutex); 2548 isert_conn->state = ISER_CONN_DOWN; 2549 mutex_unlock(&isert_conn->mutex); 2550 2551 isert_info("Destroying conn %p\n", isert_conn); 2552 isert_put_conn(isert_conn); 2553 } 2554 2555 static void 2556 isert_wait4logout(struct isert_conn *isert_conn) 2557 { 2558 struct iscsi_conn *conn = isert_conn->conn; 2559 2560 isert_info("conn %p\n", isert_conn); 2561 2562 if (isert_conn->logout_posted) { 2563 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2564 wait_for_completion_timeout(&conn->conn_logout_comp, 2565 SECONDS_FOR_LOGOUT_COMP * HZ); 2566 } 2567 } 2568 2569 static void 2570 isert_wait4cmds(struct iscsi_conn *conn) 2571 { 2572 isert_info("iscsi_conn %p\n", conn); 2573 2574 if (conn->sess) { 2575 target_sess_cmd_list_set_waiting(conn->sess->se_sess); 2576 target_wait_for_sess_cmds(conn->sess->se_sess); 2577 } 2578 } 2579 2580 /** 2581 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2582 * unsolicitate dataout 2583 * @conn: iscsi connection 2584 * 2585 * We might still have commands that are waiting for unsolicited 2586 * dataouts messages. We must put the extra reference on those 2587 * before blocking on the target_wait_for_session_cmds 2588 */ 2589 static void 2590 isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 2591 { 2592 struct iscsi_cmd *cmd, *tmp; 2593 static LIST_HEAD(drop_cmd_list); 2594 2595 spin_lock_bh(&conn->cmd_lock); 2596 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2597 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2598 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2599 (cmd->write_data_done < cmd->se_cmd.data_length)) 2600 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2601 } 2602 spin_unlock_bh(&conn->cmd_lock); 2603 2604 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2605 list_del_init(&cmd->i_conn_node); 2606 if (cmd->i_state != ISTATE_REMOVE) { 2607 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2608 2609 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2610 isert_put_cmd(isert_cmd, true); 2611 } 2612 } 2613 } 2614 2615 static void isert_wait_conn(struct iscsi_conn *conn) 2616 { 2617 struct isert_conn *isert_conn = conn->context; 2618 2619 isert_info("Starting conn %p\n", isert_conn); 2620 2621 mutex_lock(&isert_conn->mutex); 2622 isert_conn_terminate(isert_conn); 2623 mutex_unlock(&isert_conn->mutex); 2624 2625 ib_drain_qp(isert_conn->qp); 2626 isert_put_unsol_pending_cmds(conn); 2627 isert_wait4cmds(conn); 2628 isert_wait4logout(isert_conn); 2629 2630 queue_work(isert_release_wq, &isert_conn->release_work); 2631 } 2632 2633 static void isert_free_conn(struct iscsi_conn *conn) 2634 { 2635 struct isert_conn *isert_conn = conn->context; 2636 2637 ib_drain_qp(isert_conn->qp); 2638 isert_put_conn(isert_conn); 2639 } 2640 2641 static void isert_get_rx_pdu(struct iscsi_conn *conn) 2642 { 2643 struct completion comp; 2644 2645 init_completion(&comp); 2646 2647 wait_for_completion_interruptible(&comp); 2648 } 2649 2650 static struct iscsit_transport iser_target_transport = { 2651 .name = "IB/iSER", 2652 .transport_type = ISCSI_INFINIBAND, 2653 .rdma_shutdown = true, 2654 .priv_size = sizeof(struct isert_cmd), 2655 .owner = THIS_MODULE, 2656 .iscsit_setup_np = isert_setup_np, 2657 .iscsit_accept_np = isert_accept_np, 2658 .iscsit_free_np = isert_free_np, 2659 .iscsit_wait_conn = isert_wait_conn, 2660 .iscsit_free_conn = isert_free_conn, 2661 .iscsit_get_login_rx = isert_get_login_rx, 2662 .iscsit_put_login_tx = isert_put_login_tx, 2663 .iscsit_immediate_queue = isert_immediate_queue, 2664 .iscsit_response_queue = isert_response_queue, 2665 .iscsit_get_dataout = isert_get_dataout, 2666 .iscsit_queue_data_in = isert_put_datain, 2667 .iscsit_queue_status = isert_put_response, 2668 .iscsit_aborted_task = isert_aborted_task, 2669 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2670 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2671 }; 2672 2673 static int __init isert_init(void) 2674 { 2675 int ret; 2676 2677 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2678 WQ_UNBOUND | WQ_HIGHPRI, 0); 2679 if (!isert_comp_wq) { 2680 isert_err("Unable to allocate isert_comp_wq\n"); 2681 return -ENOMEM; 2682 } 2683 2684 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2685 WQ_UNBOUND_MAX_ACTIVE); 2686 if (!isert_release_wq) { 2687 isert_err("Unable to allocate isert_release_wq\n"); 2688 ret = -ENOMEM; 2689 goto destroy_comp_wq; 2690 } 2691 2692 iscsit_register_transport(&iser_target_transport); 2693 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2694 2695 return 0; 2696 2697 destroy_comp_wq: 2698 destroy_workqueue(isert_comp_wq); 2699 2700 return ret; 2701 } 2702 2703 static void __exit isert_exit(void) 2704 { 2705 flush_scheduled_work(); 2706 destroy_workqueue(isert_release_wq); 2707 destroy_workqueue(isert_comp_wq); 2708 iscsit_unregister_transport(&iser_target_transport); 2709 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2710 } 2711 2712 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2713 MODULE_VERSION("1.0"); 2714 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2715 MODULE_LICENSE("GPL"); 2716 2717 module_init(isert_init); 2718 module_exit(isert_exit); 2719