1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains iSCSI extentions for RDMA (iSER) Verbs 4 * 5 * (c) Copyright 2013 Datera, Inc. 6 * 7 * Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ****************************************************************************/ 10 11 #include <linux/string.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/socket.h> 15 #include <linux/in.h> 16 #include <linux/in6.h> 17 #include <rdma/ib_verbs.h> 18 #include <rdma/ib_cm.h> 19 #include <rdma/rdma_cm.h> 20 #include <target/target_core_base.h> 21 #include <target/target_core_fabric.h> 22 #include <target/iscsi/iscsi_transport.h> 23 #include <linux/semaphore.h> 24 25 #include "ib_isert.h" 26 27 static int isert_debug_level; 28 module_param_named(debug_level, isert_debug_level, int, 0644); 29 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 30 31 static int isert_sg_tablesize_set(const char *val, 32 const struct kernel_param *kp); 33 static const struct kernel_param_ops sg_tablesize_ops = { 34 .set = isert_sg_tablesize_set, 35 .get = param_get_int, 36 }; 37 38 static int isert_sg_tablesize = ISCSI_ISER_MIN_SG_TABLESIZE; 39 module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644); 40 MODULE_PARM_DESC(sg_tablesize, 41 "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 128, max: 4096)"); 42 43 static DEFINE_MUTEX(device_list_mutex); 44 static LIST_HEAD(device_list); 45 static struct workqueue_struct *isert_login_wq; 46 static struct workqueue_struct *isert_comp_wq; 47 static struct workqueue_struct *isert_release_wq; 48 49 static int 50 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd); 51 static int 52 isert_login_post_recv(struct isert_conn *isert_conn); 53 static int 54 isert_rdma_accept(struct isert_conn *isert_conn); 55 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 56 57 static void isert_release_work(struct work_struct *work); 58 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 59 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 61 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 62 63 static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) 64 { 65 int n = 0, ret; 66 67 ret = kstrtoint(val, 10, &n); 68 if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE || 69 n > ISCSI_ISER_MAX_SG_TABLESIZE) 70 return -EINVAL; 71 72 return param_set_int(val, kp); 73 } 74 75 static inline bool 76 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 77 { 78 return (conn->pi_support && 79 cmd->prot_op != TARGET_PROT_NORMAL); 80 } 81 82 static void 83 isert_qp_event_callback(struct ib_event *e, void *context) 84 { 85 struct isert_conn *isert_conn = context; 86 87 isert_err("%s (%d): conn %p\n", 88 ib_event_msg(e->event), e->event, isert_conn); 89 90 switch (e->event) { 91 case IB_EVENT_COMM_EST: 92 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 93 break; 94 case IB_EVENT_QP_LAST_WQE_REACHED: 95 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 96 break; 97 default: 98 break; 99 } 100 } 101 102 static struct ib_qp * 103 isert_create_qp(struct isert_conn *isert_conn, 104 struct rdma_cm_id *cma_id) 105 { 106 u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; 107 struct isert_device *device = isert_conn->device; 108 struct ib_device *ib_dev = device->ib_device; 109 struct ib_qp_init_attr attr; 110 int ret, factor; 111 112 isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); 113 if (IS_ERR(isert_conn->cq)) { 114 isert_err("Unable to allocate cq\n"); 115 ret = PTR_ERR(isert_conn->cq); 116 return ERR_PTR(ret); 117 } 118 isert_conn->cq_size = cq_size; 119 120 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 121 attr.event_handler = isert_qp_event_callback; 122 attr.qp_context = isert_conn; 123 attr.send_cq = isert_conn->cq; 124 attr.recv_cq = isert_conn->cq; 125 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 126 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 127 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 128 isert_sg_tablesize); 129 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 130 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 131 attr.cap.max_recv_sge = 1; 132 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 133 attr.qp_type = IB_QPT_RC; 134 if (device->pi_capable) 135 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 136 137 ret = rdma_create_qp(cma_id, device->pd, &attr); 138 if (ret) { 139 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 140 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 141 142 return ERR_PTR(ret); 143 } 144 145 return cma_id->qp; 146 } 147 148 static int 149 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 150 { 151 struct isert_device *device = isert_conn->device; 152 struct ib_device *ib_dev = device->ib_device; 153 struct iser_rx_desc *rx_desc; 154 struct ib_sge *rx_sg; 155 u64 dma_addr; 156 int i, j; 157 158 isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 159 sizeof(struct iser_rx_desc), 160 GFP_KERNEL); 161 if (!isert_conn->rx_descs) 162 return -ENOMEM; 163 164 rx_desc = isert_conn->rx_descs; 165 166 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 167 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf, 168 ISER_RX_SIZE, DMA_FROM_DEVICE); 169 if (ib_dma_mapping_error(ib_dev, dma_addr)) 170 goto dma_map_fail; 171 172 rx_desc->dma_addr = dma_addr; 173 174 rx_sg = &rx_desc->rx_sg; 175 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc); 176 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 177 rx_sg->lkey = device->pd->local_dma_lkey; 178 rx_desc->rx_cqe.done = isert_recv_done; 179 } 180 181 return 0; 182 183 dma_map_fail: 184 rx_desc = isert_conn->rx_descs; 185 for (j = 0; j < i; j++, rx_desc++) { 186 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 187 ISER_RX_SIZE, DMA_FROM_DEVICE); 188 } 189 kfree(isert_conn->rx_descs); 190 isert_conn->rx_descs = NULL; 191 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 192 return -ENOMEM; 193 } 194 195 static void 196 isert_free_rx_descriptors(struct isert_conn *isert_conn) 197 { 198 struct ib_device *ib_dev = isert_conn->device->ib_device; 199 struct iser_rx_desc *rx_desc; 200 int i; 201 202 if (!isert_conn->rx_descs) 203 return; 204 205 rx_desc = isert_conn->rx_descs; 206 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 207 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 208 ISER_RX_SIZE, DMA_FROM_DEVICE); 209 } 210 211 kfree(isert_conn->rx_descs); 212 isert_conn->rx_descs = NULL; 213 } 214 215 static int 216 isert_create_device_ib_res(struct isert_device *device) 217 { 218 struct ib_device *ib_dev = device->ib_device; 219 int ret; 220 221 isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 222 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 223 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 224 225 device->pd = ib_alloc_pd(ib_dev, 0); 226 if (IS_ERR(device->pd)) { 227 ret = PTR_ERR(device->pd); 228 isert_err("failed to allocate pd, device %p, ret=%d\n", 229 device, ret); 230 return ret; 231 } 232 233 /* Check signature cap */ 234 if (ib_dev->attrs.kernel_cap_flags & IBK_INTEGRITY_HANDOVER) 235 device->pi_capable = true; 236 else 237 device->pi_capable = false; 238 239 return 0; 240 } 241 242 static void 243 isert_free_device_ib_res(struct isert_device *device) 244 { 245 isert_info("device %p\n", device); 246 247 ib_dealloc_pd(device->pd); 248 } 249 250 static void 251 isert_device_put(struct isert_device *device) 252 { 253 mutex_lock(&device_list_mutex); 254 device->refcount--; 255 isert_info("device %p refcount %d\n", device, device->refcount); 256 if (!device->refcount) { 257 isert_free_device_ib_res(device); 258 list_del(&device->dev_node); 259 kfree(device); 260 } 261 mutex_unlock(&device_list_mutex); 262 } 263 264 static struct isert_device * 265 isert_device_get(struct rdma_cm_id *cma_id) 266 { 267 struct isert_device *device; 268 int ret; 269 270 mutex_lock(&device_list_mutex); 271 list_for_each_entry(device, &device_list, dev_node) { 272 if (device->ib_device->node_guid == cma_id->device->node_guid) { 273 device->refcount++; 274 isert_info("Found iser device %p refcount %d\n", 275 device, device->refcount); 276 mutex_unlock(&device_list_mutex); 277 return device; 278 } 279 } 280 281 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 282 if (!device) { 283 mutex_unlock(&device_list_mutex); 284 return ERR_PTR(-ENOMEM); 285 } 286 287 INIT_LIST_HEAD(&device->dev_node); 288 289 device->ib_device = cma_id->device; 290 ret = isert_create_device_ib_res(device); 291 if (ret) { 292 kfree(device); 293 mutex_unlock(&device_list_mutex); 294 return ERR_PTR(ret); 295 } 296 297 device->refcount++; 298 list_add_tail(&device->dev_node, &device_list); 299 isert_info("Created a new iser device %p refcount %d\n", 300 device, device->refcount); 301 mutex_unlock(&device_list_mutex); 302 303 return device; 304 } 305 306 static void 307 isert_init_conn(struct isert_conn *isert_conn) 308 { 309 isert_conn->state = ISER_CONN_INIT; 310 INIT_LIST_HEAD(&isert_conn->node); 311 init_completion(&isert_conn->login_comp); 312 init_completion(&isert_conn->login_req_comp); 313 init_waitqueue_head(&isert_conn->rem_wait); 314 kref_init(&isert_conn->kref); 315 mutex_init(&isert_conn->mutex); 316 INIT_WORK(&isert_conn->release_work, isert_release_work); 317 } 318 319 static void 320 isert_free_login_buf(struct isert_conn *isert_conn) 321 { 322 struct ib_device *ib_dev = isert_conn->device->ib_device; 323 324 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 325 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 326 kfree(isert_conn->login_rsp_buf); 327 328 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 329 ISER_RX_SIZE, DMA_FROM_DEVICE); 330 kfree(isert_conn->login_desc); 331 } 332 333 static int 334 isert_alloc_login_buf(struct isert_conn *isert_conn, 335 struct ib_device *ib_dev) 336 { 337 int ret; 338 339 isert_conn->login_desc = kzalloc(sizeof(*isert_conn->login_desc), 340 GFP_KERNEL); 341 if (!isert_conn->login_desc) 342 return -ENOMEM; 343 344 isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev, 345 isert_conn->login_desc->buf, 346 ISER_RX_SIZE, DMA_FROM_DEVICE); 347 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr); 348 if (ret) { 349 isert_err("login_desc dma mapping error: %d\n", ret); 350 isert_conn->login_desc->dma_addr = 0; 351 goto out_free_login_desc; 352 } 353 354 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 355 if (!isert_conn->login_rsp_buf) { 356 ret = -ENOMEM; 357 goto out_unmap_login_desc; 358 } 359 360 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 361 isert_conn->login_rsp_buf, 362 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 363 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 364 if (ret) { 365 isert_err("login_rsp_dma mapping error: %d\n", ret); 366 isert_conn->login_rsp_dma = 0; 367 goto out_free_login_rsp_buf; 368 } 369 370 return 0; 371 372 out_free_login_rsp_buf: 373 kfree(isert_conn->login_rsp_buf); 374 out_unmap_login_desc: 375 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 376 ISER_RX_SIZE, DMA_FROM_DEVICE); 377 out_free_login_desc: 378 kfree(isert_conn->login_desc); 379 return ret; 380 } 381 382 static void 383 isert_set_nego_params(struct isert_conn *isert_conn, 384 struct rdma_conn_param *param) 385 { 386 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 387 388 /* Set max inflight RDMA READ requests */ 389 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 390 attr->max_qp_init_rd_atom); 391 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 392 393 if (param->private_data) { 394 u8 flags = *(u8 *)param->private_data; 395 396 /* 397 * use remote invalidation if the both initiator 398 * and the HCA support it 399 */ 400 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 401 (attr->device_cap_flags & 402 IB_DEVICE_MEM_MGT_EXTENSIONS); 403 if (isert_conn->snd_w_inv) 404 isert_info("Using remote invalidation\n"); 405 } 406 } 407 408 static void 409 isert_destroy_qp(struct isert_conn *isert_conn) 410 { 411 ib_destroy_qp(isert_conn->qp); 412 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 413 } 414 415 static int 416 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 417 { 418 struct isert_np *isert_np = cma_id->context; 419 struct iscsi_np *np = isert_np->np; 420 struct isert_conn *isert_conn; 421 struct isert_device *device; 422 int ret = 0; 423 424 spin_lock_bh(&np->np_thread_lock); 425 if (!np->enabled) { 426 spin_unlock_bh(&np->np_thread_lock); 427 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 428 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 429 } 430 spin_unlock_bh(&np->np_thread_lock); 431 432 isert_dbg("cma_id: %p, portal: %p\n", 433 cma_id, cma_id->context); 434 435 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 436 if (!isert_conn) 437 return -ENOMEM; 438 439 isert_init_conn(isert_conn); 440 isert_conn->cm_id = cma_id; 441 442 device = isert_device_get(cma_id); 443 if (IS_ERR(device)) { 444 ret = PTR_ERR(device); 445 goto out; 446 } 447 isert_conn->device = device; 448 449 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 450 if (ret) 451 goto out_conn_dev; 452 453 isert_set_nego_params(isert_conn, &event->param.conn); 454 455 isert_conn->qp = isert_create_qp(isert_conn, cma_id); 456 if (IS_ERR(isert_conn->qp)) { 457 ret = PTR_ERR(isert_conn->qp); 458 goto out_rsp_dma_map; 459 } 460 461 ret = isert_login_post_recv(isert_conn); 462 if (ret) 463 goto out_destroy_qp; 464 465 ret = isert_rdma_accept(isert_conn); 466 if (ret) 467 goto out_destroy_qp; 468 469 mutex_lock(&isert_np->mutex); 470 list_add_tail(&isert_conn->node, &isert_np->accepted); 471 mutex_unlock(&isert_np->mutex); 472 473 return 0; 474 475 out_destroy_qp: 476 isert_destroy_qp(isert_conn); 477 out_rsp_dma_map: 478 isert_free_login_buf(isert_conn); 479 out_conn_dev: 480 isert_device_put(device); 481 out: 482 kfree(isert_conn); 483 rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 484 return ret; 485 } 486 487 static void 488 isert_connect_release(struct isert_conn *isert_conn) 489 { 490 struct isert_device *device = isert_conn->device; 491 492 isert_dbg("conn %p\n", isert_conn); 493 494 BUG_ON(!device); 495 496 isert_free_rx_descriptors(isert_conn); 497 if (isert_conn->cm_id && 498 !isert_conn->dev_removed) 499 rdma_destroy_id(isert_conn->cm_id); 500 501 if (isert_conn->qp) 502 isert_destroy_qp(isert_conn); 503 504 if (isert_conn->login_desc) 505 isert_free_login_buf(isert_conn); 506 507 isert_device_put(device); 508 509 if (isert_conn->dev_removed) 510 wake_up_interruptible(&isert_conn->rem_wait); 511 else 512 kfree(isert_conn); 513 } 514 515 static void 516 isert_connected_handler(struct rdma_cm_id *cma_id) 517 { 518 struct isert_conn *isert_conn = cma_id->qp->qp_context; 519 struct isert_np *isert_np = cma_id->context; 520 521 isert_info("conn %p\n", isert_conn); 522 523 mutex_lock(&isert_conn->mutex); 524 isert_conn->state = ISER_CONN_UP; 525 kref_get(&isert_conn->kref); 526 mutex_unlock(&isert_conn->mutex); 527 528 mutex_lock(&isert_np->mutex); 529 list_move_tail(&isert_conn->node, &isert_np->pending); 530 mutex_unlock(&isert_np->mutex); 531 532 isert_info("np %p: Allow accept_np to continue\n", isert_np); 533 up(&isert_np->sem); 534 } 535 536 static void 537 isert_release_kref(struct kref *kref) 538 { 539 struct isert_conn *isert_conn = container_of(kref, 540 struct isert_conn, kref); 541 542 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 543 current->pid); 544 545 isert_connect_release(isert_conn); 546 } 547 548 static void 549 isert_put_conn(struct isert_conn *isert_conn) 550 { 551 kref_put(&isert_conn->kref, isert_release_kref); 552 } 553 554 static void 555 isert_handle_unbound_conn(struct isert_conn *isert_conn) 556 { 557 struct isert_np *isert_np = isert_conn->cm_id->context; 558 559 mutex_lock(&isert_np->mutex); 560 if (!list_empty(&isert_conn->node)) { 561 /* 562 * This means iscsi doesn't know this connection 563 * so schedule a cleanup ourselves 564 */ 565 list_del_init(&isert_conn->node); 566 isert_put_conn(isert_conn); 567 queue_work(isert_release_wq, &isert_conn->release_work); 568 } 569 mutex_unlock(&isert_np->mutex); 570 } 571 572 /** 573 * isert_conn_terminate() - Initiate connection termination 574 * @isert_conn: isert connection struct 575 * 576 * Notes: 577 * In case the connection state is BOUND, move state 578 * to TEMINATING and start teardown sequence (rdma_disconnect). 579 * In case the connection state is UP, complete flush as well. 580 * 581 * This routine must be called with mutex held. Thus it is 582 * safe to call multiple times. 583 */ 584 static void 585 isert_conn_terminate(struct isert_conn *isert_conn) 586 { 587 int err; 588 589 if (isert_conn->state >= ISER_CONN_TERMINATING) 590 return; 591 592 isert_info("Terminating conn %p state %d\n", 593 isert_conn, isert_conn->state); 594 isert_conn->state = ISER_CONN_TERMINATING; 595 err = rdma_disconnect(isert_conn->cm_id); 596 if (err) 597 isert_warn("Failed rdma_disconnect isert_conn %p\n", 598 isert_conn); 599 } 600 601 static int 602 isert_np_cma_handler(struct isert_np *isert_np, 603 enum rdma_cm_event_type event) 604 { 605 isert_dbg("%s (%d): isert np %p\n", 606 rdma_event_msg(event), event, isert_np); 607 608 switch (event) { 609 case RDMA_CM_EVENT_DEVICE_REMOVAL: 610 isert_np->cm_id = NULL; 611 break; 612 case RDMA_CM_EVENT_ADDR_CHANGE: 613 isert_np->cm_id = isert_setup_id(isert_np); 614 if (IS_ERR(isert_np->cm_id)) { 615 isert_err("isert np %p setup id failed: %ld\n", 616 isert_np, PTR_ERR(isert_np->cm_id)); 617 isert_np->cm_id = NULL; 618 } 619 break; 620 default: 621 isert_err("isert np %p Unexpected event %d\n", 622 isert_np, event); 623 } 624 625 return -1; 626 } 627 628 static int 629 isert_disconnected_handler(struct rdma_cm_id *cma_id, 630 enum rdma_cm_event_type event) 631 { 632 struct isert_conn *isert_conn = cma_id->qp->qp_context; 633 634 mutex_lock(&isert_conn->mutex); 635 switch (isert_conn->state) { 636 case ISER_CONN_TERMINATING: 637 break; 638 case ISER_CONN_UP: 639 isert_conn_terminate(isert_conn); 640 ib_drain_qp(isert_conn->qp); 641 isert_handle_unbound_conn(isert_conn); 642 break; 643 case ISER_CONN_BOUND: 644 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 645 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 646 break; 647 default: 648 isert_warn("conn %p terminating in state %d\n", 649 isert_conn, isert_conn->state); 650 } 651 mutex_unlock(&isert_conn->mutex); 652 653 return 0; 654 } 655 656 static int 657 isert_connect_error(struct rdma_cm_id *cma_id) 658 { 659 struct isert_conn *isert_conn = cma_id->qp->qp_context; 660 661 ib_drain_qp(isert_conn->qp); 662 list_del_init(&isert_conn->node); 663 isert_conn->cm_id = NULL; 664 isert_put_conn(isert_conn); 665 666 return -1; 667 } 668 669 static int 670 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 671 { 672 struct isert_np *isert_np = cma_id->context; 673 struct isert_conn *isert_conn; 674 int ret = 0; 675 676 isert_info("%s (%d): status %d id %p np %p\n", 677 rdma_event_msg(event->event), event->event, 678 event->status, cma_id, cma_id->context); 679 680 if (isert_np->cm_id == cma_id) 681 return isert_np_cma_handler(cma_id->context, event->event); 682 683 switch (event->event) { 684 case RDMA_CM_EVENT_CONNECT_REQUEST: 685 ret = isert_connect_request(cma_id, event); 686 if (ret) 687 isert_err("failed handle connect request %d\n", ret); 688 break; 689 case RDMA_CM_EVENT_ESTABLISHED: 690 isert_connected_handler(cma_id); 691 break; 692 case RDMA_CM_EVENT_ADDR_CHANGE: 693 case RDMA_CM_EVENT_DISCONNECTED: 694 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 695 ret = isert_disconnected_handler(cma_id, event->event); 696 break; 697 case RDMA_CM_EVENT_DEVICE_REMOVAL: 698 isert_conn = cma_id->qp->qp_context; 699 isert_conn->dev_removed = true; 700 isert_disconnected_handler(cma_id, event->event); 701 wait_event_interruptible(isert_conn->rem_wait, 702 isert_conn->state == ISER_CONN_DOWN); 703 kfree(isert_conn); 704 /* 705 * return non-zero from the callback to destroy 706 * the rdma cm id 707 */ 708 return 1; 709 case RDMA_CM_EVENT_REJECTED: 710 isert_info("Connection rejected: %s\n", 711 rdma_reject_msg(cma_id, event->status)); 712 fallthrough; 713 case RDMA_CM_EVENT_UNREACHABLE: 714 case RDMA_CM_EVENT_CONNECT_ERROR: 715 ret = isert_connect_error(cma_id); 716 break; 717 default: 718 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 719 break; 720 } 721 722 return ret; 723 } 724 725 static int 726 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 727 { 728 struct ib_recv_wr *rx_wr; 729 int i, ret; 730 struct iser_rx_desc *rx_desc; 731 732 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 733 rx_desc = &isert_conn->rx_descs[i]; 734 735 rx_wr->wr_cqe = &rx_desc->rx_cqe; 736 rx_wr->sg_list = &rx_desc->rx_sg; 737 rx_wr->num_sge = 1; 738 rx_wr->next = rx_wr + 1; 739 rx_desc->in_use = false; 740 } 741 rx_wr--; 742 rx_wr->next = NULL; /* mark end of work requests list */ 743 744 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 745 if (ret) 746 isert_err("ib_post_recv() failed with ret: %d\n", ret); 747 748 return ret; 749 } 750 751 static int 752 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 753 { 754 struct ib_recv_wr rx_wr; 755 int ret; 756 757 if (!rx_desc->in_use) { 758 /* 759 * if the descriptor is not in-use we already reposted it 760 * for recv, so just silently return 761 */ 762 return 0; 763 } 764 765 rx_desc->in_use = false; 766 rx_wr.wr_cqe = &rx_desc->rx_cqe; 767 rx_wr.sg_list = &rx_desc->rx_sg; 768 rx_wr.num_sge = 1; 769 rx_wr.next = NULL; 770 771 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 772 if (ret) 773 isert_err("ib_post_recv() failed with ret: %d\n", ret); 774 775 return ret; 776 } 777 778 static int 779 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 780 { 781 struct ib_device *ib_dev = isert_conn->cm_id->device; 782 struct ib_send_wr send_wr; 783 int ret; 784 785 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 786 ISER_HEADERS_LEN, DMA_TO_DEVICE); 787 788 tx_desc->tx_cqe.done = isert_login_send_done; 789 790 send_wr.next = NULL; 791 send_wr.wr_cqe = &tx_desc->tx_cqe; 792 send_wr.sg_list = tx_desc->tx_sg; 793 send_wr.num_sge = tx_desc->num_sge; 794 send_wr.opcode = IB_WR_SEND; 795 send_wr.send_flags = IB_SEND_SIGNALED; 796 797 ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 798 if (ret) 799 isert_err("ib_post_send() failed, ret: %d\n", ret); 800 801 return ret; 802 } 803 804 static void 805 __isert_create_send_desc(struct isert_device *device, 806 struct iser_tx_desc *tx_desc) 807 { 808 809 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 810 tx_desc->iser_header.flags = ISCSI_CTRL; 811 812 tx_desc->num_sge = 1; 813 814 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 815 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 816 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 817 } 818 } 819 820 static void 821 isert_create_send_desc(struct isert_conn *isert_conn, 822 struct isert_cmd *isert_cmd, 823 struct iser_tx_desc *tx_desc) 824 { 825 struct isert_device *device = isert_conn->device; 826 struct ib_device *ib_dev = device->ib_device; 827 828 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 829 ISER_HEADERS_LEN, DMA_TO_DEVICE); 830 831 __isert_create_send_desc(device, tx_desc); 832 } 833 834 static int 835 isert_init_tx_hdrs(struct isert_conn *isert_conn, 836 struct iser_tx_desc *tx_desc) 837 { 838 struct isert_device *device = isert_conn->device; 839 struct ib_device *ib_dev = device->ib_device; 840 u64 dma_addr; 841 842 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 843 ISER_HEADERS_LEN, DMA_TO_DEVICE); 844 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 845 isert_err("ib_dma_mapping_error() failed\n"); 846 return -ENOMEM; 847 } 848 849 tx_desc->dma_addr = dma_addr; 850 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 851 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 852 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 853 854 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 855 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 856 tx_desc->tx_sg[0].lkey); 857 858 return 0; 859 } 860 861 static void 862 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 863 struct ib_send_wr *send_wr) 864 { 865 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 866 867 tx_desc->tx_cqe.done = isert_send_done; 868 send_wr->wr_cqe = &tx_desc->tx_cqe; 869 870 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 871 send_wr->opcode = IB_WR_SEND_WITH_INV; 872 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 873 } else { 874 send_wr->opcode = IB_WR_SEND; 875 } 876 877 send_wr->sg_list = &tx_desc->tx_sg[0]; 878 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 879 send_wr->send_flags = IB_SEND_SIGNALED; 880 } 881 882 static int 883 isert_login_post_recv(struct isert_conn *isert_conn) 884 { 885 struct ib_recv_wr rx_wr; 886 struct ib_sge sge; 887 int ret; 888 889 memset(&sge, 0, sizeof(struct ib_sge)); 890 sge.addr = isert_conn->login_desc->dma_addr + 891 isert_get_hdr_offset(isert_conn->login_desc); 892 sge.length = ISER_RX_PAYLOAD_SIZE; 893 sge.lkey = isert_conn->device->pd->local_dma_lkey; 894 895 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 896 sge.addr, sge.length, sge.lkey); 897 898 isert_conn->login_desc->rx_cqe.done = isert_login_recv_done; 899 900 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 901 rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe; 902 rx_wr.sg_list = &sge; 903 rx_wr.num_sge = 1; 904 905 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 906 if (ret) 907 isert_err("ib_post_recv() failed: %d\n", ret); 908 909 return ret; 910 } 911 912 static int 913 isert_put_login_tx(struct iscsit_conn *conn, struct iscsi_login *login, 914 u32 length) 915 { 916 struct isert_conn *isert_conn = conn->context; 917 struct isert_device *device = isert_conn->device; 918 struct ib_device *ib_dev = device->ib_device; 919 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 920 int ret; 921 922 __isert_create_send_desc(device, tx_desc); 923 924 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 925 sizeof(struct iscsi_hdr)); 926 927 isert_init_tx_hdrs(isert_conn, tx_desc); 928 929 if (length > 0) { 930 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 931 932 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 933 length, DMA_TO_DEVICE); 934 935 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 936 937 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 938 length, DMA_TO_DEVICE); 939 940 tx_dsg->addr = isert_conn->login_rsp_dma; 941 tx_dsg->length = length; 942 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 943 tx_desc->num_sge = 2; 944 } 945 if (!login->login_failed) { 946 if (login->login_complete) { 947 ret = isert_alloc_rx_descriptors(isert_conn); 948 if (ret) 949 return ret; 950 951 ret = isert_post_recvm(isert_conn, 952 ISERT_QP_MAX_RECV_DTOS); 953 if (ret) 954 return ret; 955 956 /* Now we are in FULL_FEATURE phase */ 957 mutex_lock(&isert_conn->mutex); 958 isert_conn->state = ISER_CONN_FULL_FEATURE; 959 mutex_unlock(&isert_conn->mutex); 960 goto post_send; 961 } 962 963 ret = isert_login_post_recv(isert_conn); 964 if (ret) 965 return ret; 966 } 967 post_send: 968 ret = isert_login_post_send(isert_conn, tx_desc); 969 if (ret) 970 return ret; 971 972 return 0; 973 } 974 975 static void 976 isert_rx_login_req(struct isert_conn *isert_conn) 977 { 978 struct iser_rx_desc *rx_desc = isert_conn->login_desc; 979 int rx_buflen = isert_conn->login_req_len; 980 struct iscsit_conn *conn = isert_conn->conn; 981 struct iscsi_login *login = conn->conn_login; 982 int size; 983 984 isert_info("conn %p\n", isert_conn); 985 986 WARN_ON_ONCE(!login); 987 988 if (login->first_request) { 989 struct iscsi_login_req *login_req = 990 (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc); 991 /* 992 * Setup the initial iscsi_login values from the leading 993 * login request PDU. 994 */ 995 login->leading_connection = (!login_req->tsih) ? 1 : 0; 996 login->current_stage = ISCSI_LOGIN_CURRENT_STAGE( 997 login_req->flags); 998 login->version_min = login_req->min_version; 999 login->version_max = login_req->max_version; 1000 memcpy(login->isid, login_req->isid, 6); 1001 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1002 login->init_task_tag = login_req->itt; 1003 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1004 login->cid = be16_to_cpu(login_req->cid); 1005 login->tsih = be16_to_cpu(login_req->tsih); 1006 } 1007 1008 memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN); 1009 1010 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1011 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1012 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1013 MAX_KEY_VALUE_PAIRS); 1014 memcpy(login->req_buf, isert_get_data(rx_desc), size); 1015 1016 if (login->first_request) { 1017 complete(&isert_conn->login_comp); 1018 return; 1019 } 1020 queue_delayed_work(isert_login_wq, &conn->login_work, 0); 1021 } 1022 1023 static struct iscsit_cmd 1024 *isert_allocate_cmd(struct iscsit_conn *conn, struct iser_rx_desc *rx_desc) 1025 { 1026 struct isert_conn *isert_conn = conn->context; 1027 struct isert_cmd *isert_cmd; 1028 struct iscsit_cmd *cmd; 1029 1030 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1031 if (!cmd) { 1032 isert_err("Unable to allocate iscsit_cmd + isert_cmd\n"); 1033 return NULL; 1034 } 1035 isert_cmd = iscsit_priv_cmd(cmd); 1036 isert_cmd->conn = isert_conn; 1037 isert_cmd->iscsit_cmd = cmd; 1038 isert_cmd->rx_desc = rx_desc; 1039 1040 return cmd; 1041 } 1042 1043 static int 1044 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1045 struct isert_cmd *isert_cmd, struct iscsit_cmd *cmd, 1046 struct iser_rx_desc *rx_desc, unsigned char *buf) 1047 { 1048 struct iscsit_conn *conn = isert_conn->conn; 1049 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1050 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1051 bool dump_payload = false; 1052 unsigned int data_len; 1053 1054 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1055 if (rc < 0) 1056 return rc; 1057 1058 imm_data = cmd->immediate_data; 1059 imm_data_len = cmd->first_burst_len; 1060 unsol_data = cmd->unsolicited_data; 1061 data_len = cmd->se_cmd.data_length; 1062 1063 if (imm_data && imm_data_len == data_len) 1064 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1065 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1066 if (rc < 0) { 1067 return 0; 1068 } else if (rc > 0) { 1069 dump_payload = true; 1070 goto sequence_cmd; 1071 } 1072 1073 if (!imm_data) 1074 return 0; 1075 1076 if (imm_data_len != data_len) { 1077 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1078 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1079 isert_get_data(rx_desc), imm_data_len); 1080 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1081 sg_nents, imm_data_len); 1082 } else { 1083 sg_init_table(&isert_cmd->sg, 1); 1084 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1085 cmd->se_cmd.t_data_nents = 1; 1086 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc), 1087 imm_data_len); 1088 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1089 imm_data_len); 1090 } 1091 1092 cmd->write_data_done += imm_data_len; 1093 1094 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1095 spin_lock_bh(&cmd->istate_lock); 1096 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1097 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1098 spin_unlock_bh(&cmd->istate_lock); 1099 } 1100 1101 sequence_cmd: 1102 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1103 1104 if (!rc && !dump_payload && unsol_data) 1105 iscsit_set_unsolicited_dataout(cmd); 1106 else if (dump_payload && imm_data) 1107 target_put_sess_cmd(&cmd->se_cmd); 1108 1109 return 0; 1110 } 1111 1112 static int 1113 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1114 struct iser_rx_desc *rx_desc, unsigned char *buf) 1115 { 1116 struct scatterlist *sg_start; 1117 struct iscsit_conn *conn = isert_conn->conn; 1118 struct iscsit_cmd *cmd = NULL; 1119 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1120 u32 unsol_data_len = ntoh24(hdr->dlength); 1121 int rc, sg_nents, sg_off, page_off; 1122 1123 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1124 if (rc < 0) 1125 return rc; 1126 else if (!cmd) 1127 return 0; 1128 /* 1129 * FIXME: Unexpected unsolicited_data out 1130 */ 1131 if (!cmd->unsolicited_data) { 1132 isert_err("Received unexpected solicited data payload\n"); 1133 dump_stack(); 1134 return -1; 1135 } 1136 1137 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1138 "write_data_done: %u, data_length: %u\n", 1139 unsol_data_len, cmd->write_data_done, 1140 cmd->se_cmd.data_length); 1141 1142 sg_off = cmd->write_data_done / PAGE_SIZE; 1143 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1144 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1145 page_off = cmd->write_data_done % PAGE_SIZE; 1146 /* 1147 * FIXME: Non page-aligned unsolicited_data out 1148 */ 1149 if (page_off) { 1150 isert_err("unexpected non-page aligned data payload\n"); 1151 dump_stack(); 1152 return -1; 1153 } 1154 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1155 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1156 sg_nents, isert_get_data(rx_desc), unsol_data_len); 1157 1158 sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc), 1159 unsol_data_len); 1160 1161 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1162 if (rc < 0) 1163 return rc; 1164 1165 /* 1166 * multiple data-outs on the same command can arrive - 1167 * so post the buffer before hand 1168 */ 1169 return isert_post_recv(isert_conn, rx_desc); 1170 } 1171 1172 static int 1173 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1174 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1175 unsigned char *buf) 1176 { 1177 struct iscsit_conn *conn = isert_conn->conn; 1178 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1179 int rc; 1180 1181 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1182 if (rc < 0) 1183 return rc; 1184 /* 1185 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1186 */ 1187 1188 return iscsit_process_nop_out(conn, cmd, hdr); 1189 } 1190 1191 static int 1192 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1193 struct iscsit_cmd *cmd, struct iser_rx_desc *rx_desc, 1194 struct iscsi_text *hdr) 1195 { 1196 struct iscsit_conn *conn = isert_conn->conn; 1197 u32 payload_length = ntoh24(hdr->dlength); 1198 int rc; 1199 unsigned char *text_in = NULL; 1200 1201 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1202 if (rc < 0) 1203 return rc; 1204 1205 if (payload_length) { 1206 text_in = kzalloc(payload_length, GFP_KERNEL); 1207 if (!text_in) 1208 return -ENOMEM; 1209 } 1210 cmd->text_in_ptr = text_in; 1211 1212 memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length); 1213 1214 return iscsit_process_text_cmd(conn, cmd, hdr); 1215 } 1216 1217 static int 1218 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1219 uint32_t read_stag, uint64_t read_va, 1220 uint32_t write_stag, uint64_t write_va) 1221 { 1222 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1223 struct iscsit_conn *conn = isert_conn->conn; 1224 struct iscsit_cmd *cmd; 1225 struct isert_cmd *isert_cmd; 1226 int ret = -EINVAL; 1227 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1228 1229 if (conn->sess->sess_ops->SessionType && 1230 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1231 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1232 " ignoring\n", opcode); 1233 return 0; 1234 } 1235 1236 switch (opcode) { 1237 case ISCSI_OP_SCSI_CMD: 1238 cmd = isert_allocate_cmd(conn, rx_desc); 1239 if (!cmd) 1240 break; 1241 1242 isert_cmd = iscsit_priv_cmd(cmd); 1243 isert_cmd->read_stag = read_stag; 1244 isert_cmd->read_va = read_va; 1245 isert_cmd->write_stag = write_stag; 1246 isert_cmd->write_va = write_va; 1247 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1248 1249 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1250 rx_desc, (unsigned char *)hdr); 1251 break; 1252 case ISCSI_OP_NOOP_OUT: 1253 cmd = isert_allocate_cmd(conn, rx_desc); 1254 if (!cmd) 1255 break; 1256 1257 isert_cmd = iscsit_priv_cmd(cmd); 1258 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1259 rx_desc, (unsigned char *)hdr); 1260 break; 1261 case ISCSI_OP_SCSI_DATA_OUT: 1262 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1263 (unsigned char *)hdr); 1264 break; 1265 case ISCSI_OP_SCSI_TMFUNC: 1266 cmd = isert_allocate_cmd(conn, rx_desc); 1267 if (!cmd) 1268 break; 1269 1270 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1271 (unsigned char *)hdr); 1272 break; 1273 case ISCSI_OP_LOGOUT: 1274 cmd = isert_allocate_cmd(conn, rx_desc); 1275 if (!cmd) 1276 break; 1277 1278 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1279 break; 1280 case ISCSI_OP_TEXT: 1281 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1282 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1283 else 1284 cmd = isert_allocate_cmd(conn, rx_desc); 1285 1286 if (!cmd) 1287 break; 1288 1289 isert_cmd = iscsit_priv_cmd(cmd); 1290 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1291 rx_desc, (struct iscsi_text *)hdr); 1292 break; 1293 default: 1294 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1295 dump_stack(); 1296 break; 1297 } 1298 1299 return ret; 1300 } 1301 1302 static void 1303 isert_print_wc(struct ib_wc *wc, const char *type) 1304 { 1305 if (wc->status != IB_WC_WR_FLUSH_ERR) 1306 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1307 ib_wc_status_msg(wc->status), wc->status, 1308 wc->vendor_err); 1309 else 1310 isert_dbg("%s failure: %s (%d)\n", type, 1311 ib_wc_status_msg(wc->status), wc->status); 1312 } 1313 1314 static void 1315 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1316 { 1317 struct isert_conn *isert_conn = wc->qp->qp_context; 1318 struct ib_device *ib_dev = isert_conn->cm_id->device; 1319 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1320 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1321 struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc); 1322 uint64_t read_va = 0, write_va = 0; 1323 uint32_t read_stag = 0, write_stag = 0; 1324 1325 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1326 isert_print_wc(wc, "recv"); 1327 if (wc->status != IB_WC_WR_FLUSH_ERR) 1328 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1329 return; 1330 } 1331 1332 rx_desc->in_use = true; 1333 1334 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1335 ISER_RX_SIZE, DMA_FROM_DEVICE); 1336 1337 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1338 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1339 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1340 1341 switch (iser_ctrl->flags & 0xF0) { 1342 case ISCSI_CTRL: 1343 if (iser_ctrl->flags & ISER_RSV) { 1344 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1345 read_va = be64_to_cpu(iser_ctrl->read_va); 1346 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1347 read_stag, (unsigned long long)read_va); 1348 } 1349 if (iser_ctrl->flags & ISER_WSV) { 1350 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1351 write_va = be64_to_cpu(iser_ctrl->write_va); 1352 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1353 write_stag, (unsigned long long)write_va); 1354 } 1355 1356 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1357 break; 1358 case ISER_HELLO: 1359 isert_err("iSER Hello message\n"); 1360 break; 1361 default: 1362 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1363 break; 1364 } 1365 1366 isert_rx_opcode(isert_conn, rx_desc, 1367 read_stag, read_va, write_stag, write_va); 1368 1369 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1370 ISER_RX_SIZE, DMA_FROM_DEVICE); 1371 } 1372 1373 static void 1374 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1375 { 1376 struct isert_conn *isert_conn = wc->qp->qp_context; 1377 struct ib_device *ib_dev = isert_conn->device->ib_device; 1378 1379 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1380 isert_print_wc(wc, "login recv"); 1381 return; 1382 } 1383 1384 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr, 1385 ISER_RX_SIZE, DMA_FROM_DEVICE); 1386 1387 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1388 1389 if (isert_conn->conn) { 1390 struct iscsi_login *login = isert_conn->conn->conn_login; 1391 1392 if (login && !login->first_request) 1393 isert_rx_login_req(isert_conn); 1394 } 1395 1396 mutex_lock(&isert_conn->mutex); 1397 complete(&isert_conn->login_req_comp); 1398 mutex_unlock(&isert_conn->mutex); 1399 1400 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr, 1401 ISER_RX_SIZE, DMA_FROM_DEVICE); 1402 } 1403 1404 static void 1405 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1406 { 1407 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 1408 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1409 1410 if (!cmd->rw.nr_ops) 1411 return; 1412 1413 if (isert_prot_cmd(conn, se_cmd)) { 1414 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1415 conn->cm_id->port_num, se_cmd->t_data_sg, 1416 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1417 se_cmd->t_prot_nents, dir); 1418 } else { 1419 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1420 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1421 } 1422 1423 cmd->rw.nr_ops = 0; 1424 } 1425 1426 static void 1427 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1428 { 1429 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1430 struct isert_conn *isert_conn = isert_cmd->conn; 1431 struct iscsit_conn *conn = isert_conn->conn; 1432 struct iscsi_text_rsp *hdr; 1433 1434 isert_dbg("Cmd %p\n", isert_cmd); 1435 1436 switch (cmd->iscsi_opcode) { 1437 case ISCSI_OP_SCSI_CMD: 1438 spin_lock_bh(&conn->cmd_lock); 1439 if (!list_empty(&cmd->i_conn_node)) 1440 list_del_init(&cmd->i_conn_node); 1441 spin_unlock_bh(&conn->cmd_lock); 1442 1443 if (cmd->data_direction == DMA_TO_DEVICE) { 1444 iscsit_stop_dataout_timer(cmd); 1445 /* 1446 * Check for special case during comp_err where 1447 * WRITE_PENDING has been handed off from core, 1448 * but requires an extra target_put_sess_cmd() 1449 * before transport_generic_free_cmd() below. 1450 */ 1451 if (comp_err && 1452 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1453 struct se_cmd *se_cmd = &cmd->se_cmd; 1454 1455 target_put_sess_cmd(se_cmd); 1456 } 1457 } 1458 1459 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1460 transport_generic_free_cmd(&cmd->se_cmd, 0); 1461 break; 1462 case ISCSI_OP_SCSI_TMFUNC: 1463 spin_lock_bh(&conn->cmd_lock); 1464 if (!list_empty(&cmd->i_conn_node)) 1465 list_del_init(&cmd->i_conn_node); 1466 spin_unlock_bh(&conn->cmd_lock); 1467 1468 transport_generic_free_cmd(&cmd->se_cmd, 0); 1469 break; 1470 case ISCSI_OP_REJECT: 1471 case ISCSI_OP_NOOP_OUT: 1472 case ISCSI_OP_TEXT: 1473 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1474 /* If the continue bit is on, keep the command alive */ 1475 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1476 break; 1477 1478 spin_lock_bh(&conn->cmd_lock); 1479 if (!list_empty(&cmd->i_conn_node)) 1480 list_del_init(&cmd->i_conn_node); 1481 spin_unlock_bh(&conn->cmd_lock); 1482 1483 /* 1484 * Handle special case for REJECT when iscsi_add_reject*() has 1485 * overwritten the original iscsi_opcode assignment, and the 1486 * associated cmd->se_cmd needs to be released. 1487 */ 1488 if (cmd->se_cmd.se_tfo != NULL) { 1489 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1490 cmd->iscsi_opcode); 1491 transport_generic_free_cmd(&cmd->se_cmd, 0); 1492 break; 1493 } 1494 fallthrough; 1495 default: 1496 iscsit_release_cmd(cmd); 1497 break; 1498 } 1499 } 1500 1501 static void 1502 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1503 { 1504 if (tx_desc->dma_addr != 0) { 1505 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1506 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1507 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1508 tx_desc->dma_addr = 0; 1509 } 1510 } 1511 1512 static void 1513 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1514 struct ib_device *ib_dev, bool comp_err) 1515 { 1516 if (isert_cmd->pdu_buf_dma != 0) { 1517 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1518 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1519 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1520 isert_cmd->pdu_buf_dma = 0; 1521 } 1522 1523 isert_unmap_tx_desc(tx_desc, ib_dev); 1524 isert_put_cmd(isert_cmd, comp_err); 1525 } 1526 1527 static int 1528 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1529 { 1530 struct ib_mr_status mr_status; 1531 int ret; 1532 1533 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1534 if (ret) { 1535 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1536 goto fail_mr_status; 1537 } 1538 1539 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1540 u64 sec_offset_err; 1541 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1542 1543 switch (mr_status.sig_err.err_type) { 1544 case IB_SIG_BAD_GUARD: 1545 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1546 break; 1547 case IB_SIG_BAD_REFTAG: 1548 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1549 break; 1550 case IB_SIG_BAD_APPTAG: 1551 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1552 break; 1553 } 1554 sec_offset_err = mr_status.sig_err.sig_err_offset; 1555 do_div(sec_offset_err, block_size); 1556 se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba; 1557 1558 isert_err("PI error found type %d at sector 0x%llx " 1559 "expected 0x%x vs actual 0x%x\n", 1560 mr_status.sig_err.err_type, 1561 (unsigned long long)se_cmd->sense_info, 1562 mr_status.sig_err.expected, 1563 mr_status.sig_err.actual); 1564 ret = 1; 1565 } 1566 1567 fail_mr_status: 1568 return ret; 1569 } 1570 1571 static void 1572 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1573 { 1574 struct isert_conn *isert_conn = wc->qp->qp_context; 1575 struct isert_device *device = isert_conn->device; 1576 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1577 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1578 struct se_cmd *cmd = &isert_cmd->iscsit_cmd->se_cmd; 1579 int ret = 0; 1580 1581 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1582 isert_print_wc(wc, "rdma write"); 1583 if (wc->status != IB_WC_WR_FLUSH_ERR) 1584 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1585 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1586 return; 1587 } 1588 1589 isert_dbg("Cmd %p\n", isert_cmd); 1590 1591 ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 1592 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1593 1594 if (ret) { 1595 /* 1596 * transport_generic_request_failure() expects to have 1597 * plus two references to handle queue-full, so re-add 1598 * one here as target-core will have already dropped 1599 * it after the first isert_put_datain() callback. 1600 */ 1601 kref_get(&cmd->cmd_kref); 1602 transport_generic_request_failure(cmd, cmd->pi_err); 1603 } else { 1604 /* 1605 * XXX: isert_put_response() failure is not retried. 1606 */ 1607 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsit_cmd); 1608 if (ret) 1609 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1610 } 1611 } 1612 1613 static void 1614 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1615 { 1616 struct isert_conn *isert_conn = wc->qp->qp_context; 1617 struct isert_device *device = isert_conn->device; 1618 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1619 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1620 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1621 struct se_cmd *se_cmd = &cmd->se_cmd; 1622 int ret = 0; 1623 1624 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1625 isert_print_wc(wc, "rdma read"); 1626 if (wc->status != IB_WC_WR_FLUSH_ERR) 1627 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1628 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1629 return; 1630 } 1631 1632 isert_dbg("Cmd %p\n", isert_cmd); 1633 1634 iscsit_stop_dataout_timer(cmd); 1635 1636 if (isert_prot_cmd(isert_conn, se_cmd)) 1637 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 1638 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1639 cmd->write_data_done = 0; 1640 1641 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1642 spin_lock_bh(&cmd->istate_lock); 1643 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1644 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1645 spin_unlock_bh(&cmd->istate_lock); 1646 1647 /* 1648 * transport_generic_request_failure() will drop the extra 1649 * se_cmd->cmd_kref reference after T10-PI error, and handle 1650 * any non-zero ->queue_status() callback error retries. 1651 */ 1652 if (ret) 1653 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1654 else 1655 target_execute_cmd(se_cmd); 1656 } 1657 1658 static void 1659 isert_do_control_comp(struct work_struct *work) 1660 { 1661 struct isert_cmd *isert_cmd = container_of(work, 1662 struct isert_cmd, comp_work); 1663 struct isert_conn *isert_conn = isert_cmd->conn; 1664 struct ib_device *ib_dev = isert_conn->cm_id->device; 1665 struct iscsit_cmd *cmd = isert_cmd->iscsit_cmd; 1666 1667 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1668 1669 switch (cmd->i_state) { 1670 case ISTATE_SEND_TASKMGTRSP: 1671 iscsit_tmr_post_handler(cmd, cmd->conn); 1672 fallthrough; 1673 case ISTATE_SEND_REJECT: 1674 case ISTATE_SEND_TEXTRSP: 1675 cmd->i_state = ISTATE_SENT_STATUS; 1676 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1677 ib_dev, false); 1678 break; 1679 case ISTATE_SEND_LOGOUTRSP: 1680 iscsit_logout_post_handler(cmd, cmd->conn); 1681 break; 1682 default: 1683 isert_err("Unknown i_state %d\n", cmd->i_state); 1684 dump_stack(); 1685 break; 1686 } 1687 } 1688 1689 static void 1690 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1691 { 1692 struct isert_conn *isert_conn = wc->qp->qp_context; 1693 struct ib_device *ib_dev = isert_conn->cm_id->device; 1694 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1695 1696 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1697 isert_print_wc(wc, "login send"); 1698 if (wc->status != IB_WC_WR_FLUSH_ERR) 1699 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1700 } 1701 1702 isert_unmap_tx_desc(tx_desc, ib_dev); 1703 } 1704 1705 static void 1706 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1707 { 1708 struct isert_conn *isert_conn = wc->qp->qp_context; 1709 struct ib_device *ib_dev = isert_conn->cm_id->device; 1710 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1711 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1712 1713 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1714 isert_print_wc(wc, "send"); 1715 if (wc->status != IB_WC_WR_FLUSH_ERR) 1716 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1717 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1718 return; 1719 } 1720 1721 isert_dbg("Cmd %p\n", isert_cmd); 1722 1723 switch (isert_cmd->iscsit_cmd->i_state) { 1724 case ISTATE_SEND_TASKMGTRSP: 1725 case ISTATE_SEND_LOGOUTRSP: 1726 case ISTATE_SEND_REJECT: 1727 case ISTATE_SEND_TEXTRSP: 1728 isert_unmap_tx_desc(tx_desc, ib_dev); 1729 1730 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1731 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1732 return; 1733 default: 1734 isert_cmd->iscsit_cmd->i_state = ISTATE_SENT_STATUS; 1735 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1736 break; 1737 } 1738 } 1739 1740 static int 1741 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1742 { 1743 int ret; 1744 1745 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1746 if (ret) 1747 return ret; 1748 1749 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1750 if (ret) { 1751 isert_err("ib_post_send failed with %d\n", ret); 1752 return ret; 1753 } 1754 return ret; 1755 } 1756 1757 static int 1758 isert_put_response(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1759 { 1760 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1761 struct isert_conn *isert_conn = conn->context; 1762 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1763 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1764 &isert_cmd->tx_desc.iscsi_header; 1765 1766 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1767 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1768 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1769 /* 1770 * Attach SENSE DATA payload to iSCSI Response PDU 1771 */ 1772 if (cmd->se_cmd.sense_buffer && 1773 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1774 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1775 struct isert_device *device = isert_conn->device; 1776 struct ib_device *ib_dev = device->ib_device; 1777 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1778 u32 padding, pdu_len; 1779 1780 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1781 cmd->sense_buffer); 1782 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1783 1784 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1785 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1786 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1787 1788 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1789 (void *)cmd->sense_buffer, pdu_len, 1790 DMA_TO_DEVICE); 1791 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1792 return -ENOMEM; 1793 1794 isert_cmd->pdu_buf_len = pdu_len; 1795 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1796 tx_dsg->length = pdu_len; 1797 tx_dsg->lkey = device->pd->local_dma_lkey; 1798 isert_cmd->tx_desc.num_sge = 2; 1799 } 1800 1801 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1802 1803 isert_dbg("Posting SCSI Response\n"); 1804 1805 return isert_post_response(isert_conn, isert_cmd); 1806 } 1807 1808 static void 1809 isert_aborted_task(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 1810 { 1811 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1812 struct isert_conn *isert_conn = conn->context; 1813 1814 spin_lock_bh(&conn->cmd_lock); 1815 if (!list_empty(&cmd->i_conn_node)) 1816 list_del_init(&cmd->i_conn_node); 1817 spin_unlock_bh(&conn->cmd_lock); 1818 1819 if (cmd->data_direction == DMA_TO_DEVICE) 1820 iscsit_stop_dataout_timer(cmd); 1821 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1822 } 1823 1824 static enum target_prot_op 1825 isert_get_sup_prot_ops(struct iscsit_conn *conn) 1826 { 1827 struct isert_conn *isert_conn = conn->context; 1828 struct isert_device *device = isert_conn->device; 1829 1830 if (conn->tpg->tpg_attrib.t10_pi) { 1831 if (device->pi_capable) { 1832 isert_info("conn %p PI offload enabled\n", isert_conn); 1833 isert_conn->pi_support = true; 1834 return TARGET_PROT_ALL; 1835 } 1836 } 1837 1838 isert_info("conn %p PI offload disabled\n", isert_conn); 1839 isert_conn->pi_support = false; 1840 1841 return TARGET_PROT_NORMAL; 1842 } 1843 1844 static int 1845 isert_put_nopin(struct iscsit_cmd *cmd, struct iscsit_conn *conn, 1846 bool nopout_response) 1847 { 1848 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1849 struct isert_conn *isert_conn = conn->context; 1850 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1851 1852 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1853 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1854 &isert_cmd->tx_desc.iscsi_header, 1855 nopout_response); 1856 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1857 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1858 1859 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1860 1861 return isert_post_response(isert_conn, isert_cmd); 1862 } 1863 1864 static int 1865 isert_put_logout_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1866 { 1867 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1868 struct isert_conn *isert_conn = conn->context; 1869 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1870 1871 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1872 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1873 &isert_cmd->tx_desc.iscsi_header); 1874 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1875 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1876 1877 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1878 1879 return isert_post_response(isert_conn, isert_cmd); 1880 } 1881 1882 static int 1883 isert_put_tm_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1884 { 1885 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1886 struct isert_conn *isert_conn = conn->context; 1887 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1888 1889 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1890 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1891 &isert_cmd->tx_desc.iscsi_header); 1892 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1893 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1894 1895 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1896 1897 return isert_post_response(isert_conn, isert_cmd); 1898 } 1899 1900 static int 1901 isert_put_reject(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1902 { 1903 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1904 struct isert_conn *isert_conn = conn->context; 1905 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1906 struct isert_device *device = isert_conn->device; 1907 struct ib_device *ib_dev = device->ib_device; 1908 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1909 struct iscsi_reject *hdr = 1910 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1911 1912 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1913 iscsit_build_reject(cmd, conn, hdr); 1914 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1915 1916 hton24(hdr->dlength, ISCSI_HDR_LEN); 1917 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1918 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1919 DMA_TO_DEVICE); 1920 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1921 return -ENOMEM; 1922 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1923 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1924 tx_dsg->length = ISCSI_HDR_LEN; 1925 tx_dsg->lkey = device->pd->local_dma_lkey; 1926 isert_cmd->tx_desc.num_sge = 2; 1927 1928 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1929 1930 isert_dbg("conn %p Posting Reject\n", isert_conn); 1931 1932 return isert_post_response(isert_conn, isert_cmd); 1933 } 1934 1935 static int 1936 isert_put_text_rsp(struct iscsit_cmd *cmd, struct iscsit_conn *conn) 1937 { 1938 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1939 struct isert_conn *isert_conn = conn->context; 1940 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1941 struct iscsi_text_rsp *hdr = 1942 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1943 u32 txt_rsp_len; 1944 int rc; 1945 1946 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1947 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1948 if (rc < 0) 1949 return rc; 1950 1951 txt_rsp_len = rc; 1952 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1953 1954 if (txt_rsp_len) { 1955 struct isert_device *device = isert_conn->device; 1956 struct ib_device *ib_dev = device->ib_device; 1957 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1958 void *txt_rsp_buf = cmd->buf_ptr; 1959 1960 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1961 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1962 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1963 return -ENOMEM; 1964 1965 isert_cmd->pdu_buf_len = txt_rsp_len; 1966 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1967 tx_dsg->length = txt_rsp_len; 1968 tx_dsg->lkey = device->pd->local_dma_lkey; 1969 isert_cmd->tx_desc.num_sge = 2; 1970 } 1971 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1972 1973 isert_dbg("conn %p Text Response\n", isert_conn); 1974 1975 return isert_post_response(isert_conn, isert_cmd); 1976 } 1977 1978 static inline void 1979 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 1980 { 1981 domain->sig_type = IB_SIG_TYPE_T10_DIF; 1982 domain->sig.dif.bg_type = IB_T10DIF_CRC; 1983 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 1984 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 1985 /* 1986 * At the moment we hard code those, but if in the future 1987 * the target core would like to use it, we will take it 1988 * from se_cmd. 1989 */ 1990 domain->sig.dif.apptag_check_mask = 0xffff; 1991 domain->sig.dif.app_escape = true; 1992 domain->sig.dif.ref_escape = true; 1993 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 1994 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 1995 domain->sig.dif.ref_remap = true; 1996 } 1997 1998 static int 1999 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2000 { 2001 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2002 2003 switch (se_cmd->prot_op) { 2004 case TARGET_PROT_DIN_INSERT: 2005 case TARGET_PROT_DOUT_STRIP: 2006 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2007 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2008 break; 2009 case TARGET_PROT_DOUT_INSERT: 2010 case TARGET_PROT_DIN_STRIP: 2011 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2012 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2013 break; 2014 case TARGET_PROT_DIN_PASS: 2015 case TARGET_PROT_DOUT_PASS: 2016 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2017 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2018 break; 2019 default: 2020 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2021 return -EINVAL; 2022 } 2023 2024 if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 2025 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 2026 if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 2027 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 2028 if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 2029 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 2030 2031 return 0; 2032 } 2033 2034 static int 2035 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2036 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2037 { 2038 struct se_cmd *se_cmd = &cmd->iscsit_cmd->se_cmd; 2039 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2040 u8 port_num = conn->cm_id->port_num; 2041 u64 addr; 2042 u32 rkey, offset; 2043 int ret; 2044 2045 if (cmd->ctx_init_done) 2046 goto rdma_ctx_post; 2047 2048 if (dir == DMA_FROM_DEVICE) { 2049 addr = cmd->write_va; 2050 rkey = cmd->write_stag; 2051 offset = cmd->iscsit_cmd->write_data_done; 2052 } else { 2053 addr = cmd->read_va; 2054 rkey = cmd->read_stag; 2055 offset = 0; 2056 } 2057 2058 if (isert_prot_cmd(conn, se_cmd)) { 2059 struct ib_sig_attrs sig_attrs; 2060 2061 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2062 if (ret) 2063 return ret; 2064 2065 WARN_ON_ONCE(offset); 2066 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2067 se_cmd->t_data_sg, se_cmd->t_data_nents, 2068 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2069 &sig_attrs, addr, rkey, dir); 2070 } else { 2071 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2072 se_cmd->t_data_sg, se_cmd->t_data_nents, 2073 offset, addr, rkey, dir); 2074 } 2075 2076 if (ret < 0) { 2077 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2078 return ret; 2079 } 2080 2081 cmd->ctx_init_done = true; 2082 2083 rdma_ctx_post: 2084 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2085 if (ret < 0) 2086 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2087 return ret; 2088 } 2089 2090 static int 2091 isert_put_datain(struct iscsit_conn *conn, struct iscsit_cmd *cmd) 2092 { 2093 struct se_cmd *se_cmd = &cmd->se_cmd; 2094 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2095 struct isert_conn *isert_conn = conn->context; 2096 struct ib_cqe *cqe = NULL; 2097 struct ib_send_wr *chain_wr = NULL; 2098 int rc; 2099 2100 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2101 isert_cmd, se_cmd->data_length); 2102 2103 if (isert_prot_cmd(isert_conn, se_cmd)) { 2104 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2105 cqe = &isert_cmd->tx_desc.tx_cqe; 2106 } else { 2107 /* 2108 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2109 */ 2110 isert_create_send_desc(isert_conn, isert_cmd, 2111 &isert_cmd->tx_desc); 2112 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2113 &isert_cmd->tx_desc.iscsi_header); 2114 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2115 isert_init_send_wr(isert_conn, isert_cmd, 2116 &isert_cmd->tx_desc.send_wr); 2117 2118 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2119 if (rc) 2120 return rc; 2121 2122 chain_wr = &isert_cmd->tx_desc.send_wr; 2123 } 2124 2125 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2126 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2127 isert_cmd, rc); 2128 return rc; 2129 } 2130 2131 static int 2132 isert_get_dataout(struct iscsit_conn *conn, struct iscsit_cmd *cmd, bool recovery) 2133 { 2134 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2135 int ret; 2136 2137 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2138 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2139 2140 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2141 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2142 &isert_cmd->tx_desc.tx_cqe, NULL); 2143 2144 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2145 isert_cmd, ret); 2146 return ret; 2147 } 2148 2149 static int 2150 isert_immediate_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2151 { 2152 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2153 int ret = 0; 2154 2155 switch (state) { 2156 case ISTATE_REMOVE: 2157 spin_lock_bh(&conn->cmd_lock); 2158 list_del_init(&cmd->i_conn_node); 2159 spin_unlock_bh(&conn->cmd_lock); 2160 isert_put_cmd(isert_cmd, true); 2161 break; 2162 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2163 ret = isert_put_nopin(cmd, conn, false); 2164 break; 2165 default: 2166 isert_err("Unknown immediate state: 0x%02x\n", state); 2167 ret = -EINVAL; 2168 break; 2169 } 2170 2171 return ret; 2172 } 2173 2174 static int 2175 isert_response_queue(struct iscsit_conn *conn, struct iscsit_cmd *cmd, int state) 2176 { 2177 struct isert_conn *isert_conn = conn->context; 2178 int ret; 2179 2180 switch (state) { 2181 case ISTATE_SEND_LOGOUTRSP: 2182 ret = isert_put_logout_rsp(cmd, conn); 2183 if (!ret) 2184 isert_conn->logout_posted = true; 2185 break; 2186 case ISTATE_SEND_NOPIN: 2187 ret = isert_put_nopin(cmd, conn, true); 2188 break; 2189 case ISTATE_SEND_TASKMGTRSP: 2190 ret = isert_put_tm_rsp(cmd, conn); 2191 break; 2192 case ISTATE_SEND_REJECT: 2193 ret = isert_put_reject(cmd, conn); 2194 break; 2195 case ISTATE_SEND_TEXTRSP: 2196 ret = isert_put_text_rsp(cmd, conn); 2197 break; 2198 case ISTATE_SEND_STATUS: 2199 /* 2200 * Special case for sending non GOOD SCSI status from TX thread 2201 * context during pre se_cmd excecution failure. 2202 */ 2203 ret = isert_put_response(conn, cmd); 2204 break; 2205 default: 2206 isert_err("Unknown response state: 0x%02x\n", state); 2207 ret = -EINVAL; 2208 break; 2209 } 2210 2211 return ret; 2212 } 2213 2214 struct rdma_cm_id * 2215 isert_setup_id(struct isert_np *isert_np) 2216 { 2217 struct iscsi_np *np = isert_np->np; 2218 struct rdma_cm_id *id; 2219 struct sockaddr *sa; 2220 int ret; 2221 2222 sa = (struct sockaddr *)&np->np_sockaddr; 2223 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2224 2225 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2226 RDMA_PS_TCP, IB_QPT_RC); 2227 if (IS_ERR(id)) { 2228 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2229 ret = PTR_ERR(id); 2230 goto out; 2231 } 2232 isert_dbg("id %p context %p\n", id, id->context); 2233 2234 /* 2235 * Allow both IPv4 and IPv6 sockets to bind a single port 2236 * at the same time. 2237 */ 2238 ret = rdma_set_afonly(id, 1); 2239 if (ret) { 2240 isert_err("rdma_set_afonly() failed: %d\n", ret); 2241 goto out_id; 2242 } 2243 2244 ret = rdma_bind_addr(id, sa); 2245 if (ret) { 2246 isert_err("rdma_bind_addr() failed: %d\n", ret); 2247 goto out_id; 2248 } 2249 2250 ret = rdma_listen(id, 0); 2251 if (ret) { 2252 isert_err("rdma_listen() failed: %d\n", ret); 2253 goto out_id; 2254 } 2255 2256 return id; 2257 out_id: 2258 rdma_destroy_id(id); 2259 out: 2260 return ERR_PTR(ret); 2261 } 2262 2263 static int 2264 isert_setup_np(struct iscsi_np *np, 2265 struct sockaddr_storage *ksockaddr) 2266 { 2267 struct isert_np *isert_np; 2268 struct rdma_cm_id *isert_lid; 2269 int ret; 2270 2271 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2272 if (!isert_np) 2273 return -ENOMEM; 2274 2275 sema_init(&isert_np->sem, 0); 2276 mutex_init(&isert_np->mutex); 2277 INIT_LIST_HEAD(&isert_np->accepted); 2278 INIT_LIST_HEAD(&isert_np->pending); 2279 isert_np->np = np; 2280 2281 /* 2282 * Setup the np->np_sockaddr from the passed sockaddr setup 2283 * in iscsi_target_configfs.c code.. 2284 */ 2285 memcpy(&np->np_sockaddr, ksockaddr, 2286 sizeof(struct sockaddr_storage)); 2287 2288 isert_lid = isert_setup_id(isert_np); 2289 if (IS_ERR(isert_lid)) { 2290 ret = PTR_ERR(isert_lid); 2291 goto out; 2292 } 2293 2294 isert_np->cm_id = isert_lid; 2295 np->np_context = isert_np; 2296 2297 return 0; 2298 2299 out: 2300 kfree(isert_np); 2301 2302 return ret; 2303 } 2304 2305 static int 2306 isert_rdma_accept(struct isert_conn *isert_conn) 2307 { 2308 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2309 struct rdma_conn_param cp; 2310 int ret; 2311 struct iser_cm_hdr rsp_hdr; 2312 2313 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2314 cp.initiator_depth = isert_conn->initiator_depth; 2315 cp.retry_count = 7; 2316 cp.rnr_retry_count = 7; 2317 2318 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2319 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2320 if (!isert_conn->snd_w_inv) 2321 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2322 cp.private_data = (void *)&rsp_hdr; 2323 cp.private_data_len = sizeof(rsp_hdr); 2324 2325 ret = rdma_accept(cm_id, &cp); 2326 if (ret) { 2327 isert_err("rdma_accept() failed with: %d\n", ret); 2328 return ret; 2329 } 2330 2331 return 0; 2332 } 2333 2334 static int 2335 isert_get_login_rx(struct iscsit_conn *conn, struct iscsi_login *login) 2336 { 2337 struct isert_conn *isert_conn = conn->context; 2338 int ret; 2339 2340 isert_info("before login_req comp conn: %p\n", isert_conn); 2341 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2342 if (ret) { 2343 isert_err("isert_conn %p interrupted before got login req\n", 2344 isert_conn); 2345 return ret; 2346 } 2347 reinit_completion(&isert_conn->login_req_comp); 2348 2349 /* 2350 * For login requests after the first PDU, isert_rx_login_req() will 2351 * kick queue_delayed_work(isert_login_wq, &conn->login_work) as 2352 * the packet is received, which turns this callback from 2353 * iscsi_target_do_login_rx() into a NOP. 2354 */ 2355 if (!login->first_request) 2356 return 0; 2357 2358 isert_rx_login_req(isert_conn); 2359 2360 isert_info("before login_comp conn: %p\n", conn); 2361 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2362 if (ret) 2363 return ret; 2364 2365 isert_info("processing login->req: %p\n", login->req); 2366 2367 return 0; 2368 } 2369 2370 static void 2371 isert_set_conn_info(struct iscsi_np *np, struct iscsit_conn *conn, 2372 struct isert_conn *isert_conn) 2373 { 2374 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2375 struct rdma_route *cm_route = &cm_id->route; 2376 2377 conn->login_family = np->np_sockaddr.ss_family; 2378 2379 conn->login_sockaddr = cm_route->addr.dst_addr; 2380 conn->local_sockaddr = cm_route->addr.src_addr; 2381 } 2382 2383 static int 2384 isert_accept_np(struct iscsi_np *np, struct iscsit_conn *conn) 2385 { 2386 struct isert_np *isert_np = np->np_context; 2387 struct isert_conn *isert_conn; 2388 int ret; 2389 2390 accept_wait: 2391 ret = down_interruptible(&isert_np->sem); 2392 if (ret) 2393 return -ENODEV; 2394 2395 spin_lock_bh(&np->np_thread_lock); 2396 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2397 spin_unlock_bh(&np->np_thread_lock); 2398 isert_dbg("np_thread_state %d\n", 2399 np->np_thread_state); 2400 /* 2401 * No point in stalling here when np_thread 2402 * is in state RESET/SHUTDOWN/EXIT - bail 2403 */ 2404 return -ENODEV; 2405 } 2406 spin_unlock_bh(&np->np_thread_lock); 2407 2408 mutex_lock(&isert_np->mutex); 2409 if (list_empty(&isert_np->pending)) { 2410 mutex_unlock(&isert_np->mutex); 2411 goto accept_wait; 2412 } 2413 isert_conn = list_first_entry(&isert_np->pending, 2414 struct isert_conn, node); 2415 list_del_init(&isert_conn->node); 2416 mutex_unlock(&isert_np->mutex); 2417 2418 conn->context = isert_conn; 2419 isert_conn->conn = conn; 2420 isert_conn->state = ISER_CONN_BOUND; 2421 2422 isert_set_conn_info(np, conn, isert_conn); 2423 2424 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2425 2426 return 0; 2427 } 2428 2429 static void 2430 isert_free_np(struct iscsi_np *np) 2431 { 2432 struct isert_np *isert_np = np->np_context; 2433 struct isert_conn *isert_conn, *n; 2434 2435 if (isert_np->cm_id) 2436 rdma_destroy_id(isert_np->cm_id); 2437 2438 /* 2439 * FIXME: At this point we don't have a good way to insure 2440 * that at this point we don't have hanging connections that 2441 * completed RDMA establishment but didn't start iscsi login 2442 * process. So work-around this by cleaning up what ever piled 2443 * up in accepted and pending lists. 2444 */ 2445 mutex_lock(&isert_np->mutex); 2446 if (!list_empty(&isert_np->pending)) { 2447 isert_info("Still have isert pending connections\n"); 2448 list_for_each_entry_safe(isert_conn, n, 2449 &isert_np->pending, 2450 node) { 2451 isert_info("cleaning isert_conn %p state (%d)\n", 2452 isert_conn, isert_conn->state); 2453 isert_connect_release(isert_conn); 2454 } 2455 } 2456 2457 if (!list_empty(&isert_np->accepted)) { 2458 isert_info("Still have isert accepted connections\n"); 2459 list_for_each_entry_safe(isert_conn, n, 2460 &isert_np->accepted, 2461 node) { 2462 isert_info("cleaning isert_conn %p state (%d)\n", 2463 isert_conn, isert_conn->state); 2464 isert_connect_release(isert_conn); 2465 } 2466 } 2467 mutex_unlock(&isert_np->mutex); 2468 2469 np->np_context = NULL; 2470 kfree(isert_np); 2471 } 2472 2473 static void isert_release_work(struct work_struct *work) 2474 { 2475 struct isert_conn *isert_conn = container_of(work, 2476 struct isert_conn, 2477 release_work); 2478 2479 isert_info("Starting release conn %p\n", isert_conn); 2480 2481 mutex_lock(&isert_conn->mutex); 2482 isert_conn->state = ISER_CONN_DOWN; 2483 mutex_unlock(&isert_conn->mutex); 2484 2485 isert_info("Destroying conn %p\n", isert_conn); 2486 isert_put_conn(isert_conn); 2487 } 2488 2489 static void 2490 isert_wait4logout(struct isert_conn *isert_conn) 2491 { 2492 struct iscsit_conn *conn = isert_conn->conn; 2493 2494 isert_info("conn %p\n", isert_conn); 2495 2496 if (isert_conn->logout_posted) { 2497 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2498 wait_for_completion_timeout(&conn->conn_logout_comp, 2499 SECONDS_FOR_LOGOUT_COMP * HZ); 2500 } 2501 } 2502 2503 static void 2504 isert_wait4cmds(struct iscsit_conn *conn) 2505 { 2506 isert_info("iscsit_conn %p\n", conn); 2507 2508 if (conn->sess) { 2509 target_stop_session(conn->sess->se_sess); 2510 target_wait_for_sess_cmds(conn->sess->se_sess); 2511 } 2512 } 2513 2514 /** 2515 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2516 * unsolicitate dataout 2517 * @conn: iscsi connection 2518 * 2519 * We might still have commands that are waiting for unsolicited 2520 * dataouts messages. We must put the extra reference on those 2521 * before blocking on the target_wait_for_session_cmds 2522 */ 2523 static void 2524 isert_put_unsol_pending_cmds(struct iscsit_conn *conn) 2525 { 2526 struct iscsit_cmd *cmd, *tmp; 2527 static LIST_HEAD(drop_cmd_list); 2528 2529 spin_lock_bh(&conn->cmd_lock); 2530 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2531 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2532 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2533 (cmd->write_data_done < cmd->se_cmd.data_length)) 2534 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2535 } 2536 spin_unlock_bh(&conn->cmd_lock); 2537 2538 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2539 list_del_init(&cmd->i_conn_node); 2540 if (cmd->i_state != ISTATE_REMOVE) { 2541 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2542 2543 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2544 isert_put_cmd(isert_cmd, true); 2545 } 2546 } 2547 } 2548 2549 static void isert_wait_conn(struct iscsit_conn *conn) 2550 { 2551 struct isert_conn *isert_conn = conn->context; 2552 2553 isert_info("Starting conn %p\n", isert_conn); 2554 2555 mutex_lock(&isert_conn->mutex); 2556 isert_conn_terminate(isert_conn); 2557 mutex_unlock(&isert_conn->mutex); 2558 2559 ib_drain_qp(isert_conn->qp); 2560 isert_put_unsol_pending_cmds(conn); 2561 isert_wait4cmds(conn); 2562 isert_wait4logout(isert_conn); 2563 2564 queue_work(isert_release_wq, &isert_conn->release_work); 2565 } 2566 2567 static void isert_free_conn(struct iscsit_conn *conn) 2568 { 2569 struct isert_conn *isert_conn = conn->context; 2570 2571 ib_drain_qp(isert_conn->qp); 2572 isert_put_conn(isert_conn); 2573 } 2574 2575 static void isert_get_rx_pdu(struct iscsit_conn *conn) 2576 { 2577 struct completion comp; 2578 2579 init_completion(&comp); 2580 2581 wait_for_completion_interruptible(&comp); 2582 } 2583 2584 static struct iscsit_transport iser_target_transport = { 2585 .name = "IB/iSER", 2586 .transport_type = ISCSI_INFINIBAND, 2587 .rdma_shutdown = true, 2588 .priv_size = sizeof(struct isert_cmd), 2589 .owner = THIS_MODULE, 2590 .iscsit_setup_np = isert_setup_np, 2591 .iscsit_accept_np = isert_accept_np, 2592 .iscsit_free_np = isert_free_np, 2593 .iscsit_wait_conn = isert_wait_conn, 2594 .iscsit_free_conn = isert_free_conn, 2595 .iscsit_get_login_rx = isert_get_login_rx, 2596 .iscsit_put_login_tx = isert_put_login_tx, 2597 .iscsit_immediate_queue = isert_immediate_queue, 2598 .iscsit_response_queue = isert_response_queue, 2599 .iscsit_get_dataout = isert_get_dataout, 2600 .iscsit_queue_data_in = isert_put_datain, 2601 .iscsit_queue_status = isert_put_response, 2602 .iscsit_aborted_task = isert_aborted_task, 2603 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2604 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2605 }; 2606 2607 static int __init isert_init(void) 2608 { 2609 isert_login_wq = alloc_workqueue("isert_login_wq", 0, 0); 2610 if (!isert_login_wq) { 2611 isert_err("Unable to allocate isert_login_wq\n"); 2612 return -ENOMEM; 2613 } 2614 2615 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2616 WQ_UNBOUND | WQ_HIGHPRI, 0); 2617 if (!isert_comp_wq) { 2618 isert_err("Unable to allocate isert_comp_wq\n"); 2619 goto destroy_login_wq; 2620 } 2621 2622 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2623 WQ_UNBOUND_MAX_ACTIVE); 2624 if (!isert_release_wq) { 2625 isert_err("Unable to allocate isert_release_wq\n"); 2626 goto destroy_comp_wq; 2627 } 2628 2629 iscsit_register_transport(&iser_target_transport); 2630 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2631 2632 return 0; 2633 2634 destroy_comp_wq: 2635 destroy_workqueue(isert_comp_wq); 2636 destroy_login_wq: 2637 destroy_workqueue(isert_login_wq); 2638 2639 return -ENOMEM; 2640 } 2641 2642 static void __exit isert_exit(void) 2643 { 2644 flush_workqueue(isert_login_wq); 2645 destroy_workqueue(isert_release_wq); 2646 destroy_workqueue(isert_comp_wq); 2647 iscsit_unregister_transport(&iser_target_transport); 2648 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2649 destroy_workqueue(isert_login_wq); 2650 } 2651 2652 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2653 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2654 MODULE_LICENSE("GPL"); 2655 2656 module_init(isert_init); 2657 module_exit(isert_exit); 2658