1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /******************************************************************************* 3 * This file contains iSCSI extentions for RDMA (iSER) Verbs 4 * 5 * (c) Copyright 2013 Datera, Inc. 6 * 7 * Nicholas A. Bellinger <nab@linux-iscsi.org> 8 * 9 ****************************************************************************/ 10 11 #include <linux/string.h> 12 #include <linux/module.h> 13 #include <linux/scatterlist.h> 14 #include <linux/socket.h> 15 #include <linux/in.h> 16 #include <linux/in6.h> 17 #include <rdma/ib_verbs.h> 18 #include <rdma/ib_cm.h> 19 #include <rdma/rdma_cm.h> 20 #include <target/target_core_base.h> 21 #include <target/target_core_fabric.h> 22 #include <target/iscsi/iscsi_transport.h> 23 #include <linux/semaphore.h> 24 25 #include "ib_isert.h" 26 27 static int isert_debug_level; 28 module_param_named(debug_level, isert_debug_level, int, 0644); 29 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)"); 30 31 static int isert_sg_tablesize_set(const char *val, 32 const struct kernel_param *kp); 33 static const struct kernel_param_ops sg_tablesize_ops = { 34 .set = isert_sg_tablesize_set, 35 .get = param_get_int, 36 }; 37 38 static int isert_sg_tablesize = ISCSI_ISER_DEF_SG_TABLESIZE; 39 module_param_cb(sg_tablesize, &sg_tablesize_ops, &isert_sg_tablesize, 0644); 40 MODULE_PARM_DESC(sg_tablesize, 41 "Number of gather/scatter entries in a single scsi command, should >= 128 (default: 256, max: 4096)"); 42 43 static DEFINE_MUTEX(device_list_mutex); 44 static LIST_HEAD(device_list); 45 static struct workqueue_struct *isert_comp_wq; 46 static struct workqueue_struct *isert_release_wq; 47 48 static int 49 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd); 50 static int 51 isert_login_post_recv(struct isert_conn *isert_conn); 52 static int 53 isert_rdma_accept(struct isert_conn *isert_conn); 54 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np); 55 56 static void isert_release_work(struct work_struct *work); 57 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc); 58 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc); 59 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc); 60 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc); 61 62 static int isert_sg_tablesize_set(const char *val, const struct kernel_param *kp) 63 { 64 int n = 0, ret; 65 66 ret = kstrtoint(val, 10, &n); 67 if (ret != 0 || n < ISCSI_ISER_MIN_SG_TABLESIZE || 68 n > ISCSI_ISER_MAX_SG_TABLESIZE) 69 return -EINVAL; 70 71 return param_set_int(val, kp); 72 } 73 74 static inline bool 75 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd) 76 { 77 return (conn->pi_support && 78 cmd->prot_op != TARGET_PROT_NORMAL); 79 } 80 81 static void 82 isert_qp_event_callback(struct ib_event *e, void *context) 83 { 84 struct isert_conn *isert_conn = context; 85 86 isert_err("%s (%d): conn %p\n", 87 ib_event_msg(e->event), e->event, isert_conn); 88 89 switch (e->event) { 90 case IB_EVENT_COMM_EST: 91 rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST); 92 break; 93 case IB_EVENT_QP_LAST_WQE_REACHED: 94 isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n"); 95 break; 96 default: 97 break; 98 } 99 } 100 101 static struct ib_qp * 102 isert_create_qp(struct isert_conn *isert_conn, 103 struct rdma_cm_id *cma_id) 104 { 105 u32 cq_size = ISERT_QP_MAX_REQ_DTOS + ISERT_QP_MAX_RECV_DTOS + 2; 106 struct isert_device *device = isert_conn->device; 107 struct ib_device *ib_dev = device->ib_device; 108 struct ib_qp_init_attr attr; 109 int ret, factor; 110 111 isert_conn->cq = ib_cq_pool_get(ib_dev, cq_size, -1, IB_POLL_WORKQUEUE); 112 if (IS_ERR(isert_conn->cq)) { 113 isert_err("Unable to allocate cq\n"); 114 ret = PTR_ERR(isert_conn->cq); 115 return ERR_PTR(ret); 116 } 117 isert_conn->cq_size = cq_size; 118 119 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 120 attr.event_handler = isert_qp_event_callback; 121 attr.qp_context = isert_conn; 122 attr.send_cq = isert_conn->cq; 123 attr.recv_cq = isert_conn->cq; 124 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1; 125 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1; 126 factor = rdma_rw_mr_factor(device->ib_device, cma_id->port_num, 127 isert_sg_tablesize); 128 attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX * factor; 129 attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge; 130 attr.cap.max_recv_sge = 1; 131 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 132 attr.qp_type = IB_QPT_RC; 133 if (device->pi_capable) 134 attr.create_flags |= IB_QP_CREATE_INTEGRITY_EN; 135 136 ret = rdma_create_qp(cma_id, device->pd, &attr); 137 if (ret) { 138 isert_err("rdma_create_qp failed for cma_id %d\n", ret); 139 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 140 141 return ERR_PTR(ret); 142 } 143 144 return cma_id->qp; 145 } 146 147 static int 148 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 149 { 150 struct isert_device *device = isert_conn->device; 151 struct ib_device *ib_dev = device->ib_device; 152 struct iser_rx_desc *rx_desc; 153 struct ib_sge *rx_sg; 154 u64 dma_addr; 155 int i, j; 156 157 isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS, 158 sizeof(struct iser_rx_desc), 159 GFP_KERNEL); 160 if (!isert_conn->rx_descs) 161 return -ENOMEM; 162 163 rx_desc = isert_conn->rx_descs; 164 165 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 166 dma_addr = ib_dma_map_single(ib_dev, rx_desc->buf, 167 ISER_RX_SIZE, DMA_FROM_DEVICE); 168 if (ib_dma_mapping_error(ib_dev, dma_addr)) 169 goto dma_map_fail; 170 171 rx_desc->dma_addr = dma_addr; 172 173 rx_sg = &rx_desc->rx_sg; 174 rx_sg->addr = rx_desc->dma_addr + isert_get_hdr_offset(rx_desc); 175 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 176 rx_sg->lkey = device->pd->local_dma_lkey; 177 rx_desc->rx_cqe.done = isert_recv_done; 178 } 179 180 return 0; 181 182 dma_map_fail: 183 rx_desc = isert_conn->rx_descs; 184 for (j = 0; j < i; j++, rx_desc++) { 185 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 186 ISER_RX_SIZE, DMA_FROM_DEVICE); 187 } 188 kfree(isert_conn->rx_descs); 189 isert_conn->rx_descs = NULL; 190 isert_err("conn %p failed to allocate rx descriptors\n", isert_conn); 191 return -ENOMEM; 192 } 193 194 static void 195 isert_free_rx_descriptors(struct isert_conn *isert_conn) 196 { 197 struct ib_device *ib_dev = isert_conn->device->ib_device; 198 struct iser_rx_desc *rx_desc; 199 int i; 200 201 if (!isert_conn->rx_descs) 202 return; 203 204 rx_desc = isert_conn->rx_descs; 205 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 206 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 207 ISER_RX_SIZE, DMA_FROM_DEVICE); 208 } 209 210 kfree(isert_conn->rx_descs); 211 isert_conn->rx_descs = NULL; 212 } 213 214 static int 215 isert_create_device_ib_res(struct isert_device *device) 216 { 217 struct ib_device *ib_dev = device->ib_device; 218 int ret; 219 220 isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n", 221 ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge); 222 isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd); 223 224 device->pd = ib_alloc_pd(ib_dev, 0); 225 if (IS_ERR(device->pd)) { 226 ret = PTR_ERR(device->pd); 227 isert_err("failed to allocate pd, device %p, ret=%d\n", 228 device, ret); 229 return ret; 230 } 231 232 /* Check signature cap */ 233 if (ib_dev->attrs.device_cap_flags & IB_DEVICE_INTEGRITY_HANDOVER) 234 device->pi_capable = true; 235 else 236 device->pi_capable = false; 237 238 return 0; 239 } 240 241 static void 242 isert_free_device_ib_res(struct isert_device *device) 243 { 244 isert_info("device %p\n", device); 245 246 ib_dealloc_pd(device->pd); 247 } 248 249 static void 250 isert_device_put(struct isert_device *device) 251 { 252 mutex_lock(&device_list_mutex); 253 device->refcount--; 254 isert_info("device %p refcount %d\n", device, device->refcount); 255 if (!device->refcount) { 256 isert_free_device_ib_res(device); 257 list_del(&device->dev_node); 258 kfree(device); 259 } 260 mutex_unlock(&device_list_mutex); 261 } 262 263 static struct isert_device * 264 isert_device_get(struct rdma_cm_id *cma_id) 265 { 266 struct isert_device *device; 267 int ret; 268 269 mutex_lock(&device_list_mutex); 270 list_for_each_entry(device, &device_list, dev_node) { 271 if (device->ib_device->node_guid == cma_id->device->node_guid) { 272 device->refcount++; 273 isert_info("Found iser device %p refcount %d\n", 274 device, device->refcount); 275 mutex_unlock(&device_list_mutex); 276 return device; 277 } 278 } 279 280 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 281 if (!device) { 282 mutex_unlock(&device_list_mutex); 283 return ERR_PTR(-ENOMEM); 284 } 285 286 INIT_LIST_HEAD(&device->dev_node); 287 288 device->ib_device = cma_id->device; 289 ret = isert_create_device_ib_res(device); 290 if (ret) { 291 kfree(device); 292 mutex_unlock(&device_list_mutex); 293 return ERR_PTR(ret); 294 } 295 296 device->refcount++; 297 list_add_tail(&device->dev_node, &device_list); 298 isert_info("Created a new iser device %p refcount %d\n", 299 device, device->refcount); 300 mutex_unlock(&device_list_mutex); 301 302 return device; 303 } 304 305 static void 306 isert_init_conn(struct isert_conn *isert_conn) 307 { 308 isert_conn->state = ISER_CONN_INIT; 309 INIT_LIST_HEAD(&isert_conn->node); 310 init_completion(&isert_conn->login_comp); 311 init_completion(&isert_conn->login_req_comp); 312 init_waitqueue_head(&isert_conn->rem_wait); 313 kref_init(&isert_conn->kref); 314 mutex_init(&isert_conn->mutex); 315 INIT_WORK(&isert_conn->release_work, isert_release_work); 316 } 317 318 static void 319 isert_free_login_buf(struct isert_conn *isert_conn) 320 { 321 struct ib_device *ib_dev = isert_conn->device->ib_device; 322 323 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 324 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 325 kfree(isert_conn->login_rsp_buf); 326 327 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 328 ISER_RX_SIZE, DMA_FROM_DEVICE); 329 kfree(isert_conn->login_desc); 330 } 331 332 static int 333 isert_alloc_login_buf(struct isert_conn *isert_conn, 334 struct ib_device *ib_dev) 335 { 336 int ret; 337 338 isert_conn->login_desc = kzalloc(sizeof(*isert_conn->login_desc), 339 GFP_KERNEL); 340 if (!isert_conn->login_desc) 341 return -ENOMEM; 342 343 isert_conn->login_desc->dma_addr = ib_dma_map_single(ib_dev, 344 isert_conn->login_desc->buf, 345 ISER_RX_SIZE, DMA_FROM_DEVICE); 346 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_desc->dma_addr); 347 if (ret) { 348 isert_err("login_desc dma mapping error: %d\n", ret); 349 isert_conn->login_desc->dma_addr = 0; 350 goto out_free_login_desc; 351 } 352 353 isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL); 354 if (!isert_conn->login_rsp_buf) { 355 ret = -ENOMEM; 356 goto out_unmap_login_desc; 357 } 358 359 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 360 isert_conn->login_rsp_buf, 361 ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE); 362 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 363 if (ret) { 364 isert_err("login_rsp_dma mapping error: %d\n", ret); 365 isert_conn->login_rsp_dma = 0; 366 goto out_free_login_rsp_buf; 367 } 368 369 return 0; 370 371 out_free_login_rsp_buf: 372 kfree(isert_conn->login_rsp_buf); 373 out_unmap_login_desc: 374 ib_dma_unmap_single(ib_dev, isert_conn->login_desc->dma_addr, 375 ISER_RX_SIZE, DMA_FROM_DEVICE); 376 out_free_login_desc: 377 kfree(isert_conn->login_desc); 378 return ret; 379 } 380 381 static void 382 isert_set_nego_params(struct isert_conn *isert_conn, 383 struct rdma_conn_param *param) 384 { 385 struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs; 386 387 /* Set max inflight RDMA READ requests */ 388 isert_conn->initiator_depth = min_t(u8, param->initiator_depth, 389 attr->max_qp_init_rd_atom); 390 isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth); 391 392 if (param->private_data) { 393 u8 flags = *(u8 *)param->private_data; 394 395 /* 396 * use remote invalidation if the both initiator 397 * and the HCA support it 398 */ 399 isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) && 400 (attr->device_cap_flags & 401 IB_DEVICE_MEM_MGT_EXTENSIONS); 402 if (isert_conn->snd_w_inv) 403 isert_info("Using remote invalidation\n"); 404 } 405 } 406 407 static void 408 isert_destroy_qp(struct isert_conn *isert_conn) 409 { 410 ib_destroy_qp(isert_conn->qp); 411 ib_cq_pool_put(isert_conn->cq, isert_conn->cq_size); 412 } 413 414 static int 415 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 416 { 417 struct isert_np *isert_np = cma_id->context; 418 struct iscsi_np *np = isert_np->np; 419 struct isert_conn *isert_conn; 420 struct isert_device *device; 421 int ret = 0; 422 423 spin_lock_bh(&np->np_thread_lock); 424 if (!np->enabled) { 425 spin_unlock_bh(&np->np_thread_lock); 426 isert_dbg("iscsi_np is not enabled, reject connect request\n"); 427 return rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 428 } 429 spin_unlock_bh(&np->np_thread_lock); 430 431 isert_dbg("cma_id: %p, portal: %p\n", 432 cma_id, cma_id->context); 433 434 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 435 if (!isert_conn) 436 return -ENOMEM; 437 438 isert_init_conn(isert_conn); 439 isert_conn->cm_id = cma_id; 440 441 ret = isert_alloc_login_buf(isert_conn, cma_id->device); 442 if (ret) 443 goto out; 444 445 device = isert_device_get(cma_id); 446 if (IS_ERR(device)) { 447 ret = PTR_ERR(device); 448 goto out_rsp_dma_map; 449 } 450 isert_conn->device = device; 451 452 isert_set_nego_params(isert_conn, &event->param.conn); 453 454 isert_conn->qp = isert_create_qp(isert_conn, cma_id); 455 if (IS_ERR(isert_conn->qp)) { 456 ret = PTR_ERR(isert_conn->qp); 457 goto out_conn_dev; 458 } 459 460 ret = isert_login_post_recv(isert_conn); 461 if (ret) 462 goto out_destroy_qp; 463 464 ret = isert_rdma_accept(isert_conn); 465 if (ret) 466 goto out_destroy_qp; 467 468 mutex_lock(&isert_np->mutex); 469 list_add_tail(&isert_conn->node, &isert_np->accepted); 470 mutex_unlock(&isert_np->mutex); 471 472 return 0; 473 474 out_destroy_qp: 475 isert_destroy_qp(isert_conn); 476 out_conn_dev: 477 isert_device_put(device); 478 out_rsp_dma_map: 479 isert_free_login_buf(isert_conn); 480 out: 481 kfree(isert_conn); 482 rdma_reject(cma_id, NULL, 0, IB_CM_REJ_CONSUMER_DEFINED); 483 return ret; 484 } 485 486 static void 487 isert_connect_release(struct isert_conn *isert_conn) 488 { 489 struct isert_device *device = isert_conn->device; 490 491 isert_dbg("conn %p\n", isert_conn); 492 493 BUG_ON(!device); 494 495 isert_free_rx_descriptors(isert_conn); 496 if (isert_conn->cm_id && 497 !isert_conn->dev_removed) 498 rdma_destroy_id(isert_conn->cm_id); 499 500 if (isert_conn->qp) 501 isert_destroy_qp(isert_conn); 502 503 if (isert_conn->login_desc) 504 isert_free_login_buf(isert_conn); 505 506 isert_device_put(device); 507 508 if (isert_conn->dev_removed) 509 wake_up_interruptible(&isert_conn->rem_wait); 510 else 511 kfree(isert_conn); 512 } 513 514 static void 515 isert_connected_handler(struct rdma_cm_id *cma_id) 516 { 517 struct isert_conn *isert_conn = cma_id->qp->qp_context; 518 struct isert_np *isert_np = cma_id->context; 519 520 isert_info("conn %p\n", isert_conn); 521 522 mutex_lock(&isert_conn->mutex); 523 isert_conn->state = ISER_CONN_UP; 524 kref_get(&isert_conn->kref); 525 mutex_unlock(&isert_conn->mutex); 526 527 mutex_lock(&isert_np->mutex); 528 list_move_tail(&isert_conn->node, &isert_np->pending); 529 mutex_unlock(&isert_np->mutex); 530 531 isert_info("np %p: Allow accept_np to continue\n", isert_np); 532 up(&isert_np->sem); 533 } 534 535 static void 536 isert_release_kref(struct kref *kref) 537 { 538 struct isert_conn *isert_conn = container_of(kref, 539 struct isert_conn, kref); 540 541 isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm, 542 current->pid); 543 544 isert_connect_release(isert_conn); 545 } 546 547 static void 548 isert_put_conn(struct isert_conn *isert_conn) 549 { 550 kref_put(&isert_conn->kref, isert_release_kref); 551 } 552 553 static void 554 isert_handle_unbound_conn(struct isert_conn *isert_conn) 555 { 556 struct isert_np *isert_np = isert_conn->cm_id->context; 557 558 mutex_lock(&isert_np->mutex); 559 if (!list_empty(&isert_conn->node)) { 560 /* 561 * This means iscsi doesn't know this connection 562 * so schedule a cleanup ourselves 563 */ 564 list_del_init(&isert_conn->node); 565 isert_put_conn(isert_conn); 566 queue_work(isert_release_wq, &isert_conn->release_work); 567 } 568 mutex_unlock(&isert_np->mutex); 569 } 570 571 /** 572 * isert_conn_terminate() - Initiate connection termination 573 * @isert_conn: isert connection struct 574 * 575 * Notes: 576 * In case the connection state is BOUND, move state 577 * to TEMINATING and start teardown sequence (rdma_disconnect). 578 * In case the connection state is UP, complete flush as well. 579 * 580 * This routine must be called with mutex held. Thus it is 581 * safe to call multiple times. 582 */ 583 static void 584 isert_conn_terminate(struct isert_conn *isert_conn) 585 { 586 int err; 587 588 if (isert_conn->state >= ISER_CONN_TERMINATING) 589 return; 590 591 isert_info("Terminating conn %p state %d\n", 592 isert_conn, isert_conn->state); 593 isert_conn->state = ISER_CONN_TERMINATING; 594 err = rdma_disconnect(isert_conn->cm_id); 595 if (err) 596 isert_warn("Failed rdma_disconnect isert_conn %p\n", 597 isert_conn); 598 } 599 600 static int 601 isert_np_cma_handler(struct isert_np *isert_np, 602 enum rdma_cm_event_type event) 603 { 604 isert_dbg("%s (%d): isert np %p\n", 605 rdma_event_msg(event), event, isert_np); 606 607 switch (event) { 608 case RDMA_CM_EVENT_DEVICE_REMOVAL: 609 isert_np->cm_id = NULL; 610 break; 611 case RDMA_CM_EVENT_ADDR_CHANGE: 612 isert_np->cm_id = isert_setup_id(isert_np); 613 if (IS_ERR(isert_np->cm_id)) { 614 isert_err("isert np %p setup id failed: %ld\n", 615 isert_np, PTR_ERR(isert_np->cm_id)); 616 isert_np->cm_id = NULL; 617 } 618 break; 619 default: 620 isert_err("isert np %p Unexpected event %d\n", 621 isert_np, event); 622 } 623 624 return -1; 625 } 626 627 static int 628 isert_disconnected_handler(struct rdma_cm_id *cma_id, 629 enum rdma_cm_event_type event) 630 { 631 struct isert_conn *isert_conn = cma_id->qp->qp_context; 632 633 mutex_lock(&isert_conn->mutex); 634 switch (isert_conn->state) { 635 case ISER_CONN_TERMINATING: 636 break; 637 case ISER_CONN_UP: 638 isert_conn_terminate(isert_conn); 639 ib_drain_qp(isert_conn->qp); 640 isert_handle_unbound_conn(isert_conn); 641 break; 642 case ISER_CONN_BOUND: 643 case ISER_CONN_FULL_FEATURE: /* FALLTHRU */ 644 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 645 break; 646 default: 647 isert_warn("conn %p terminating in state %d\n", 648 isert_conn, isert_conn->state); 649 } 650 mutex_unlock(&isert_conn->mutex); 651 652 return 0; 653 } 654 655 static int 656 isert_connect_error(struct rdma_cm_id *cma_id) 657 { 658 struct isert_conn *isert_conn = cma_id->qp->qp_context; 659 660 ib_drain_qp(isert_conn->qp); 661 list_del_init(&isert_conn->node); 662 isert_conn->cm_id = NULL; 663 isert_put_conn(isert_conn); 664 665 return -1; 666 } 667 668 static int 669 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 670 { 671 struct isert_np *isert_np = cma_id->context; 672 struct isert_conn *isert_conn; 673 int ret = 0; 674 675 isert_info("%s (%d): status %d id %p np %p\n", 676 rdma_event_msg(event->event), event->event, 677 event->status, cma_id, cma_id->context); 678 679 if (isert_np->cm_id == cma_id) 680 return isert_np_cma_handler(cma_id->context, event->event); 681 682 switch (event->event) { 683 case RDMA_CM_EVENT_CONNECT_REQUEST: 684 ret = isert_connect_request(cma_id, event); 685 if (ret) 686 isert_err("failed handle connect request %d\n", ret); 687 break; 688 case RDMA_CM_EVENT_ESTABLISHED: 689 isert_connected_handler(cma_id); 690 break; 691 case RDMA_CM_EVENT_ADDR_CHANGE: 692 case RDMA_CM_EVENT_DISCONNECTED: 693 case RDMA_CM_EVENT_TIMEWAIT_EXIT: /* FALLTHRU */ 694 ret = isert_disconnected_handler(cma_id, event->event); 695 break; 696 case RDMA_CM_EVENT_DEVICE_REMOVAL: 697 isert_conn = cma_id->qp->qp_context; 698 isert_conn->dev_removed = true; 699 isert_disconnected_handler(cma_id, event->event); 700 wait_event_interruptible(isert_conn->rem_wait, 701 isert_conn->state == ISER_CONN_DOWN); 702 kfree(isert_conn); 703 /* 704 * return non-zero from the callback to destroy 705 * the rdma cm id 706 */ 707 return 1; 708 case RDMA_CM_EVENT_REJECTED: 709 isert_info("Connection rejected: %s\n", 710 rdma_reject_msg(cma_id, event->status)); 711 fallthrough; 712 case RDMA_CM_EVENT_UNREACHABLE: 713 case RDMA_CM_EVENT_CONNECT_ERROR: 714 ret = isert_connect_error(cma_id); 715 break; 716 default: 717 isert_err("Unhandled RDMA CMA event: %d\n", event->event); 718 break; 719 } 720 721 return ret; 722 } 723 724 static int 725 isert_post_recvm(struct isert_conn *isert_conn, u32 count) 726 { 727 struct ib_recv_wr *rx_wr; 728 int i, ret; 729 struct iser_rx_desc *rx_desc; 730 731 for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) { 732 rx_desc = &isert_conn->rx_descs[i]; 733 734 rx_wr->wr_cqe = &rx_desc->rx_cqe; 735 rx_wr->sg_list = &rx_desc->rx_sg; 736 rx_wr->num_sge = 1; 737 rx_wr->next = rx_wr + 1; 738 rx_desc->in_use = false; 739 } 740 rx_wr--; 741 rx_wr->next = NULL; /* mark end of work requests list */ 742 743 ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL); 744 if (ret) 745 isert_err("ib_post_recv() failed with ret: %d\n", ret); 746 747 return ret; 748 } 749 750 static int 751 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc) 752 { 753 struct ib_recv_wr rx_wr; 754 int ret; 755 756 if (!rx_desc->in_use) { 757 /* 758 * if the descriptor is not in-use we already reposted it 759 * for recv, so just silently return 760 */ 761 return 0; 762 } 763 764 rx_desc->in_use = false; 765 rx_wr.wr_cqe = &rx_desc->rx_cqe; 766 rx_wr.sg_list = &rx_desc->rx_sg; 767 rx_wr.num_sge = 1; 768 rx_wr.next = NULL; 769 770 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 771 if (ret) 772 isert_err("ib_post_recv() failed with ret: %d\n", ret); 773 774 return ret; 775 } 776 777 static int 778 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 779 { 780 struct ib_device *ib_dev = isert_conn->cm_id->device; 781 struct ib_send_wr send_wr; 782 int ret; 783 784 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 785 ISER_HEADERS_LEN, DMA_TO_DEVICE); 786 787 tx_desc->tx_cqe.done = isert_login_send_done; 788 789 send_wr.next = NULL; 790 send_wr.wr_cqe = &tx_desc->tx_cqe; 791 send_wr.sg_list = tx_desc->tx_sg; 792 send_wr.num_sge = tx_desc->num_sge; 793 send_wr.opcode = IB_WR_SEND; 794 send_wr.send_flags = IB_SEND_SIGNALED; 795 796 ret = ib_post_send(isert_conn->qp, &send_wr, NULL); 797 if (ret) 798 isert_err("ib_post_send() failed, ret: %d\n", ret); 799 800 return ret; 801 } 802 803 static void 804 __isert_create_send_desc(struct isert_device *device, 805 struct iser_tx_desc *tx_desc) 806 { 807 808 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 809 tx_desc->iser_header.flags = ISCSI_CTRL; 810 811 tx_desc->num_sge = 1; 812 813 if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) { 814 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 815 isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc); 816 } 817 } 818 819 static void 820 isert_create_send_desc(struct isert_conn *isert_conn, 821 struct isert_cmd *isert_cmd, 822 struct iser_tx_desc *tx_desc) 823 { 824 struct isert_device *device = isert_conn->device; 825 struct ib_device *ib_dev = device->ib_device; 826 827 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 828 ISER_HEADERS_LEN, DMA_TO_DEVICE); 829 830 __isert_create_send_desc(device, tx_desc); 831 } 832 833 static int 834 isert_init_tx_hdrs(struct isert_conn *isert_conn, 835 struct iser_tx_desc *tx_desc) 836 { 837 struct isert_device *device = isert_conn->device; 838 struct ib_device *ib_dev = device->ib_device; 839 u64 dma_addr; 840 841 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 842 ISER_HEADERS_LEN, DMA_TO_DEVICE); 843 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 844 isert_err("ib_dma_mapping_error() failed\n"); 845 return -ENOMEM; 846 } 847 848 tx_desc->dma_addr = dma_addr; 849 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 850 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 851 tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey; 852 853 isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n", 854 tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length, 855 tx_desc->tx_sg[0].lkey); 856 857 return 0; 858 } 859 860 static void 861 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 862 struct ib_send_wr *send_wr) 863 { 864 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 865 866 tx_desc->tx_cqe.done = isert_send_done; 867 send_wr->wr_cqe = &tx_desc->tx_cqe; 868 869 if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) { 870 send_wr->opcode = IB_WR_SEND_WITH_INV; 871 send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey; 872 } else { 873 send_wr->opcode = IB_WR_SEND; 874 } 875 876 send_wr->sg_list = &tx_desc->tx_sg[0]; 877 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 878 send_wr->send_flags = IB_SEND_SIGNALED; 879 } 880 881 static int 882 isert_login_post_recv(struct isert_conn *isert_conn) 883 { 884 struct ib_recv_wr rx_wr; 885 struct ib_sge sge; 886 int ret; 887 888 memset(&sge, 0, sizeof(struct ib_sge)); 889 sge.addr = isert_conn->login_desc->dma_addr + 890 isert_get_hdr_offset(isert_conn->login_desc); 891 sge.length = ISER_RX_PAYLOAD_SIZE; 892 sge.lkey = isert_conn->device->pd->local_dma_lkey; 893 894 isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n", 895 sge.addr, sge.length, sge.lkey); 896 897 isert_conn->login_desc->rx_cqe.done = isert_login_recv_done; 898 899 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 900 rx_wr.wr_cqe = &isert_conn->login_desc->rx_cqe; 901 rx_wr.sg_list = &sge; 902 rx_wr.num_sge = 1; 903 904 ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL); 905 if (ret) 906 isert_err("ib_post_recv() failed: %d\n", ret); 907 908 return ret; 909 } 910 911 static int 912 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 913 u32 length) 914 { 915 struct isert_conn *isert_conn = conn->context; 916 struct isert_device *device = isert_conn->device; 917 struct ib_device *ib_dev = device->ib_device; 918 struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc; 919 int ret; 920 921 __isert_create_send_desc(device, tx_desc); 922 923 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 924 sizeof(struct iscsi_hdr)); 925 926 isert_init_tx_hdrs(isert_conn, tx_desc); 927 928 if (length > 0) { 929 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 930 931 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 932 length, DMA_TO_DEVICE); 933 934 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 935 936 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 937 length, DMA_TO_DEVICE); 938 939 tx_dsg->addr = isert_conn->login_rsp_dma; 940 tx_dsg->length = length; 941 tx_dsg->lkey = isert_conn->device->pd->local_dma_lkey; 942 tx_desc->num_sge = 2; 943 } 944 if (!login->login_failed) { 945 if (login->login_complete) { 946 ret = isert_alloc_rx_descriptors(isert_conn); 947 if (ret) 948 return ret; 949 950 ret = isert_post_recvm(isert_conn, 951 ISERT_QP_MAX_RECV_DTOS); 952 if (ret) 953 return ret; 954 955 /* Now we are in FULL_FEATURE phase */ 956 mutex_lock(&isert_conn->mutex); 957 isert_conn->state = ISER_CONN_FULL_FEATURE; 958 mutex_unlock(&isert_conn->mutex); 959 goto post_send; 960 } 961 962 ret = isert_login_post_recv(isert_conn); 963 if (ret) 964 return ret; 965 } 966 post_send: 967 ret = isert_login_post_send(isert_conn, tx_desc); 968 if (ret) 969 return ret; 970 971 return 0; 972 } 973 974 static void 975 isert_rx_login_req(struct isert_conn *isert_conn) 976 { 977 struct iser_rx_desc *rx_desc = isert_conn->login_desc; 978 int rx_buflen = isert_conn->login_req_len; 979 struct iscsi_conn *conn = isert_conn->conn; 980 struct iscsi_login *login = conn->conn_login; 981 int size; 982 983 isert_info("conn %p\n", isert_conn); 984 985 WARN_ON_ONCE(!login); 986 987 if (login->first_request) { 988 struct iscsi_login_req *login_req = 989 (struct iscsi_login_req *)isert_get_iscsi_hdr(rx_desc); 990 /* 991 * Setup the initial iscsi_login values from the leading 992 * login request PDU. 993 */ 994 login->leading_connection = (!login_req->tsih) ? 1 : 0; 995 login->current_stage = 996 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 997 >> 2; 998 login->version_min = login_req->min_version; 999 login->version_max = login_req->max_version; 1000 memcpy(login->isid, login_req->isid, 6); 1001 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1002 login->init_task_tag = login_req->itt; 1003 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1004 login->cid = be16_to_cpu(login_req->cid); 1005 login->tsih = be16_to_cpu(login_req->tsih); 1006 } 1007 1008 memcpy(&login->req[0], isert_get_iscsi_hdr(rx_desc), ISCSI_HDR_LEN); 1009 1010 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1011 isert_dbg("Using login payload size: %d, rx_buflen: %d " 1012 "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen, 1013 MAX_KEY_VALUE_PAIRS); 1014 memcpy(login->req_buf, isert_get_data(rx_desc), size); 1015 1016 if (login->first_request) { 1017 complete(&isert_conn->login_comp); 1018 return; 1019 } 1020 schedule_delayed_work(&conn->login_work, 0); 1021 } 1022 1023 static struct iscsi_cmd 1024 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc) 1025 { 1026 struct isert_conn *isert_conn = conn->context; 1027 struct isert_cmd *isert_cmd; 1028 struct iscsi_cmd *cmd; 1029 1030 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1031 if (!cmd) { 1032 isert_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1033 return NULL; 1034 } 1035 isert_cmd = iscsit_priv_cmd(cmd); 1036 isert_cmd->conn = isert_conn; 1037 isert_cmd->iscsi_cmd = cmd; 1038 isert_cmd->rx_desc = rx_desc; 1039 1040 return cmd; 1041 } 1042 1043 static int 1044 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1045 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1046 struct iser_rx_desc *rx_desc, unsigned char *buf) 1047 { 1048 struct iscsi_conn *conn = isert_conn->conn; 1049 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1050 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1051 bool dump_payload = false; 1052 unsigned int data_len; 1053 1054 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1055 if (rc < 0) 1056 return rc; 1057 1058 imm_data = cmd->immediate_data; 1059 imm_data_len = cmd->first_burst_len; 1060 unsol_data = cmd->unsolicited_data; 1061 data_len = cmd->se_cmd.data_length; 1062 1063 if (imm_data && imm_data_len == data_len) 1064 cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC; 1065 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1066 if (rc < 0) { 1067 return 0; 1068 } else if (rc > 0) { 1069 dump_payload = true; 1070 goto sequence_cmd; 1071 } 1072 1073 if (!imm_data) 1074 return 0; 1075 1076 if (imm_data_len != data_len) { 1077 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1078 sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents, 1079 isert_get_data(rx_desc), imm_data_len); 1080 isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n", 1081 sg_nents, imm_data_len); 1082 } else { 1083 sg_init_table(&isert_cmd->sg, 1); 1084 cmd->se_cmd.t_data_sg = &isert_cmd->sg; 1085 cmd->se_cmd.t_data_nents = 1; 1086 sg_set_buf(&isert_cmd->sg, isert_get_data(rx_desc), 1087 imm_data_len); 1088 isert_dbg("Transfer Immediate imm_data_len: %d\n", 1089 imm_data_len); 1090 } 1091 1092 cmd->write_data_done += imm_data_len; 1093 1094 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1095 spin_lock_bh(&cmd->istate_lock); 1096 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1097 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1098 spin_unlock_bh(&cmd->istate_lock); 1099 } 1100 1101 sequence_cmd: 1102 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1103 1104 if (!rc && !dump_payload && unsol_data) 1105 iscsit_set_unsolicited_dataout(cmd); 1106 else if (dump_payload && imm_data) 1107 target_put_sess_cmd(&cmd->se_cmd); 1108 1109 return 0; 1110 } 1111 1112 static int 1113 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1114 struct iser_rx_desc *rx_desc, unsigned char *buf) 1115 { 1116 struct scatterlist *sg_start; 1117 struct iscsi_conn *conn = isert_conn->conn; 1118 struct iscsi_cmd *cmd = NULL; 1119 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1120 u32 unsol_data_len = ntoh24(hdr->dlength); 1121 int rc, sg_nents, sg_off, page_off; 1122 1123 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1124 if (rc < 0) 1125 return rc; 1126 else if (!cmd) 1127 return 0; 1128 /* 1129 * FIXME: Unexpected unsolicited_data out 1130 */ 1131 if (!cmd->unsolicited_data) { 1132 isert_err("Received unexpected solicited data payload\n"); 1133 dump_stack(); 1134 return -1; 1135 } 1136 1137 isert_dbg("Unsolicited DataOut unsol_data_len: %u, " 1138 "write_data_done: %u, data_length: %u\n", 1139 unsol_data_len, cmd->write_data_done, 1140 cmd->se_cmd.data_length); 1141 1142 sg_off = cmd->write_data_done / PAGE_SIZE; 1143 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1144 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1145 page_off = cmd->write_data_done % PAGE_SIZE; 1146 /* 1147 * FIXME: Non page-aligned unsolicited_data out 1148 */ 1149 if (page_off) { 1150 isert_err("unexpected non-page aligned data payload\n"); 1151 dump_stack(); 1152 return -1; 1153 } 1154 isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u " 1155 "sg_nents: %u from %p %u\n", sg_start, sg_off, 1156 sg_nents, isert_get_data(rx_desc), unsol_data_len); 1157 1158 sg_copy_from_buffer(sg_start, sg_nents, isert_get_data(rx_desc), 1159 unsol_data_len); 1160 1161 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1162 if (rc < 0) 1163 return rc; 1164 1165 /* 1166 * multiple data-outs on the same command can arrive - 1167 * so post the buffer before hand 1168 */ 1169 return isert_post_recv(isert_conn, rx_desc); 1170 } 1171 1172 static int 1173 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1174 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1175 unsigned char *buf) 1176 { 1177 struct iscsi_conn *conn = isert_conn->conn; 1178 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1179 int rc; 1180 1181 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1182 if (rc < 0) 1183 return rc; 1184 /* 1185 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1186 */ 1187 1188 return iscsit_process_nop_out(conn, cmd, hdr); 1189 } 1190 1191 static int 1192 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1193 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1194 struct iscsi_text *hdr) 1195 { 1196 struct iscsi_conn *conn = isert_conn->conn; 1197 u32 payload_length = ntoh24(hdr->dlength); 1198 int rc; 1199 unsigned char *text_in = NULL; 1200 1201 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1202 if (rc < 0) 1203 return rc; 1204 1205 if (payload_length) { 1206 text_in = kzalloc(payload_length, GFP_KERNEL); 1207 if (!text_in) 1208 return -ENOMEM; 1209 } 1210 cmd->text_in_ptr = text_in; 1211 1212 memcpy(cmd->text_in_ptr, isert_get_data(rx_desc), payload_length); 1213 1214 return iscsit_process_text_cmd(conn, cmd, hdr); 1215 } 1216 1217 static int 1218 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1219 uint32_t read_stag, uint64_t read_va, 1220 uint32_t write_stag, uint64_t write_va) 1221 { 1222 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1223 struct iscsi_conn *conn = isert_conn->conn; 1224 struct iscsi_cmd *cmd; 1225 struct isert_cmd *isert_cmd; 1226 int ret = -EINVAL; 1227 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1228 1229 if (conn->sess->sess_ops->SessionType && 1230 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1231 isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1232 " ignoring\n", opcode); 1233 return 0; 1234 } 1235 1236 switch (opcode) { 1237 case ISCSI_OP_SCSI_CMD: 1238 cmd = isert_allocate_cmd(conn, rx_desc); 1239 if (!cmd) 1240 break; 1241 1242 isert_cmd = iscsit_priv_cmd(cmd); 1243 isert_cmd->read_stag = read_stag; 1244 isert_cmd->read_va = read_va; 1245 isert_cmd->write_stag = write_stag; 1246 isert_cmd->write_va = write_va; 1247 isert_cmd->inv_rkey = read_stag ? read_stag : write_stag; 1248 1249 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1250 rx_desc, (unsigned char *)hdr); 1251 break; 1252 case ISCSI_OP_NOOP_OUT: 1253 cmd = isert_allocate_cmd(conn, rx_desc); 1254 if (!cmd) 1255 break; 1256 1257 isert_cmd = iscsit_priv_cmd(cmd); 1258 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1259 rx_desc, (unsigned char *)hdr); 1260 break; 1261 case ISCSI_OP_SCSI_DATA_OUT: 1262 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1263 (unsigned char *)hdr); 1264 break; 1265 case ISCSI_OP_SCSI_TMFUNC: 1266 cmd = isert_allocate_cmd(conn, rx_desc); 1267 if (!cmd) 1268 break; 1269 1270 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1271 (unsigned char *)hdr); 1272 break; 1273 case ISCSI_OP_LOGOUT: 1274 cmd = isert_allocate_cmd(conn, rx_desc); 1275 if (!cmd) 1276 break; 1277 1278 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1279 break; 1280 case ISCSI_OP_TEXT: 1281 if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF) 1282 cmd = iscsit_find_cmd_from_itt(conn, hdr->itt); 1283 else 1284 cmd = isert_allocate_cmd(conn, rx_desc); 1285 1286 if (!cmd) 1287 break; 1288 1289 isert_cmd = iscsit_priv_cmd(cmd); 1290 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1291 rx_desc, (struct iscsi_text *)hdr); 1292 break; 1293 default: 1294 isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1295 dump_stack(); 1296 break; 1297 } 1298 1299 return ret; 1300 } 1301 1302 static void 1303 isert_print_wc(struct ib_wc *wc, const char *type) 1304 { 1305 if (wc->status != IB_WC_WR_FLUSH_ERR) 1306 isert_err("%s failure: %s (%d) vend_err %x\n", type, 1307 ib_wc_status_msg(wc->status), wc->status, 1308 wc->vendor_err); 1309 else 1310 isert_dbg("%s failure: %s (%d)\n", type, 1311 ib_wc_status_msg(wc->status), wc->status); 1312 } 1313 1314 static void 1315 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1316 { 1317 struct isert_conn *isert_conn = wc->qp->qp_context; 1318 struct ib_device *ib_dev = isert_conn->cm_id->device; 1319 struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe); 1320 struct iscsi_hdr *hdr = isert_get_iscsi_hdr(rx_desc); 1321 struct iser_ctrl *iser_ctrl = isert_get_iser_hdr(rx_desc); 1322 uint64_t read_va = 0, write_va = 0; 1323 uint32_t read_stag = 0, write_stag = 0; 1324 1325 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1326 isert_print_wc(wc, "recv"); 1327 if (wc->status != IB_WC_WR_FLUSH_ERR) 1328 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1329 return; 1330 } 1331 1332 rx_desc->in_use = true; 1333 1334 ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr, 1335 ISER_RX_SIZE, DMA_FROM_DEVICE); 1336 1337 isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1338 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags, 1339 (int)(wc->byte_len - ISER_HEADERS_LEN)); 1340 1341 switch (iser_ctrl->flags & 0xF0) { 1342 case ISCSI_CTRL: 1343 if (iser_ctrl->flags & ISER_RSV) { 1344 read_stag = be32_to_cpu(iser_ctrl->read_stag); 1345 read_va = be64_to_cpu(iser_ctrl->read_va); 1346 isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n", 1347 read_stag, (unsigned long long)read_va); 1348 } 1349 if (iser_ctrl->flags & ISER_WSV) { 1350 write_stag = be32_to_cpu(iser_ctrl->write_stag); 1351 write_va = be64_to_cpu(iser_ctrl->write_va); 1352 isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n", 1353 write_stag, (unsigned long long)write_va); 1354 } 1355 1356 isert_dbg("ISER ISCSI_CTRL PDU\n"); 1357 break; 1358 case ISER_HELLO: 1359 isert_err("iSER Hello message\n"); 1360 break; 1361 default: 1362 isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags); 1363 break; 1364 } 1365 1366 isert_rx_opcode(isert_conn, rx_desc, 1367 read_stag, read_va, write_stag, write_va); 1368 1369 ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr, 1370 ISER_RX_SIZE, DMA_FROM_DEVICE); 1371 } 1372 1373 static void 1374 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1375 { 1376 struct isert_conn *isert_conn = wc->qp->qp_context; 1377 struct ib_device *ib_dev = isert_conn->device->ib_device; 1378 1379 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1380 isert_print_wc(wc, "login recv"); 1381 return; 1382 } 1383 1384 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_desc->dma_addr, 1385 ISER_RX_SIZE, DMA_FROM_DEVICE); 1386 1387 isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN; 1388 1389 if (isert_conn->conn) { 1390 struct iscsi_login *login = isert_conn->conn->conn_login; 1391 1392 if (login && !login->first_request) 1393 isert_rx_login_req(isert_conn); 1394 } 1395 1396 mutex_lock(&isert_conn->mutex); 1397 complete(&isert_conn->login_req_comp); 1398 mutex_unlock(&isert_conn->mutex); 1399 1400 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_desc->dma_addr, 1401 ISER_RX_SIZE, DMA_FROM_DEVICE); 1402 } 1403 1404 static void 1405 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn) 1406 { 1407 struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 1408 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 1409 1410 if (!cmd->rw.nr_ops) 1411 return; 1412 1413 if (isert_prot_cmd(conn, se_cmd)) { 1414 rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp, 1415 conn->cm_id->port_num, se_cmd->t_data_sg, 1416 se_cmd->t_data_nents, se_cmd->t_prot_sg, 1417 se_cmd->t_prot_nents, dir); 1418 } else { 1419 rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num, 1420 se_cmd->t_data_sg, se_cmd->t_data_nents, dir); 1421 } 1422 1423 cmd->rw.nr_ops = 0; 1424 } 1425 1426 static void 1427 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err) 1428 { 1429 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1430 struct isert_conn *isert_conn = isert_cmd->conn; 1431 struct iscsi_conn *conn = isert_conn->conn; 1432 struct iscsi_text_rsp *hdr; 1433 1434 isert_dbg("Cmd %p\n", isert_cmd); 1435 1436 switch (cmd->iscsi_opcode) { 1437 case ISCSI_OP_SCSI_CMD: 1438 spin_lock_bh(&conn->cmd_lock); 1439 if (!list_empty(&cmd->i_conn_node)) 1440 list_del_init(&cmd->i_conn_node); 1441 spin_unlock_bh(&conn->cmd_lock); 1442 1443 if (cmd->data_direction == DMA_TO_DEVICE) { 1444 iscsit_stop_dataout_timer(cmd); 1445 /* 1446 * Check for special case during comp_err where 1447 * WRITE_PENDING has been handed off from core, 1448 * but requires an extra target_put_sess_cmd() 1449 * before transport_generic_free_cmd() below. 1450 */ 1451 if (comp_err && 1452 cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) { 1453 struct se_cmd *se_cmd = &cmd->se_cmd; 1454 1455 target_put_sess_cmd(se_cmd); 1456 } 1457 } 1458 1459 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1460 transport_generic_free_cmd(&cmd->se_cmd, 0); 1461 break; 1462 case ISCSI_OP_SCSI_TMFUNC: 1463 spin_lock_bh(&conn->cmd_lock); 1464 if (!list_empty(&cmd->i_conn_node)) 1465 list_del_init(&cmd->i_conn_node); 1466 spin_unlock_bh(&conn->cmd_lock); 1467 1468 transport_generic_free_cmd(&cmd->se_cmd, 0); 1469 break; 1470 case ISCSI_OP_REJECT: 1471 case ISCSI_OP_NOOP_OUT: 1472 case ISCSI_OP_TEXT: 1473 hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1474 /* If the continue bit is on, keep the command alive */ 1475 if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE) 1476 break; 1477 1478 spin_lock_bh(&conn->cmd_lock); 1479 if (!list_empty(&cmd->i_conn_node)) 1480 list_del_init(&cmd->i_conn_node); 1481 spin_unlock_bh(&conn->cmd_lock); 1482 1483 /* 1484 * Handle special case for REJECT when iscsi_add_reject*() has 1485 * overwritten the original iscsi_opcode assignment, and the 1486 * associated cmd->se_cmd needs to be released. 1487 */ 1488 if (cmd->se_cmd.se_tfo != NULL) { 1489 isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n", 1490 cmd->iscsi_opcode); 1491 transport_generic_free_cmd(&cmd->se_cmd, 0); 1492 break; 1493 } 1494 fallthrough; 1495 default: 1496 iscsit_release_cmd(cmd); 1497 break; 1498 } 1499 } 1500 1501 static void 1502 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1503 { 1504 if (tx_desc->dma_addr != 0) { 1505 isert_dbg("unmap single for tx_desc->dma_addr\n"); 1506 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1507 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1508 tx_desc->dma_addr = 0; 1509 } 1510 } 1511 1512 static void 1513 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1514 struct ib_device *ib_dev, bool comp_err) 1515 { 1516 if (isert_cmd->pdu_buf_dma != 0) { 1517 isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n"); 1518 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1519 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1520 isert_cmd->pdu_buf_dma = 0; 1521 } 1522 1523 isert_unmap_tx_desc(tx_desc, ib_dev); 1524 isert_put_cmd(isert_cmd, comp_err); 1525 } 1526 1527 static int 1528 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr) 1529 { 1530 struct ib_mr_status mr_status; 1531 int ret; 1532 1533 ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status); 1534 if (ret) { 1535 isert_err("ib_check_mr_status failed, ret %d\n", ret); 1536 goto fail_mr_status; 1537 } 1538 1539 if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) { 1540 u64 sec_offset_err; 1541 u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8; 1542 1543 switch (mr_status.sig_err.err_type) { 1544 case IB_SIG_BAD_GUARD: 1545 se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED; 1546 break; 1547 case IB_SIG_BAD_REFTAG: 1548 se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED; 1549 break; 1550 case IB_SIG_BAD_APPTAG: 1551 se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED; 1552 break; 1553 } 1554 sec_offset_err = mr_status.sig_err.sig_err_offset; 1555 do_div(sec_offset_err, block_size); 1556 se_cmd->sense_info = sec_offset_err + se_cmd->t_task_lba; 1557 1558 isert_err("PI error found type %d at sector 0x%llx " 1559 "expected 0x%x vs actual 0x%x\n", 1560 mr_status.sig_err.err_type, 1561 (unsigned long long)se_cmd->sense_info, 1562 mr_status.sig_err.expected, 1563 mr_status.sig_err.actual); 1564 ret = 1; 1565 } 1566 1567 fail_mr_status: 1568 return ret; 1569 } 1570 1571 static void 1572 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc) 1573 { 1574 struct isert_conn *isert_conn = wc->qp->qp_context; 1575 struct isert_device *device = isert_conn->device; 1576 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1577 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1578 struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd; 1579 int ret = 0; 1580 1581 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1582 isert_print_wc(wc, "rdma write"); 1583 if (wc->status != IB_WC_WR_FLUSH_ERR) 1584 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1585 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1586 return; 1587 } 1588 1589 isert_dbg("Cmd %p\n", isert_cmd); 1590 1591 ret = isert_check_pi_status(cmd, isert_cmd->rw.reg->mr); 1592 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1593 1594 if (ret) { 1595 /* 1596 * transport_generic_request_failure() expects to have 1597 * plus two references to handle queue-full, so re-add 1598 * one here as target-core will have already dropped 1599 * it after the first isert_put_datain() callback. 1600 */ 1601 kref_get(&cmd->cmd_kref); 1602 transport_generic_request_failure(cmd, cmd->pi_err); 1603 } else { 1604 /* 1605 * XXX: isert_put_response() failure is not retried. 1606 */ 1607 ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd); 1608 if (ret) 1609 pr_warn_ratelimited("isert_put_response() ret: %d\n", ret); 1610 } 1611 } 1612 1613 static void 1614 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc) 1615 { 1616 struct isert_conn *isert_conn = wc->qp->qp_context; 1617 struct isert_device *device = isert_conn->device; 1618 struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe); 1619 struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc); 1620 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1621 struct se_cmd *se_cmd = &cmd->se_cmd; 1622 int ret = 0; 1623 1624 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1625 isert_print_wc(wc, "rdma read"); 1626 if (wc->status != IB_WC_WR_FLUSH_ERR) 1627 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1628 isert_completion_put(desc, isert_cmd, device->ib_device, true); 1629 return; 1630 } 1631 1632 isert_dbg("Cmd %p\n", isert_cmd); 1633 1634 iscsit_stop_dataout_timer(cmd); 1635 1636 if (isert_prot_cmd(isert_conn, se_cmd)) 1637 ret = isert_check_pi_status(se_cmd, isert_cmd->rw.reg->mr); 1638 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1639 cmd->write_data_done = 0; 1640 1641 isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1642 spin_lock_bh(&cmd->istate_lock); 1643 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1644 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1645 spin_unlock_bh(&cmd->istate_lock); 1646 1647 /* 1648 * transport_generic_request_failure() will drop the extra 1649 * se_cmd->cmd_kref reference after T10-PI error, and handle 1650 * any non-zero ->queue_status() callback error retries. 1651 */ 1652 if (ret) 1653 transport_generic_request_failure(se_cmd, se_cmd->pi_err); 1654 else 1655 target_execute_cmd(se_cmd); 1656 } 1657 1658 static void 1659 isert_do_control_comp(struct work_struct *work) 1660 { 1661 struct isert_cmd *isert_cmd = container_of(work, 1662 struct isert_cmd, comp_work); 1663 struct isert_conn *isert_conn = isert_cmd->conn; 1664 struct ib_device *ib_dev = isert_conn->cm_id->device; 1665 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1666 1667 isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state); 1668 1669 switch (cmd->i_state) { 1670 case ISTATE_SEND_TASKMGTRSP: 1671 iscsit_tmr_post_handler(cmd, cmd->conn); 1672 fallthrough; 1673 case ISTATE_SEND_REJECT: 1674 case ISTATE_SEND_TEXTRSP: 1675 cmd->i_state = ISTATE_SENT_STATUS; 1676 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, 1677 ib_dev, false); 1678 break; 1679 case ISTATE_SEND_LOGOUTRSP: 1680 iscsit_logout_post_handler(cmd, cmd->conn); 1681 break; 1682 default: 1683 isert_err("Unknown i_state %d\n", cmd->i_state); 1684 dump_stack(); 1685 break; 1686 } 1687 } 1688 1689 static void 1690 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc) 1691 { 1692 struct isert_conn *isert_conn = wc->qp->qp_context; 1693 struct ib_device *ib_dev = isert_conn->cm_id->device; 1694 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1695 1696 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1697 isert_print_wc(wc, "login send"); 1698 if (wc->status != IB_WC_WR_FLUSH_ERR) 1699 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1700 } 1701 1702 isert_unmap_tx_desc(tx_desc, ib_dev); 1703 } 1704 1705 static void 1706 isert_send_done(struct ib_cq *cq, struct ib_wc *wc) 1707 { 1708 struct isert_conn *isert_conn = wc->qp->qp_context; 1709 struct ib_device *ib_dev = isert_conn->cm_id->device; 1710 struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe); 1711 struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc); 1712 1713 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1714 isert_print_wc(wc, "send"); 1715 if (wc->status != IB_WC_WR_FLUSH_ERR) 1716 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 1717 isert_completion_put(tx_desc, isert_cmd, ib_dev, true); 1718 return; 1719 } 1720 1721 isert_dbg("Cmd %p\n", isert_cmd); 1722 1723 switch (isert_cmd->iscsi_cmd->i_state) { 1724 case ISTATE_SEND_TASKMGTRSP: 1725 case ISTATE_SEND_LOGOUTRSP: 1726 case ISTATE_SEND_REJECT: 1727 case ISTATE_SEND_TEXTRSP: 1728 isert_unmap_tx_desc(tx_desc, ib_dev); 1729 1730 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1731 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1732 return; 1733 default: 1734 isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS; 1735 isert_completion_put(tx_desc, isert_cmd, ib_dev, false); 1736 break; 1737 } 1738 } 1739 1740 static int 1741 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1742 { 1743 int ret; 1744 1745 ret = isert_post_recv(isert_conn, isert_cmd->rx_desc); 1746 if (ret) 1747 return ret; 1748 1749 ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL); 1750 if (ret) { 1751 isert_err("ib_post_send failed with %d\n", ret); 1752 return ret; 1753 } 1754 return ret; 1755 } 1756 1757 static int 1758 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1759 { 1760 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1761 struct isert_conn *isert_conn = conn->context; 1762 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1763 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1764 &isert_cmd->tx_desc.iscsi_header; 1765 1766 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1767 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1768 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1769 /* 1770 * Attach SENSE DATA payload to iSCSI Response PDU 1771 */ 1772 if (cmd->se_cmd.sense_buffer && 1773 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1774 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1775 struct isert_device *device = isert_conn->device; 1776 struct ib_device *ib_dev = device->ib_device; 1777 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1778 u32 padding, pdu_len; 1779 1780 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1781 cmd->sense_buffer); 1782 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1783 1784 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1785 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1786 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1787 1788 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1789 (void *)cmd->sense_buffer, pdu_len, 1790 DMA_TO_DEVICE); 1791 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1792 return -ENOMEM; 1793 1794 isert_cmd->pdu_buf_len = pdu_len; 1795 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1796 tx_dsg->length = pdu_len; 1797 tx_dsg->lkey = device->pd->local_dma_lkey; 1798 isert_cmd->tx_desc.num_sge = 2; 1799 } 1800 1801 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1802 1803 isert_dbg("Posting SCSI Response\n"); 1804 1805 return isert_post_response(isert_conn, isert_cmd); 1806 } 1807 1808 static void 1809 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1810 { 1811 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1812 struct isert_conn *isert_conn = conn->context; 1813 1814 spin_lock_bh(&conn->cmd_lock); 1815 if (!list_empty(&cmd->i_conn_node)) 1816 list_del_init(&cmd->i_conn_node); 1817 spin_unlock_bh(&conn->cmd_lock); 1818 1819 if (cmd->data_direction == DMA_TO_DEVICE) 1820 iscsit_stop_dataout_timer(cmd); 1821 isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn); 1822 } 1823 1824 static enum target_prot_op 1825 isert_get_sup_prot_ops(struct iscsi_conn *conn) 1826 { 1827 struct isert_conn *isert_conn = conn->context; 1828 struct isert_device *device = isert_conn->device; 1829 1830 if (conn->tpg->tpg_attrib.t10_pi) { 1831 if (device->pi_capable) { 1832 isert_info("conn %p PI offload enabled\n", isert_conn); 1833 isert_conn->pi_support = true; 1834 return TARGET_PROT_ALL; 1835 } 1836 } 1837 1838 isert_info("conn %p PI offload disabled\n", isert_conn); 1839 isert_conn->pi_support = false; 1840 1841 return TARGET_PROT_NORMAL; 1842 } 1843 1844 static int 1845 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1846 bool nopout_response) 1847 { 1848 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1849 struct isert_conn *isert_conn = conn->context; 1850 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1851 1852 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1853 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1854 &isert_cmd->tx_desc.iscsi_header, 1855 nopout_response); 1856 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1857 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1858 1859 isert_dbg("conn %p Posting NOPIN Response\n", isert_conn); 1860 1861 return isert_post_response(isert_conn, isert_cmd); 1862 } 1863 1864 static int 1865 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1866 { 1867 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1868 struct isert_conn *isert_conn = conn->context; 1869 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1870 1871 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1872 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1873 &isert_cmd->tx_desc.iscsi_header); 1874 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1875 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1876 1877 isert_dbg("conn %p Posting Logout Response\n", isert_conn); 1878 1879 return isert_post_response(isert_conn, isert_cmd); 1880 } 1881 1882 static int 1883 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1884 { 1885 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1886 struct isert_conn *isert_conn = conn->context; 1887 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1888 1889 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1890 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1891 &isert_cmd->tx_desc.iscsi_header); 1892 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1893 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1894 1895 isert_dbg("conn %p Posting Task Management Response\n", isert_conn); 1896 1897 return isert_post_response(isert_conn, isert_cmd); 1898 } 1899 1900 static int 1901 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1902 { 1903 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1904 struct isert_conn *isert_conn = conn->context; 1905 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1906 struct isert_device *device = isert_conn->device; 1907 struct ib_device *ib_dev = device->ib_device; 1908 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1909 struct iscsi_reject *hdr = 1910 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1911 1912 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1913 iscsit_build_reject(cmd, conn, hdr); 1914 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1915 1916 hton24(hdr->dlength, ISCSI_HDR_LEN); 1917 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1918 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1919 DMA_TO_DEVICE); 1920 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1921 return -ENOMEM; 1922 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1923 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1924 tx_dsg->length = ISCSI_HDR_LEN; 1925 tx_dsg->lkey = device->pd->local_dma_lkey; 1926 isert_cmd->tx_desc.num_sge = 2; 1927 1928 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1929 1930 isert_dbg("conn %p Posting Reject\n", isert_conn); 1931 1932 return isert_post_response(isert_conn, isert_cmd); 1933 } 1934 1935 static int 1936 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1937 { 1938 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1939 struct isert_conn *isert_conn = conn->context; 1940 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1941 struct iscsi_text_rsp *hdr = 1942 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1943 u32 txt_rsp_len; 1944 int rc; 1945 1946 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1947 rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND); 1948 if (rc < 0) 1949 return rc; 1950 1951 txt_rsp_len = rc; 1952 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1953 1954 if (txt_rsp_len) { 1955 struct isert_device *device = isert_conn->device; 1956 struct ib_device *ib_dev = device->ib_device; 1957 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1958 void *txt_rsp_buf = cmd->buf_ptr; 1959 1960 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1961 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1962 if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma)) 1963 return -ENOMEM; 1964 1965 isert_cmd->pdu_buf_len = txt_rsp_len; 1966 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1967 tx_dsg->length = txt_rsp_len; 1968 tx_dsg->lkey = device->pd->local_dma_lkey; 1969 isert_cmd->tx_desc.num_sge = 2; 1970 } 1971 isert_init_send_wr(isert_conn, isert_cmd, send_wr); 1972 1973 isert_dbg("conn %p Text Response\n", isert_conn); 1974 1975 return isert_post_response(isert_conn, isert_cmd); 1976 } 1977 1978 static inline void 1979 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_domain *domain) 1980 { 1981 domain->sig_type = IB_SIG_TYPE_T10_DIF; 1982 domain->sig.dif.bg_type = IB_T10DIF_CRC; 1983 domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size; 1984 domain->sig.dif.ref_tag = se_cmd->reftag_seed; 1985 /* 1986 * At the moment we hard code those, but if in the future 1987 * the target core would like to use it, we will take it 1988 * from se_cmd. 1989 */ 1990 domain->sig.dif.apptag_check_mask = 0xffff; 1991 domain->sig.dif.app_escape = true; 1992 domain->sig.dif.ref_escape = true; 1993 if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT || 1994 se_cmd->prot_type == TARGET_DIF_TYPE2_PROT) 1995 domain->sig.dif.ref_remap = true; 1996 } 1997 1998 static int 1999 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs) 2000 { 2001 memset(sig_attrs, 0, sizeof(*sig_attrs)); 2002 2003 switch (se_cmd->prot_op) { 2004 case TARGET_PROT_DIN_INSERT: 2005 case TARGET_PROT_DOUT_STRIP: 2006 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 2007 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2008 break; 2009 case TARGET_PROT_DOUT_INSERT: 2010 case TARGET_PROT_DIN_STRIP: 2011 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 2012 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2013 break; 2014 case TARGET_PROT_DIN_PASS: 2015 case TARGET_PROT_DOUT_PASS: 2016 isert_set_dif_domain(se_cmd, &sig_attrs->wire); 2017 isert_set_dif_domain(se_cmd, &sig_attrs->mem); 2018 break; 2019 default: 2020 isert_err("Unsupported PI operation %d\n", se_cmd->prot_op); 2021 return -EINVAL; 2022 } 2023 2024 if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD) 2025 sig_attrs->check_mask |= IB_SIG_CHECK_GUARD; 2026 if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG) 2027 sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG; 2028 if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG) 2029 sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG; 2030 2031 return 0; 2032 } 2033 2034 static int 2035 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn, 2036 struct ib_cqe *cqe, struct ib_send_wr *chain_wr) 2037 { 2038 struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd; 2039 enum dma_data_direction dir = target_reverse_dma_direction(se_cmd); 2040 u8 port_num = conn->cm_id->port_num; 2041 u64 addr; 2042 u32 rkey, offset; 2043 int ret; 2044 2045 if (cmd->ctx_init_done) 2046 goto rdma_ctx_post; 2047 2048 if (dir == DMA_FROM_DEVICE) { 2049 addr = cmd->write_va; 2050 rkey = cmd->write_stag; 2051 offset = cmd->iscsi_cmd->write_data_done; 2052 } else { 2053 addr = cmd->read_va; 2054 rkey = cmd->read_stag; 2055 offset = 0; 2056 } 2057 2058 if (isert_prot_cmd(conn, se_cmd)) { 2059 struct ib_sig_attrs sig_attrs; 2060 2061 ret = isert_set_sig_attrs(se_cmd, &sig_attrs); 2062 if (ret) 2063 return ret; 2064 2065 WARN_ON_ONCE(offset); 2066 ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num, 2067 se_cmd->t_data_sg, se_cmd->t_data_nents, 2068 se_cmd->t_prot_sg, se_cmd->t_prot_nents, 2069 &sig_attrs, addr, rkey, dir); 2070 } else { 2071 ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num, 2072 se_cmd->t_data_sg, se_cmd->t_data_nents, 2073 offset, addr, rkey, dir); 2074 } 2075 2076 if (ret < 0) { 2077 isert_err("Cmd: %p failed to prepare RDMA res\n", cmd); 2078 return ret; 2079 } 2080 2081 cmd->ctx_init_done = true; 2082 2083 rdma_ctx_post: 2084 ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr); 2085 if (ret < 0) 2086 isert_err("Cmd: %p failed to post RDMA res\n", cmd); 2087 return ret; 2088 } 2089 2090 static int 2091 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2092 { 2093 struct se_cmd *se_cmd = &cmd->se_cmd; 2094 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2095 struct isert_conn *isert_conn = conn->context; 2096 struct ib_cqe *cqe = NULL; 2097 struct ib_send_wr *chain_wr = NULL; 2098 int rc; 2099 2100 isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n", 2101 isert_cmd, se_cmd->data_length); 2102 2103 if (isert_prot_cmd(isert_conn, se_cmd)) { 2104 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done; 2105 cqe = &isert_cmd->tx_desc.tx_cqe; 2106 } else { 2107 /* 2108 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2109 */ 2110 isert_create_send_desc(isert_conn, isert_cmd, 2111 &isert_cmd->tx_desc); 2112 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2113 &isert_cmd->tx_desc.iscsi_header); 2114 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2115 isert_init_send_wr(isert_conn, isert_cmd, 2116 &isert_cmd->tx_desc.send_wr); 2117 2118 rc = isert_post_recv(isert_conn, isert_cmd->rx_desc); 2119 if (rc) 2120 return rc; 2121 2122 chain_wr = &isert_cmd->tx_desc.send_wr; 2123 } 2124 2125 rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr); 2126 isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n", 2127 isert_cmd, rc); 2128 return rc; 2129 } 2130 2131 static int 2132 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2133 { 2134 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2135 int ret; 2136 2137 isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2138 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done); 2139 2140 isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done; 2141 ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context, 2142 &isert_cmd->tx_desc.tx_cqe, NULL); 2143 2144 isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n", 2145 isert_cmd, ret); 2146 return ret; 2147 } 2148 2149 static int 2150 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2151 { 2152 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2153 int ret = 0; 2154 2155 switch (state) { 2156 case ISTATE_REMOVE: 2157 spin_lock_bh(&conn->cmd_lock); 2158 list_del_init(&cmd->i_conn_node); 2159 spin_unlock_bh(&conn->cmd_lock); 2160 isert_put_cmd(isert_cmd, true); 2161 break; 2162 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2163 ret = isert_put_nopin(cmd, conn, false); 2164 break; 2165 default: 2166 isert_err("Unknown immediate state: 0x%02x\n", state); 2167 ret = -EINVAL; 2168 break; 2169 } 2170 2171 return ret; 2172 } 2173 2174 static int 2175 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2176 { 2177 struct isert_conn *isert_conn = conn->context; 2178 int ret; 2179 2180 switch (state) { 2181 case ISTATE_SEND_LOGOUTRSP: 2182 ret = isert_put_logout_rsp(cmd, conn); 2183 if (!ret) 2184 isert_conn->logout_posted = true; 2185 break; 2186 case ISTATE_SEND_NOPIN: 2187 ret = isert_put_nopin(cmd, conn, true); 2188 break; 2189 case ISTATE_SEND_TASKMGTRSP: 2190 ret = isert_put_tm_rsp(cmd, conn); 2191 break; 2192 case ISTATE_SEND_REJECT: 2193 ret = isert_put_reject(cmd, conn); 2194 break; 2195 case ISTATE_SEND_TEXTRSP: 2196 ret = isert_put_text_rsp(cmd, conn); 2197 break; 2198 case ISTATE_SEND_STATUS: 2199 /* 2200 * Special case for sending non GOOD SCSI status from TX thread 2201 * context during pre se_cmd excecution failure. 2202 */ 2203 ret = isert_put_response(conn, cmd); 2204 break; 2205 default: 2206 isert_err("Unknown response state: 0x%02x\n", state); 2207 ret = -EINVAL; 2208 break; 2209 } 2210 2211 return ret; 2212 } 2213 2214 struct rdma_cm_id * 2215 isert_setup_id(struct isert_np *isert_np) 2216 { 2217 struct iscsi_np *np = isert_np->np; 2218 struct rdma_cm_id *id; 2219 struct sockaddr *sa; 2220 int ret; 2221 2222 sa = (struct sockaddr *)&np->np_sockaddr; 2223 isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa); 2224 2225 id = rdma_create_id(&init_net, isert_cma_handler, isert_np, 2226 RDMA_PS_TCP, IB_QPT_RC); 2227 if (IS_ERR(id)) { 2228 isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id)); 2229 ret = PTR_ERR(id); 2230 goto out; 2231 } 2232 isert_dbg("id %p context %p\n", id, id->context); 2233 2234 ret = rdma_bind_addr(id, sa); 2235 if (ret) { 2236 isert_err("rdma_bind_addr() failed: %d\n", ret); 2237 goto out_id; 2238 } 2239 2240 ret = rdma_listen(id, 0); 2241 if (ret) { 2242 isert_err("rdma_listen() failed: %d\n", ret); 2243 goto out_id; 2244 } 2245 2246 return id; 2247 out_id: 2248 rdma_destroy_id(id); 2249 out: 2250 return ERR_PTR(ret); 2251 } 2252 2253 static int 2254 isert_setup_np(struct iscsi_np *np, 2255 struct sockaddr_storage *ksockaddr) 2256 { 2257 struct isert_np *isert_np; 2258 struct rdma_cm_id *isert_lid; 2259 int ret; 2260 2261 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2262 if (!isert_np) 2263 return -ENOMEM; 2264 2265 sema_init(&isert_np->sem, 0); 2266 mutex_init(&isert_np->mutex); 2267 INIT_LIST_HEAD(&isert_np->accepted); 2268 INIT_LIST_HEAD(&isert_np->pending); 2269 isert_np->np = np; 2270 2271 /* 2272 * Setup the np->np_sockaddr from the passed sockaddr setup 2273 * in iscsi_target_configfs.c code.. 2274 */ 2275 memcpy(&np->np_sockaddr, ksockaddr, 2276 sizeof(struct sockaddr_storage)); 2277 2278 isert_lid = isert_setup_id(isert_np); 2279 if (IS_ERR(isert_lid)) { 2280 ret = PTR_ERR(isert_lid); 2281 goto out; 2282 } 2283 2284 isert_np->cm_id = isert_lid; 2285 np->np_context = isert_np; 2286 2287 return 0; 2288 2289 out: 2290 kfree(isert_np); 2291 2292 return ret; 2293 } 2294 2295 static int 2296 isert_rdma_accept(struct isert_conn *isert_conn) 2297 { 2298 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2299 struct rdma_conn_param cp; 2300 int ret; 2301 struct iser_cm_hdr rsp_hdr; 2302 2303 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2304 cp.initiator_depth = isert_conn->initiator_depth; 2305 cp.retry_count = 7; 2306 cp.rnr_retry_count = 7; 2307 2308 memset(&rsp_hdr, 0, sizeof(rsp_hdr)); 2309 rsp_hdr.flags = ISERT_ZBVA_NOT_USED; 2310 if (!isert_conn->snd_w_inv) 2311 rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED; 2312 cp.private_data = (void *)&rsp_hdr; 2313 cp.private_data_len = sizeof(rsp_hdr); 2314 2315 ret = rdma_accept(cm_id, &cp); 2316 if (ret) { 2317 isert_err("rdma_accept() failed with: %d\n", ret); 2318 return ret; 2319 } 2320 2321 return 0; 2322 } 2323 2324 static int 2325 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2326 { 2327 struct isert_conn *isert_conn = conn->context; 2328 int ret; 2329 2330 isert_info("before login_req comp conn: %p\n", isert_conn); 2331 ret = wait_for_completion_interruptible(&isert_conn->login_req_comp); 2332 if (ret) { 2333 isert_err("isert_conn %p interrupted before got login req\n", 2334 isert_conn); 2335 return ret; 2336 } 2337 reinit_completion(&isert_conn->login_req_comp); 2338 2339 /* 2340 * For login requests after the first PDU, isert_rx_login_req() will 2341 * kick schedule_delayed_work(&conn->login_work) as the packet is 2342 * received, which turns this callback from iscsi_target_do_login_rx() 2343 * into a NOP. 2344 */ 2345 if (!login->first_request) 2346 return 0; 2347 2348 isert_rx_login_req(isert_conn); 2349 2350 isert_info("before login_comp conn: %p\n", conn); 2351 ret = wait_for_completion_interruptible(&isert_conn->login_comp); 2352 if (ret) 2353 return ret; 2354 2355 isert_info("processing login->req: %p\n", login->req); 2356 2357 return 0; 2358 } 2359 2360 static void 2361 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2362 struct isert_conn *isert_conn) 2363 { 2364 struct rdma_cm_id *cm_id = isert_conn->cm_id; 2365 struct rdma_route *cm_route = &cm_id->route; 2366 2367 conn->login_family = np->np_sockaddr.ss_family; 2368 2369 conn->login_sockaddr = cm_route->addr.dst_addr; 2370 conn->local_sockaddr = cm_route->addr.src_addr; 2371 } 2372 2373 static int 2374 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2375 { 2376 struct isert_np *isert_np = np->np_context; 2377 struct isert_conn *isert_conn; 2378 int ret; 2379 2380 accept_wait: 2381 ret = down_interruptible(&isert_np->sem); 2382 if (ret) 2383 return -ENODEV; 2384 2385 spin_lock_bh(&np->np_thread_lock); 2386 if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) { 2387 spin_unlock_bh(&np->np_thread_lock); 2388 isert_dbg("np_thread_state %d\n", 2389 np->np_thread_state); 2390 /** 2391 * No point in stalling here when np_thread 2392 * is in state RESET/SHUTDOWN/EXIT - bail 2393 **/ 2394 return -ENODEV; 2395 } 2396 spin_unlock_bh(&np->np_thread_lock); 2397 2398 mutex_lock(&isert_np->mutex); 2399 if (list_empty(&isert_np->pending)) { 2400 mutex_unlock(&isert_np->mutex); 2401 goto accept_wait; 2402 } 2403 isert_conn = list_first_entry(&isert_np->pending, 2404 struct isert_conn, node); 2405 list_del_init(&isert_conn->node); 2406 mutex_unlock(&isert_np->mutex); 2407 2408 conn->context = isert_conn; 2409 isert_conn->conn = conn; 2410 isert_conn->state = ISER_CONN_BOUND; 2411 2412 isert_set_conn_info(np, conn, isert_conn); 2413 2414 isert_dbg("Processing isert_conn: %p\n", isert_conn); 2415 2416 return 0; 2417 } 2418 2419 static void 2420 isert_free_np(struct iscsi_np *np) 2421 { 2422 struct isert_np *isert_np = np->np_context; 2423 struct isert_conn *isert_conn, *n; 2424 2425 if (isert_np->cm_id) 2426 rdma_destroy_id(isert_np->cm_id); 2427 2428 /* 2429 * FIXME: At this point we don't have a good way to insure 2430 * that at this point we don't have hanging connections that 2431 * completed RDMA establishment but didn't start iscsi login 2432 * process. So work-around this by cleaning up what ever piled 2433 * up in accepted and pending lists. 2434 */ 2435 mutex_lock(&isert_np->mutex); 2436 if (!list_empty(&isert_np->pending)) { 2437 isert_info("Still have isert pending connections\n"); 2438 list_for_each_entry_safe(isert_conn, n, 2439 &isert_np->pending, 2440 node) { 2441 isert_info("cleaning isert_conn %p state (%d)\n", 2442 isert_conn, isert_conn->state); 2443 isert_connect_release(isert_conn); 2444 } 2445 } 2446 2447 if (!list_empty(&isert_np->accepted)) { 2448 isert_info("Still have isert accepted connections\n"); 2449 list_for_each_entry_safe(isert_conn, n, 2450 &isert_np->accepted, 2451 node) { 2452 isert_info("cleaning isert_conn %p state (%d)\n", 2453 isert_conn, isert_conn->state); 2454 isert_connect_release(isert_conn); 2455 } 2456 } 2457 mutex_unlock(&isert_np->mutex); 2458 2459 np->np_context = NULL; 2460 kfree(isert_np); 2461 } 2462 2463 static void isert_release_work(struct work_struct *work) 2464 { 2465 struct isert_conn *isert_conn = container_of(work, 2466 struct isert_conn, 2467 release_work); 2468 2469 isert_info("Starting release conn %p\n", isert_conn); 2470 2471 mutex_lock(&isert_conn->mutex); 2472 isert_conn->state = ISER_CONN_DOWN; 2473 mutex_unlock(&isert_conn->mutex); 2474 2475 isert_info("Destroying conn %p\n", isert_conn); 2476 isert_put_conn(isert_conn); 2477 } 2478 2479 static void 2480 isert_wait4logout(struct isert_conn *isert_conn) 2481 { 2482 struct iscsi_conn *conn = isert_conn->conn; 2483 2484 isert_info("conn %p\n", isert_conn); 2485 2486 if (isert_conn->logout_posted) { 2487 isert_info("conn %p wait for conn_logout_comp\n", isert_conn); 2488 wait_for_completion_timeout(&conn->conn_logout_comp, 2489 SECONDS_FOR_LOGOUT_COMP * HZ); 2490 } 2491 } 2492 2493 static void 2494 isert_wait4cmds(struct iscsi_conn *conn) 2495 { 2496 isert_info("iscsi_conn %p\n", conn); 2497 2498 if (conn->sess) { 2499 target_stop_session(conn->sess->se_sess); 2500 target_wait_for_sess_cmds(conn->sess->se_sess); 2501 } 2502 } 2503 2504 /** 2505 * isert_put_unsol_pending_cmds() - Drop commands waiting for 2506 * unsolicitate dataout 2507 * @conn: iscsi connection 2508 * 2509 * We might still have commands that are waiting for unsolicited 2510 * dataouts messages. We must put the extra reference on those 2511 * before blocking on the target_wait_for_session_cmds 2512 */ 2513 static void 2514 isert_put_unsol_pending_cmds(struct iscsi_conn *conn) 2515 { 2516 struct iscsi_cmd *cmd, *tmp; 2517 static LIST_HEAD(drop_cmd_list); 2518 2519 spin_lock_bh(&conn->cmd_lock); 2520 list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) { 2521 if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) && 2522 (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) && 2523 (cmd->write_data_done < cmd->se_cmd.data_length)) 2524 list_move_tail(&cmd->i_conn_node, &drop_cmd_list); 2525 } 2526 spin_unlock_bh(&conn->cmd_lock); 2527 2528 list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) { 2529 list_del_init(&cmd->i_conn_node); 2530 if (cmd->i_state != ISTATE_REMOVE) { 2531 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2532 2533 isert_info("conn %p dropping cmd %p\n", conn, cmd); 2534 isert_put_cmd(isert_cmd, true); 2535 } 2536 } 2537 } 2538 2539 static void isert_wait_conn(struct iscsi_conn *conn) 2540 { 2541 struct isert_conn *isert_conn = conn->context; 2542 2543 isert_info("Starting conn %p\n", isert_conn); 2544 2545 mutex_lock(&isert_conn->mutex); 2546 isert_conn_terminate(isert_conn); 2547 mutex_unlock(&isert_conn->mutex); 2548 2549 ib_drain_qp(isert_conn->qp); 2550 isert_put_unsol_pending_cmds(conn); 2551 isert_wait4cmds(conn); 2552 isert_wait4logout(isert_conn); 2553 2554 queue_work(isert_release_wq, &isert_conn->release_work); 2555 } 2556 2557 static void isert_free_conn(struct iscsi_conn *conn) 2558 { 2559 struct isert_conn *isert_conn = conn->context; 2560 2561 ib_drain_qp(isert_conn->qp); 2562 isert_put_conn(isert_conn); 2563 } 2564 2565 static void isert_get_rx_pdu(struct iscsi_conn *conn) 2566 { 2567 struct completion comp; 2568 2569 init_completion(&comp); 2570 2571 wait_for_completion_interruptible(&comp); 2572 } 2573 2574 static struct iscsit_transport iser_target_transport = { 2575 .name = "IB/iSER", 2576 .transport_type = ISCSI_INFINIBAND, 2577 .rdma_shutdown = true, 2578 .priv_size = sizeof(struct isert_cmd), 2579 .owner = THIS_MODULE, 2580 .iscsit_setup_np = isert_setup_np, 2581 .iscsit_accept_np = isert_accept_np, 2582 .iscsit_free_np = isert_free_np, 2583 .iscsit_wait_conn = isert_wait_conn, 2584 .iscsit_free_conn = isert_free_conn, 2585 .iscsit_get_login_rx = isert_get_login_rx, 2586 .iscsit_put_login_tx = isert_put_login_tx, 2587 .iscsit_immediate_queue = isert_immediate_queue, 2588 .iscsit_response_queue = isert_response_queue, 2589 .iscsit_get_dataout = isert_get_dataout, 2590 .iscsit_queue_data_in = isert_put_datain, 2591 .iscsit_queue_status = isert_put_response, 2592 .iscsit_aborted_task = isert_aborted_task, 2593 .iscsit_get_rx_pdu = isert_get_rx_pdu, 2594 .iscsit_get_sup_prot_ops = isert_get_sup_prot_ops, 2595 }; 2596 2597 static int __init isert_init(void) 2598 { 2599 int ret; 2600 2601 isert_comp_wq = alloc_workqueue("isert_comp_wq", 2602 WQ_UNBOUND | WQ_HIGHPRI, 0); 2603 if (!isert_comp_wq) { 2604 isert_err("Unable to allocate isert_comp_wq\n"); 2605 return -ENOMEM; 2606 } 2607 2608 isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND, 2609 WQ_UNBOUND_MAX_ACTIVE); 2610 if (!isert_release_wq) { 2611 isert_err("Unable to allocate isert_release_wq\n"); 2612 ret = -ENOMEM; 2613 goto destroy_comp_wq; 2614 } 2615 2616 iscsit_register_transport(&iser_target_transport); 2617 isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2618 2619 return 0; 2620 2621 destroy_comp_wq: 2622 destroy_workqueue(isert_comp_wq); 2623 2624 return ret; 2625 } 2626 2627 static void __exit isert_exit(void) 2628 { 2629 flush_scheduled_work(); 2630 destroy_workqueue(isert_release_wq); 2631 destroy_workqueue(isert_comp_wq); 2632 iscsit_unregister_transport(&iser_target_transport); 2633 isert_info("iSER_TARGET[0] - Released iser_target_transport\n"); 2634 } 2635 2636 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2637 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2638 MODULE_LICENSE("GPL"); 2639 2640 module_init(isert_init); 2641 module_exit(isert_exit); 2642