1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <linux/llist.h> 26 #include <rdma/ib_verbs.h> 27 #include <rdma/rdma_cm.h> 28 #include <target/target_core_base.h> 29 #include <target/target_core_fabric.h> 30 #include <target/iscsi/iscsi_transport.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 39 static DEFINE_MUTEX(device_list_mutex); 40 static LIST_HEAD(device_list); 41 static struct workqueue_struct *isert_rx_wq; 42 static struct workqueue_struct *isert_comp_wq; 43 44 static void 45 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 46 static int 47 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 48 struct isert_rdma_wr *wr); 49 static void 50 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51 static int 52 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 53 struct isert_rdma_wr *wr); 54 55 static void 56 isert_qp_event_callback(struct ib_event *e, void *context) 57 { 58 struct isert_conn *isert_conn = (struct isert_conn *)context; 59 60 pr_err("isert_qp_event_callback event: %d\n", e->event); 61 switch (e->event) { 62 case IB_EVENT_COMM_EST: 63 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 64 break; 65 case IB_EVENT_QP_LAST_WQE_REACHED: 66 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 67 break; 68 default: 69 break; 70 } 71 } 72 73 static int 74 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 75 { 76 int ret; 77 78 ret = ib_query_device(ib_dev, devattr); 79 if (ret) { 80 pr_err("ib_query_device() failed: %d\n", ret); 81 return ret; 82 } 83 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 84 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 85 86 return 0; 87 } 88 89 static int 90 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 91 { 92 struct isert_device *device = isert_conn->conn_device; 93 struct ib_qp_init_attr attr; 94 int ret, index, min_index = 0; 95 96 mutex_lock(&device_list_mutex); 97 for (index = 0; index < device->cqs_used; index++) 98 if (device->cq_active_qps[index] < 99 device->cq_active_qps[min_index]) 100 min_index = index; 101 device->cq_active_qps[min_index]++; 102 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 103 mutex_unlock(&device_list_mutex); 104 105 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 106 attr.event_handler = isert_qp_event_callback; 107 attr.qp_context = isert_conn; 108 attr.send_cq = device->dev_tx_cq[min_index]; 109 attr.recv_cq = device->dev_rx_cq[min_index]; 110 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 111 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 112 /* 113 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 114 * work-around for RDMA_READ.. 115 */ 116 attr.cap.max_send_sge = device->dev_attr.max_sge - 2; 117 isert_conn->max_sge = attr.cap.max_send_sge; 118 119 attr.cap.max_recv_sge = 1; 120 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 121 attr.qp_type = IB_QPT_RC; 122 123 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 124 cma_id->device); 125 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 126 isert_conn->conn_pd->device); 127 128 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 129 if (ret) { 130 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 131 return ret; 132 } 133 isert_conn->conn_qp = cma_id->qp; 134 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 135 136 return 0; 137 } 138 139 static void 140 isert_cq_event_callback(struct ib_event *e, void *context) 141 { 142 pr_debug("isert_cq_event_callback event: %d\n", e->event); 143 } 144 145 static int 146 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 147 { 148 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 149 struct iser_rx_desc *rx_desc; 150 struct ib_sge *rx_sg; 151 u64 dma_addr; 152 int i, j; 153 154 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 155 sizeof(struct iser_rx_desc), GFP_KERNEL); 156 if (!isert_conn->conn_rx_descs) 157 goto fail; 158 159 rx_desc = isert_conn->conn_rx_descs; 160 161 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 162 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 163 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 164 if (ib_dma_mapping_error(ib_dev, dma_addr)) 165 goto dma_map_fail; 166 167 rx_desc->dma_addr = dma_addr; 168 169 rx_sg = &rx_desc->rx_sg; 170 rx_sg->addr = rx_desc->dma_addr; 171 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 172 rx_sg->lkey = isert_conn->conn_mr->lkey; 173 } 174 175 isert_conn->conn_rx_desc_head = 0; 176 return 0; 177 178 dma_map_fail: 179 rx_desc = isert_conn->conn_rx_descs; 180 for (j = 0; j < i; j++, rx_desc++) { 181 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 182 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 183 } 184 kfree(isert_conn->conn_rx_descs); 185 isert_conn->conn_rx_descs = NULL; 186 fail: 187 return -ENOMEM; 188 } 189 190 static void 191 isert_free_rx_descriptors(struct isert_conn *isert_conn) 192 { 193 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 194 struct iser_rx_desc *rx_desc; 195 int i; 196 197 if (!isert_conn->conn_rx_descs) 198 return; 199 200 rx_desc = isert_conn->conn_rx_descs; 201 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 202 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 203 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 204 } 205 206 kfree(isert_conn->conn_rx_descs); 207 isert_conn->conn_rx_descs = NULL; 208 } 209 210 static void isert_cq_tx_work(struct work_struct *); 211 static void isert_cq_tx_callback(struct ib_cq *, void *); 212 static void isert_cq_rx_work(struct work_struct *); 213 static void isert_cq_rx_callback(struct ib_cq *, void *); 214 215 static int 216 isert_create_device_ib_res(struct isert_device *device) 217 { 218 struct ib_device *ib_dev = device->ib_device; 219 struct isert_cq_desc *cq_desc; 220 struct ib_device_attr *dev_attr; 221 int ret = 0, i, j; 222 223 dev_attr = &device->dev_attr; 224 ret = isert_query_device(ib_dev, dev_attr); 225 if (ret) 226 return ret; 227 228 /* asign function handlers */ 229 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) { 230 device->use_fastreg = 1; 231 device->reg_rdma_mem = isert_reg_rdma; 232 device->unreg_rdma_mem = isert_unreg_rdma; 233 } else { 234 device->use_fastreg = 0; 235 device->reg_rdma_mem = isert_map_rdma; 236 device->unreg_rdma_mem = isert_unmap_cmd; 237 } 238 239 device->cqs_used = min_t(int, num_online_cpus(), 240 device->ib_device->num_comp_vectors); 241 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 242 pr_debug("Using %d CQs, device %s supports %d vectors support " 243 "Fast registration %d\n", 244 device->cqs_used, device->ib_device->name, 245 device->ib_device->num_comp_vectors, device->use_fastreg); 246 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 247 device->cqs_used, GFP_KERNEL); 248 if (!device->cq_desc) { 249 pr_err("Unable to allocate device->cq_desc\n"); 250 return -ENOMEM; 251 } 252 cq_desc = device->cq_desc; 253 254 for (i = 0; i < device->cqs_used; i++) { 255 cq_desc[i].device = device; 256 cq_desc[i].cq_index = i; 257 258 INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 259 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 260 isert_cq_rx_callback, 261 isert_cq_event_callback, 262 (void *)&cq_desc[i], 263 ISER_MAX_RX_CQ_LEN, i); 264 if (IS_ERR(device->dev_rx_cq[i])) { 265 ret = PTR_ERR(device->dev_rx_cq[i]); 266 device->dev_rx_cq[i] = NULL; 267 goto out_cq; 268 } 269 270 INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 271 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 272 isert_cq_tx_callback, 273 isert_cq_event_callback, 274 (void *)&cq_desc[i], 275 ISER_MAX_TX_CQ_LEN, i); 276 if (IS_ERR(device->dev_tx_cq[i])) { 277 ret = PTR_ERR(device->dev_tx_cq[i]); 278 device->dev_tx_cq[i] = NULL; 279 goto out_cq; 280 } 281 282 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 283 if (ret) 284 goto out_cq; 285 286 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 287 if (ret) 288 goto out_cq; 289 } 290 291 return 0; 292 293 out_cq: 294 for (j = 0; j < i; j++) { 295 cq_desc = &device->cq_desc[j]; 296 297 if (device->dev_rx_cq[j]) { 298 cancel_work_sync(&cq_desc->cq_rx_work); 299 ib_destroy_cq(device->dev_rx_cq[j]); 300 } 301 if (device->dev_tx_cq[j]) { 302 cancel_work_sync(&cq_desc->cq_tx_work); 303 ib_destroy_cq(device->dev_tx_cq[j]); 304 } 305 } 306 kfree(device->cq_desc); 307 308 return ret; 309 } 310 311 static void 312 isert_free_device_ib_res(struct isert_device *device) 313 { 314 struct isert_cq_desc *cq_desc; 315 int i; 316 317 for (i = 0; i < device->cqs_used; i++) { 318 cq_desc = &device->cq_desc[i]; 319 320 cancel_work_sync(&cq_desc->cq_rx_work); 321 cancel_work_sync(&cq_desc->cq_tx_work); 322 ib_destroy_cq(device->dev_rx_cq[i]); 323 ib_destroy_cq(device->dev_tx_cq[i]); 324 device->dev_rx_cq[i] = NULL; 325 device->dev_tx_cq[i] = NULL; 326 } 327 328 kfree(device->cq_desc); 329 } 330 331 static void 332 isert_device_try_release(struct isert_device *device) 333 { 334 mutex_lock(&device_list_mutex); 335 device->refcount--; 336 if (!device->refcount) { 337 isert_free_device_ib_res(device); 338 list_del(&device->dev_node); 339 kfree(device); 340 } 341 mutex_unlock(&device_list_mutex); 342 } 343 344 static struct isert_device * 345 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 346 { 347 struct isert_device *device; 348 int ret; 349 350 mutex_lock(&device_list_mutex); 351 list_for_each_entry(device, &device_list, dev_node) { 352 if (device->ib_device->node_guid == cma_id->device->node_guid) { 353 device->refcount++; 354 mutex_unlock(&device_list_mutex); 355 return device; 356 } 357 } 358 359 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 360 if (!device) { 361 mutex_unlock(&device_list_mutex); 362 return ERR_PTR(-ENOMEM); 363 } 364 365 INIT_LIST_HEAD(&device->dev_node); 366 367 device->ib_device = cma_id->device; 368 ret = isert_create_device_ib_res(device); 369 if (ret) { 370 kfree(device); 371 mutex_unlock(&device_list_mutex); 372 return ERR_PTR(ret); 373 } 374 375 device->refcount++; 376 list_add_tail(&device->dev_node, &device_list); 377 mutex_unlock(&device_list_mutex); 378 379 return device; 380 } 381 382 static void 383 isert_conn_free_fastreg_pool(struct isert_conn *isert_conn) 384 { 385 struct fast_reg_descriptor *fr_desc, *tmp; 386 int i = 0; 387 388 if (list_empty(&isert_conn->conn_fr_pool)) 389 return; 390 391 pr_debug("Freeing conn %p fastreg pool", isert_conn); 392 393 list_for_each_entry_safe(fr_desc, tmp, 394 &isert_conn->conn_fr_pool, list) { 395 list_del(&fr_desc->list); 396 ib_free_fast_reg_page_list(fr_desc->data_frpl); 397 ib_dereg_mr(fr_desc->data_mr); 398 kfree(fr_desc); 399 ++i; 400 } 401 402 if (i < isert_conn->conn_fr_pool_size) 403 pr_warn("Pool still has %d regions registered\n", 404 isert_conn->conn_fr_pool_size - i); 405 } 406 407 static int 408 isert_create_fr_desc(struct ib_device *ib_device, struct ib_pd *pd, 409 struct fast_reg_descriptor *fr_desc) 410 { 411 fr_desc->data_frpl = ib_alloc_fast_reg_page_list(ib_device, 412 ISCSI_ISER_SG_TABLESIZE); 413 if (IS_ERR(fr_desc->data_frpl)) { 414 pr_err("Failed to allocate data frpl err=%ld\n", 415 PTR_ERR(fr_desc->data_frpl)); 416 return PTR_ERR(fr_desc->data_frpl); 417 } 418 419 fr_desc->data_mr = ib_alloc_fast_reg_mr(pd, ISCSI_ISER_SG_TABLESIZE); 420 if (IS_ERR(fr_desc->data_mr)) { 421 pr_err("Failed to allocate data frmr err=%ld\n", 422 PTR_ERR(fr_desc->data_mr)); 423 ib_free_fast_reg_page_list(fr_desc->data_frpl); 424 return PTR_ERR(fr_desc->data_mr); 425 } 426 pr_debug("Create fr_desc %p page_list %p\n", 427 fr_desc, fr_desc->data_frpl->page_list); 428 429 fr_desc->valid = true; 430 431 return 0; 432 } 433 434 static int 435 isert_conn_create_fastreg_pool(struct isert_conn *isert_conn) 436 { 437 struct fast_reg_descriptor *fr_desc; 438 struct isert_device *device = isert_conn->conn_device; 439 int i, ret; 440 441 INIT_LIST_HEAD(&isert_conn->conn_fr_pool); 442 isert_conn->conn_fr_pool_size = 0; 443 for (i = 0; i < ISCSI_DEF_XMIT_CMDS_MAX; i++) { 444 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 445 if (!fr_desc) { 446 pr_err("Failed to allocate fast_reg descriptor\n"); 447 ret = -ENOMEM; 448 goto err; 449 } 450 451 ret = isert_create_fr_desc(device->ib_device, 452 isert_conn->conn_pd, fr_desc); 453 if (ret) { 454 pr_err("Failed to create fastreg descriptor err=%d\n", 455 ret); 456 kfree(fr_desc); 457 goto err; 458 } 459 460 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 461 isert_conn->conn_fr_pool_size++; 462 } 463 464 pr_debug("Creating conn %p fastreg pool size=%d", 465 isert_conn, isert_conn->conn_fr_pool_size); 466 467 return 0; 468 469 err: 470 isert_conn_free_fastreg_pool(isert_conn); 471 return ret; 472 } 473 474 static int 475 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 476 { 477 struct iscsi_np *np = cma_id->context; 478 struct isert_np *isert_np = np->np_context; 479 struct isert_conn *isert_conn; 480 struct isert_device *device; 481 struct ib_device *ib_dev = cma_id->device; 482 int ret = 0; 483 484 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 485 cma_id, cma_id->context); 486 487 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 488 if (!isert_conn) { 489 pr_err("Unable to allocate isert_conn\n"); 490 return -ENOMEM; 491 } 492 isert_conn->state = ISER_CONN_INIT; 493 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 494 init_completion(&isert_conn->conn_login_comp); 495 init_waitqueue_head(&isert_conn->conn_wait); 496 init_waitqueue_head(&isert_conn->conn_wait_comp_err); 497 kref_init(&isert_conn->conn_kref); 498 kref_get(&isert_conn->conn_kref); 499 mutex_init(&isert_conn->conn_mutex); 500 mutex_init(&isert_conn->conn_comp_mutex); 501 spin_lock_init(&isert_conn->conn_lock); 502 503 cma_id->context = isert_conn; 504 isert_conn->conn_cm_id = cma_id; 505 isert_conn->responder_resources = event->param.conn.responder_resources; 506 isert_conn->initiator_depth = event->param.conn.initiator_depth; 507 pr_debug("Using responder_resources: %u initiator_depth: %u\n", 508 isert_conn->responder_resources, isert_conn->initiator_depth); 509 510 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 511 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 512 if (!isert_conn->login_buf) { 513 pr_err("Unable to allocate isert_conn->login_buf\n"); 514 ret = -ENOMEM; 515 goto out; 516 } 517 518 isert_conn->login_req_buf = isert_conn->login_buf; 519 isert_conn->login_rsp_buf = isert_conn->login_buf + 520 ISCSI_DEF_MAX_RECV_SEG_LEN; 521 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 522 isert_conn->login_buf, isert_conn->login_req_buf, 523 isert_conn->login_rsp_buf); 524 525 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 526 (void *)isert_conn->login_req_buf, 527 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 528 529 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 530 if (ret) { 531 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 532 ret); 533 isert_conn->login_req_dma = 0; 534 goto out_login_buf; 535 } 536 537 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 538 (void *)isert_conn->login_rsp_buf, 539 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 540 541 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 542 if (ret) { 543 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 544 ret); 545 isert_conn->login_rsp_dma = 0; 546 goto out_req_dma_map; 547 } 548 549 device = isert_device_find_by_ib_dev(cma_id); 550 if (IS_ERR(device)) { 551 ret = PTR_ERR(device); 552 goto out_rsp_dma_map; 553 } 554 555 isert_conn->conn_device = device; 556 isert_conn->conn_pd = ib_alloc_pd(isert_conn->conn_device->ib_device); 557 if (IS_ERR(isert_conn->conn_pd)) { 558 ret = PTR_ERR(isert_conn->conn_pd); 559 pr_err("ib_alloc_pd failed for conn %p: ret=%d\n", 560 isert_conn, ret); 561 goto out_pd; 562 } 563 564 isert_conn->conn_mr = ib_get_dma_mr(isert_conn->conn_pd, 565 IB_ACCESS_LOCAL_WRITE); 566 if (IS_ERR(isert_conn->conn_mr)) { 567 ret = PTR_ERR(isert_conn->conn_mr); 568 pr_err("ib_get_dma_mr failed for conn %p: ret=%d\n", 569 isert_conn, ret); 570 goto out_mr; 571 } 572 573 if (device->use_fastreg) { 574 ret = isert_conn_create_fastreg_pool(isert_conn); 575 if (ret) { 576 pr_err("Conn: %p failed to create fastreg pool\n", 577 isert_conn); 578 goto out_fastreg; 579 } 580 } 581 582 ret = isert_conn_setup_qp(isert_conn, cma_id); 583 if (ret) 584 goto out_conn_dev; 585 586 mutex_lock(&isert_np->np_accept_mutex); 587 list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 588 mutex_unlock(&isert_np->np_accept_mutex); 589 590 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 591 wake_up(&isert_np->np_accept_wq); 592 return 0; 593 594 out_conn_dev: 595 if (device->use_fastreg) 596 isert_conn_free_fastreg_pool(isert_conn); 597 out_fastreg: 598 ib_dereg_mr(isert_conn->conn_mr); 599 out_mr: 600 ib_dealloc_pd(isert_conn->conn_pd); 601 out_pd: 602 isert_device_try_release(device); 603 out_rsp_dma_map: 604 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 605 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 606 out_req_dma_map: 607 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 608 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 609 out_login_buf: 610 kfree(isert_conn->login_buf); 611 out: 612 kfree(isert_conn); 613 return ret; 614 } 615 616 static void 617 isert_connect_release(struct isert_conn *isert_conn) 618 { 619 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 620 struct isert_device *device = isert_conn->conn_device; 621 int cq_index; 622 623 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 624 625 if (device && device->use_fastreg) 626 isert_conn_free_fastreg_pool(isert_conn); 627 628 if (isert_conn->conn_qp) { 629 cq_index = ((struct isert_cq_desc *) 630 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 631 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 632 isert_conn->conn_device->cq_active_qps[cq_index]--; 633 634 rdma_destroy_qp(isert_conn->conn_cm_id); 635 } 636 637 isert_free_rx_descriptors(isert_conn); 638 rdma_destroy_id(isert_conn->conn_cm_id); 639 640 ib_dereg_mr(isert_conn->conn_mr); 641 ib_dealloc_pd(isert_conn->conn_pd); 642 643 if (isert_conn->login_buf) { 644 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 645 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 646 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 647 ISCSI_DEF_MAX_RECV_SEG_LEN, 648 DMA_FROM_DEVICE); 649 kfree(isert_conn->login_buf); 650 } 651 kfree(isert_conn); 652 653 if (device) 654 isert_device_try_release(device); 655 656 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 657 } 658 659 static void 660 isert_connected_handler(struct rdma_cm_id *cma_id) 661 { 662 return; 663 } 664 665 static void 666 isert_release_conn_kref(struct kref *kref) 667 { 668 struct isert_conn *isert_conn = container_of(kref, 669 struct isert_conn, conn_kref); 670 671 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 672 current->comm, current->pid); 673 674 isert_connect_release(isert_conn); 675 } 676 677 static void 678 isert_put_conn(struct isert_conn *isert_conn) 679 { 680 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 681 } 682 683 static void 684 isert_disconnect_work(struct work_struct *work) 685 { 686 struct isert_conn *isert_conn = container_of(work, 687 struct isert_conn, conn_logout_work); 688 689 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 690 mutex_lock(&isert_conn->conn_mutex); 691 isert_conn->state = ISER_CONN_DOWN; 692 693 if (isert_conn->post_recv_buf_count == 0 && 694 atomic_read(&isert_conn->post_send_buf_count) == 0) { 695 pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); 696 mutex_unlock(&isert_conn->conn_mutex); 697 goto wake_up; 698 } 699 if (!isert_conn->conn_cm_id) { 700 mutex_unlock(&isert_conn->conn_mutex); 701 isert_put_conn(isert_conn); 702 return; 703 } 704 if (!isert_conn->logout_posted) { 705 pr_debug("Calling rdma_disconnect for !logout_posted from" 706 " isert_disconnect_work\n"); 707 rdma_disconnect(isert_conn->conn_cm_id); 708 mutex_unlock(&isert_conn->conn_mutex); 709 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 710 goto wake_up; 711 } 712 mutex_unlock(&isert_conn->conn_mutex); 713 714 wake_up: 715 wake_up(&isert_conn->conn_wait); 716 isert_put_conn(isert_conn); 717 } 718 719 static void 720 isert_disconnected_handler(struct rdma_cm_id *cma_id) 721 { 722 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 723 724 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 725 schedule_work(&isert_conn->conn_logout_work); 726 } 727 728 static int 729 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 730 { 731 int ret = 0; 732 733 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 734 event->event, event->status, cma_id->context, cma_id); 735 736 switch (event->event) { 737 case RDMA_CM_EVENT_CONNECT_REQUEST: 738 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 739 ret = isert_connect_request(cma_id, event); 740 break; 741 case RDMA_CM_EVENT_ESTABLISHED: 742 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 743 isert_connected_handler(cma_id); 744 break; 745 case RDMA_CM_EVENT_DISCONNECTED: 746 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 747 isert_disconnected_handler(cma_id); 748 break; 749 case RDMA_CM_EVENT_DEVICE_REMOVAL: 750 case RDMA_CM_EVENT_ADDR_CHANGE: 751 break; 752 case RDMA_CM_EVENT_CONNECT_ERROR: 753 default: 754 pr_err("Unknown RDMA CMA event: %d\n", event->event); 755 break; 756 } 757 758 if (ret != 0) { 759 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 760 event->event, ret); 761 dump_stack(); 762 } 763 764 return ret; 765 } 766 767 static int 768 isert_post_recv(struct isert_conn *isert_conn, u32 count) 769 { 770 struct ib_recv_wr *rx_wr, *rx_wr_failed; 771 int i, ret; 772 unsigned int rx_head = isert_conn->conn_rx_desc_head; 773 struct iser_rx_desc *rx_desc; 774 775 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 776 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 777 rx_wr->wr_id = (unsigned long)rx_desc; 778 rx_wr->sg_list = &rx_desc->rx_sg; 779 rx_wr->num_sge = 1; 780 rx_wr->next = rx_wr + 1; 781 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 782 } 783 784 rx_wr--; 785 rx_wr->next = NULL; /* mark end of work requests list */ 786 787 isert_conn->post_recv_buf_count += count; 788 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 789 &rx_wr_failed); 790 if (ret) { 791 pr_err("ib_post_recv() failed with ret: %d\n", ret); 792 isert_conn->post_recv_buf_count -= count; 793 } else { 794 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 795 isert_conn->conn_rx_desc_head = rx_head; 796 } 797 return ret; 798 } 799 800 static int 801 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 802 { 803 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 804 struct ib_send_wr send_wr, *send_wr_failed; 805 int ret; 806 807 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 808 ISER_HEADERS_LEN, DMA_TO_DEVICE); 809 810 send_wr.next = NULL; 811 send_wr.wr_id = (unsigned long)tx_desc; 812 send_wr.sg_list = tx_desc->tx_sg; 813 send_wr.num_sge = tx_desc->num_sge; 814 send_wr.opcode = IB_WR_SEND; 815 send_wr.send_flags = IB_SEND_SIGNALED; 816 817 atomic_inc(&isert_conn->post_send_buf_count); 818 819 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 820 if (ret) { 821 pr_err("ib_post_send() failed, ret: %d\n", ret); 822 atomic_dec(&isert_conn->post_send_buf_count); 823 } 824 825 return ret; 826 } 827 828 static void 829 isert_create_send_desc(struct isert_conn *isert_conn, 830 struct isert_cmd *isert_cmd, 831 struct iser_tx_desc *tx_desc) 832 { 833 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 834 835 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 836 ISER_HEADERS_LEN, DMA_TO_DEVICE); 837 838 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 839 tx_desc->iser_header.flags = ISER_VER; 840 841 tx_desc->num_sge = 1; 842 tx_desc->isert_cmd = isert_cmd; 843 844 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 845 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 846 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 847 } 848 } 849 850 static int 851 isert_init_tx_hdrs(struct isert_conn *isert_conn, 852 struct iser_tx_desc *tx_desc) 853 { 854 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 855 u64 dma_addr; 856 857 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 858 ISER_HEADERS_LEN, DMA_TO_DEVICE); 859 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 860 pr_err("ib_dma_mapping_error() failed\n"); 861 return -ENOMEM; 862 } 863 864 tx_desc->dma_addr = dma_addr; 865 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 866 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 867 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 868 869 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 870 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 871 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 872 873 return 0; 874 } 875 876 static void 877 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 878 struct ib_send_wr *send_wr, bool coalesce) 879 { 880 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 881 882 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 883 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 884 send_wr->opcode = IB_WR_SEND; 885 send_wr->sg_list = &tx_desc->tx_sg[0]; 886 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 887 /* 888 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 889 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 890 */ 891 mutex_lock(&isert_conn->conn_comp_mutex); 892 if (coalesce && 893 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 894 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 895 mutex_unlock(&isert_conn->conn_comp_mutex); 896 return; 897 } 898 isert_conn->conn_comp_batch = 0; 899 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 900 mutex_unlock(&isert_conn->conn_comp_mutex); 901 902 send_wr->send_flags = IB_SEND_SIGNALED; 903 } 904 905 static int 906 isert_rdma_post_recvl(struct isert_conn *isert_conn) 907 { 908 struct ib_recv_wr rx_wr, *rx_wr_fail; 909 struct ib_sge sge; 910 int ret; 911 912 memset(&sge, 0, sizeof(struct ib_sge)); 913 sge.addr = isert_conn->login_req_dma; 914 sge.length = ISER_RX_LOGIN_SIZE; 915 sge.lkey = isert_conn->conn_mr->lkey; 916 917 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 918 sge.addr, sge.length, sge.lkey); 919 920 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 921 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 922 rx_wr.sg_list = &sge; 923 rx_wr.num_sge = 1; 924 925 isert_conn->post_recv_buf_count++; 926 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 927 if (ret) { 928 pr_err("ib_post_recv() failed: %d\n", ret); 929 isert_conn->post_recv_buf_count--; 930 } 931 932 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 933 return ret; 934 } 935 936 static int 937 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 938 u32 length) 939 { 940 struct isert_conn *isert_conn = conn->context; 941 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 942 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 943 int ret; 944 945 isert_create_send_desc(isert_conn, NULL, tx_desc); 946 947 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 948 sizeof(struct iscsi_hdr)); 949 950 isert_init_tx_hdrs(isert_conn, tx_desc); 951 952 if (length > 0) { 953 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 954 955 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 956 length, DMA_TO_DEVICE); 957 958 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 959 960 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 961 length, DMA_TO_DEVICE); 962 963 tx_dsg->addr = isert_conn->login_rsp_dma; 964 tx_dsg->length = length; 965 tx_dsg->lkey = isert_conn->conn_mr->lkey; 966 tx_desc->num_sge = 2; 967 } 968 if (!login->login_failed) { 969 if (login->login_complete) { 970 ret = isert_alloc_rx_descriptors(isert_conn); 971 if (ret) 972 return ret; 973 974 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 975 if (ret) 976 return ret; 977 978 isert_conn->state = ISER_CONN_UP; 979 goto post_send; 980 } 981 982 ret = isert_rdma_post_recvl(isert_conn); 983 if (ret) 984 return ret; 985 } 986 post_send: 987 ret = isert_post_send(isert_conn, tx_desc); 988 if (ret) 989 return ret; 990 991 return 0; 992 } 993 994 static void 995 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 996 struct isert_conn *isert_conn) 997 { 998 struct iscsi_conn *conn = isert_conn->conn; 999 struct iscsi_login *login = conn->conn_login; 1000 int size; 1001 1002 if (!login) { 1003 pr_err("conn->conn_login is NULL\n"); 1004 dump_stack(); 1005 return; 1006 } 1007 1008 if (login->first_request) { 1009 struct iscsi_login_req *login_req = 1010 (struct iscsi_login_req *)&rx_desc->iscsi_header; 1011 /* 1012 * Setup the initial iscsi_login values from the leading 1013 * login request PDU. 1014 */ 1015 login->leading_connection = (!login_req->tsih) ? 1 : 0; 1016 login->current_stage = 1017 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1018 >> 2; 1019 login->version_min = login_req->min_version; 1020 login->version_max = login_req->max_version; 1021 memcpy(login->isid, login_req->isid, 6); 1022 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1023 login->init_task_tag = login_req->itt; 1024 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1025 login->cid = be16_to_cpu(login_req->cid); 1026 login->tsih = be16_to_cpu(login_req->tsih); 1027 } 1028 1029 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1030 1031 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1032 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1033 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1034 memcpy(login->req_buf, &rx_desc->data[0], size); 1035 1036 if (login->first_request) { 1037 complete(&isert_conn->conn_login_comp); 1038 return; 1039 } 1040 schedule_delayed_work(&conn->login_work, 0); 1041 } 1042 1043 static struct iscsi_cmd 1044 *isert_allocate_cmd(struct iscsi_conn *conn) 1045 { 1046 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1047 struct isert_cmd *isert_cmd; 1048 struct iscsi_cmd *cmd; 1049 1050 cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE); 1051 if (!cmd) { 1052 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1053 return NULL; 1054 } 1055 isert_cmd = iscsit_priv_cmd(cmd); 1056 isert_cmd->conn = isert_conn; 1057 isert_cmd->iscsi_cmd = cmd; 1058 1059 return cmd; 1060 } 1061 1062 static int 1063 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1064 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1065 struct iser_rx_desc *rx_desc, unsigned char *buf) 1066 { 1067 struct iscsi_conn *conn = isert_conn->conn; 1068 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1069 struct scatterlist *sg; 1070 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1071 bool dump_payload = false; 1072 1073 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1074 if (rc < 0) 1075 return rc; 1076 1077 imm_data = cmd->immediate_data; 1078 imm_data_len = cmd->first_burst_len; 1079 unsol_data = cmd->unsolicited_data; 1080 1081 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1082 if (rc < 0) { 1083 return 0; 1084 } else if (rc > 0) { 1085 dump_payload = true; 1086 goto sequence_cmd; 1087 } 1088 1089 if (!imm_data) 1090 return 0; 1091 1092 sg = &cmd->se_cmd.t_data_sg[0]; 1093 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1094 1095 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1096 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1097 1098 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1099 1100 cmd->write_data_done += imm_data_len; 1101 1102 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1103 spin_lock_bh(&cmd->istate_lock); 1104 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1105 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1106 spin_unlock_bh(&cmd->istate_lock); 1107 } 1108 1109 sequence_cmd: 1110 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1111 1112 if (!rc && dump_payload == false && unsol_data) 1113 iscsit_set_unsoliticed_dataout(cmd); 1114 1115 return 0; 1116 } 1117 1118 static int 1119 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1120 struct iser_rx_desc *rx_desc, unsigned char *buf) 1121 { 1122 struct scatterlist *sg_start; 1123 struct iscsi_conn *conn = isert_conn->conn; 1124 struct iscsi_cmd *cmd = NULL; 1125 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1126 u32 unsol_data_len = ntoh24(hdr->dlength); 1127 int rc, sg_nents, sg_off, page_off; 1128 1129 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1130 if (rc < 0) 1131 return rc; 1132 else if (!cmd) 1133 return 0; 1134 /* 1135 * FIXME: Unexpected unsolicited_data out 1136 */ 1137 if (!cmd->unsolicited_data) { 1138 pr_err("Received unexpected solicited data payload\n"); 1139 dump_stack(); 1140 return -1; 1141 } 1142 1143 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1144 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1145 1146 sg_off = cmd->write_data_done / PAGE_SIZE; 1147 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1148 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1149 page_off = cmd->write_data_done % PAGE_SIZE; 1150 /* 1151 * FIXME: Non page-aligned unsolicited_data out 1152 */ 1153 if (page_off) { 1154 pr_err("Received unexpected non-page aligned data payload\n"); 1155 dump_stack(); 1156 return -1; 1157 } 1158 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1159 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1160 1161 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1162 unsol_data_len); 1163 1164 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1165 if (rc < 0) 1166 return rc; 1167 1168 return 0; 1169 } 1170 1171 static int 1172 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1173 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1174 unsigned char *buf) 1175 { 1176 struct iscsi_conn *conn = isert_conn->conn; 1177 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1178 int rc; 1179 1180 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1181 if (rc < 0) 1182 return rc; 1183 /* 1184 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1185 */ 1186 1187 return iscsit_process_nop_out(conn, cmd, hdr); 1188 } 1189 1190 static int 1191 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1192 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1193 struct iscsi_text *hdr) 1194 { 1195 struct iscsi_conn *conn = isert_conn->conn; 1196 u32 payload_length = ntoh24(hdr->dlength); 1197 int rc; 1198 unsigned char *text_in; 1199 1200 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1201 if (rc < 0) 1202 return rc; 1203 1204 text_in = kzalloc(payload_length, GFP_KERNEL); 1205 if (!text_in) { 1206 pr_err("Unable to allocate text_in of payload_length: %u\n", 1207 payload_length); 1208 return -ENOMEM; 1209 } 1210 cmd->text_in_ptr = text_in; 1211 1212 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1213 1214 return iscsit_process_text_cmd(conn, cmd, hdr); 1215 } 1216 1217 static int 1218 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1219 uint32_t read_stag, uint64_t read_va, 1220 uint32_t write_stag, uint64_t write_va) 1221 { 1222 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1223 struct iscsi_conn *conn = isert_conn->conn; 1224 struct iscsi_session *sess = conn->sess; 1225 struct iscsi_cmd *cmd; 1226 struct isert_cmd *isert_cmd; 1227 int ret = -EINVAL; 1228 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1229 1230 if (sess->sess_ops->SessionType && 1231 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1232 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1233 " ignoring\n", opcode); 1234 return 0; 1235 } 1236 1237 switch (opcode) { 1238 case ISCSI_OP_SCSI_CMD: 1239 cmd = isert_allocate_cmd(conn); 1240 if (!cmd) 1241 break; 1242 1243 isert_cmd = iscsit_priv_cmd(cmd); 1244 isert_cmd->read_stag = read_stag; 1245 isert_cmd->read_va = read_va; 1246 isert_cmd->write_stag = write_stag; 1247 isert_cmd->write_va = write_va; 1248 1249 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1250 rx_desc, (unsigned char *)hdr); 1251 break; 1252 case ISCSI_OP_NOOP_OUT: 1253 cmd = isert_allocate_cmd(conn); 1254 if (!cmd) 1255 break; 1256 1257 isert_cmd = iscsit_priv_cmd(cmd); 1258 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1259 rx_desc, (unsigned char *)hdr); 1260 break; 1261 case ISCSI_OP_SCSI_DATA_OUT: 1262 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1263 (unsigned char *)hdr); 1264 break; 1265 case ISCSI_OP_SCSI_TMFUNC: 1266 cmd = isert_allocate_cmd(conn); 1267 if (!cmd) 1268 break; 1269 1270 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1271 (unsigned char *)hdr); 1272 break; 1273 case ISCSI_OP_LOGOUT: 1274 cmd = isert_allocate_cmd(conn); 1275 if (!cmd) 1276 break; 1277 1278 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1279 if (ret > 0) 1280 wait_for_completion_timeout(&conn->conn_logout_comp, 1281 SECONDS_FOR_LOGOUT_COMP * 1282 HZ); 1283 break; 1284 case ISCSI_OP_TEXT: 1285 cmd = isert_allocate_cmd(conn); 1286 if (!cmd) 1287 break; 1288 1289 isert_cmd = iscsit_priv_cmd(cmd); 1290 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1291 rx_desc, (struct iscsi_text *)hdr); 1292 break; 1293 default: 1294 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1295 dump_stack(); 1296 break; 1297 } 1298 1299 return ret; 1300 } 1301 1302 static void 1303 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1304 { 1305 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1306 uint64_t read_va = 0, write_va = 0; 1307 uint32_t read_stag = 0, write_stag = 0; 1308 int rc; 1309 1310 switch (iser_hdr->flags & 0xF0) { 1311 case ISCSI_CTRL: 1312 if (iser_hdr->flags & ISER_RSV) { 1313 read_stag = be32_to_cpu(iser_hdr->read_stag); 1314 read_va = be64_to_cpu(iser_hdr->read_va); 1315 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1316 read_stag, (unsigned long long)read_va); 1317 } 1318 if (iser_hdr->flags & ISER_WSV) { 1319 write_stag = be32_to_cpu(iser_hdr->write_stag); 1320 write_va = be64_to_cpu(iser_hdr->write_va); 1321 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1322 write_stag, (unsigned long long)write_va); 1323 } 1324 1325 pr_debug("ISER ISCSI_CTRL PDU\n"); 1326 break; 1327 case ISER_HELLO: 1328 pr_err("iSER Hello message\n"); 1329 break; 1330 default: 1331 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1332 break; 1333 } 1334 1335 rc = isert_rx_opcode(isert_conn, rx_desc, 1336 read_stag, read_va, write_stag, write_va); 1337 } 1338 1339 static void 1340 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1341 unsigned long xfer_len) 1342 { 1343 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1344 struct iscsi_hdr *hdr; 1345 u64 rx_dma; 1346 int rx_buflen, outstanding; 1347 1348 if ((char *)desc == isert_conn->login_req_buf) { 1349 rx_dma = isert_conn->login_req_dma; 1350 rx_buflen = ISER_RX_LOGIN_SIZE; 1351 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1352 rx_dma, rx_buflen); 1353 } else { 1354 rx_dma = desc->dma_addr; 1355 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1356 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1357 rx_dma, rx_buflen); 1358 } 1359 1360 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1361 1362 hdr = &desc->iscsi_header; 1363 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1364 hdr->opcode, hdr->itt, hdr->flags, 1365 (int)(xfer_len - ISER_HEADERS_LEN)); 1366 1367 if ((char *)desc == isert_conn->login_req_buf) 1368 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1369 isert_conn); 1370 else 1371 isert_rx_do_work(desc, isert_conn); 1372 1373 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1374 DMA_FROM_DEVICE); 1375 1376 isert_conn->post_recv_buf_count--; 1377 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1378 isert_conn->post_recv_buf_count); 1379 1380 if ((char *)desc == isert_conn->login_req_buf) 1381 return; 1382 1383 outstanding = isert_conn->post_recv_buf_count; 1384 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1385 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1386 ISERT_MIN_POSTED_RX); 1387 err = isert_post_recv(isert_conn, count); 1388 if (err) { 1389 pr_err("isert_post_recv() count: %d failed, %d\n", 1390 count, err); 1391 } 1392 } 1393 } 1394 1395 static void 1396 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1397 { 1398 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1399 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1400 1401 pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1402 if (wr->sge) { 1403 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1404 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1405 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1406 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1407 wr->sge = NULL; 1408 } 1409 1410 if (wr->send_wr) { 1411 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1412 kfree(wr->send_wr); 1413 wr->send_wr = NULL; 1414 } 1415 1416 if (wr->ib_sge) { 1417 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 1418 kfree(wr->ib_sge); 1419 wr->ib_sge = NULL; 1420 } 1421 } 1422 1423 static void 1424 isert_unreg_rdma(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1425 { 1426 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1427 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1428 LIST_HEAD(unmap_list); 1429 1430 pr_debug("unreg_fastreg_cmd: %p\n", isert_cmd); 1431 1432 if (wr->fr_desc) { 1433 pr_debug("unreg_fastreg_cmd: %p free fr_desc %p\n", 1434 isert_cmd, wr->fr_desc); 1435 spin_lock_bh(&isert_conn->conn_lock); 1436 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_fr_pool); 1437 spin_unlock_bh(&isert_conn->conn_lock); 1438 wr->fr_desc = NULL; 1439 } 1440 1441 if (wr->sge) { 1442 pr_debug("unreg_fastreg_cmd: %p unmap_sg op\n", isert_cmd); 1443 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1444 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1445 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1446 wr->sge = NULL; 1447 } 1448 1449 wr->ib_sge = NULL; 1450 wr->send_wr = NULL; 1451 } 1452 1453 static void 1454 isert_put_cmd(struct isert_cmd *isert_cmd) 1455 { 1456 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1457 struct isert_conn *isert_conn = isert_cmd->conn; 1458 struct iscsi_conn *conn = isert_conn->conn; 1459 struct isert_device *device = isert_conn->conn_device; 1460 1461 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1462 1463 switch (cmd->iscsi_opcode) { 1464 case ISCSI_OP_SCSI_CMD: 1465 spin_lock_bh(&conn->cmd_lock); 1466 if (!list_empty(&cmd->i_conn_node)) 1467 list_del(&cmd->i_conn_node); 1468 spin_unlock_bh(&conn->cmd_lock); 1469 1470 if (cmd->data_direction == DMA_TO_DEVICE) 1471 iscsit_stop_dataout_timer(cmd); 1472 1473 device->unreg_rdma_mem(isert_cmd, isert_conn); 1474 transport_generic_free_cmd(&cmd->se_cmd, 0); 1475 break; 1476 case ISCSI_OP_SCSI_TMFUNC: 1477 spin_lock_bh(&conn->cmd_lock); 1478 if (!list_empty(&cmd->i_conn_node)) 1479 list_del(&cmd->i_conn_node); 1480 spin_unlock_bh(&conn->cmd_lock); 1481 1482 transport_generic_free_cmd(&cmd->se_cmd, 0); 1483 break; 1484 case ISCSI_OP_REJECT: 1485 case ISCSI_OP_NOOP_OUT: 1486 case ISCSI_OP_TEXT: 1487 spin_lock_bh(&conn->cmd_lock); 1488 if (!list_empty(&cmd->i_conn_node)) 1489 list_del(&cmd->i_conn_node); 1490 spin_unlock_bh(&conn->cmd_lock); 1491 1492 /* 1493 * Handle special case for REJECT when iscsi_add_reject*() has 1494 * overwritten the original iscsi_opcode assignment, and the 1495 * associated cmd->se_cmd needs to be released. 1496 */ 1497 if (cmd->se_cmd.se_tfo != NULL) { 1498 pr_debug("Calling transport_generic_free_cmd from" 1499 " isert_put_cmd for 0x%02x\n", 1500 cmd->iscsi_opcode); 1501 transport_generic_free_cmd(&cmd->se_cmd, 0); 1502 break; 1503 } 1504 /* 1505 * Fall-through 1506 */ 1507 default: 1508 iscsit_release_cmd(cmd); 1509 break; 1510 } 1511 } 1512 1513 static void 1514 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1515 { 1516 if (tx_desc->dma_addr != 0) { 1517 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1518 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1519 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1520 tx_desc->dma_addr = 0; 1521 } 1522 } 1523 1524 static void 1525 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1526 struct ib_device *ib_dev) 1527 { 1528 if (isert_cmd->pdu_buf_dma != 0) { 1529 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1530 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1531 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1532 isert_cmd->pdu_buf_dma = 0; 1533 } 1534 1535 isert_unmap_tx_desc(tx_desc, ib_dev); 1536 isert_put_cmd(isert_cmd); 1537 } 1538 1539 static void 1540 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1541 struct isert_cmd *isert_cmd) 1542 { 1543 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1544 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1545 struct se_cmd *se_cmd = &cmd->se_cmd; 1546 struct isert_conn *isert_conn = isert_cmd->conn; 1547 struct isert_device *device = isert_conn->conn_device; 1548 1549 iscsit_stop_dataout_timer(cmd); 1550 device->unreg_rdma_mem(isert_cmd, isert_conn); 1551 cmd->write_data_done = wr->cur_rdma_length; 1552 1553 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1554 spin_lock_bh(&cmd->istate_lock); 1555 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1556 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1557 spin_unlock_bh(&cmd->istate_lock); 1558 1559 target_execute_cmd(se_cmd); 1560 } 1561 1562 static void 1563 isert_do_control_comp(struct work_struct *work) 1564 { 1565 struct isert_cmd *isert_cmd = container_of(work, 1566 struct isert_cmd, comp_work); 1567 struct isert_conn *isert_conn = isert_cmd->conn; 1568 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1569 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1570 1571 switch (cmd->i_state) { 1572 case ISTATE_SEND_TASKMGTRSP: 1573 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1574 1575 atomic_dec(&isert_conn->post_send_buf_count); 1576 iscsit_tmr_post_handler(cmd, cmd->conn); 1577 1578 cmd->i_state = ISTATE_SENT_STATUS; 1579 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1580 break; 1581 case ISTATE_SEND_REJECT: 1582 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1583 atomic_dec(&isert_conn->post_send_buf_count); 1584 1585 cmd->i_state = ISTATE_SENT_STATUS; 1586 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1587 break; 1588 case ISTATE_SEND_LOGOUTRSP: 1589 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1590 /* 1591 * Call atomic_dec(&isert_conn->post_send_buf_count) 1592 * from isert_free_conn() 1593 */ 1594 isert_conn->logout_posted = true; 1595 iscsit_logout_post_handler(cmd, cmd->conn); 1596 break; 1597 case ISTATE_SEND_TEXTRSP: 1598 atomic_dec(&isert_conn->post_send_buf_count); 1599 cmd->i_state = ISTATE_SENT_STATUS; 1600 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1601 break; 1602 default: 1603 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1604 dump_stack(); 1605 break; 1606 } 1607 } 1608 1609 static void 1610 isert_response_completion(struct iser_tx_desc *tx_desc, 1611 struct isert_cmd *isert_cmd, 1612 struct isert_conn *isert_conn, 1613 struct ib_device *ib_dev) 1614 { 1615 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1616 1617 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1618 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1619 cmd->i_state == ISTATE_SEND_REJECT || 1620 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1621 isert_unmap_tx_desc(tx_desc, ib_dev); 1622 1623 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1624 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1625 return; 1626 } 1627 atomic_dec(&isert_conn->post_send_buf_count); 1628 1629 cmd->i_state = ISTATE_SENT_STATUS; 1630 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1631 } 1632 1633 static void 1634 __isert_send_completion(struct iser_tx_desc *tx_desc, 1635 struct isert_conn *isert_conn) 1636 { 1637 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1638 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1639 struct isert_rdma_wr *wr; 1640 1641 if (!isert_cmd) { 1642 atomic_dec(&isert_conn->post_send_buf_count); 1643 isert_unmap_tx_desc(tx_desc, ib_dev); 1644 return; 1645 } 1646 wr = &isert_cmd->rdma_wr; 1647 1648 switch (wr->iser_ib_op) { 1649 case ISER_IB_RECV: 1650 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1651 dump_stack(); 1652 break; 1653 case ISER_IB_SEND: 1654 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1655 isert_response_completion(tx_desc, isert_cmd, 1656 isert_conn, ib_dev); 1657 break; 1658 case ISER_IB_RDMA_WRITE: 1659 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1660 dump_stack(); 1661 break; 1662 case ISER_IB_RDMA_READ: 1663 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1664 1665 atomic_dec(&isert_conn->post_send_buf_count); 1666 isert_completion_rdma_read(tx_desc, isert_cmd); 1667 break; 1668 default: 1669 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1670 dump_stack(); 1671 break; 1672 } 1673 } 1674 1675 static void 1676 isert_send_completion(struct iser_tx_desc *tx_desc, 1677 struct isert_conn *isert_conn) 1678 { 1679 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1680 struct iser_tx_desc *t; 1681 /* 1682 * Drain coalesced completion llist starting from comp_llnode_batch 1683 * setup in isert_init_send_wr(), and then complete trailing tx_desc. 1684 */ 1685 while (llnode) { 1686 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1687 llnode = llist_next(llnode); 1688 __isert_send_completion(t, isert_conn); 1689 } 1690 __isert_send_completion(tx_desc, isert_conn); 1691 } 1692 1693 static void 1694 isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1695 { 1696 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1697 1698 if (tx_desc) { 1699 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1700 1701 if (!isert_cmd) 1702 isert_unmap_tx_desc(tx_desc, ib_dev); 1703 else 1704 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1705 } 1706 1707 if (isert_conn->post_recv_buf_count == 0 && 1708 atomic_read(&isert_conn->post_send_buf_count) == 0) { 1709 pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 1710 pr_debug("Calling wake_up from isert_cq_comp_err\n"); 1711 1712 mutex_lock(&isert_conn->conn_mutex); 1713 if (isert_conn->state != ISER_CONN_DOWN) 1714 isert_conn->state = ISER_CONN_TERMINATING; 1715 mutex_unlock(&isert_conn->conn_mutex); 1716 1717 wake_up(&isert_conn->conn_wait_comp_err); 1718 } 1719 } 1720 1721 static void 1722 isert_cq_tx_work(struct work_struct *work) 1723 { 1724 struct isert_cq_desc *cq_desc = container_of(work, 1725 struct isert_cq_desc, cq_tx_work); 1726 struct isert_device *device = cq_desc->device; 1727 int cq_index = cq_desc->cq_index; 1728 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 1729 struct isert_conn *isert_conn; 1730 struct iser_tx_desc *tx_desc; 1731 struct ib_wc wc; 1732 1733 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 1734 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 1735 isert_conn = wc.qp->qp_context; 1736 1737 if (wc.status == IB_WC_SUCCESS) { 1738 isert_send_completion(tx_desc, isert_conn); 1739 } else { 1740 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1741 pr_debug("TX wc.status: 0x%08x\n", wc.status); 1742 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 1743 atomic_dec(&isert_conn->post_send_buf_count); 1744 isert_cq_comp_err(tx_desc, isert_conn); 1745 } 1746 } 1747 1748 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 1749 } 1750 1751 static void 1752 isert_cq_tx_callback(struct ib_cq *cq, void *context) 1753 { 1754 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1755 1756 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 1757 } 1758 1759 static void 1760 isert_cq_rx_work(struct work_struct *work) 1761 { 1762 struct isert_cq_desc *cq_desc = container_of(work, 1763 struct isert_cq_desc, cq_rx_work); 1764 struct isert_device *device = cq_desc->device; 1765 int cq_index = cq_desc->cq_index; 1766 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 1767 struct isert_conn *isert_conn; 1768 struct iser_rx_desc *rx_desc; 1769 struct ib_wc wc; 1770 unsigned long xfer_len; 1771 1772 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 1773 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 1774 isert_conn = wc.qp->qp_context; 1775 1776 if (wc.status == IB_WC_SUCCESS) { 1777 xfer_len = (unsigned long)wc.byte_len; 1778 isert_rx_completion(rx_desc, isert_conn, xfer_len); 1779 } else { 1780 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1781 if (wc.status != IB_WC_WR_FLUSH_ERR) { 1782 pr_debug("RX wc.status: 0x%08x\n", wc.status); 1783 pr_debug("RX wc.vendor_err: 0x%08x\n", 1784 wc.vendor_err); 1785 } 1786 isert_conn->post_recv_buf_count--; 1787 isert_cq_comp_err(NULL, isert_conn); 1788 } 1789 } 1790 1791 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 1792 } 1793 1794 static void 1795 isert_cq_rx_callback(struct ib_cq *cq, void *context) 1796 { 1797 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1798 1799 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 1800 } 1801 1802 static int 1803 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1804 { 1805 struct ib_send_wr *wr_failed; 1806 int ret; 1807 1808 atomic_inc(&isert_conn->post_send_buf_count); 1809 1810 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 1811 &wr_failed); 1812 if (ret) { 1813 pr_err("ib_post_send failed with %d\n", ret); 1814 atomic_dec(&isert_conn->post_send_buf_count); 1815 return ret; 1816 } 1817 return ret; 1818 } 1819 1820 static int 1821 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1822 { 1823 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1824 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1825 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1826 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1827 &isert_cmd->tx_desc.iscsi_header; 1828 1829 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1830 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1831 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1832 /* 1833 * Attach SENSE DATA payload to iSCSI Response PDU 1834 */ 1835 if (cmd->se_cmd.sense_buffer && 1836 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1837 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1838 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1839 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1840 u32 padding, pdu_len; 1841 1842 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1843 cmd->sense_buffer); 1844 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1845 1846 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1847 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1848 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1849 1850 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1851 (void *)cmd->sense_buffer, pdu_len, 1852 DMA_TO_DEVICE); 1853 1854 isert_cmd->pdu_buf_len = pdu_len; 1855 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1856 tx_dsg->length = pdu_len; 1857 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1858 isert_cmd->tx_desc.num_sge = 2; 1859 } 1860 1861 isert_init_send_wr(isert_conn, isert_cmd, send_wr, true); 1862 1863 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1864 1865 return isert_post_response(isert_conn, isert_cmd); 1866 } 1867 1868 static int 1869 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1870 bool nopout_response) 1871 { 1872 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1873 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1874 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1875 1876 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1877 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1878 &isert_cmd->tx_desc.iscsi_header, 1879 nopout_response); 1880 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1881 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1882 1883 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1884 1885 return isert_post_response(isert_conn, isert_cmd); 1886 } 1887 1888 static int 1889 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1890 { 1891 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1892 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1893 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1894 1895 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1896 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1897 &isert_cmd->tx_desc.iscsi_header); 1898 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1899 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1900 1901 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1902 1903 return isert_post_response(isert_conn, isert_cmd); 1904 } 1905 1906 static int 1907 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1908 { 1909 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1910 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1911 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1912 1913 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1914 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1915 &isert_cmd->tx_desc.iscsi_header); 1916 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1917 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1918 1919 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1920 1921 return isert_post_response(isert_conn, isert_cmd); 1922 } 1923 1924 static int 1925 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1926 { 1927 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1928 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1929 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1930 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1931 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1932 struct iscsi_reject *hdr = 1933 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1934 1935 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1936 iscsit_build_reject(cmd, conn, hdr); 1937 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1938 1939 hton24(hdr->dlength, ISCSI_HDR_LEN); 1940 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1941 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1942 DMA_TO_DEVICE); 1943 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1944 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1945 tx_dsg->length = ISCSI_HDR_LEN; 1946 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1947 isert_cmd->tx_desc.num_sge = 2; 1948 1949 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1950 1951 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1952 1953 return isert_post_response(isert_conn, isert_cmd); 1954 } 1955 1956 static int 1957 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1958 { 1959 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1960 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1961 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1962 struct iscsi_text_rsp *hdr = 1963 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1964 u32 txt_rsp_len; 1965 int rc; 1966 1967 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1968 rc = iscsit_build_text_rsp(cmd, conn, hdr); 1969 if (rc < 0) 1970 return rc; 1971 1972 txt_rsp_len = rc; 1973 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1974 1975 if (txt_rsp_len) { 1976 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1977 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1978 void *txt_rsp_buf = cmd->buf_ptr; 1979 1980 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1981 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1982 1983 isert_cmd->pdu_buf_len = txt_rsp_len; 1984 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1985 tx_dsg->length = txt_rsp_len; 1986 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1987 isert_cmd->tx_desc.num_sge = 2; 1988 } 1989 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1990 1991 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1992 1993 return isert_post_response(isert_conn, isert_cmd); 1994 } 1995 1996 static int 1997 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1998 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 1999 u32 data_left, u32 offset) 2000 { 2001 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2002 struct scatterlist *sg_start, *tmp_sg; 2003 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2004 u32 sg_off, page_off; 2005 int i = 0, sg_nents; 2006 2007 sg_off = offset / PAGE_SIZE; 2008 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2009 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 2010 page_off = offset % PAGE_SIZE; 2011 2012 send_wr->sg_list = ib_sge; 2013 send_wr->num_sge = sg_nents; 2014 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2015 /* 2016 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2017 */ 2018 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2019 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2020 (unsigned long long)tmp_sg->dma_address, 2021 tmp_sg->length, page_off); 2022 2023 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2024 ib_sge->length = min_t(u32, data_left, 2025 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2026 ib_sge->lkey = isert_conn->conn_mr->lkey; 2027 2028 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2029 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2030 page_off = 0; 2031 data_left -= ib_sge->length; 2032 ib_sge++; 2033 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2034 } 2035 2036 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2037 send_wr->sg_list, send_wr->num_sge); 2038 2039 return sg_nents; 2040 } 2041 2042 static int 2043 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2044 struct isert_rdma_wr *wr) 2045 { 2046 struct se_cmd *se_cmd = &cmd->se_cmd; 2047 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2048 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2049 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2050 struct ib_send_wr *send_wr; 2051 struct ib_sge *ib_sge; 2052 struct scatterlist *sg_start; 2053 u32 sg_off = 0, sg_nents; 2054 u32 offset = 0, data_len, data_left, rdma_write_max, va_offset = 0; 2055 int ret = 0, count, i, ib_sge_cnt; 2056 2057 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2058 data_left = se_cmd->data_length; 2059 } else { 2060 sg_off = cmd->write_data_done / PAGE_SIZE; 2061 data_left = se_cmd->data_length - cmd->write_data_done; 2062 offset = cmd->write_data_done; 2063 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2064 } 2065 2066 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2067 sg_nents = se_cmd->t_data_nents - sg_off; 2068 2069 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2070 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2071 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2072 if (unlikely(!count)) { 2073 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2074 return -EINVAL; 2075 } 2076 wr->sge = sg_start; 2077 wr->num_sge = sg_nents; 2078 wr->cur_rdma_length = data_left; 2079 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2080 isert_cmd, count, sg_start, sg_nents, data_left); 2081 2082 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 2083 if (!ib_sge) { 2084 pr_warn("Unable to allocate ib_sge\n"); 2085 ret = -ENOMEM; 2086 goto unmap_sg; 2087 } 2088 wr->ib_sge = ib_sge; 2089 2090 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 2091 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2092 GFP_KERNEL); 2093 if (!wr->send_wr) { 2094 pr_debug("Unable to allocate wr->send_wr\n"); 2095 ret = -ENOMEM; 2096 goto unmap_sg; 2097 } 2098 2099 wr->isert_cmd = isert_cmd; 2100 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2101 2102 for (i = 0; i < wr->send_wr_num; i++) { 2103 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2104 data_len = min(data_left, rdma_write_max); 2105 2106 send_wr->send_flags = 0; 2107 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2108 send_wr->opcode = IB_WR_RDMA_WRITE; 2109 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2110 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2111 if (i + 1 == wr->send_wr_num) 2112 send_wr->next = &isert_cmd->tx_desc.send_wr; 2113 else 2114 send_wr->next = &wr->send_wr[i + 1]; 2115 } else { 2116 send_wr->opcode = IB_WR_RDMA_READ; 2117 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2118 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2119 if (i + 1 == wr->send_wr_num) 2120 send_wr->send_flags = IB_SEND_SIGNALED; 2121 else 2122 send_wr->next = &wr->send_wr[i + 1]; 2123 } 2124 2125 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2126 send_wr, data_len, offset); 2127 ib_sge += ib_sge_cnt; 2128 2129 offset += data_len; 2130 va_offset += data_len; 2131 data_left -= data_len; 2132 } 2133 2134 return 0; 2135 unmap_sg: 2136 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2137 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2138 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2139 return ret; 2140 } 2141 2142 static int 2143 isert_map_fr_pagelist(struct ib_device *ib_dev, 2144 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2145 { 2146 u64 start_addr, end_addr, page, chunk_start = 0; 2147 struct scatterlist *tmp_sg; 2148 int i = 0, new_chunk, last_ent, n_pages; 2149 2150 n_pages = 0; 2151 new_chunk = 1; 2152 last_ent = sg_nents - 1; 2153 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2154 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2155 if (new_chunk) 2156 chunk_start = start_addr; 2157 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2158 2159 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 2160 i, (unsigned long long)tmp_sg->dma_address, 2161 tmp_sg->length); 2162 2163 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2164 new_chunk = 0; 2165 continue; 2166 } 2167 new_chunk = 1; 2168 2169 page = chunk_start & PAGE_MASK; 2170 do { 2171 fr_pl[n_pages++] = page; 2172 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 2173 n_pages - 1, page); 2174 page += PAGE_SIZE; 2175 } while (page < end_addr); 2176 } 2177 2178 return n_pages; 2179 } 2180 2181 static int 2182 isert_fast_reg_mr(struct fast_reg_descriptor *fr_desc, 2183 struct isert_conn *isert_conn, struct scatterlist *sg_start, 2184 struct ib_sge *ib_sge, u32 sg_nents, u32 offset, 2185 unsigned int data_len) 2186 { 2187 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2188 struct ib_send_wr fr_wr, inv_wr; 2189 struct ib_send_wr *bad_wr, *wr = NULL; 2190 int ret, pagelist_len; 2191 u32 page_off; 2192 u8 key; 2193 2194 sg_nents = min_t(unsigned int, sg_nents, ISCSI_ISER_SG_TABLESIZE); 2195 page_off = offset % PAGE_SIZE; 2196 2197 pr_debug("Use fr_desc %p sg_nents %d offset %u\n", 2198 fr_desc, sg_nents, offset); 2199 2200 pagelist_len = isert_map_fr_pagelist(ib_dev, sg_start, sg_nents, 2201 &fr_desc->data_frpl->page_list[0]); 2202 2203 if (!fr_desc->valid) { 2204 memset(&inv_wr, 0, sizeof(inv_wr)); 2205 inv_wr.opcode = IB_WR_LOCAL_INV; 2206 inv_wr.ex.invalidate_rkey = fr_desc->data_mr->rkey; 2207 wr = &inv_wr; 2208 /* Bump the key */ 2209 key = (u8)(fr_desc->data_mr->rkey & 0x000000FF); 2210 ib_update_fast_reg_key(fr_desc->data_mr, ++key); 2211 } 2212 2213 /* Prepare FASTREG WR */ 2214 memset(&fr_wr, 0, sizeof(fr_wr)); 2215 fr_wr.opcode = IB_WR_FAST_REG_MR; 2216 fr_wr.wr.fast_reg.iova_start = 2217 fr_desc->data_frpl->page_list[0] + page_off; 2218 fr_wr.wr.fast_reg.page_list = fr_desc->data_frpl; 2219 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2220 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2221 fr_wr.wr.fast_reg.length = data_len; 2222 fr_wr.wr.fast_reg.rkey = fr_desc->data_mr->rkey; 2223 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2224 2225 if (!wr) 2226 wr = &fr_wr; 2227 else 2228 wr->next = &fr_wr; 2229 2230 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2231 if (ret) { 2232 pr_err("fast registration failed, ret:%d\n", ret); 2233 return ret; 2234 } 2235 fr_desc->valid = false; 2236 2237 ib_sge->lkey = fr_desc->data_mr->lkey; 2238 ib_sge->addr = fr_desc->data_frpl->page_list[0] + page_off; 2239 ib_sge->length = data_len; 2240 2241 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2242 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2243 2244 return ret; 2245 } 2246 2247 static int 2248 isert_reg_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2249 struct isert_rdma_wr *wr) 2250 { 2251 struct se_cmd *se_cmd = &cmd->se_cmd; 2252 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2253 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2254 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2255 struct ib_send_wr *send_wr; 2256 struct ib_sge *ib_sge; 2257 struct scatterlist *sg_start; 2258 struct fast_reg_descriptor *fr_desc; 2259 u32 sg_off = 0, sg_nents; 2260 u32 offset = 0, data_len, data_left, rdma_write_max; 2261 int ret = 0, count; 2262 unsigned long flags; 2263 2264 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2265 data_left = se_cmd->data_length; 2266 } else { 2267 offset = cmd->write_data_done; 2268 sg_off = offset / PAGE_SIZE; 2269 data_left = se_cmd->data_length - cmd->write_data_done; 2270 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2271 } 2272 2273 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2274 sg_nents = se_cmd->t_data_nents - sg_off; 2275 2276 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2277 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2278 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2279 if (unlikely(!count)) { 2280 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2281 return -EINVAL; 2282 } 2283 wr->sge = sg_start; 2284 wr->num_sge = sg_nents; 2285 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2286 isert_cmd, count, sg_start, sg_nents, data_left); 2287 2288 memset(&wr->s_ib_sge, 0, sizeof(*ib_sge)); 2289 ib_sge = &wr->s_ib_sge; 2290 wr->ib_sge = ib_sge; 2291 2292 wr->send_wr_num = 1; 2293 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2294 wr->send_wr = &wr->s_send_wr; 2295 2296 wr->isert_cmd = isert_cmd; 2297 rdma_write_max = ISCSI_ISER_SG_TABLESIZE * PAGE_SIZE; 2298 2299 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2300 send_wr->sg_list = ib_sge; 2301 send_wr->num_sge = 1; 2302 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2303 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2304 send_wr->opcode = IB_WR_RDMA_WRITE; 2305 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2306 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2307 send_wr->send_flags = 0; 2308 send_wr->next = &isert_cmd->tx_desc.send_wr; 2309 } else { 2310 send_wr->opcode = IB_WR_RDMA_READ; 2311 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2312 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2313 send_wr->send_flags = IB_SEND_SIGNALED; 2314 } 2315 2316 data_len = min(data_left, rdma_write_max); 2317 wr->cur_rdma_length = data_len; 2318 2319 /* if there is a single dma entry, dma mr is sufficient */ 2320 if (count == 1) { 2321 ib_sge->addr = ib_sg_dma_address(ib_dev, &sg_start[0]); 2322 ib_sge->length = ib_sg_dma_len(ib_dev, &sg_start[0]); 2323 ib_sge->lkey = isert_conn->conn_mr->lkey; 2324 wr->fr_desc = NULL; 2325 } else { 2326 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2327 fr_desc = list_first_entry(&isert_conn->conn_fr_pool, 2328 struct fast_reg_descriptor, list); 2329 list_del(&fr_desc->list); 2330 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2331 wr->fr_desc = fr_desc; 2332 2333 ret = isert_fast_reg_mr(fr_desc, isert_conn, sg_start, 2334 ib_sge, sg_nents, offset, data_len); 2335 if (ret) { 2336 list_add_tail(&fr_desc->list, &isert_conn->conn_fr_pool); 2337 goto unmap_sg; 2338 } 2339 } 2340 2341 return 0; 2342 2343 unmap_sg: 2344 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2345 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2346 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2347 return ret; 2348 } 2349 2350 static int 2351 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2352 { 2353 struct se_cmd *se_cmd = &cmd->se_cmd; 2354 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2355 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2356 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2357 struct isert_device *device = isert_conn->conn_device; 2358 struct ib_send_wr *wr_failed; 2359 int rc; 2360 2361 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 2362 isert_cmd, se_cmd->data_length); 2363 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2364 rc = device->reg_rdma_mem(conn, cmd, wr); 2365 if (rc) { 2366 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2367 return rc; 2368 } 2369 2370 /* 2371 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2372 */ 2373 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2374 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2375 &isert_cmd->tx_desc.iscsi_header); 2376 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2377 isert_init_send_wr(isert_conn, isert_cmd, 2378 &isert_cmd->tx_desc.send_wr, true); 2379 2380 atomic_inc(&isert_conn->post_send_buf_count); 2381 2382 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2383 if (rc) { 2384 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2385 atomic_dec(&isert_conn->post_send_buf_count); 2386 } 2387 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data READ\n", 2388 isert_cmd); 2389 2390 return 1; 2391 } 2392 2393 static int 2394 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2395 { 2396 struct se_cmd *se_cmd = &cmd->se_cmd; 2397 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2398 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2399 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2400 struct isert_device *device = isert_conn->conn_device; 2401 struct ib_send_wr *wr_failed; 2402 int rc; 2403 2404 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2405 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2406 wr->iser_ib_op = ISER_IB_RDMA_READ; 2407 rc = device->reg_rdma_mem(conn, cmd, wr); 2408 if (rc) { 2409 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2410 return rc; 2411 } 2412 2413 atomic_inc(&isert_conn->post_send_buf_count); 2414 2415 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2416 if (rc) { 2417 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2418 atomic_dec(&isert_conn->post_send_buf_count); 2419 } 2420 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2421 isert_cmd); 2422 2423 return 0; 2424 } 2425 2426 static int 2427 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2428 { 2429 int ret; 2430 2431 switch (state) { 2432 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2433 ret = isert_put_nopin(cmd, conn, false); 2434 break; 2435 default: 2436 pr_err("Unknown immediate state: 0x%02x\n", state); 2437 ret = -EINVAL; 2438 break; 2439 } 2440 2441 return ret; 2442 } 2443 2444 static int 2445 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2446 { 2447 int ret; 2448 2449 switch (state) { 2450 case ISTATE_SEND_LOGOUTRSP: 2451 ret = isert_put_logout_rsp(cmd, conn); 2452 if (!ret) { 2453 pr_debug("Returning iSER Logout -EAGAIN\n"); 2454 ret = -EAGAIN; 2455 } 2456 break; 2457 case ISTATE_SEND_NOPIN: 2458 ret = isert_put_nopin(cmd, conn, true); 2459 break; 2460 case ISTATE_SEND_TASKMGTRSP: 2461 ret = isert_put_tm_rsp(cmd, conn); 2462 break; 2463 case ISTATE_SEND_REJECT: 2464 ret = isert_put_reject(cmd, conn); 2465 break; 2466 case ISTATE_SEND_TEXTRSP: 2467 ret = isert_put_text_rsp(cmd, conn); 2468 break; 2469 case ISTATE_SEND_STATUS: 2470 /* 2471 * Special case for sending non GOOD SCSI status from TX thread 2472 * context during pre se_cmd excecution failure. 2473 */ 2474 ret = isert_put_response(conn, cmd); 2475 break; 2476 default: 2477 pr_err("Unknown response state: 0x%02x\n", state); 2478 ret = -EINVAL; 2479 break; 2480 } 2481 2482 return ret; 2483 } 2484 2485 static int 2486 isert_setup_np(struct iscsi_np *np, 2487 struct __kernel_sockaddr_storage *ksockaddr) 2488 { 2489 struct isert_np *isert_np; 2490 struct rdma_cm_id *isert_lid; 2491 struct sockaddr *sa; 2492 int ret; 2493 2494 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2495 if (!isert_np) { 2496 pr_err("Unable to allocate struct isert_np\n"); 2497 return -ENOMEM; 2498 } 2499 init_waitqueue_head(&isert_np->np_accept_wq); 2500 mutex_init(&isert_np->np_accept_mutex); 2501 INIT_LIST_HEAD(&isert_np->np_accept_list); 2502 init_completion(&isert_np->np_login_comp); 2503 2504 sa = (struct sockaddr *)ksockaddr; 2505 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 2506 /* 2507 * Setup the np->np_sockaddr from the passed sockaddr setup 2508 * in iscsi_target_configfs.c code.. 2509 */ 2510 memcpy(&np->np_sockaddr, ksockaddr, 2511 sizeof(struct __kernel_sockaddr_storage)); 2512 2513 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 2514 IB_QPT_RC); 2515 if (IS_ERR(isert_lid)) { 2516 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 2517 PTR_ERR(isert_lid)); 2518 ret = PTR_ERR(isert_lid); 2519 goto out; 2520 } 2521 2522 ret = rdma_bind_addr(isert_lid, sa); 2523 if (ret) { 2524 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 2525 goto out_lid; 2526 } 2527 2528 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 2529 if (ret) { 2530 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 2531 goto out_lid; 2532 } 2533 2534 isert_np->np_cm_id = isert_lid; 2535 np->np_context = isert_np; 2536 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 2537 2538 return 0; 2539 2540 out_lid: 2541 rdma_destroy_id(isert_lid); 2542 out: 2543 kfree(isert_np); 2544 return ret; 2545 } 2546 2547 static int 2548 isert_check_accept_queue(struct isert_np *isert_np) 2549 { 2550 int empty; 2551 2552 mutex_lock(&isert_np->np_accept_mutex); 2553 empty = list_empty(&isert_np->np_accept_list); 2554 mutex_unlock(&isert_np->np_accept_mutex); 2555 2556 return empty; 2557 } 2558 2559 static int 2560 isert_rdma_accept(struct isert_conn *isert_conn) 2561 { 2562 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2563 struct rdma_conn_param cp; 2564 int ret; 2565 2566 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2567 cp.responder_resources = isert_conn->responder_resources; 2568 cp.initiator_depth = isert_conn->initiator_depth; 2569 cp.retry_count = 7; 2570 cp.rnr_retry_count = 7; 2571 2572 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 2573 2574 ret = rdma_accept(cm_id, &cp); 2575 if (ret) { 2576 pr_err("rdma_accept() failed with: %d\n", ret); 2577 return ret; 2578 } 2579 2580 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 2581 2582 return 0; 2583 } 2584 2585 static int 2586 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2587 { 2588 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2589 int ret; 2590 2591 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 2592 /* 2593 * For login requests after the first PDU, isert_rx_login_req() will 2594 * kick schedule_delayed_work(&conn->login_work) as the packet is 2595 * received, which turns this callback from iscsi_target_do_login_rx() 2596 * into a NOP. 2597 */ 2598 if (!login->first_request) 2599 return 0; 2600 2601 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 2602 if (ret) 2603 return ret; 2604 2605 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 2606 return 0; 2607 } 2608 2609 static void 2610 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2611 struct isert_conn *isert_conn) 2612 { 2613 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2614 struct rdma_route *cm_route = &cm_id->route; 2615 struct sockaddr_in *sock_in; 2616 struct sockaddr_in6 *sock_in6; 2617 2618 conn->login_family = np->np_sockaddr.ss_family; 2619 2620 if (np->np_sockaddr.ss_family == AF_INET6) { 2621 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 2622 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 2623 &sock_in6->sin6_addr.in6_u); 2624 conn->login_port = ntohs(sock_in6->sin6_port); 2625 2626 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 2627 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 2628 &sock_in6->sin6_addr.in6_u); 2629 conn->local_port = ntohs(sock_in6->sin6_port); 2630 } else { 2631 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 2632 sprintf(conn->login_ip, "%pI4", 2633 &sock_in->sin_addr.s_addr); 2634 conn->login_port = ntohs(sock_in->sin_port); 2635 2636 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 2637 sprintf(conn->local_ip, "%pI4", 2638 &sock_in->sin_addr.s_addr); 2639 conn->local_port = ntohs(sock_in->sin_port); 2640 } 2641 } 2642 2643 static int 2644 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2645 { 2646 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2647 struct isert_conn *isert_conn; 2648 int max_accept = 0, ret; 2649 2650 accept_wait: 2651 ret = wait_event_interruptible(isert_np->np_accept_wq, 2652 !isert_check_accept_queue(isert_np) || 2653 np->np_thread_state == ISCSI_NP_THREAD_RESET); 2654 if (max_accept > 5) 2655 return -ENODEV; 2656 2657 spin_lock_bh(&np->np_thread_lock); 2658 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 2659 spin_unlock_bh(&np->np_thread_lock); 2660 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 2661 return -ENODEV; 2662 } 2663 spin_unlock_bh(&np->np_thread_lock); 2664 2665 mutex_lock(&isert_np->np_accept_mutex); 2666 if (list_empty(&isert_np->np_accept_list)) { 2667 mutex_unlock(&isert_np->np_accept_mutex); 2668 max_accept++; 2669 goto accept_wait; 2670 } 2671 isert_conn = list_first_entry(&isert_np->np_accept_list, 2672 struct isert_conn, conn_accept_node); 2673 list_del_init(&isert_conn->conn_accept_node); 2674 mutex_unlock(&isert_np->np_accept_mutex); 2675 2676 conn->context = isert_conn; 2677 isert_conn->conn = conn; 2678 max_accept = 0; 2679 2680 ret = isert_rdma_post_recvl(isert_conn); 2681 if (ret) 2682 return ret; 2683 2684 ret = isert_rdma_accept(isert_conn); 2685 if (ret) 2686 return ret; 2687 2688 isert_set_conn_info(np, conn, isert_conn); 2689 2690 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 2691 return 0; 2692 } 2693 2694 static void 2695 isert_free_np(struct iscsi_np *np) 2696 { 2697 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2698 2699 rdma_destroy_id(isert_np->np_cm_id); 2700 2701 np->np_context = NULL; 2702 kfree(isert_np); 2703 } 2704 2705 static int isert_check_state(struct isert_conn *isert_conn, int state) 2706 { 2707 int ret; 2708 2709 mutex_lock(&isert_conn->conn_mutex); 2710 ret = (isert_conn->state == state); 2711 mutex_unlock(&isert_conn->conn_mutex); 2712 2713 return ret; 2714 } 2715 2716 static void isert_free_conn(struct iscsi_conn *conn) 2717 { 2718 struct isert_conn *isert_conn = conn->context; 2719 2720 pr_debug("isert_free_conn: Starting \n"); 2721 /* 2722 * Decrement post_send_buf_count for special case when called 2723 * from isert_do_control_comp() -> iscsit_logout_post_handler() 2724 */ 2725 mutex_lock(&isert_conn->conn_mutex); 2726 if (isert_conn->logout_posted) 2727 atomic_dec(&isert_conn->post_send_buf_count); 2728 2729 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 2730 pr_debug("Calling rdma_disconnect from isert_free_conn\n"); 2731 rdma_disconnect(isert_conn->conn_cm_id); 2732 } 2733 /* 2734 * Only wait for conn_wait_comp_err if the isert_conn made it 2735 * into full feature phase.. 2736 */ 2737 if (isert_conn->state == ISER_CONN_UP) { 2738 pr_debug("isert_free_conn: Before wait_event comp_err %d\n", 2739 isert_conn->state); 2740 mutex_unlock(&isert_conn->conn_mutex); 2741 2742 wait_event(isert_conn->conn_wait_comp_err, 2743 (isert_check_state(isert_conn, ISER_CONN_TERMINATING))); 2744 2745 wait_event(isert_conn->conn_wait, 2746 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2747 2748 isert_put_conn(isert_conn); 2749 return; 2750 } 2751 if (isert_conn->state == ISER_CONN_INIT) { 2752 mutex_unlock(&isert_conn->conn_mutex); 2753 isert_put_conn(isert_conn); 2754 return; 2755 } 2756 pr_debug("isert_free_conn: wait_event conn_wait %d\n", 2757 isert_conn->state); 2758 mutex_unlock(&isert_conn->conn_mutex); 2759 2760 wait_event(isert_conn->conn_wait, 2761 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2762 2763 isert_put_conn(isert_conn); 2764 } 2765 2766 static struct iscsit_transport iser_target_transport = { 2767 .name = "IB/iSER", 2768 .transport_type = ISCSI_INFINIBAND, 2769 .priv_size = sizeof(struct isert_cmd), 2770 .owner = THIS_MODULE, 2771 .iscsit_setup_np = isert_setup_np, 2772 .iscsit_accept_np = isert_accept_np, 2773 .iscsit_free_np = isert_free_np, 2774 .iscsit_free_conn = isert_free_conn, 2775 .iscsit_get_login_rx = isert_get_login_rx, 2776 .iscsit_put_login_tx = isert_put_login_tx, 2777 .iscsit_immediate_queue = isert_immediate_queue, 2778 .iscsit_response_queue = isert_response_queue, 2779 .iscsit_get_dataout = isert_get_dataout, 2780 .iscsit_queue_data_in = isert_put_datain, 2781 .iscsit_queue_status = isert_put_response, 2782 }; 2783 2784 static int __init isert_init(void) 2785 { 2786 int ret; 2787 2788 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 2789 if (!isert_rx_wq) { 2790 pr_err("Unable to allocate isert_rx_wq\n"); 2791 return -ENOMEM; 2792 } 2793 2794 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 2795 if (!isert_comp_wq) { 2796 pr_err("Unable to allocate isert_comp_wq\n"); 2797 ret = -ENOMEM; 2798 goto destroy_rx_wq; 2799 } 2800 2801 iscsit_register_transport(&iser_target_transport); 2802 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2803 return 0; 2804 2805 destroy_rx_wq: 2806 destroy_workqueue(isert_rx_wq); 2807 return ret; 2808 } 2809 2810 static void __exit isert_exit(void) 2811 { 2812 destroy_workqueue(isert_comp_wq); 2813 destroy_workqueue(isert_rx_wq); 2814 iscsit_unregister_transport(&iser_target_transport); 2815 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 2816 } 2817 2818 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2819 MODULE_VERSION("0.1"); 2820 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2821 MODULE_LICENSE("GPL"); 2822 2823 module_init(isert_init); 2824 module_exit(isert_exit); 2825