1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <linux/llist.h> 26 #include <rdma/ib_verbs.h> 27 #include <rdma/rdma_cm.h> 28 #include <target/target_core_base.h> 29 #include <target/target_core_fabric.h> 30 #include <target/iscsi/iscsi_transport.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 39 static DEFINE_MUTEX(device_list_mutex); 40 static LIST_HEAD(device_list); 41 static struct workqueue_struct *isert_rx_wq; 42 static struct workqueue_struct *isert_comp_wq; 43 44 static void 45 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 46 static int 47 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 48 struct isert_rdma_wr *wr); 49 static void 50 isert_unreg_rdma_frwr(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51 static int 52 isert_reg_rdma_frwr(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 53 struct isert_rdma_wr *wr); 54 55 static void 56 isert_qp_event_callback(struct ib_event *e, void *context) 57 { 58 struct isert_conn *isert_conn = (struct isert_conn *)context; 59 60 pr_err("isert_qp_event_callback event: %d\n", e->event); 61 switch (e->event) { 62 case IB_EVENT_COMM_EST: 63 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 64 break; 65 case IB_EVENT_QP_LAST_WQE_REACHED: 66 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 67 break; 68 default: 69 break; 70 } 71 } 72 73 static int 74 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 75 { 76 int ret; 77 78 ret = ib_query_device(ib_dev, devattr); 79 if (ret) { 80 pr_err("ib_query_device() failed: %d\n", ret); 81 return ret; 82 } 83 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 84 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 85 86 return 0; 87 } 88 89 static int 90 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 91 { 92 struct isert_device *device = isert_conn->conn_device; 93 struct ib_qp_init_attr attr; 94 int ret, index, min_index = 0; 95 96 mutex_lock(&device_list_mutex); 97 for (index = 0; index < device->cqs_used; index++) 98 if (device->cq_active_qps[index] < 99 device->cq_active_qps[min_index]) 100 min_index = index; 101 device->cq_active_qps[min_index]++; 102 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 103 mutex_unlock(&device_list_mutex); 104 105 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 106 attr.event_handler = isert_qp_event_callback; 107 attr.qp_context = isert_conn; 108 attr.send_cq = device->dev_tx_cq[min_index]; 109 attr.recv_cq = device->dev_rx_cq[min_index]; 110 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 111 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 112 /* 113 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 114 * work-around for RDMA_READ.. 115 */ 116 attr.cap.max_send_sge = device->dev_attr.max_sge - 2; 117 isert_conn->max_sge = attr.cap.max_send_sge; 118 119 attr.cap.max_recv_sge = 1; 120 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 121 attr.qp_type = IB_QPT_RC; 122 123 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 124 cma_id->device); 125 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 126 isert_conn->conn_pd->device); 127 128 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 129 if (ret) { 130 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 131 return ret; 132 } 133 isert_conn->conn_qp = cma_id->qp; 134 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 135 136 return 0; 137 } 138 139 static void 140 isert_cq_event_callback(struct ib_event *e, void *context) 141 { 142 pr_debug("isert_cq_event_callback event: %d\n", e->event); 143 } 144 145 static int 146 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 147 { 148 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 149 struct iser_rx_desc *rx_desc; 150 struct ib_sge *rx_sg; 151 u64 dma_addr; 152 int i, j; 153 154 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 155 sizeof(struct iser_rx_desc), GFP_KERNEL); 156 if (!isert_conn->conn_rx_descs) 157 goto fail; 158 159 rx_desc = isert_conn->conn_rx_descs; 160 161 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 162 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 163 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 164 if (ib_dma_mapping_error(ib_dev, dma_addr)) 165 goto dma_map_fail; 166 167 rx_desc->dma_addr = dma_addr; 168 169 rx_sg = &rx_desc->rx_sg; 170 rx_sg->addr = rx_desc->dma_addr; 171 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 172 rx_sg->lkey = isert_conn->conn_mr->lkey; 173 } 174 175 isert_conn->conn_rx_desc_head = 0; 176 return 0; 177 178 dma_map_fail: 179 rx_desc = isert_conn->conn_rx_descs; 180 for (j = 0; j < i; j++, rx_desc++) { 181 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 182 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 183 } 184 kfree(isert_conn->conn_rx_descs); 185 isert_conn->conn_rx_descs = NULL; 186 fail: 187 return -ENOMEM; 188 } 189 190 static void 191 isert_free_rx_descriptors(struct isert_conn *isert_conn) 192 { 193 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 194 struct iser_rx_desc *rx_desc; 195 int i; 196 197 if (!isert_conn->conn_rx_descs) 198 return; 199 200 rx_desc = isert_conn->conn_rx_descs; 201 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 202 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 203 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 204 } 205 206 kfree(isert_conn->conn_rx_descs); 207 isert_conn->conn_rx_descs = NULL; 208 } 209 210 static void isert_cq_tx_callback(struct ib_cq *, void *); 211 static void isert_cq_rx_callback(struct ib_cq *, void *); 212 213 static int 214 isert_create_device_ib_res(struct isert_device *device) 215 { 216 struct ib_device *ib_dev = device->ib_device; 217 struct isert_cq_desc *cq_desc; 218 struct ib_device_attr *dev_attr; 219 int ret = 0, i, j; 220 221 dev_attr = &device->dev_attr; 222 ret = isert_query_device(ib_dev, dev_attr); 223 if (ret) 224 return ret; 225 226 /* asign function handlers */ 227 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) { 228 device->use_frwr = 1; 229 device->reg_rdma_mem = isert_reg_rdma_frwr; 230 device->unreg_rdma_mem = isert_unreg_rdma_frwr; 231 } else { 232 device->use_frwr = 0; 233 device->reg_rdma_mem = isert_map_rdma; 234 device->unreg_rdma_mem = isert_unmap_cmd; 235 } 236 237 device->cqs_used = min_t(int, num_online_cpus(), 238 device->ib_device->num_comp_vectors); 239 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 240 pr_debug("Using %d CQs, device %s supports %d vectors support FRWR %d\n", 241 device->cqs_used, device->ib_device->name, 242 device->ib_device->num_comp_vectors, device->use_frwr); 243 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 244 device->cqs_used, GFP_KERNEL); 245 if (!device->cq_desc) { 246 pr_err("Unable to allocate device->cq_desc\n"); 247 return -ENOMEM; 248 } 249 cq_desc = device->cq_desc; 250 251 device->dev_pd = ib_alloc_pd(ib_dev); 252 if (IS_ERR(device->dev_pd)) { 253 ret = PTR_ERR(device->dev_pd); 254 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret); 255 goto out_cq_desc; 256 } 257 258 for (i = 0; i < device->cqs_used; i++) { 259 cq_desc[i].device = device; 260 cq_desc[i].cq_index = i; 261 262 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 263 isert_cq_rx_callback, 264 isert_cq_event_callback, 265 (void *)&cq_desc[i], 266 ISER_MAX_RX_CQ_LEN, i); 267 if (IS_ERR(device->dev_rx_cq[i])) 268 goto out_cq; 269 270 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 271 isert_cq_tx_callback, 272 isert_cq_event_callback, 273 (void *)&cq_desc[i], 274 ISER_MAX_TX_CQ_LEN, i); 275 if (IS_ERR(device->dev_tx_cq[i])) 276 goto out_cq; 277 278 if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP)) 279 goto out_cq; 280 281 if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP)) 282 goto out_cq; 283 } 284 285 device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE); 286 if (IS_ERR(device->dev_mr)) { 287 ret = PTR_ERR(device->dev_mr); 288 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret); 289 goto out_cq; 290 } 291 292 return 0; 293 294 out_cq: 295 for (j = 0; j < i; j++) { 296 cq_desc = &device->cq_desc[j]; 297 298 if (device->dev_rx_cq[j]) { 299 cancel_work_sync(&cq_desc->cq_rx_work); 300 ib_destroy_cq(device->dev_rx_cq[j]); 301 } 302 if (device->dev_tx_cq[j]) { 303 cancel_work_sync(&cq_desc->cq_tx_work); 304 ib_destroy_cq(device->dev_tx_cq[j]); 305 } 306 } 307 ib_dealloc_pd(device->dev_pd); 308 309 out_cq_desc: 310 kfree(device->cq_desc); 311 312 return ret; 313 } 314 315 static void 316 isert_free_device_ib_res(struct isert_device *device) 317 { 318 struct isert_cq_desc *cq_desc; 319 int i; 320 321 for (i = 0; i < device->cqs_used; i++) { 322 cq_desc = &device->cq_desc[i]; 323 324 cancel_work_sync(&cq_desc->cq_rx_work); 325 cancel_work_sync(&cq_desc->cq_tx_work); 326 ib_destroy_cq(device->dev_rx_cq[i]); 327 ib_destroy_cq(device->dev_tx_cq[i]); 328 device->dev_rx_cq[i] = NULL; 329 device->dev_tx_cq[i] = NULL; 330 } 331 332 ib_dereg_mr(device->dev_mr); 333 ib_dealloc_pd(device->dev_pd); 334 kfree(device->cq_desc); 335 } 336 337 static void 338 isert_device_try_release(struct isert_device *device) 339 { 340 mutex_lock(&device_list_mutex); 341 device->refcount--; 342 if (!device->refcount) { 343 isert_free_device_ib_res(device); 344 list_del(&device->dev_node); 345 kfree(device); 346 } 347 mutex_unlock(&device_list_mutex); 348 } 349 350 static struct isert_device * 351 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 352 { 353 struct isert_device *device; 354 int ret; 355 356 mutex_lock(&device_list_mutex); 357 list_for_each_entry(device, &device_list, dev_node) { 358 if (device->ib_device->node_guid == cma_id->device->node_guid) { 359 device->refcount++; 360 mutex_unlock(&device_list_mutex); 361 return device; 362 } 363 } 364 365 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 366 if (!device) { 367 mutex_unlock(&device_list_mutex); 368 return ERR_PTR(-ENOMEM); 369 } 370 371 INIT_LIST_HEAD(&device->dev_node); 372 373 device->ib_device = cma_id->device; 374 ret = isert_create_device_ib_res(device); 375 if (ret) { 376 kfree(device); 377 mutex_unlock(&device_list_mutex); 378 return ERR_PTR(ret); 379 } 380 381 device->refcount++; 382 list_add_tail(&device->dev_node, &device_list); 383 mutex_unlock(&device_list_mutex); 384 385 return device; 386 } 387 388 static void 389 isert_conn_free_frwr_pool(struct isert_conn *isert_conn) 390 { 391 struct fast_reg_descriptor *fr_desc, *tmp; 392 int i = 0; 393 394 if (list_empty(&isert_conn->conn_frwr_pool)) 395 return; 396 397 pr_debug("Freeing conn %p frwr pool", isert_conn); 398 399 list_for_each_entry_safe(fr_desc, tmp, 400 &isert_conn->conn_frwr_pool, list) { 401 list_del(&fr_desc->list); 402 ib_free_fast_reg_page_list(fr_desc->data_frpl); 403 ib_dereg_mr(fr_desc->data_mr); 404 kfree(fr_desc); 405 ++i; 406 } 407 408 if (i < isert_conn->conn_frwr_pool_size) 409 pr_warn("Pool still has %d regions registered\n", 410 isert_conn->conn_frwr_pool_size - i); 411 } 412 413 static int 414 isert_conn_create_frwr_pool(struct isert_conn *isert_conn) 415 { 416 struct fast_reg_descriptor *fr_desc; 417 struct isert_device *device = isert_conn->conn_device; 418 int i, ret; 419 420 INIT_LIST_HEAD(&isert_conn->conn_frwr_pool); 421 isert_conn->conn_frwr_pool_size = 0; 422 for (i = 0; i < ISCSI_DEF_XMIT_CMDS_MAX; i++) { 423 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 424 if (!fr_desc) { 425 pr_err("Failed to allocate fast_reg descriptor\n"); 426 ret = -ENOMEM; 427 goto err; 428 } 429 430 fr_desc->data_frpl = 431 ib_alloc_fast_reg_page_list(device->ib_device, 432 ISCSI_ISER_SG_TABLESIZE); 433 if (IS_ERR(fr_desc->data_frpl)) { 434 pr_err("Failed to allocate fr_pg_list err=%ld\n", 435 PTR_ERR(fr_desc->data_frpl)); 436 ret = PTR_ERR(fr_desc->data_frpl); 437 goto err; 438 } 439 440 fr_desc->data_mr = ib_alloc_fast_reg_mr(device->dev_pd, 441 ISCSI_ISER_SG_TABLESIZE); 442 if (IS_ERR(fr_desc->data_mr)) { 443 pr_err("Failed to allocate frmr err=%ld\n", 444 PTR_ERR(fr_desc->data_mr)); 445 ret = PTR_ERR(fr_desc->data_mr); 446 ib_free_fast_reg_page_list(fr_desc->data_frpl); 447 goto err; 448 } 449 pr_debug("Create fr_desc %p page_list %p\n", 450 fr_desc, fr_desc->data_frpl->page_list); 451 452 fr_desc->valid = true; 453 list_add_tail(&fr_desc->list, &isert_conn->conn_frwr_pool); 454 isert_conn->conn_frwr_pool_size++; 455 } 456 457 pr_debug("Creating conn %p frwr pool size=%d", 458 isert_conn, isert_conn->conn_frwr_pool_size); 459 460 return 0; 461 462 err: 463 isert_conn_free_frwr_pool(isert_conn); 464 return ret; 465 } 466 467 static int 468 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 469 { 470 struct iscsi_np *np = cma_id->context; 471 struct isert_np *isert_np = np->np_context; 472 struct isert_conn *isert_conn; 473 struct isert_device *device; 474 struct ib_device *ib_dev = cma_id->device; 475 int ret = 0; 476 477 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 478 cma_id, cma_id->context); 479 480 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 481 if (!isert_conn) { 482 pr_err("Unable to allocate isert_conn\n"); 483 return -ENOMEM; 484 } 485 isert_conn->state = ISER_CONN_INIT; 486 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 487 init_completion(&isert_conn->conn_login_comp); 488 init_waitqueue_head(&isert_conn->conn_wait); 489 init_waitqueue_head(&isert_conn->conn_wait_comp_err); 490 kref_init(&isert_conn->conn_kref); 491 kref_get(&isert_conn->conn_kref); 492 mutex_init(&isert_conn->conn_mutex); 493 mutex_init(&isert_conn->conn_comp_mutex); 494 spin_lock_init(&isert_conn->conn_lock); 495 496 cma_id->context = isert_conn; 497 isert_conn->conn_cm_id = cma_id; 498 isert_conn->responder_resources = event->param.conn.responder_resources; 499 isert_conn->initiator_depth = event->param.conn.initiator_depth; 500 pr_debug("Using responder_resources: %u initiator_depth: %u\n", 501 isert_conn->responder_resources, isert_conn->initiator_depth); 502 503 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 504 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 505 if (!isert_conn->login_buf) { 506 pr_err("Unable to allocate isert_conn->login_buf\n"); 507 ret = -ENOMEM; 508 goto out; 509 } 510 511 isert_conn->login_req_buf = isert_conn->login_buf; 512 isert_conn->login_rsp_buf = isert_conn->login_buf + 513 ISCSI_DEF_MAX_RECV_SEG_LEN; 514 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 515 isert_conn->login_buf, isert_conn->login_req_buf, 516 isert_conn->login_rsp_buf); 517 518 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 519 (void *)isert_conn->login_req_buf, 520 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 521 522 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 523 if (ret) { 524 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 525 ret); 526 isert_conn->login_req_dma = 0; 527 goto out_login_buf; 528 } 529 530 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 531 (void *)isert_conn->login_rsp_buf, 532 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 533 534 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 535 if (ret) { 536 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 537 ret); 538 isert_conn->login_rsp_dma = 0; 539 goto out_req_dma_map; 540 } 541 542 device = isert_device_find_by_ib_dev(cma_id); 543 if (IS_ERR(device)) { 544 ret = PTR_ERR(device); 545 goto out_rsp_dma_map; 546 } 547 548 isert_conn->conn_device = device; 549 isert_conn->conn_pd = device->dev_pd; 550 isert_conn->conn_mr = device->dev_mr; 551 552 if (device->use_frwr) { 553 ret = isert_conn_create_frwr_pool(isert_conn); 554 if (ret) { 555 pr_err("Conn: %p failed to create frwr_pool\n", isert_conn); 556 goto out_frwr; 557 } 558 } 559 560 ret = isert_conn_setup_qp(isert_conn, cma_id); 561 if (ret) 562 goto out_conn_dev; 563 564 mutex_lock(&isert_np->np_accept_mutex); 565 list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 566 mutex_unlock(&isert_np->np_accept_mutex); 567 568 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 569 wake_up(&isert_np->np_accept_wq); 570 return 0; 571 572 out_conn_dev: 573 if (device->use_frwr) 574 isert_conn_free_frwr_pool(isert_conn); 575 out_frwr: 576 isert_device_try_release(device); 577 out_rsp_dma_map: 578 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 579 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 580 out_req_dma_map: 581 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 582 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 583 out_login_buf: 584 kfree(isert_conn->login_buf); 585 out: 586 kfree(isert_conn); 587 return ret; 588 } 589 590 static void 591 isert_connect_release(struct isert_conn *isert_conn) 592 { 593 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 594 struct isert_device *device = isert_conn->conn_device; 595 int cq_index; 596 597 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 598 599 if (device && device->use_frwr) 600 isert_conn_free_frwr_pool(isert_conn); 601 602 if (isert_conn->conn_qp) { 603 cq_index = ((struct isert_cq_desc *) 604 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 605 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 606 isert_conn->conn_device->cq_active_qps[cq_index]--; 607 608 rdma_destroy_qp(isert_conn->conn_cm_id); 609 } 610 611 isert_free_rx_descriptors(isert_conn); 612 rdma_destroy_id(isert_conn->conn_cm_id); 613 614 if (isert_conn->login_buf) { 615 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 616 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 617 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 618 ISCSI_DEF_MAX_RECV_SEG_LEN, 619 DMA_FROM_DEVICE); 620 kfree(isert_conn->login_buf); 621 } 622 kfree(isert_conn); 623 624 if (device) 625 isert_device_try_release(device); 626 627 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 628 } 629 630 static void 631 isert_connected_handler(struct rdma_cm_id *cma_id) 632 { 633 return; 634 } 635 636 static void 637 isert_release_conn_kref(struct kref *kref) 638 { 639 struct isert_conn *isert_conn = container_of(kref, 640 struct isert_conn, conn_kref); 641 642 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 643 current->comm, current->pid); 644 645 isert_connect_release(isert_conn); 646 } 647 648 static void 649 isert_put_conn(struct isert_conn *isert_conn) 650 { 651 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 652 } 653 654 static void 655 isert_disconnect_work(struct work_struct *work) 656 { 657 struct isert_conn *isert_conn = container_of(work, 658 struct isert_conn, conn_logout_work); 659 660 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 661 mutex_lock(&isert_conn->conn_mutex); 662 isert_conn->state = ISER_CONN_DOWN; 663 664 if (isert_conn->post_recv_buf_count == 0 && 665 atomic_read(&isert_conn->post_send_buf_count) == 0) { 666 pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); 667 mutex_unlock(&isert_conn->conn_mutex); 668 goto wake_up; 669 } 670 if (!isert_conn->conn_cm_id) { 671 mutex_unlock(&isert_conn->conn_mutex); 672 isert_put_conn(isert_conn); 673 return; 674 } 675 if (!isert_conn->logout_posted) { 676 pr_debug("Calling rdma_disconnect for !logout_posted from" 677 " isert_disconnect_work\n"); 678 rdma_disconnect(isert_conn->conn_cm_id); 679 mutex_unlock(&isert_conn->conn_mutex); 680 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 681 goto wake_up; 682 } 683 mutex_unlock(&isert_conn->conn_mutex); 684 685 wake_up: 686 wake_up(&isert_conn->conn_wait); 687 isert_put_conn(isert_conn); 688 } 689 690 static void 691 isert_disconnected_handler(struct rdma_cm_id *cma_id) 692 { 693 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 694 695 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 696 schedule_work(&isert_conn->conn_logout_work); 697 } 698 699 static int 700 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 701 { 702 int ret = 0; 703 704 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 705 event->event, event->status, cma_id->context, cma_id); 706 707 switch (event->event) { 708 case RDMA_CM_EVENT_CONNECT_REQUEST: 709 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 710 ret = isert_connect_request(cma_id, event); 711 break; 712 case RDMA_CM_EVENT_ESTABLISHED: 713 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 714 isert_connected_handler(cma_id); 715 break; 716 case RDMA_CM_EVENT_DISCONNECTED: 717 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 718 isert_disconnected_handler(cma_id); 719 break; 720 case RDMA_CM_EVENT_DEVICE_REMOVAL: 721 case RDMA_CM_EVENT_ADDR_CHANGE: 722 break; 723 case RDMA_CM_EVENT_CONNECT_ERROR: 724 default: 725 pr_err("Unknown RDMA CMA event: %d\n", event->event); 726 break; 727 } 728 729 if (ret != 0) { 730 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 731 event->event, ret); 732 dump_stack(); 733 } 734 735 return ret; 736 } 737 738 static int 739 isert_post_recv(struct isert_conn *isert_conn, u32 count) 740 { 741 struct ib_recv_wr *rx_wr, *rx_wr_failed; 742 int i, ret; 743 unsigned int rx_head = isert_conn->conn_rx_desc_head; 744 struct iser_rx_desc *rx_desc; 745 746 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 747 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 748 rx_wr->wr_id = (unsigned long)rx_desc; 749 rx_wr->sg_list = &rx_desc->rx_sg; 750 rx_wr->num_sge = 1; 751 rx_wr->next = rx_wr + 1; 752 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 753 } 754 755 rx_wr--; 756 rx_wr->next = NULL; /* mark end of work requests list */ 757 758 isert_conn->post_recv_buf_count += count; 759 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 760 &rx_wr_failed); 761 if (ret) { 762 pr_err("ib_post_recv() failed with ret: %d\n", ret); 763 isert_conn->post_recv_buf_count -= count; 764 } else { 765 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 766 isert_conn->conn_rx_desc_head = rx_head; 767 } 768 return ret; 769 } 770 771 static int 772 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 773 { 774 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 775 struct ib_send_wr send_wr, *send_wr_failed; 776 int ret; 777 778 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 779 ISER_HEADERS_LEN, DMA_TO_DEVICE); 780 781 send_wr.next = NULL; 782 send_wr.wr_id = (unsigned long)tx_desc; 783 send_wr.sg_list = tx_desc->tx_sg; 784 send_wr.num_sge = tx_desc->num_sge; 785 send_wr.opcode = IB_WR_SEND; 786 send_wr.send_flags = IB_SEND_SIGNALED; 787 788 atomic_inc(&isert_conn->post_send_buf_count); 789 790 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 791 if (ret) { 792 pr_err("ib_post_send() failed, ret: %d\n", ret); 793 atomic_dec(&isert_conn->post_send_buf_count); 794 } 795 796 return ret; 797 } 798 799 static void 800 isert_create_send_desc(struct isert_conn *isert_conn, 801 struct isert_cmd *isert_cmd, 802 struct iser_tx_desc *tx_desc) 803 { 804 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 805 806 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 807 ISER_HEADERS_LEN, DMA_TO_DEVICE); 808 809 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 810 tx_desc->iser_header.flags = ISER_VER; 811 812 tx_desc->num_sge = 1; 813 tx_desc->isert_cmd = isert_cmd; 814 815 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 816 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 817 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 818 } 819 } 820 821 static int 822 isert_init_tx_hdrs(struct isert_conn *isert_conn, 823 struct iser_tx_desc *tx_desc) 824 { 825 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 826 u64 dma_addr; 827 828 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 829 ISER_HEADERS_LEN, DMA_TO_DEVICE); 830 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 831 pr_err("ib_dma_mapping_error() failed\n"); 832 return -ENOMEM; 833 } 834 835 tx_desc->dma_addr = dma_addr; 836 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 837 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 838 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 839 840 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 841 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 842 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 843 844 return 0; 845 } 846 847 static void 848 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 849 struct ib_send_wr *send_wr, bool coalesce) 850 { 851 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 852 853 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 854 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 855 send_wr->opcode = IB_WR_SEND; 856 send_wr->sg_list = &tx_desc->tx_sg[0]; 857 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 858 /* 859 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 860 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 861 */ 862 mutex_lock(&isert_conn->conn_comp_mutex); 863 if (coalesce && 864 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 865 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 866 mutex_unlock(&isert_conn->conn_comp_mutex); 867 return; 868 } 869 isert_conn->conn_comp_batch = 0; 870 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 871 mutex_unlock(&isert_conn->conn_comp_mutex); 872 873 send_wr->send_flags = IB_SEND_SIGNALED; 874 } 875 876 static int 877 isert_rdma_post_recvl(struct isert_conn *isert_conn) 878 { 879 struct ib_recv_wr rx_wr, *rx_wr_fail; 880 struct ib_sge sge; 881 int ret; 882 883 memset(&sge, 0, sizeof(struct ib_sge)); 884 sge.addr = isert_conn->login_req_dma; 885 sge.length = ISER_RX_LOGIN_SIZE; 886 sge.lkey = isert_conn->conn_mr->lkey; 887 888 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 889 sge.addr, sge.length, sge.lkey); 890 891 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 892 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 893 rx_wr.sg_list = &sge; 894 rx_wr.num_sge = 1; 895 896 isert_conn->post_recv_buf_count++; 897 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 898 if (ret) { 899 pr_err("ib_post_recv() failed: %d\n", ret); 900 isert_conn->post_recv_buf_count--; 901 } 902 903 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 904 return ret; 905 } 906 907 static int 908 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 909 u32 length) 910 { 911 struct isert_conn *isert_conn = conn->context; 912 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 913 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 914 int ret; 915 916 isert_create_send_desc(isert_conn, NULL, tx_desc); 917 918 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 919 sizeof(struct iscsi_hdr)); 920 921 isert_init_tx_hdrs(isert_conn, tx_desc); 922 923 if (length > 0) { 924 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 925 926 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 927 length, DMA_TO_DEVICE); 928 929 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 930 931 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 932 length, DMA_TO_DEVICE); 933 934 tx_dsg->addr = isert_conn->login_rsp_dma; 935 tx_dsg->length = length; 936 tx_dsg->lkey = isert_conn->conn_mr->lkey; 937 tx_desc->num_sge = 2; 938 } 939 if (!login->login_failed) { 940 if (login->login_complete) { 941 ret = isert_alloc_rx_descriptors(isert_conn); 942 if (ret) 943 return ret; 944 945 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 946 if (ret) 947 return ret; 948 949 isert_conn->state = ISER_CONN_UP; 950 goto post_send; 951 } 952 953 ret = isert_rdma_post_recvl(isert_conn); 954 if (ret) 955 return ret; 956 } 957 post_send: 958 ret = isert_post_send(isert_conn, tx_desc); 959 if (ret) 960 return ret; 961 962 return 0; 963 } 964 965 static void 966 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 967 struct isert_conn *isert_conn) 968 { 969 struct iscsi_conn *conn = isert_conn->conn; 970 struct iscsi_login *login = conn->conn_login; 971 int size; 972 973 if (!login) { 974 pr_err("conn->conn_login is NULL\n"); 975 dump_stack(); 976 return; 977 } 978 979 if (login->first_request) { 980 struct iscsi_login_req *login_req = 981 (struct iscsi_login_req *)&rx_desc->iscsi_header; 982 /* 983 * Setup the initial iscsi_login values from the leading 984 * login request PDU. 985 */ 986 login->leading_connection = (!login_req->tsih) ? 1 : 0; 987 login->current_stage = 988 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 989 >> 2; 990 login->version_min = login_req->min_version; 991 login->version_max = login_req->max_version; 992 memcpy(login->isid, login_req->isid, 6); 993 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 994 login->init_task_tag = login_req->itt; 995 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 996 login->cid = be16_to_cpu(login_req->cid); 997 login->tsih = be16_to_cpu(login_req->tsih); 998 } 999 1000 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1001 1002 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1003 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1004 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1005 memcpy(login->req_buf, &rx_desc->data[0], size); 1006 1007 if (login->first_request) { 1008 complete(&isert_conn->conn_login_comp); 1009 return; 1010 } 1011 schedule_delayed_work(&conn->login_work, 0); 1012 } 1013 1014 static struct iscsi_cmd 1015 *isert_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp) 1016 { 1017 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1018 struct isert_cmd *isert_cmd; 1019 struct iscsi_cmd *cmd; 1020 1021 cmd = iscsit_allocate_cmd(conn, gfp); 1022 if (!cmd) { 1023 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1024 return NULL; 1025 } 1026 isert_cmd = iscsit_priv_cmd(cmd); 1027 isert_cmd->conn = isert_conn; 1028 isert_cmd->iscsi_cmd = cmd; 1029 1030 return cmd; 1031 } 1032 1033 static int 1034 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1035 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1036 struct iser_rx_desc *rx_desc, unsigned char *buf) 1037 { 1038 struct iscsi_conn *conn = isert_conn->conn; 1039 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1040 struct scatterlist *sg; 1041 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1042 bool dump_payload = false; 1043 1044 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1045 if (rc < 0) 1046 return rc; 1047 1048 imm_data = cmd->immediate_data; 1049 imm_data_len = cmd->first_burst_len; 1050 unsol_data = cmd->unsolicited_data; 1051 1052 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1053 if (rc < 0) { 1054 return 0; 1055 } else if (rc > 0) { 1056 dump_payload = true; 1057 goto sequence_cmd; 1058 } 1059 1060 if (!imm_data) 1061 return 0; 1062 1063 sg = &cmd->se_cmd.t_data_sg[0]; 1064 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1065 1066 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1067 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1068 1069 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1070 1071 cmd->write_data_done += imm_data_len; 1072 1073 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1074 spin_lock_bh(&cmd->istate_lock); 1075 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1076 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1077 spin_unlock_bh(&cmd->istate_lock); 1078 } 1079 1080 sequence_cmd: 1081 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1082 1083 if (!rc && dump_payload == false && unsol_data) 1084 iscsit_set_unsoliticed_dataout(cmd); 1085 1086 return 0; 1087 } 1088 1089 static int 1090 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1091 struct iser_rx_desc *rx_desc, unsigned char *buf) 1092 { 1093 struct scatterlist *sg_start; 1094 struct iscsi_conn *conn = isert_conn->conn; 1095 struct iscsi_cmd *cmd = NULL; 1096 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1097 u32 unsol_data_len = ntoh24(hdr->dlength); 1098 int rc, sg_nents, sg_off, page_off; 1099 1100 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1101 if (rc < 0) 1102 return rc; 1103 else if (!cmd) 1104 return 0; 1105 /* 1106 * FIXME: Unexpected unsolicited_data out 1107 */ 1108 if (!cmd->unsolicited_data) { 1109 pr_err("Received unexpected solicited data payload\n"); 1110 dump_stack(); 1111 return -1; 1112 } 1113 1114 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1115 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1116 1117 sg_off = cmd->write_data_done / PAGE_SIZE; 1118 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1119 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1120 page_off = cmd->write_data_done % PAGE_SIZE; 1121 /* 1122 * FIXME: Non page-aligned unsolicited_data out 1123 */ 1124 if (page_off) { 1125 pr_err("Received unexpected non-page aligned data payload\n"); 1126 dump_stack(); 1127 return -1; 1128 } 1129 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1130 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1131 1132 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1133 unsol_data_len); 1134 1135 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1136 if (rc < 0) 1137 return rc; 1138 1139 return 0; 1140 } 1141 1142 static int 1143 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1144 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1145 unsigned char *buf) 1146 { 1147 struct iscsi_conn *conn = isert_conn->conn; 1148 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1149 int rc; 1150 1151 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1152 if (rc < 0) 1153 return rc; 1154 /* 1155 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1156 */ 1157 1158 return iscsit_process_nop_out(conn, cmd, hdr); 1159 } 1160 1161 static int 1162 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1163 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1164 struct iscsi_text *hdr) 1165 { 1166 struct iscsi_conn *conn = isert_conn->conn; 1167 u32 payload_length = ntoh24(hdr->dlength); 1168 int rc; 1169 unsigned char *text_in; 1170 1171 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1172 if (rc < 0) 1173 return rc; 1174 1175 text_in = kzalloc(payload_length, GFP_KERNEL); 1176 if (!text_in) { 1177 pr_err("Unable to allocate text_in of payload_length: %u\n", 1178 payload_length); 1179 return -ENOMEM; 1180 } 1181 cmd->text_in_ptr = text_in; 1182 1183 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1184 1185 return iscsit_process_text_cmd(conn, cmd, hdr); 1186 } 1187 1188 static int 1189 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1190 uint32_t read_stag, uint64_t read_va, 1191 uint32_t write_stag, uint64_t write_va) 1192 { 1193 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1194 struct iscsi_conn *conn = isert_conn->conn; 1195 struct iscsi_session *sess = conn->sess; 1196 struct iscsi_cmd *cmd; 1197 struct isert_cmd *isert_cmd; 1198 int ret = -EINVAL; 1199 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1200 1201 if (sess->sess_ops->SessionType && 1202 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1203 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1204 " ignoring\n", opcode); 1205 return 0; 1206 } 1207 1208 switch (opcode) { 1209 case ISCSI_OP_SCSI_CMD: 1210 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1211 if (!cmd) 1212 break; 1213 1214 isert_cmd = iscsit_priv_cmd(cmd); 1215 isert_cmd->read_stag = read_stag; 1216 isert_cmd->read_va = read_va; 1217 isert_cmd->write_stag = write_stag; 1218 isert_cmd->write_va = write_va; 1219 1220 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1221 rx_desc, (unsigned char *)hdr); 1222 break; 1223 case ISCSI_OP_NOOP_OUT: 1224 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1225 if (!cmd) 1226 break; 1227 1228 isert_cmd = iscsit_priv_cmd(cmd); 1229 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1230 rx_desc, (unsigned char *)hdr); 1231 break; 1232 case ISCSI_OP_SCSI_DATA_OUT: 1233 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1234 (unsigned char *)hdr); 1235 break; 1236 case ISCSI_OP_SCSI_TMFUNC: 1237 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1238 if (!cmd) 1239 break; 1240 1241 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1242 (unsigned char *)hdr); 1243 break; 1244 case ISCSI_OP_LOGOUT: 1245 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1246 if (!cmd) 1247 break; 1248 1249 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1250 if (ret > 0) 1251 wait_for_completion_timeout(&conn->conn_logout_comp, 1252 SECONDS_FOR_LOGOUT_COMP * 1253 HZ); 1254 break; 1255 case ISCSI_OP_TEXT: 1256 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1257 if (!cmd) 1258 break; 1259 1260 isert_cmd = iscsit_priv_cmd(cmd); 1261 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1262 rx_desc, (struct iscsi_text *)hdr); 1263 break; 1264 default: 1265 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1266 dump_stack(); 1267 break; 1268 } 1269 1270 return ret; 1271 } 1272 1273 static void 1274 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1275 { 1276 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1277 uint64_t read_va = 0, write_va = 0; 1278 uint32_t read_stag = 0, write_stag = 0; 1279 int rc; 1280 1281 switch (iser_hdr->flags & 0xF0) { 1282 case ISCSI_CTRL: 1283 if (iser_hdr->flags & ISER_RSV) { 1284 read_stag = be32_to_cpu(iser_hdr->read_stag); 1285 read_va = be64_to_cpu(iser_hdr->read_va); 1286 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1287 read_stag, (unsigned long long)read_va); 1288 } 1289 if (iser_hdr->flags & ISER_WSV) { 1290 write_stag = be32_to_cpu(iser_hdr->write_stag); 1291 write_va = be64_to_cpu(iser_hdr->write_va); 1292 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1293 write_stag, (unsigned long long)write_va); 1294 } 1295 1296 pr_debug("ISER ISCSI_CTRL PDU\n"); 1297 break; 1298 case ISER_HELLO: 1299 pr_err("iSER Hello message\n"); 1300 break; 1301 default: 1302 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1303 break; 1304 } 1305 1306 rc = isert_rx_opcode(isert_conn, rx_desc, 1307 read_stag, read_va, write_stag, write_va); 1308 } 1309 1310 static void 1311 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1312 unsigned long xfer_len) 1313 { 1314 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1315 struct iscsi_hdr *hdr; 1316 u64 rx_dma; 1317 int rx_buflen, outstanding; 1318 1319 if ((char *)desc == isert_conn->login_req_buf) { 1320 rx_dma = isert_conn->login_req_dma; 1321 rx_buflen = ISER_RX_LOGIN_SIZE; 1322 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1323 rx_dma, rx_buflen); 1324 } else { 1325 rx_dma = desc->dma_addr; 1326 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1327 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1328 rx_dma, rx_buflen); 1329 } 1330 1331 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1332 1333 hdr = &desc->iscsi_header; 1334 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1335 hdr->opcode, hdr->itt, hdr->flags, 1336 (int)(xfer_len - ISER_HEADERS_LEN)); 1337 1338 if ((char *)desc == isert_conn->login_req_buf) 1339 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1340 isert_conn); 1341 else 1342 isert_rx_do_work(desc, isert_conn); 1343 1344 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1345 DMA_FROM_DEVICE); 1346 1347 isert_conn->post_recv_buf_count--; 1348 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1349 isert_conn->post_recv_buf_count); 1350 1351 if ((char *)desc == isert_conn->login_req_buf) 1352 return; 1353 1354 outstanding = isert_conn->post_recv_buf_count; 1355 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1356 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1357 ISERT_MIN_POSTED_RX); 1358 err = isert_post_recv(isert_conn, count); 1359 if (err) { 1360 pr_err("isert_post_recv() count: %d failed, %d\n", 1361 count, err); 1362 } 1363 } 1364 } 1365 1366 static void 1367 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1368 { 1369 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1370 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1371 1372 pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1373 if (wr->sge) { 1374 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1375 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1376 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1377 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1378 wr->sge = NULL; 1379 } 1380 1381 if (wr->send_wr) { 1382 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1383 kfree(wr->send_wr); 1384 wr->send_wr = NULL; 1385 } 1386 1387 if (wr->ib_sge) { 1388 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 1389 kfree(wr->ib_sge); 1390 wr->ib_sge = NULL; 1391 } 1392 } 1393 1394 static void 1395 isert_unreg_rdma_frwr(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1396 { 1397 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1398 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1399 LIST_HEAD(unmap_list); 1400 1401 pr_debug("unreg_frwr_cmd: %p\n", isert_cmd); 1402 1403 if (wr->fr_desc) { 1404 pr_debug("unreg_frwr_cmd: %p free fr_desc %p\n", 1405 isert_cmd, wr->fr_desc); 1406 spin_lock_bh(&isert_conn->conn_lock); 1407 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_frwr_pool); 1408 spin_unlock_bh(&isert_conn->conn_lock); 1409 wr->fr_desc = NULL; 1410 } 1411 1412 if (wr->sge) { 1413 pr_debug("unreg_frwr_cmd: %p unmap_sg op\n", isert_cmd); 1414 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1415 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1416 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1417 wr->sge = NULL; 1418 } 1419 1420 wr->ib_sge = NULL; 1421 wr->send_wr = NULL; 1422 } 1423 1424 static void 1425 isert_put_cmd(struct isert_cmd *isert_cmd) 1426 { 1427 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1428 struct isert_conn *isert_conn = isert_cmd->conn; 1429 struct iscsi_conn *conn = isert_conn->conn; 1430 struct isert_device *device = isert_conn->conn_device; 1431 1432 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1433 1434 switch (cmd->iscsi_opcode) { 1435 case ISCSI_OP_SCSI_CMD: 1436 spin_lock_bh(&conn->cmd_lock); 1437 if (!list_empty(&cmd->i_conn_node)) 1438 list_del(&cmd->i_conn_node); 1439 spin_unlock_bh(&conn->cmd_lock); 1440 1441 if (cmd->data_direction == DMA_TO_DEVICE) 1442 iscsit_stop_dataout_timer(cmd); 1443 1444 device->unreg_rdma_mem(isert_cmd, isert_conn); 1445 transport_generic_free_cmd(&cmd->se_cmd, 0); 1446 break; 1447 case ISCSI_OP_SCSI_TMFUNC: 1448 spin_lock_bh(&conn->cmd_lock); 1449 if (!list_empty(&cmd->i_conn_node)) 1450 list_del(&cmd->i_conn_node); 1451 spin_unlock_bh(&conn->cmd_lock); 1452 1453 transport_generic_free_cmd(&cmd->se_cmd, 0); 1454 break; 1455 case ISCSI_OP_REJECT: 1456 case ISCSI_OP_NOOP_OUT: 1457 case ISCSI_OP_TEXT: 1458 spin_lock_bh(&conn->cmd_lock); 1459 if (!list_empty(&cmd->i_conn_node)) 1460 list_del(&cmd->i_conn_node); 1461 spin_unlock_bh(&conn->cmd_lock); 1462 1463 /* 1464 * Handle special case for REJECT when iscsi_add_reject*() has 1465 * overwritten the original iscsi_opcode assignment, and the 1466 * associated cmd->se_cmd needs to be released. 1467 */ 1468 if (cmd->se_cmd.se_tfo != NULL) { 1469 pr_debug("Calling transport_generic_free_cmd from" 1470 " isert_put_cmd for 0x%02x\n", 1471 cmd->iscsi_opcode); 1472 transport_generic_free_cmd(&cmd->se_cmd, 0); 1473 break; 1474 } 1475 /* 1476 * Fall-through 1477 */ 1478 default: 1479 iscsit_release_cmd(cmd); 1480 break; 1481 } 1482 } 1483 1484 static void 1485 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1486 { 1487 if (tx_desc->dma_addr != 0) { 1488 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1489 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1490 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1491 tx_desc->dma_addr = 0; 1492 } 1493 } 1494 1495 static void 1496 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1497 struct ib_device *ib_dev) 1498 { 1499 if (isert_cmd->pdu_buf_dma != 0) { 1500 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1501 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1502 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1503 isert_cmd->pdu_buf_dma = 0; 1504 } 1505 1506 isert_unmap_tx_desc(tx_desc, ib_dev); 1507 isert_put_cmd(isert_cmd); 1508 } 1509 1510 static void 1511 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1512 struct isert_cmd *isert_cmd) 1513 { 1514 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1515 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1516 struct se_cmd *se_cmd = &cmd->se_cmd; 1517 struct isert_conn *isert_conn = isert_cmd->conn; 1518 struct isert_device *device = isert_conn->conn_device; 1519 1520 iscsit_stop_dataout_timer(cmd); 1521 device->unreg_rdma_mem(isert_cmd, isert_conn); 1522 cmd->write_data_done = wr->cur_rdma_length; 1523 1524 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1525 spin_lock_bh(&cmd->istate_lock); 1526 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1527 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1528 spin_unlock_bh(&cmd->istate_lock); 1529 1530 target_execute_cmd(se_cmd); 1531 } 1532 1533 static void 1534 isert_do_control_comp(struct work_struct *work) 1535 { 1536 struct isert_cmd *isert_cmd = container_of(work, 1537 struct isert_cmd, comp_work); 1538 struct isert_conn *isert_conn = isert_cmd->conn; 1539 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1540 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1541 1542 switch (cmd->i_state) { 1543 case ISTATE_SEND_TASKMGTRSP: 1544 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1545 1546 atomic_dec(&isert_conn->post_send_buf_count); 1547 iscsit_tmr_post_handler(cmd, cmd->conn); 1548 1549 cmd->i_state = ISTATE_SENT_STATUS; 1550 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1551 break; 1552 case ISTATE_SEND_REJECT: 1553 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1554 atomic_dec(&isert_conn->post_send_buf_count); 1555 1556 cmd->i_state = ISTATE_SENT_STATUS; 1557 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1558 break; 1559 case ISTATE_SEND_LOGOUTRSP: 1560 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1561 /* 1562 * Call atomic_dec(&isert_conn->post_send_buf_count) 1563 * from isert_free_conn() 1564 */ 1565 isert_conn->logout_posted = true; 1566 iscsit_logout_post_handler(cmd, cmd->conn); 1567 break; 1568 case ISTATE_SEND_TEXTRSP: 1569 atomic_dec(&isert_conn->post_send_buf_count); 1570 cmd->i_state = ISTATE_SENT_STATUS; 1571 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1572 break; 1573 default: 1574 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1575 dump_stack(); 1576 break; 1577 } 1578 } 1579 1580 static void 1581 isert_response_completion(struct iser_tx_desc *tx_desc, 1582 struct isert_cmd *isert_cmd, 1583 struct isert_conn *isert_conn, 1584 struct ib_device *ib_dev) 1585 { 1586 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1587 1588 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1589 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1590 cmd->i_state == ISTATE_SEND_REJECT || 1591 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1592 isert_unmap_tx_desc(tx_desc, ib_dev); 1593 1594 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1595 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1596 return; 1597 } 1598 atomic_dec(&isert_conn->post_send_buf_count); 1599 1600 cmd->i_state = ISTATE_SENT_STATUS; 1601 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1602 } 1603 1604 static void 1605 __isert_send_completion(struct iser_tx_desc *tx_desc, 1606 struct isert_conn *isert_conn) 1607 { 1608 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1609 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1610 struct isert_rdma_wr *wr; 1611 1612 if (!isert_cmd) { 1613 atomic_dec(&isert_conn->post_send_buf_count); 1614 isert_unmap_tx_desc(tx_desc, ib_dev); 1615 return; 1616 } 1617 wr = &isert_cmd->rdma_wr; 1618 1619 switch (wr->iser_ib_op) { 1620 case ISER_IB_RECV: 1621 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1622 dump_stack(); 1623 break; 1624 case ISER_IB_SEND: 1625 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1626 isert_response_completion(tx_desc, isert_cmd, 1627 isert_conn, ib_dev); 1628 break; 1629 case ISER_IB_RDMA_WRITE: 1630 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1631 dump_stack(); 1632 break; 1633 case ISER_IB_RDMA_READ: 1634 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1635 1636 atomic_dec(&isert_conn->post_send_buf_count); 1637 isert_completion_rdma_read(tx_desc, isert_cmd); 1638 break; 1639 default: 1640 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1641 dump_stack(); 1642 break; 1643 } 1644 } 1645 1646 static void 1647 isert_send_completion(struct iser_tx_desc *tx_desc, 1648 struct isert_conn *isert_conn) 1649 { 1650 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1651 struct iser_tx_desc *t; 1652 /* 1653 * Drain coalesced completion llist starting from comp_llnode_batch 1654 * setup in isert_init_send_wr(), and then complete trailing tx_desc. 1655 */ 1656 while (llnode) { 1657 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1658 llnode = llist_next(llnode); 1659 __isert_send_completion(t, isert_conn); 1660 } 1661 __isert_send_completion(tx_desc, isert_conn); 1662 } 1663 1664 static void 1665 isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1666 { 1667 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1668 1669 if (tx_desc) { 1670 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1671 1672 if (!isert_cmd) 1673 isert_unmap_tx_desc(tx_desc, ib_dev); 1674 else 1675 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1676 } 1677 1678 if (isert_conn->post_recv_buf_count == 0 && 1679 atomic_read(&isert_conn->post_send_buf_count) == 0) { 1680 pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 1681 pr_debug("Calling wake_up from isert_cq_comp_err\n"); 1682 1683 mutex_lock(&isert_conn->conn_mutex); 1684 if (isert_conn->state != ISER_CONN_DOWN) 1685 isert_conn->state = ISER_CONN_TERMINATING; 1686 mutex_unlock(&isert_conn->conn_mutex); 1687 1688 wake_up(&isert_conn->conn_wait_comp_err); 1689 } 1690 } 1691 1692 static void 1693 isert_cq_tx_work(struct work_struct *work) 1694 { 1695 struct isert_cq_desc *cq_desc = container_of(work, 1696 struct isert_cq_desc, cq_tx_work); 1697 struct isert_device *device = cq_desc->device; 1698 int cq_index = cq_desc->cq_index; 1699 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 1700 struct isert_conn *isert_conn; 1701 struct iser_tx_desc *tx_desc; 1702 struct ib_wc wc; 1703 1704 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 1705 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 1706 isert_conn = wc.qp->qp_context; 1707 1708 if (wc.status == IB_WC_SUCCESS) { 1709 isert_send_completion(tx_desc, isert_conn); 1710 } else { 1711 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1712 pr_debug("TX wc.status: 0x%08x\n", wc.status); 1713 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 1714 atomic_dec(&isert_conn->post_send_buf_count); 1715 isert_cq_comp_err(tx_desc, isert_conn); 1716 } 1717 } 1718 1719 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 1720 } 1721 1722 static void 1723 isert_cq_tx_callback(struct ib_cq *cq, void *context) 1724 { 1725 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1726 1727 INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work); 1728 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 1729 } 1730 1731 static void 1732 isert_cq_rx_work(struct work_struct *work) 1733 { 1734 struct isert_cq_desc *cq_desc = container_of(work, 1735 struct isert_cq_desc, cq_rx_work); 1736 struct isert_device *device = cq_desc->device; 1737 int cq_index = cq_desc->cq_index; 1738 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 1739 struct isert_conn *isert_conn; 1740 struct iser_rx_desc *rx_desc; 1741 struct ib_wc wc; 1742 unsigned long xfer_len; 1743 1744 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 1745 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 1746 isert_conn = wc.qp->qp_context; 1747 1748 if (wc.status == IB_WC_SUCCESS) { 1749 xfer_len = (unsigned long)wc.byte_len; 1750 isert_rx_completion(rx_desc, isert_conn, xfer_len); 1751 } else { 1752 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1753 if (wc.status != IB_WC_WR_FLUSH_ERR) { 1754 pr_debug("RX wc.status: 0x%08x\n", wc.status); 1755 pr_debug("RX wc.vendor_err: 0x%08x\n", 1756 wc.vendor_err); 1757 } 1758 isert_conn->post_recv_buf_count--; 1759 isert_cq_comp_err(NULL, isert_conn); 1760 } 1761 } 1762 1763 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 1764 } 1765 1766 static void 1767 isert_cq_rx_callback(struct ib_cq *cq, void *context) 1768 { 1769 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1770 1771 INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work); 1772 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 1773 } 1774 1775 static int 1776 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1777 { 1778 struct ib_send_wr *wr_failed; 1779 int ret; 1780 1781 atomic_inc(&isert_conn->post_send_buf_count); 1782 1783 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 1784 &wr_failed); 1785 if (ret) { 1786 pr_err("ib_post_send failed with %d\n", ret); 1787 atomic_dec(&isert_conn->post_send_buf_count); 1788 return ret; 1789 } 1790 return ret; 1791 } 1792 1793 static int 1794 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1795 { 1796 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1797 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1798 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1799 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1800 &isert_cmd->tx_desc.iscsi_header; 1801 1802 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1803 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1804 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1805 /* 1806 * Attach SENSE DATA payload to iSCSI Response PDU 1807 */ 1808 if (cmd->se_cmd.sense_buffer && 1809 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1810 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1811 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1812 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1813 u32 padding, pdu_len; 1814 1815 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1816 cmd->sense_buffer); 1817 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1818 1819 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1820 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1821 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1822 1823 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1824 (void *)cmd->sense_buffer, pdu_len, 1825 DMA_TO_DEVICE); 1826 1827 isert_cmd->pdu_buf_len = pdu_len; 1828 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1829 tx_dsg->length = pdu_len; 1830 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1831 isert_cmd->tx_desc.num_sge = 2; 1832 } 1833 1834 isert_init_send_wr(isert_conn, isert_cmd, send_wr, true); 1835 1836 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1837 1838 return isert_post_response(isert_conn, isert_cmd); 1839 } 1840 1841 static int 1842 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1843 bool nopout_response) 1844 { 1845 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1846 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1847 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1848 1849 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1850 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1851 &isert_cmd->tx_desc.iscsi_header, 1852 nopout_response); 1853 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1854 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1855 1856 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1857 1858 return isert_post_response(isert_conn, isert_cmd); 1859 } 1860 1861 static int 1862 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1863 { 1864 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1865 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1866 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1867 1868 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1869 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1870 &isert_cmd->tx_desc.iscsi_header); 1871 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1872 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1873 1874 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1875 1876 return isert_post_response(isert_conn, isert_cmd); 1877 } 1878 1879 static int 1880 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1881 { 1882 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1883 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1884 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1885 1886 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1887 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1888 &isert_cmd->tx_desc.iscsi_header); 1889 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1890 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1891 1892 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1893 1894 return isert_post_response(isert_conn, isert_cmd); 1895 } 1896 1897 static int 1898 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1899 { 1900 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1901 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1902 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1903 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1904 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1905 struct iscsi_reject *hdr = 1906 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1907 1908 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1909 iscsit_build_reject(cmd, conn, hdr); 1910 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1911 1912 hton24(hdr->dlength, ISCSI_HDR_LEN); 1913 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1914 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1915 DMA_TO_DEVICE); 1916 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1917 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1918 tx_dsg->length = ISCSI_HDR_LEN; 1919 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1920 isert_cmd->tx_desc.num_sge = 2; 1921 1922 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1923 1924 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1925 1926 return isert_post_response(isert_conn, isert_cmd); 1927 } 1928 1929 static int 1930 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1931 { 1932 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1933 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1934 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1935 struct iscsi_text_rsp *hdr = 1936 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1937 u32 txt_rsp_len; 1938 int rc; 1939 1940 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1941 rc = iscsit_build_text_rsp(cmd, conn, hdr); 1942 if (rc < 0) 1943 return rc; 1944 1945 txt_rsp_len = rc; 1946 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1947 1948 if (txt_rsp_len) { 1949 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1950 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1951 void *txt_rsp_buf = cmd->buf_ptr; 1952 1953 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1954 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1955 1956 isert_cmd->pdu_buf_len = txt_rsp_len; 1957 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1958 tx_dsg->length = txt_rsp_len; 1959 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1960 isert_cmd->tx_desc.num_sge = 2; 1961 } 1962 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1963 1964 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1965 1966 return isert_post_response(isert_conn, isert_cmd); 1967 } 1968 1969 static int 1970 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1971 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 1972 u32 data_left, u32 offset) 1973 { 1974 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1975 struct scatterlist *sg_start, *tmp_sg; 1976 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1977 u32 sg_off, page_off; 1978 int i = 0, sg_nents; 1979 1980 sg_off = offset / PAGE_SIZE; 1981 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1982 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 1983 page_off = offset % PAGE_SIZE; 1984 1985 send_wr->sg_list = ib_sge; 1986 send_wr->num_sge = sg_nents; 1987 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 1988 /* 1989 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 1990 */ 1991 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 1992 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 1993 (unsigned long long)tmp_sg->dma_address, 1994 tmp_sg->length, page_off); 1995 1996 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 1997 ib_sge->length = min_t(u32, data_left, 1998 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 1999 ib_sge->lkey = isert_conn->conn_mr->lkey; 2000 2001 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2002 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2003 page_off = 0; 2004 data_left -= ib_sge->length; 2005 ib_sge++; 2006 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2007 } 2008 2009 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2010 send_wr->sg_list, send_wr->num_sge); 2011 2012 return sg_nents; 2013 } 2014 2015 static int 2016 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2017 struct isert_rdma_wr *wr) 2018 { 2019 struct se_cmd *se_cmd = &cmd->se_cmd; 2020 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2021 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2022 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2023 struct ib_send_wr *send_wr; 2024 struct ib_sge *ib_sge; 2025 struct scatterlist *sg_start; 2026 u32 sg_off = 0, sg_nents; 2027 u32 offset = 0, data_len, data_left, rdma_write_max, va_offset = 0; 2028 int ret = 0, count, i, ib_sge_cnt; 2029 2030 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2031 data_left = se_cmd->data_length; 2032 } else { 2033 sg_off = cmd->write_data_done / PAGE_SIZE; 2034 data_left = se_cmd->data_length - cmd->write_data_done; 2035 offset = cmd->write_data_done; 2036 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2037 } 2038 2039 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2040 sg_nents = se_cmd->t_data_nents - sg_off; 2041 2042 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2043 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2044 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2045 if (unlikely(!count)) { 2046 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2047 return -EINVAL; 2048 } 2049 wr->sge = sg_start; 2050 wr->num_sge = sg_nents; 2051 wr->cur_rdma_length = data_left; 2052 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2053 isert_cmd, count, sg_start, sg_nents, data_left); 2054 2055 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 2056 if (!ib_sge) { 2057 pr_warn("Unable to allocate ib_sge\n"); 2058 ret = -ENOMEM; 2059 goto unmap_sg; 2060 } 2061 wr->ib_sge = ib_sge; 2062 2063 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 2064 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2065 GFP_KERNEL); 2066 if (!wr->send_wr) { 2067 pr_debug("Unable to allocate wr->send_wr\n"); 2068 ret = -ENOMEM; 2069 goto unmap_sg; 2070 } 2071 2072 wr->isert_cmd = isert_cmd; 2073 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2074 2075 for (i = 0; i < wr->send_wr_num; i++) { 2076 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2077 data_len = min(data_left, rdma_write_max); 2078 2079 send_wr->send_flags = 0; 2080 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2081 send_wr->opcode = IB_WR_RDMA_WRITE; 2082 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2083 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2084 if (i + 1 == wr->send_wr_num) 2085 send_wr->next = &isert_cmd->tx_desc.send_wr; 2086 else 2087 send_wr->next = &wr->send_wr[i + 1]; 2088 } else { 2089 send_wr->opcode = IB_WR_RDMA_READ; 2090 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2091 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2092 if (i + 1 == wr->send_wr_num) 2093 send_wr->send_flags = IB_SEND_SIGNALED; 2094 else 2095 send_wr->next = &wr->send_wr[i + 1]; 2096 } 2097 2098 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2099 send_wr, data_len, offset); 2100 ib_sge += ib_sge_cnt; 2101 2102 offset += data_len; 2103 va_offset += data_len; 2104 data_left -= data_len; 2105 } 2106 2107 return 0; 2108 unmap_sg: 2109 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2110 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2111 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2112 return ret; 2113 } 2114 2115 static int 2116 isert_map_fr_pagelist(struct ib_device *ib_dev, 2117 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2118 { 2119 u64 start_addr, end_addr, page, chunk_start = 0; 2120 struct scatterlist *tmp_sg; 2121 int i = 0, new_chunk, last_ent, n_pages; 2122 2123 n_pages = 0; 2124 new_chunk = 1; 2125 last_ent = sg_nents - 1; 2126 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2127 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2128 if (new_chunk) 2129 chunk_start = start_addr; 2130 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2131 2132 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 2133 i, (unsigned long long)tmp_sg->dma_address, 2134 tmp_sg->length); 2135 2136 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2137 new_chunk = 0; 2138 continue; 2139 } 2140 new_chunk = 1; 2141 2142 page = chunk_start & PAGE_MASK; 2143 do { 2144 fr_pl[n_pages++] = page; 2145 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 2146 n_pages - 1, page); 2147 page += PAGE_SIZE; 2148 } while (page < end_addr); 2149 } 2150 2151 return n_pages; 2152 } 2153 2154 static int 2155 isert_fast_reg_mr(struct fast_reg_descriptor *fr_desc, 2156 struct isert_cmd *isert_cmd, struct isert_conn *isert_conn, 2157 struct ib_sge *ib_sge, u32 offset, unsigned int data_len) 2158 { 2159 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2160 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2161 struct scatterlist *sg_start; 2162 u32 sg_off, page_off; 2163 struct ib_send_wr fr_wr, inv_wr; 2164 struct ib_send_wr *bad_wr, *wr = NULL; 2165 u8 key; 2166 int ret, sg_nents, pagelist_len; 2167 2168 sg_off = offset / PAGE_SIZE; 2169 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2170 sg_nents = min_t(unsigned int, cmd->se_cmd.t_data_nents - sg_off, 2171 ISCSI_ISER_SG_TABLESIZE); 2172 page_off = offset % PAGE_SIZE; 2173 2174 pr_debug("Cmd: %p use fr_desc %p sg_nents %d sg_off %d offset %u\n", 2175 isert_cmd, fr_desc, sg_nents, sg_off, offset); 2176 2177 pagelist_len = isert_map_fr_pagelist(ib_dev, sg_start, sg_nents, 2178 &fr_desc->data_frpl->page_list[0]); 2179 2180 if (!fr_desc->valid) { 2181 memset(&inv_wr, 0, sizeof(inv_wr)); 2182 inv_wr.opcode = IB_WR_LOCAL_INV; 2183 inv_wr.ex.invalidate_rkey = fr_desc->data_mr->rkey; 2184 wr = &inv_wr; 2185 /* Bump the key */ 2186 key = (u8)(fr_desc->data_mr->rkey & 0x000000FF); 2187 ib_update_fast_reg_key(fr_desc->data_mr, ++key); 2188 } 2189 2190 /* Prepare FASTREG WR */ 2191 memset(&fr_wr, 0, sizeof(fr_wr)); 2192 fr_wr.opcode = IB_WR_FAST_REG_MR; 2193 fr_wr.wr.fast_reg.iova_start = 2194 fr_desc->data_frpl->page_list[0] + page_off; 2195 fr_wr.wr.fast_reg.page_list = fr_desc->data_frpl; 2196 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2197 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2198 fr_wr.wr.fast_reg.length = data_len; 2199 fr_wr.wr.fast_reg.rkey = fr_desc->data_mr->rkey; 2200 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2201 2202 if (!wr) 2203 wr = &fr_wr; 2204 else 2205 wr->next = &fr_wr; 2206 2207 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2208 if (ret) { 2209 pr_err("fast registration failed, ret:%d\n", ret); 2210 return ret; 2211 } 2212 fr_desc->valid = false; 2213 2214 ib_sge->lkey = fr_desc->data_mr->lkey; 2215 ib_sge->addr = fr_desc->data_frpl->page_list[0] + page_off; 2216 ib_sge->length = data_len; 2217 2218 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2219 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2220 2221 return ret; 2222 } 2223 2224 static int 2225 isert_reg_rdma_frwr(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2226 struct isert_rdma_wr *wr) 2227 { 2228 struct se_cmd *se_cmd = &cmd->se_cmd; 2229 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2230 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2231 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2232 struct ib_send_wr *send_wr; 2233 struct ib_sge *ib_sge; 2234 struct scatterlist *sg_start; 2235 struct fast_reg_descriptor *fr_desc; 2236 u32 sg_off = 0, sg_nents; 2237 u32 offset = 0, data_len, data_left, rdma_write_max; 2238 int ret = 0, count; 2239 unsigned long flags; 2240 2241 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2242 data_left = se_cmd->data_length; 2243 } else { 2244 sg_off = cmd->write_data_done / PAGE_SIZE; 2245 data_left = se_cmd->data_length - cmd->write_data_done; 2246 offset = cmd->write_data_done; 2247 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2248 } 2249 2250 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2251 sg_nents = se_cmd->t_data_nents - sg_off; 2252 2253 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2254 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2255 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2256 if (unlikely(!count)) { 2257 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2258 return -EINVAL; 2259 } 2260 wr->sge = sg_start; 2261 wr->num_sge = sg_nents; 2262 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2263 isert_cmd, count, sg_start, sg_nents, data_left); 2264 2265 memset(&wr->s_ib_sge, 0, sizeof(*ib_sge)); 2266 ib_sge = &wr->s_ib_sge; 2267 wr->ib_sge = ib_sge; 2268 2269 wr->send_wr_num = 1; 2270 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2271 wr->send_wr = &wr->s_send_wr; 2272 2273 wr->isert_cmd = isert_cmd; 2274 rdma_write_max = ISCSI_ISER_SG_TABLESIZE * PAGE_SIZE; 2275 2276 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2277 send_wr->sg_list = ib_sge; 2278 send_wr->num_sge = 1; 2279 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2280 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2281 send_wr->opcode = IB_WR_RDMA_WRITE; 2282 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2283 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2284 send_wr->send_flags = 0; 2285 send_wr->next = &isert_cmd->tx_desc.send_wr; 2286 } else { 2287 send_wr->opcode = IB_WR_RDMA_READ; 2288 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2289 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2290 send_wr->send_flags = IB_SEND_SIGNALED; 2291 } 2292 2293 data_len = min(data_left, rdma_write_max); 2294 wr->cur_rdma_length = data_len; 2295 2296 /* if there is a single dma entry, dma mr is sufficient */ 2297 if (count == 1) { 2298 ib_sge->addr = ib_sg_dma_address(ib_dev, &sg_start[0]); 2299 ib_sge->length = ib_sg_dma_len(ib_dev, &sg_start[0]); 2300 ib_sge->lkey = isert_conn->conn_mr->lkey; 2301 wr->fr_desc = NULL; 2302 } else { 2303 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2304 fr_desc = list_first_entry(&isert_conn->conn_frwr_pool, 2305 struct fast_reg_descriptor, list); 2306 list_del(&fr_desc->list); 2307 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2308 wr->fr_desc = fr_desc; 2309 2310 ret = isert_fast_reg_mr(fr_desc, isert_cmd, isert_conn, 2311 ib_sge, offset, data_len); 2312 if (ret) { 2313 list_add_tail(&fr_desc->list, &isert_conn->conn_frwr_pool); 2314 goto unmap_sg; 2315 } 2316 } 2317 2318 return 0; 2319 2320 unmap_sg: 2321 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2322 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2323 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2324 return ret; 2325 } 2326 2327 static int 2328 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2329 { 2330 struct se_cmd *se_cmd = &cmd->se_cmd; 2331 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2332 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2333 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2334 struct isert_device *device = isert_conn->conn_device; 2335 struct ib_send_wr *wr_failed; 2336 int rc; 2337 2338 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 2339 isert_cmd, se_cmd->data_length); 2340 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2341 rc = device->reg_rdma_mem(conn, cmd, wr); 2342 if (rc) { 2343 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2344 return rc; 2345 } 2346 2347 /* 2348 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2349 */ 2350 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2351 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2352 &isert_cmd->tx_desc.iscsi_header); 2353 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2354 isert_init_send_wr(isert_conn, isert_cmd, 2355 &isert_cmd->tx_desc.send_wr, true); 2356 2357 atomic_inc(&isert_conn->post_send_buf_count); 2358 2359 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2360 if (rc) { 2361 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2362 atomic_dec(&isert_conn->post_send_buf_count); 2363 } 2364 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data READ\n", 2365 isert_cmd); 2366 2367 return 1; 2368 } 2369 2370 static int 2371 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2372 { 2373 struct se_cmd *se_cmd = &cmd->se_cmd; 2374 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2375 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2376 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2377 struct isert_device *device = isert_conn->conn_device; 2378 struct ib_send_wr *wr_failed; 2379 int rc; 2380 2381 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2382 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2383 wr->iser_ib_op = ISER_IB_RDMA_READ; 2384 rc = device->reg_rdma_mem(conn, cmd, wr); 2385 if (rc) { 2386 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2387 return rc; 2388 } 2389 2390 atomic_inc(&isert_conn->post_send_buf_count); 2391 2392 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2393 if (rc) { 2394 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2395 atomic_dec(&isert_conn->post_send_buf_count); 2396 } 2397 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2398 isert_cmd); 2399 2400 return 0; 2401 } 2402 2403 static int 2404 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2405 { 2406 int ret; 2407 2408 switch (state) { 2409 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2410 ret = isert_put_nopin(cmd, conn, false); 2411 break; 2412 default: 2413 pr_err("Unknown immediate state: 0x%02x\n", state); 2414 ret = -EINVAL; 2415 break; 2416 } 2417 2418 return ret; 2419 } 2420 2421 static int 2422 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2423 { 2424 int ret; 2425 2426 switch (state) { 2427 case ISTATE_SEND_LOGOUTRSP: 2428 ret = isert_put_logout_rsp(cmd, conn); 2429 if (!ret) { 2430 pr_debug("Returning iSER Logout -EAGAIN\n"); 2431 ret = -EAGAIN; 2432 } 2433 break; 2434 case ISTATE_SEND_NOPIN: 2435 ret = isert_put_nopin(cmd, conn, true); 2436 break; 2437 case ISTATE_SEND_TASKMGTRSP: 2438 ret = isert_put_tm_rsp(cmd, conn); 2439 break; 2440 case ISTATE_SEND_REJECT: 2441 ret = isert_put_reject(cmd, conn); 2442 break; 2443 case ISTATE_SEND_TEXTRSP: 2444 ret = isert_put_text_rsp(cmd, conn); 2445 break; 2446 case ISTATE_SEND_STATUS: 2447 /* 2448 * Special case for sending non GOOD SCSI status from TX thread 2449 * context during pre se_cmd excecution failure. 2450 */ 2451 ret = isert_put_response(conn, cmd); 2452 break; 2453 default: 2454 pr_err("Unknown response state: 0x%02x\n", state); 2455 ret = -EINVAL; 2456 break; 2457 } 2458 2459 return ret; 2460 } 2461 2462 static int 2463 isert_setup_np(struct iscsi_np *np, 2464 struct __kernel_sockaddr_storage *ksockaddr) 2465 { 2466 struct isert_np *isert_np; 2467 struct rdma_cm_id *isert_lid; 2468 struct sockaddr *sa; 2469 int ret; 2470 2471 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2472 if (!isert_np) { 2473 pr_err("Unable to allocate struct isert_np\n"); 2474 return -ENOMEM; 2475 } 2476 init_waitqueue_head(&isert_np->np_accept_wq); 2477 mutex_init(&isert_np->np_accept_mutex); 2478 INIT_LIST_HEAD(&isert_np->np_accept_list); 2479 init_completion(&isert_np->np_login_comp); 2480 2481 sa = (struct sockaddr *)ksockaddr; 2482 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 2483 /* 2484 * Setup the np->np_sockaddr from the passed sockaddr setup 2485 * in iscsi_target_configfs.c code.. 2486 */ 2487 memcpy(&np->np_sockaddr, ksockaddr, 2488 sizeof(struct __kernel_sockaddr_storage)); 2489 2490 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 2491 IB_QPT_RC); 2492 if (IS_ERR(isert_lid)) { 2493 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 2494 PTR_ERR(isert_lid)); 2495 ret = PTR_ERR(isert_lid); 2496 goto out; 2497 } 2498 2499 ret = rdma_bind_addr(isert_lid, sa); 2500 if (ret) { 2501 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 2502 goto out_lid; 2503 } 2504 2505 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 2506 if (ret) { 2507 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 2508 goto out_lid; 2509 } 2510 2511 isert_np->np_cm_id = isert_lid; 2512 np->np_context = isert_np; 2513 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 2514 2515 return 0; 2516 2517 out_lid: 2518 rdma_destroy_id(isert_lid); 2519 out: 2520 kfree(isert_np); 2521 return ret; 2522 } 2523 2524 static int 2525 isert_check_accept_queue(struct isert_np *isert_np) 2526 { 2527 int empty; 2528 2529 mutex_lock(&isert_np->np_accept_mutex); 2530 empty = list_empty(&isert_np->np_accept_list); 2531 mutex_unlock(&isert_np->np_accept_mutex); 2532 2533 return empty; 2534 } 2535 2536 static int 2537 isert_rdma_accept(struct isert_conn *isert_conn) 2538 { 2539 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2540 struct rdma_conn_param cp; 2541 int ret; 2542 2543 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2544 cp.responder_resources = isert_conn->responder_resources; 2545 cp.initiator_depth = isert_conn->initiator_depth; 2546 cp.retry_count = 7; 2547 cp.rnr_retry_count = 7; 2548 2549 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 2550 2551 ret = rdma_accept(cm_id, &cp); 2552 if (ret) { 2553 pr_err("rdma_accept() failed with: %d\n", ret); 2554 return ret; 2555 } 2556 2557 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 2558 2559 return 0; 2560 } 2561 2562 static int 2563 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2564 { 2565 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2566 int ret; 2567 2568 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 2569 /* 2570 * For login requests after the first PDU, isert_rx_login_req() will 2571 * kick schedule_delayed_work(&conn->login_work) as the packet is 2572 * received, which turns this callback from iscsi_target_do_login_rx() 2573 * into a NOP. 2574 */ 2575 if (!login->first_request) 2576 return 0; 2577 2578 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 2579 if (ret) 2580 return ret; 2581 2582 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 2583 return 0; 2584 } 2585 2586 static void 2587 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2588 struct isert_conn *isert_conn) 2589 { 2590 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2591 struct rdma_route *cm_route = &cm_id->route; 2592 struct sockaddr_in *sock_in; 2593 struct sockaddr_in6 *sock_in6; 2594 2595 conn->login_family = np->np_sockaddr.ss_family; 2596 2597 if (np->np_sockaddr.ss_family == AF_INET6) { 2598 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 2599 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 2600 &sock_in6->sin6_addr.in6_u); 2601 conn->login_port = ntohs(sock_in6->sin6_port); 2602 2603 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 2604 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 2605 &sock_in6->sin6_addr.in6_u); 2606 conn->local_port = ntohs(sock_in6->sin6_port); 2607 } else { 2608 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 2609 sprintf(conn->login_ip, "%pI4", 2610 &sock_in->sin_addr.s_addr); 2611 conn->login_port = ntohs(sock_in->sin_port); 2612 2613 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 2614 sprintf(conn->local_ip, "%pI4", 2615 &sock_in->sin_addr.s_addr); 2616 conn->local_port = ntohs(sock_in->sin_port); 2617 } 2618 } 2619 2620 static int 2621 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2622 { 2623 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2624 struct isert_conn *isert_conn; 2625 int max_accept = 0, ret; 2626 2627 accept_wait: 2628 ret = wait_event_interruptible(isert_np->np_accept_wq, 2629 !isert_check_accept_queue(isert_np) || 2630 np->np_thread_state == ISCSI_NP_THREAD_RESET); 2631 if (max_accept > 5) 2632 return -ENODEV; 2633 2634 spin_lock_bh(&np->np_thread_lock); 2635 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 2636 spin_unlock_bh(&np->np_thread_lock); 2637 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 2638 return -ENODEV; 2639 } 2640 spin_unlock_bh(&np->np_thread_lock); 2641 2642 mutex_lock(&isert_np->np_accept_mutex); 2643 if (list_empty(&isert_np->np_accept_list)) { 2644 mutex_unlock(&isert_np->np_accept_mutex); 2645 max_accept++; 2646 goto accept_wait; 2647 } 2648 isert_conn = list_first_entry(&isert_np->np_accept_list, 2649 struct isert_conn, conn_accept_node); 2650 list_del_init(&isert_conn->conn_accept_node); 2651 mutex_unlock(&isert_np->np_accept_mutex); 2652 2653 conn->context = isert_conn; 2654 isert_conn->conn = conn; 2655 max_accept = 0; 2656 2657 ret = isert_rdma_post_recvl(isert_conn); 2658 if (ret) 2659 return ret; 2660 2661 ret = isert_rdma_accept(isert_conn); 2662 if (ret) 2663 return ret; 2664 2665 isert_set_conn_info(np, conn, isert_conn); 2666 2667 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 2668 return 0; 2669 } 2670 2671 static void 2672 isert_free_np(struct iscsi_np *np) 2673 { 2674 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2675 2676 rdma_destroy_id(isert_np->np_cm_id); 2677 2678 np->np_context = NULL; 2679 kfree(isert_np); 2680 } 2681 2682 static int isert_check_state(struct isert_conn *isert_conn, int state) 2683 { 2684 int ret; 2685 2686 mutex_lock(&isert_conn->conn_mutex); 2687 ret = (isert_conn->state == state); 2688 mutex_unlock(&isert_conn->conn_mutex); 2689 2690 return ret; 2691 } 2692 2693 static void isert_free_conn(struct iscsi_conn *conn) 2694 { 2695 struct isert_conn *isert_conn = conn->context; 2696 2697 pr_debug("isert_free_conn: Starting \n"); 2698 /* 2699 * Decrement post_send_buf_count for special case when called 2700 * from isert_do_control_comp() -> iscsit_logout_post_handler() 2701 */ 2702 mutex_lock(&isert_conn->conn_mutex); 2703 if (isert_conn->logout_posted) 2704 atomic_dec(&isert_conn->post_send_buf_count); 2705 2706 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 2707 pr_debug("Calling rdma_disconnect from isert_free_conn\n"); 2708 rdma_disconnect(isert_conn->conn_cm_id); 2709 } 2710 /* 2711 * Only wait for conn_wait_comp_err if the isert_conn made it 2712 * into full feature phase.. 2713 */ 2714 if (isert_conn->state == ISER_CONN_UP) { 2715 pr_debug("isert_free_conn: Before wait_event comp_err %d\n", 2716 isert_conn->state); 2717 mutex_unlock(&isert_conn->conn_mutex); 2718 2719 wait_event(isert_conn->conn_wait_comp_err, 2720 (isert_check_state(isert_conn, ISER_CONN_TERMINATING))); 2721 2722 wait_event(isert_conn->conn_wait, 2723 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2724 2725 isert_put_conn(isert_conn); 2726 return; 2727 } 2728 if (isert_conn->state == ISER_CONN_INIT) { 2729 mutex_unlock(&isert_conn->conn_mutex); 2730 isert_put_conn(isert_conn); 2731 return; 2732 } 2733 pr_debug("isert_free_conn: wait_event conn_wait %d\n", 2734 isert_conn->state); 2735 mutex_unlock(&isert_conn->conn_mutex); 2736 2737 wait_event(isert_conn->conn_wait, 2738 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2739 2740 isert_put_conn(isert_conn); 2741 } 2742 2743 static struct iscsit_transport iser_target_transport = { 2744 .name = "IB/iSER", 2745 .transport_type = ISCSI_INFINIBAND, 2746 .priv_size = sizeof(struct isert_cmd), 2747 .owner = THIS_MODULE, 2748 .iscsit_setup_np = isert_setup_np, 2749 .iscsit_accept_np = isert_accept_np, 2750 .iscsit_free_np = isert_free_np, 2751 .iscsit_free_conn = isert_free_conn, 2752 .iscsit_get_login_rx = isert_get_login_rx, 2753 .iscsit_put_login_tx = isert_put_login_tx, 2754 .iscsit_immediate_queue = isert_immediate_queue, 2755 .iscsit_response_queue = isert_response_queue, 2756 .iscsit_get_dataout = isert_get_dataout, 2757 .iscsit_queue_data_in = isert_put_datain, 2758 .iscsit_queue_status = isert_put_response, 2759 }; 2760 2761 static int __init isert_init(void) 2762 { 2763 int ret; 2764 2765 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 2766 if (!isert_rx_wq) { 2767 pr_err("Unable to allocate isert_rx_wq\n"); 2768 return -ENOMEM; 2769 } 2770 2771 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 2772 if (!isert_comp_wq) { 2773 pr_err("Unable to allocate isert_comp_wq\n"); 2774 ret = -ENOMEM; 2775 goto destroy_rx_wq; 2776 } 2777 2778 iscsit_register_transport(&iser_target_transport); 2779 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2780 return 0; 2781 2782 destroy_rx_wq: 2783 destroy_workqueue(isert_rx_wq); 2784 return ret; 2785 } 2786 2787 static void __exit isert_exit(void) 2788 { 2789 destroy_workqueue(isert_comp_wq); 2790 destroy_workqueue(isert_rx_wq); 2791 iscsit_unregister_transport(&iser_target_transport); 2792 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 2793 } 2794 2795 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2796 MODULE_VERSION("0.1"); 2797 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2798 MODULE_LICENSE("GPL"); 2799 2800 module_init(isert_init); 2801 module_exit(isert_exit); 2802