1 /******************************************************************************* 2 * This file contains iSCSI extentions for RDMA (iSER) Verbs 3 * 4 * (c) Copyright 2013 Datera, Inc. 5 * 6 * Nicholas A. Bellinger <nab@linux-iscsi.org> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 * 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 ****************************************************************************/ 18 19 #include <linux/string.h> 20 #include <linux/module.h> 21 #include <linux/scatterlist.h> 22 #include <linux/socket.h> 23 #include <linux/in.h> 24 #include <linux/in6.h> 25 #include <linux/llist.h> 26 #include <rdma/ib_verbs.h> 27 #include <rdma/rdma_cm.h> 28 #include <target/target_core_base.h> 29 #include <target/target_core_fabric.h> 30 #include <target/iscsi/iscsi_transport.h> 31 32 #include "isert_proto.h" 33 #include "ib_isert.h" 34 35 #define ISERT_MAX_CONN 8 36 #define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) 37 #define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) 38 39 static DEFINE_MUTEX(device_list_mutex); 40 static LIST_HEAD(device_list); 41 static struct workqueue_struct *isert_rx_wq; 42 static struct workqueue_struct *isert_comp_wq; 43 44 static void 45 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 46 static int 47 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 48 struct isert_rdma_wr *wr); 49 static void 50 isert_unreg_rdma_frwr(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn); 51 static int 52 isert_reg_rdma_frwr(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 53 struct isert_rdma_wr *wr); 54 55 static void 56 isert_qp_event_callback(struct ib_event *e, void *context) 57 { 58 struct isert_conn *isert_conn = (struct isert_conn *)context; 59 60 pr_err("isert_qp_event_callback event: %d\n", e->event); 61 switch (e->event) { 62 case IB_EVENT_COMM_EST: 63 rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); 64 break; 65 case IB_EVENT_QP_LAST_WQE_REACHED: 66 pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); 67 break; 68 default: 69 break; 70 } 71 } 72 73 static int 74 isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) 75 { 76 int ret; 77 78 ret = ib_query_device(ib_dev, devattr); 79 if (ret) { 80 pr_err("ib_query_device() failed: %d\n", ret); 81 return ret; 82 } 83 pr_debug("devattr->max_sge: %d\n", devattr->max_sge); 84 pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); 85 86 return 0; 87 } 88 89 static int 90 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) 91 { 92 struct isert_device *device = isert_conn->conn_device; 93 struct ib_qp_init_attr attr; 94 int ret, index, min_index = 0; 95 96 mutex_lock(&device_list_mutex); 97 for (index = 0; index < device->cqs_used; index++) 98 if (device->cq_active_qps[index] < 99 device->cq_active_qps[min_index]) 100 min_index = index; 101 device->cq_active_qps[min_index]++; 102 pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); 103 mutex_unlock(&device_list_mutex); 104 105 memset(&attr, 0, sizeof(struct ib_qp_init_attr)); 106 attr.event_handler = isert_qp_event_callback; 107 attr.qp_context = isert_conn; 108 attr.send_cq = device->dev_tx_cq[min_index]; 109 attr.recv_cq = device->dev_rx_cq[min_index]; 110 attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; 111 attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; 112 /* 113 * FIXME: Use devattr.max_sge - 2 for max_send_sge as 114 * work-around for RDMA_READ.. 115 */ 116 attr.cap.max_send_sge = device->dev_attr.max_sge - 2; 117 isert_conn->max_sge = attr.cap.max_send_sge; 118 119 attr.cap.max_recv_sge = 1; 120 attr.sq_sig_type = IB_SIGNAL_REQ_WR; 121 attr.qp_type = IB_QPT_RC; 122 123 pr_debug("isert_conn_setup_qp cma_id->device: %p\n", 124 cma_id->device); 125 pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", 126 isert_conn->conn_pd->device); 127 128 ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); 129 if (ret) { 130 pr_err("rdma_create_qp failed for cma_id %d\n", ret); 131 return ret; 132 } 133 isert_conn->conn_qp = cma_id->qp; 134 pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); 135 136 return 0; 137 } 138 139 static void 140 isert_cq_event_callback(struct ib_event *e, void *context) 141 { 142 pr_debug("isert_cq_event_callback event: %d\n", e->event); 143 } 144 145 static int 146 isert_alloc_rx_descriptors(struct isert_conn *isert_conn) 147 { 148 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 149 struct iser_rx_desc *rx_desc; 150 struct ib_sge *rx_sg; 151 u64 dma_addr; 152 int i, j; 153 154 isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * 155 sizeof(struct iser_rx_desc), GFP_KERNEL); 156 if (!isert_conn->conn_rx_descs) 157 goto fail; 158 159 rx_desc = isert_conn->conn_rx_descs; 160 161 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 162 dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, 163 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 164 if (ib_dma_mapping_error(ib_dev, dma_addr)) 165 goto dma_map_fail; 166 167 rx_desc->dma_addr = dma_addr; 168 169 rx_sg = &rx_desc->rx_sg; 170 rx_sg->addr = rx_desc->dma_addr; 171 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 172 rx_sg->lkey = isert_conn->conn_mr->lkey; 173 } 174 175 isert_conn->conn_rx_desc_head = 0; 176 return 0; 177 178 dma_map_fail: 179 rx_desc = isert_conn->conn_rx_descs; 180 for (j = 0; j < i; j++, rx_desc++) { 181 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 182 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 183 } 184 kfree(isert_conn->conn_rx_descs); 185 isert_conn->conn_rx_descs = NULL; 186 fail: 187 return -ENOMEM; 188 } 189 190 static void 191 isert_free_rx_descriptors(struct isert_conn *isert_conn) 192 { 193 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 194 struct iser_rx_desc *rx_desc; 195 int i; 196 197 if (!isert_conn->conn_rx_descs) 198 return; 199 200 rx_desc = isert_conn->conn_rx_descs; 201 for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { 202 ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, 203 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 204 } 205 206 kfree(isert_conn->conn_rx_descs); 207 isert_conn->conn_rx_descs = NULL; 208 } 209 210 static void isert_cq_tx_work(struct work_struct *); 211 static void isert_cq_tx_callback(struct ib_cq *, void *); 212 static void isert_cq_rx_work(struct work_struct *); 213 static void isert_cq_rx_callback(struct ib_cq *, void *); 214 215 static int 216 isert_create_device_ib_res(struct isert_device *device) 217 { 218 struct ib_device *ib_dev = device->ib_device; 219 struct isert_cq_desc *cq_desc; 220 struct ib_device_attr *dev_attr; 221 int ret = 0, i, j; 222 223 dev_attr = &device->dev_attr; 224 ret = isert_query_device(ib_dev, dev_attr); 225 if (ret) 226 return ret; 227 228 /* asign function handlers */ 229 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) { 230 device->use_frwr = 1; 231 device->reg_rdma_mem = isert_reg_rdma_frwr; 232 device->unreg_rdma_mem = isert_unreg_rdma_frwr; 233 } else { 234 device->use_frwr = 0; 235 device->reg_rdma_mem = isert_map_rdma; 236 device->unreg_rdma_mem = isert_unmap_cmd; 237 } 238 239 device->cqs_used = min_t(int, num_online_cpus(), 240 device->ib_device->num_comp_vectors); 241 device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); 242 pr_debug("Using %d CQs, device %s supports %d vectors support FRWR %d\n", 243 device->cqs_used, device->ib_device->name, 244 device->ib_device->num_comp_vectors, device->use_frwr); 245 device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * 246 device->cqs_used, GFP_KERNEL); 247 if (!device->cq_desc) { 248 pr_err("Unable to allocate device->cq_desc\n"); 249 return -ENOMEM; 250 } 251 cq_desc = device->cq_desc; 252 253 device->dev_pd = ib_alloc_pd(ib_dev); 254 if (IS_ERR(device->dev_pd)) { 255 ret = PTR_ERR(device->dev_pd); 256 pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret); 257 goto out_cq_desc; 258 } 259 260 for (i = 0; i < device->cqs_used; i++) { 261 cq_desc[i].device = device; 262 cq_desc[i].cq_index = i; 263 264 INIT_WORK(&cq_desc[i].cq_rx_work, isert_cq_rx_work); 265 device->dev_rx_cq[i] = ib_create_cq(device->ib_device, 266 isert_cq_rx_callback, 267 isert_cq_event_callback, 268 (void *)&cq_desc[i], 269 ISER_MAX_RX_CQ_LEN, i); 270 if (IS_ERR(device->dev_rx_cq[i])) { 271 ret = PTR_ERR(device->dev_rx_cq[i]); 272 device->dev_rx_cq[i] = NULL; 273 goto out_cq; 274 } 275 276 INIT_WORK(&cq_desc[i].cq_tx_work, isert_cq_tx_work); 277 device->dev_tx_cq[i] = ib_create_cq(device->ib_device, 278 isert_cq_tx_callback, 279 isert_cq_event_callback, 280 (void *)&cq_desc[i], 281 ISER_MAX_TX_CQ_LEN, i); 282 if (IS_ERR(device->dev_tx_cq[i])) { 283 ret = PTR_ERR(device->dev_tx_cq[i]); 284 device->dev_tx_cq[i] = NULL; 285 goto out_cq; 286 } 287 288 ret = ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP); 289 if (ret) 290 goto out_cq; 291 292 ret = ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP); 293 if (ret) 294 goto out_cq; 295 } 296 297 device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE); 298 if (IS_ERR(device->dev_mr)) { 299 ret = PTR_ERR(device->dev_mr); 300 pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret); 301 goto out_cq; 302 } 303 304 return 0; 305 306 out_cq: 307 for (j = 0; j < i; j++) { 308 cq_desc = &device->cq_desc[j]; 309 310 if (device->dev_rx_cq[j]) { 311 cancel_work_sync(&cq_desc->cq_rx_work); 312 ib_destroy_cq(device->dev_rx_cq[j]); 313 } 314 if (device->dev_tx_cq[j]) { 315 cancel_work_sync(&cq_desc->cq_tx_work); 316 ib_destroy_cq(device->dev_tx_cq[j]); 317 } 318 } 319 ib_dealloc_pd(device->dev_pd); 320 321 out_cq_desc: 322 kfree(device->cq_desc); 323 324 return ret; 325 } 326 327 static void 328 isert_free_device_ib_res(struct isert_device *device) 329 { 330 struct isert_cq_desc *cq_desc; 331 int i; 332 333 for (i = 0; i < device->cqs_used; i++) { 334 cq_desc = &device->cq_desc[i]; 335 336 cancel_work_sync(&cq_desc->cq_rx_work); 337 cancel_work_sync(&cq_desc->cq_tx_work); 338 ib_destroy_cq(device->dev_rx_cq[i]); 339 ib_destroy_cq(device->dev_tx_cq[i]); 340 device->dev_rx_cq[i] = NULL; 341 device->dev_tx_cq[i] = NULL; 342 } 343 344 ib_dereg_mr(device->dev_mr); 345 ib_dealloc_pd(device->dev_pd); 346 kfree(device->cq_desc); 347 } 348 349 static void 350 isert_device_try_release(struct isert_device *device) 351 { 352 mutex_lock(&device_list_mutex); 353 device->refcount--; 354 if (!device->refcount) { 355 isert_free_device_ib_res(device); 356 list_del(&device->dev_node); 357 kfree(device); 358 } 359 mutex_unlock(&device_list_mutex); 360 } 361 362 static struct isert_device * 363 isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) 364 { 365 struct isert_device *device; 366 int ret; 367 368 mutex_lock(&device_list_mutex); 369 list_for_each_entry(device, &device_list, dev_node) { 370 if (device->ib_device->node_guid == cma_id->device->node_guid) { 371 device->refcount++; 372 mutex_unlock(&device_list_mutex); 373 return device; 374 } 375 } 376 377 device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); 378 if (!device) { 379 mutex_unlock(&device_list_mutex); 380 return ERR_PTR(-ENOMEM); 381 } 382 383 INIT_LIST_HEAD(&device->dev_node); 384 385 device->ib_device = cma_id->device; 386 ret = isert_create_device_ib_res(device); 387 if (ret) { 388 kfree(device); 389 mutex_unlock(&device_list_mutex); 390 return ERR_PTR(ret); 391 } 392 393 device->refcount++; 394 list_add_tail(&device->dev_node, &device_list); 395 mutex_unlock(&device_list_mutex); 396 397 return device; 398 } 399 400 static void 401 isert_conn_free_frwr_pool(struct isert_conn *isert_conn) 402 { 403 struct fast_reg_descriptor *fr_desc, *tmp; 404 int i = 0; 405 406 if (list_empty(&isert_conn->conn_frwr_pool)) 407 return; 408 409 pr_debug("Freeing conn %p frwr pool", isert_conn); 410 411 list_for_each_entry_safe(fr_desc, tmp, 412 &isert_conn->conn_frwr_pool, list) { 413 list_del(&fr_desc->list); 414 ib_free_fast_reg_page_list(fr_desc->data_frpl); 415 ib_dereg_mr(fr_desc->data_mr); 416 kfree(fr_desc); 417 ++i; 418 } 419 420 if (i < isert_conn->conn_frwr_pool_size) 421 pr_warn("Pool still has %d regions registered\n", 422 isert_conn->conn_frwr_pool_size - i); 423 } 424 425 static int 426 isert_conn_create_frwr_pool(struct isert_conn *isert_conn) 427 { 428 struct fast_reg_descriptor *fr_desc; 429 struct isert_device *device = isert_conn->conn_device; 430 int i, ret; 431 432 INIT_LIST_HEAD(&isert_conn->conn_frwr_pool); 433 isert_conn->conn_frwr_pool_size = 0; 434 for (i = 0; i < ISCSI_DEF_XMIT_CMDS_MAX; i++) { 435 fr_desc = kzalloc(sizeof(*fr_desc), GFP_KERNEL); 436 if (!fr_desc) { 437 pr_err("Failed to allocate fast_reg descriptor\n"); 438 ret = -ENOMEM; 439 goto err; 440 } 441 442 fr_desc->data_frpl = 443 ib_alloc_fast_reg_page_list(device->ib_device, 444 ISCSI_ISER_SG_TABLESIZE); 445 if (IS_ERR(fr_desc->data_frpl)) { 446 pr_err("Failed to allocate fr_pg_list err=%ld\n", 447 PTR_ERR(fr_desc->data_frpl)); 448 ret = PTR_ERR(fr_desc->data_frpl); 449 goto err; 450 } 451 452 fr_desc->data_mr = ib_alloc_fast_reg_mr(device->dev_pd, 453 ISCSI_ISER_SG_TABLESIZE); 454 if (IS_ERR(fr_desc->data_mr)) { 455 pr_err("Failed to allocate frmr err=%ld\n", 456 PTR_ERR(fr_desc->data_mr)); 457 ret = PTR_ERR(fr_desc->data_mr); 458 ib_free_fast_reg_page_list(fr_desc->data_frpl); 459 goto err; 460 } 461 pr_debug("Create fr_desc %p page_list %p\n", 462 fr_desc, fr_desc->data_frpl->page_list); 463 464 fr_desc->valid = true; 465 list_add_tail(&fr_desc->list, &isert_conn->conn_frwr_pool); 466 isert_conn->conn_frwr_pool_size++; 467 } 468 469 pr_debug("Creating conn %p frwr pool size=%d", 470 isert_conn, isert_conn->conn_frwr_pool_size); 471 472 return 0; 473 474 err: 475 isert_conn_free_frwr_pool(isert_conn); 476 return ret; 477 } 478 479 static int 480 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 481 { 482 struct iscsi_np *np = cma_id->context; 483 struct isert_np *isert_np = np->np_context; 484 struct isert_conn *isert_conn; 485 struct isert_device *device; 486 struct ib_device *ib_dev = cma_id->device; 487 int ret = 0; 488 489 pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", 490 cma_id, cma_id->context); 491 492 isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); 493 if (!isert_conn) { 494 pr_err("Unable to allocate isert_conn\n"); 495 return -ENOMEM; 496 } 497 isert_conn->state = ISER_CONN_INIT; 498 INIT_LIST_HEAD(&isert_conn->conn_accept_node); 499 init_completion(&isert_conn->conn_login_comp); 500 init_waitqueue_head(&isert_conn->conn_wait); 501 init_waitqueue_head(&isert_conn->conn_wait_comp_err); 502 kref_init(&isert_conn->conn_kref); 503 kref_get(&isert_conn->conn_kref); 504 mutex_init(&isert_conn->conn_mutex); 505 mutex_init(&isert_conn->conn_comp_mutex); 506 spin_lock_init(&isert_conn->conn_lock); 507 508 cma_id->context = isert_conn; 509 isert_conn->conn_cm_id = cma_id; 510 isert_conn->responder_resources = event->param.conn.responder_resources; 511 isert_conn->initiator_depth = event->param.conn.initiator_depth; 512 pr_debug("Using responder_resources: %u initiator_depth: %u\n", 513 isert_conn->responder_resources, isert_conn->initiator_depth); 514 515 isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + 516 ISER_RX_LOGIN_SIZE, GFP_KERNEL); 517 if (!isert_conn->login_buf) { 518 pr_err("Unable to allocate isert_conn->login_buf\n"); 519 ret = -ENOMEM; 520 goto out; 521 } 522 523 isert_conn->login_req_buf = isert_conn->login_buf; 524 isert_conn->login_rsp_buf = isert_conn->login_buf + 525 ISCSI_DEF_MAX_RECV_SEG_LEN; 526 pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", 527 isert_conn->login_buf, isert_conn->login_req_buf, 528 isert_conn->login_rsp_buf); 529 530 isert_conn->login_req_dma = ib_dma_map_single(ib_dev, 531 (void *)isert_conn->login_req_buf, 532 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 533 534 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); 535 if (ret) { 536 pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", 537 ret); 538 isert_conn->login_req_dma = 0; 539 goto out_login_buf; 540 } 541 542 isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, 543 (void *)isert_conn->login_rsp_buf, 544 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 545 546 ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); 547 if (ret) { 548 pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", 549 ret); 550 isert_conn->login_rsp_dma = 0; 551 goto out_req_dma_map; 552 } 553 554 device = isert_device_find_by_ib_dev(cma_id); 555 if (IS_ERR(device)) { 556 ret = PTR_ERR(device); 557 goto out_rsp_dma_map; 558 } 559 560 isert_conn->conn_device = device; 561 isert_conn->conn_pd = device->dev_pd; 562 isert_conn->conn_mr = device->dev_mr; 563 564 if (device->use_frwr) { 565 ret = isert_conn_create_frwr_pool(isert_conn); 566 if (ret) { 567 pr_err("Conn: %p failed to create frwr_pool\n", isert_conn); 568 goto out_frwr; 569 } 570 } 571 572 ret = isert_conn_setup_qp(isert_conn, cma_id); 573 if (ret) 574 goto out_conn_dev; 575 576 mutex_lock(&isert_np->np_accept_mutex); 577 list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); 578 mutex_unlock(&isert_np->np_accept_mutex); 579 580 pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); 581 wake_up(&isert_np->np_accept_wq); 582 return 0; 583 584 out_conn_dev: 585 if (device->use_frwr) 586 isert_conn_free_frwr_pool(isert_conn); 587 out_frwr: 588 isert_device_try_release(device); 589 out_rsp_dma_map: 590 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 591 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 592 out_req_dma_map: 593 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 594 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); 595 out_login_buf: 596 kfree(isert_conn->login_buf); 597 out: 598 kfree(isert_conn); 599 return ret; 600 } 601 602 static void 603 isert_connect_release(struct isert_conn *isert_conn) 604 { 605 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 606 struct isert_device *device = isert_conn->conn_device; 607 int cq_index; 608 609 pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 610 611 if (device && device->use_frwr) 612 isert_conn_free_frwr_pool(isert_conn); 613 614 if (isert_conn->conn_qp) { 615 cq_index = ((struct isert_cq_desc *) 616 isert_conn->conn_qp->recv_cq->cq_context)->cq_index; 617 pr_debug("isert_connect_release: cq_index: %d\n", cq_index); 618 isert_conn->conn_device->cq_active_qps[cq_index]--; 619 620 rdma_destroy_qp(isert_conn->conn_cm_id); 621 } 622 623 isert_free_rx_descriptors(isert_conn); 624 rdma_destroy_id(isert_conn->conn_cm_id); 625 626 if (isert_conn->login_buf) { 627 ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, 628 ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); 629 ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, 630 ISCSI_DEF_MAX_RECV_SEG_LEN, 631 DMA_FROM_DEVICE); 632 kfree(isert_conn->login_buf); 633 } 634 kfree(isert_conn); 635 636 if (device) 637 isert_device_try_release(device); 638 639 pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); 640 } 641 642 static void 643 isert_connected_handler(struct rdma_cm_id *cma_id) 644 { 645 return; 646 } 647 648 static void 649 isert_release_conn_kref(struct kref *kref) 650 { 651 struct isert_conn *isert_conn = container_of(kref, 652 struct isert_conn, conn_kref); 653 654 pr_debug("Calling isert_connect_release for final kref %s/%d\n", 655 current->comm, current->pid); 656 657 isert_connect_release(isert_conn); 658 } 659 660 static void 661 isert_put_conn(struct isert_conn *isert_conn) 662 { 663 kref_put(&isert_conn->conn_kref, isert_release_conn_kref); 664 } 665 666 static void 667 isert_disconnect_work(struct work_struct *work) 668 { 669 struct isert_conn *isert_conn = container_of(work, 670 struct isert_conn, conn_logout_work); 671 672 pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 673 mutex_lock(&isert_conn->conn_mutex); 674 isert_conn->state = ISER_CONN_DOWN; 675 676 if (isert_conn->post_recv_buf_count == 0 && 677 atomic_read(&isert_conn->post_send_buf_count) == 0) { 678 pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); 679 mutex_unlock(&isert_conn->conn_mutex); 680 goto wake_up; 681 } 682 if (!isert_conn->conn_cm_id) { 683 mutex_unlock(&isert_conn->conn_mutex); 684 isert_put_conn(isert_conn); 685 return; 686 } 687 if (!isert_conn->logout_posted) { 688 pr_debug("Calling rdma_disconnect for !logout_posted from" 689 " isert_disconnect_work\n"); 690 rdma_disconnect(isert_conn->conn_cm_id); 691 mutex_unlock(&isert_conn->conn_mutex); 692 iscsit_cause_connection_reinstatement(isert_conn->conn, 0); 693 goto wake_up; 694 } 695 mutex_unlock(&isert_conn->conn_mutex); 696 697 wake_up: 698 wake_up(&isert_conn->conn_wait); 699 isert_put_conn(isert_conn); 700 } 701 702 static void 703 isert_disconnected_handler(struct rdma_cm_id *cma_id) 704 { 705 struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; 706 707 INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); 708 schedule_work(&isert_conn->conn_logout_work); 709 } 710 711 static int 712 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) 713 { 714 int ret = 0; 715 716 pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", 717 event->event, event->status, cma_id->context, cma_id); 718 719 switch (event->event) { 720 case RDMA_CM_EVENT_CONNECT_REQUEST: 721 pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); 722 ret = isert_connect_request(cma_id, event); 723 break; 724 case RDMA_CM_EVENT_ESTABLISHED: 725 pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); 726 isert_connected_handler(cma_id); 727 break; 728 case RDMA_CM_EVENT_DISCONNECTED: 729 pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); 730 isert_disconnected_handler(cma_id); 731 break; 732 case RDMA_CM_EVENT_DEVICE_REMOVAL: 733 case RDMA_CM_EVENT_ADDR_CHANGE: 734 break; 735 case RDMA_CM_EVENT_CONNECT_ERROR: 736 default: 737 pr_err("Unknown RDMA CMA event: %d\n", event->event); 738 break; 739 } 740 741 if (ret != 0) { 742 pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", 743 event->event, ret); 744 dump_stack(); 745 } 746 747 return ret; 748 } 749 750 static int 751 isert_post_recv(struct isert_conn *isert_conn, u32 count) 752 { 753 struct ib_recv_wr *rx_wr, *rx_wr_failed; 754 int i, ret; 755 unsigned int rx_head = isert_conn->conn_rx_desc_head; 756 struct iser_rx_desc *rx_desc; 757 758 for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { 759 rx_desc = &isert_conn->conn_rx_descs[rx_head]; 760 rx_wr->wr_id = (unsigned long)rx_desc; 761 rx_wr->sg_list = &rx_desc->rx_sg; 762 rx_wr->num_sge = 1; 763 rx_wr->next = rx_wr + 1; 764 rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); 765 } 766 767 rx_wr--; 768 rx_wr->next = NULL; /* mark end of work requests list */ 769 770 isert_conn->post_recv_buf_count += count; 771 ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, 772 &rx_wr_failed); 773 if (ret) { 774 pr_err("ib_post_recv() failed with ret: %d\n", ret); 775 isert_conn->post_recv_buf_count -= count; 776 } else { 777 pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); 778 isert_conn->conn_rx_desc_head = rx_head; 779 } 780 return ret; 781 } 782 783 static int 784 isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) 785 { 786 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 787 struct ib_send_wr send_wr, *send_wr_failed; 788 int ret; 789 790 ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, 791 ISER_HEADERS_LEN, DMA_TO_DEVICE); 792 793 send_wr.next = NULL; 794 send_wr.wr_id = (unsigned long)tx_desc; 795 send_wr.sg_list = tx_desc->tx_sg; 796 send_wr.num_sge = tx_desc->num_sge; 797 send_wr.opcode = IB_WR_SEND; 798 send_wr.send_flags = IB_SEND_SIGNALED; 799 800 atomic_inc(&isert_conn->post_send_buf_count); 801 802 ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); 803 if (ret) { 804 pr_err("ib_post_send() failed, ret: %d\n", ret); 805 atomic_dec(&isert_conn->post_send_buf_count); 806 } 807 808 return ret; 809 } 810 811 static void 812 isert_create_send_desc(struct isert_conn *isert_conn, 813 struct isert_cmd *isert_cmd, 814 struct iser_tx_desc *tx_desc) 815 { 816 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 817 818 ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, 819 ISER_HEADERS_LEN, DMA_TO_DEVICE); 820 821 memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); 822 tx_desc->iser_header.flags = ISER_VER; 823 824 tx_desc->num_sge = 1; 825 tx_desc->isert_cmd = isert_cmd; 826 827 if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { 828 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 829 pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); 830 } 831 } 832 833 static int 834 isert_init_tx_hdrs(struct isert_conn *isert_conn, 835 struct iser_tx_desc *tx_desc) 836 { 837 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 838 u64 dma_addr; 839 840 dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, 841 ISER_HEADERS_LEN, DMA_TO_DEVICE); 842 if (ib_dma_mapping_error(ib_dev, dma_addr)) { 843 pr_err("ib_dma_mapping_error() failed\n"); 844 return -ENOMEM; 845 } 846 847 tx_desc->dma_addr = dma_addr; 848 tx_desc->tx_sg[0].addr = tx_desc->dma_addr; 849 tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; 850 tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; 851 852 pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" 853 " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, 854 tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); 855 856 return 0; 857 } 858 859 static void 860 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 861 struct ib_send_wr *send_wr, bool coalesce) 862 { 863 struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc; 864 865 isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; 866 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 867 send_wr->opcode = IB_WR_SEND; 868 send_wr->sg_list = &tx_desc->tx_sg[0]; 869 send_wr->num_sge = isert_cmd->tx_desc.num_sge; 870 /* 871 * Coalesce send completion interrupts by only setting IB_SEND_SIGNALED 872 * bit for every ISERT_COMP_BATCH_COUNT number of ib_post_send() calls. 873 */ 874 mutex_lock(&isert_conn->conn_comp_mutex); 875 if (coalesce && 876 ++isert_conn->conn_comp_batch < ISERT_COMP_BATCH_COUNT) { 877 llist_add(&tx_desc->comp_llnode, &isert_conn->conn_comp_llist); 878 mutex_unlock(&isert_conn->conn_comp_mutex); 879 return; 880 } 881 isert_conn->conn_comp_batch = 0; 882 tx_desc->comp_llnode_batch = llist_del_all(&isert_conn->conn_comp_llist); 883 mutex_unlock(&isert_conn->conn_comp_mutex); 884 885 send_wr->send_flags = IB_SEND_SIGNALED; 886 } 887 888 static int 889 isert_rdma_post_recvl(struct isert_conn *isert_conn) 890 { 891 struct ib_recv_wr rx_wr, *rx_wr_fail; 892 struct ib_sge sge; 893 int ret; 894 895 memset(&sge, 0, sizeof(struct ib_sge)); 896 sge.addr = isert_conn->login_req_dma; 897 sge.length = ISER_RX_LOGIN_SIZE; 898 sge.lkey = isert_conn->conn_mr->lkey; 899 900 pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", 901 sge.addr, sge.length, sge.lkey); 902 903 memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); 904 rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; 905 rx_wr.sg_list = &sge; 906 rx_wr.num_sge = 1; 907 908 isert_conn->post_recv_buf_count++; 909 ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); 910 if (ret) { 911 pr_err("ib_post_recv() failed: %d\n", ret); 912 isert_conn->post_recv_buf_count--; 913 } 914 915 pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); 916 return ret; 917 } 918 919 static int 920 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, 921 u32 length) 922 { 923 struct isert_conn *isert_conn = conn->context; 924 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 925 struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; 926 int ret; 927 928 isert_create_send_desc(isert_conn, NULL, tx_desc); 929 930 memcpy(&tx_desc->iscsi_header, &login->rsp[0], 931 sizeof(struct iscsi_hdr)); 932 933 isert_init_tx_hdrs(isert_conn, tx_desc); 934 935 if (length > 0) { 936 struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; 937 938 ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, 939 length, DMA_TO_DEVICE); 940 941 memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); 942 943 ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, 944 length, DMA_TO_DEVICE); 945 946 tx_dsg->addr = isert_conn->login_rsp_dma; 947 tx_dsg->length = length; 948 tx_dsg->lkey = isert_conn->conn_mr->lkey; 949 tx_desc->num_sge = 2; 950 } 951 if (!login->login_failed) { 952 if (login->login_complete) { 953 ret = isert_alloc_rx_descriptors(isert_conn); 954 if (ret) 955 return ret; 956 957 ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); 958 if (ret) 959 return ret; 960 961 isert_conn->state = ISER_CONN_UP; 962 goto post_send; 963 } 964 965 ret = isert_rdma_post_recvl(isert_conn); 966 if (ret) 967 return ret; 968 } 969 post_send: 970 ret = isert_post_send(isert_conn, tx_desc); 971 if (ret) 972 return ret; 973 974 return 0; 975 } 976 977 static void 978 isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, 979 struct isert_conn *isert_conn) 980 { 981 struct iscsi_conn *conn = isert_conn->conn; 982 struct iscsi_login *login = conn->conn_login; 983 int size; 984 985 if (!login) { 986 pr_err("conn->conn_login is NULL\n"); 987 dump_stack(); 988 return; 989 } 990 991 if (login->first_request) { 992 struct iscsi_login_req *login_req = 993 (struct iscsi_login_req *)&rx_desc->iscsi_header; 994 /* 995 * Setup the initial iscsi_login values from the leading 996 * login request PDU. 997 */ 998 login->leading_connection = (!login_req->tsih) ? 1 : 0; 999 login->current_stage = 1000 (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) 1001 >> 2; 1002 login->version_min = login_req->min_version; 1003 login->version_max = login_req->max_version; 1004 memcpy(login->isid, login_req->isid, 6); 1005 login->cmd_sn = be32_to_cpu(login_req->cmdsn); 1006 login->init_task_tag = login_req->itt; 1007 login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); 1008 login->cid = be16_to_cpu(login_req->cid); 1009 login->tsih = be16_to_cpu(login_req->tsih); 1010 } 1011 1012 memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); 1013 1014 size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); 1015 pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", 1016 size, rx_buflen, MAX_KEY_VALUE_PAIRS); 1017 memcpy(login->req_buf, &rx_desc->data[0], size); 1018 1019 if (login->first_request) { 1020 complete(&isert_conn->conn_login_comp); 1021 return; 1022 } 1023 schedule_delayed_work(&conn->login_work, 0); 1024 } 1025 1026 static struct iscsi_cmd 1027 *isert_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp) 1028 { 1029 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1030 struct isert_cmd *isert_cmd; 1031 struct iscsi_cmd *cmd; 1032 1033 cmd = iscsit_allocate_cmd(conn, gfp); 1034 if (!cmd) { 1035 pr_err("Unable to allocate iscsi_cmd + isert_cmd\n"); 1036 return NULL; 1037 } 1038 isert_cmd = iscsit_priv_cmd(cmd); 1039 isert_cmd->conn = isert_conn; 1040 isert_cmd->iscsi_cmd = cmd; 1041 1042 return cmd; 1043 } 1044 1045 static int 1046 isert_handle_scsi_cmd(struct isert_conn *isert_conn, 1047 struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd, 1048 struct iser_rx_desc *rx_desc, unsigned char *buf) 1049 { 1050 struct iscsi_conn *conn = isert_conn->conn; 1051 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; 1052 struct scatterlist *sg; 1053 int imm_data, imm_data_len, unsol_data, sg_nents, rc; 1054 bool dump_payload = false; 1055 1056 rc = iscsit_setup_scsi_cmd(conn, cmd, buf); 1057 if (rc < 0) 1058 return rc; 1059 1060 imm_data = cmd->immediate_data; 1061 imm_data_len = cmd->first_burst_len; 1062 unsol_data = cmd->unsolicited_data; 1063 1064 rc = iscsit_process_scsi_cmd(conn, cmd, hdr); 1065 if (rc < 0) { 1066 return 0; 1067 } else if (rc > 0) { 1068 dump_payload = true; 1069 goto sequence_cmd; 1070 } 1071 1072 if (!imm_data) 1073 return 0; 1074 1075 sg = &cmd->se_cmd.t_data_sg[0]; 1076 sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); 1077 1078 pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", 1079 sg, sg_nents, &rx_desc->data[0], imm_data_len); 1080 1081 sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); 1082 1083 cmd->write_data_done += imm_data_len; 1084 1085 if (cmd->write_data_done == cmd->se_cmd.data_length) { 1086 spin_lock_bh(&cmd->istate_lock); 1087 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1088 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1089 spin_unlock_bh(&cmd->istate_lock); 1090 } 1091 1092 sequence_cmd: 1093 rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn); 1094 1095 if (!rc && dump_payload == false && unsol_data) 1096 iscsit_set_unsoliticed_dataout(cmd); 1097 1098 return 0; 1099 } 1100 1101 static int 1102 isert_handle_iscsi_dataout(struct isert_conn *isert_conn, 1103 struct iser_rx_desc *rx_desc, unsigned char *buf) 1104 { 1105 struct scatterlist *sg_start; 1106 struct iscsi_conn *conn = isert_conn->conn; 1107 struct iscsi_cmd *cmd = NULL; 1108 struct iscsi_data *hdr = (struct iscsi_data *)buf; 1109 u32 unsol_data_len = ntoh24(hdr->dlength); 1110 int rc, sg_nents, sg_off, page_off; 1111 1112 rc = iscsit_check_dataout_hdr(conn, buf, &cmd); 1113 if (rc < 0) 1114 return rc; 1115 else if (!cmd) 1116 return 0; 1117 /* 1118 * FIXME: Unexpected unsolicited_data out 1119 */ 1120 if (!cmd->unsolicited_data) { 1121 pr_err("Received unexpected solicited data payload\n"); 1122 dump_stack(); 1123 return -1; 1124 } 1125 1126 pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", 1127 unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); 1128 1129 sg_off = cmd->write_data_done / PAGE_SIZE; 1130 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1131 sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); 1132 page_off = cmd->write_data_done % PAGE_SIZE; 1133 /* 1134 * FIXME: Non page-aligned unsolicited_data out 1135 */ 1136 if (page_off) { 1137 pr_err("Received unexpected non-page aligned data payload\n"); 1138 dump_stack(); 1139 return -1; 1140 } 1141 pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", 1142 sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); 1143 1144 sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], 1145 unsol_data_len); 1146 1147 rc = iscsit_check_dataout_payload(cmd, hdr, false); 1148 if (rc < 0) 1149 return rc; 1150 1151 return 0; 1152 } 1153 1154 static int 1155 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1156 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1157 unsigned char *buf) 1158 { 1159 struct iscsi_conn *conn = isert_conn->conn; 1160 struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf; 1161 int rc; 1162 1163 rc = iscsit_setup_nop_out(conn, cmd, hdr); 1164 if (rc < 0) 1165 return rc; 1166 /* 1167 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload 1168 */ 1169 1170 return iscsit_process_nop_out(conn, cmd, hdr); 1171 } 1172 1173 static int 1174 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1175 struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc, 1176 struct iscsi_text *hdr) 1177 { 1178 struct iscsi_conn *conn = isert_conn->conn; 1179 u32 payload_length = ntoh24(hdr->dlength); 1180 int rc; 1181 unsigned char *text_in; 1182 1183 rc = iscsit_setup_text_cmd(conn, cmd, hdr); 1184 if (rc < 0) 1185 return rc; 1186 1187 text_in = kzalloc(payload_length, GFP_KERNEL); 1188 if (!text_in) { 1189 pr_err("Unable to allocate text_in of payload_length: %u\n", 1190 payload_length); 1191 return -ENOMEM; 1192 } 1193 cmd->text_in_ptr = text_in; 1194 1195 memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length); 1196 1197 return iscsit_process_text_cmd(conn, cmd, hdr); 1198 } 1199 1200 static int 1201 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, 1202 uint32_t read_stag, uint64_t read_va, 1203 uint32_t write_stag, uint64_t write_va) 1204 { 1205 struct iscsi_hdr *hdr = &rx_desc->iscsi_header; 1206 struct iscsi_conn *conn = isert_conn->conn; 1207 struct iscsi_session *sess = conn->sess; 1208 struct iscsi_cmd *cmd; 1209 struct isert_cmd *isert_cmd; 1210 int ret = -EINVAL; 1211 u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); 1212 1213 if (sess->sess_ops->SessionType && 1214 (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) { 1215 pr_err("Got illegal opcode: 0x%02x in SessionType=Discovery," 1216 " ignoring\n", opcode); 1217 return 0; 1218 } 1219 1220 switch (opcode) { 1221 case ISCSI_OP_SCSI_CMD: 1222 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1223 if (!cmd) 1224 break; 1225 1226 isert_cmd = iscsit_priv_cmd(cmd); 1227 isert_cmd->read_stag = read_stag; 1228 isert_cmd->read_va = read_va; 1229 isert_cmd->write_stag = write_stag; 1230 isert_cmd->write_va = write_va; 1231 1232 ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd, 1233 rx_desc, (unsigned char *)hdr); 1234 break; 1235 case ISCSI_OP_NOOP_OUT: 1236 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1237 if (!cmd) 1238 break; 1239 1240 isert_cmd = iscsit_priv_cmd(cmd); 1241 ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd, 1242 rx_desc, (unsigned char *)hdr); 1243 break; 1244 case ISCSI_OP_SCSI_DATA_OUT: 1245 ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, 1246 (unsigned char *)hdr); 1247 break; 1248 case ISCSI_OP_SCSI_TMFUNC: 1249 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1250 if (!cmd) 1251 break; 1252 1253 ret = iscsit_handle_task_mgt_cmd(conn, cmd, 1254 (unsigned char *)hdr); 1255 break; 1256 case ISCSI_OP_LOGOUT: 1257 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1258 if (!cmd) 1259 break; 1260 1261 ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); 1262 if (ret > 0) 1263 wait_for_completion_timeout(&conn->conn_logout_comp, 1264 SECONDS_FOR_LOGOUT_COMP * 1265 HZ); 1266 break; 1267 case ISCSI_OP_TEXT: 1268 cmd = isert_allocate_cmd(conn, GFP_KERNEL); 1269 if (!cmd) 1270 break; 1271 1272 isert_cmd = iscsit_priv_cmd(cmd); 1273 ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd, 1274 rx_desc, (struct iscsi_text *)hdr); 1275 break; 1276 default: 1277 pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); 1278 dump_stack(); 1279 break; 1280 } 1281 1282 return ret; 1283 } 1284 1285 static void 1286 isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) 1287 { 1288 struct iser_hdr *iser_hdr = &rx_desc->iser_header; 1289 uint64_t read_va = 0, write_va = 0; 1290 uint32_t read_stag = 0, write_stag = 0; 1291 int rc; 1292 1293 switch (iser_hdr->flags & 0xF0) { 1294 case ISCSI_CTRL: 1295 if (iser_hdr->flags & ISER_RSV) { 1296 read_stag = be32_to_cpu(iser_hdr->read_stag); 1297 read_va = be64_to_cpu(iser_hdr->read_va); 1298 pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", 1299 read_stag, (unsigned long long)read_va); 1300 } 1301 if (iser_hdr->flags & ISER_WSV) { 1302 write_stag = be32_to_cpu(iser_hdr->write_stag); 1303 write_va = be64_to_cpu(iser_hdr->write_va); 1304 pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", 1305 write_stag, (unsigned long long)write_va); 1306 } 1307 1308 pr_debug("ISER ISCSI_CTRL PDU\n"); 1309 break; 1310 case ISER_HELLO: 1311 pr_err("iSER Hello message\n"); 1312 break; 1313 default: 1314 pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); 1315 break; 1316 } 1317 1318 rc = isert_rx_opcode(isert_conn, rx_desc, 1319 read_stag, read_va, write_stag, write_va); 1320 } 1321 1322 static void 1323 isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, 1324 unsigned long xfer_len) 1325 { 1326 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1327 struct iscsi_hdr *hdr; 1328 u64 rx_dma; 1329 int rx_buflen, outstanding; 1330 1331 if ((char *)desc == isert_conn->login_req_buf) { 1332 rx_dma = isert_conn->login_req_dma; 1333 rx_buflen = ISER_RX_LOGIN_SIZE; 1334 pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1335 rx_dma, rx_buflen); 1336 } else { 1337 rx_dma = desc->dma_addr; 1338 rx_buflen = ISER_RX_PAYLOAD_SIZE; 1339 pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", 1340 rx_dma, rx_buflen); 1341 } 1342 1343 ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); 1344 1345 hdr = &desc->iscsi_header; 1346 pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", 1347 hdr->opcode, hdr->itt, hdr->flags, 1348 (int)(xfer_len - ISER_HEADERS_LEN)); 1349 1350 if ((char *)desc == isert_conn->login_req_buf) 1351 isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, 1352 isert_conn); 1353 else 1354 isert_rx_do_work(desc, isert_conn); 1355 1356 ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, 1357 DMA_FROM_DEVICE); 1358 1359 isert_conn->post_recv_buf_count--; 1360 pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", 1361 isert_conn->post_recv_buf_count); 1362 1363 if ((char *)desc == isert_conn->login_req_buf) 1364 return; 1365 1366 outstanding = isert_conn->post_recv_buf_count; 1367 if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { 1368 int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, 1369 ISERT_MIN_POSTED_RX); 1370 err = isert_post_recv(isert_conn, count); 1371 if (err) { 1372 pr_err("isert_post_recv() count: %d failed, %d\n", 1373 count, err); 1374 } 1375 } 1376 } 1377 1378 static void 1379 isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1380 { 1381 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1382 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1383 1384 pr_debug("isert_unmap_cmd: %p\n", isert_cmd); 1385 if (wr->sge) { 1386 pr_debug("isert_unmap_cmd: %p unmap_sg op\n", isert_cmd); 1387 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1388 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1389 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1390 wr->sge = NULL; 1391 } 1392 1393 if (wr->send_wr) { 1394 pr_debug("isert_unmap_cmd: %p free send_wr\n", isert_cmd); 1395 kfree(wr->send_wr); 1396 wr->send_wr = NULL; 1397 } 1398 1399 if (wr->ib_sge) { 1400 pr_debug("isert_unmap_cmd: %p free ib_sge\n", isert_cmd); 1401 kfree(wr->ib_sge); 1402 wr->ib_sge = NULL; 1403 } 1404 } 1405 1406 static void 1407 isert_unreg_rdma_frwr(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) 1408 { 1409 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1410 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1411 LIST_HEAD(unmap_list); 1412 1413 pr_debug("unreg_frwr_cmd: %p\n", isert_cmd); 1414 1415 if (wr->fr_desc) { 1416 pr_debug("unreg_frwr_cmd: %p free fr_desc %p\n", 1417 isert_cmd, wr->fr_desc); 1418 spin_lock_bh(&isert_conn->conn_lock); 1419 list_add_tail(&wr->fr_desc->list, &isert_conn->conn_frwr_pool); 1420 spin_unlock_bh(&isert_conn->conn_lock); 1421 wr->fr_desc = NULL; 1422 } 1423 1424 if (wr->sge) { 1425 pr_debug("unreg_frwr_cmd: %p unmap_sg op\n", isert_cmd); 1426 ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, 1427 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 1428 DMA_TO_DEVICE : DMA_FROM_DEVICE); 1429 wr->sge = NULL; 1430 } 1431 1432 wr->ib_sge = NULL; 1433 wr->send_wr = NULL; 1434 } 1435 1436 static void 1437 isert_put_cmd(struct isert_cmd *isert_cmd) 1438 { 1439 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1440 struct isert_conn *isert_conn = isert_cmd->conn; 1441 struct iscsi_conn *conn = isert_conn->conn; 1442 struct isert_device *device = isert_conn->conn_device; 1443 1444 pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); 1445 1446 switch (cmd->iscsi_opcode) { 1447 case ISCSI_OP_SCSI_CMD: 1448 spin_lock_bh(&conn->cmd_lock); 1449 if (!list_empty(&cmd->i_conn_node)) 1450 list_del(&cmd->i_conn_node); 1451 spin_unlock_bh(&conn->cmd_lock); 1452 1453 if (cmd->data_direction == DMA_TO_DEVICE) 1454 iscsit_stop_dataout_timer(cmd); 1455 1456 device->unreg_rdma_mem(isert_cmd, isert_conn); 1457 transport_generic_free_cmd(&cmd->se_cmd, 0); 1458 break; 1459 case ISCSI_OP_SCSI_TMFUNC: 1460 spin_lock_bh(&conn->cmd_lock); 1461 if (!list_empty(&cmd->i_conn_node)) 1462 list_del(&cmd->i_conn_node); 1463 spin_unlock_bh(&conn->cmd_lock); 1464 1465 transport_generic_free_cmd(&cmd->se_cmd, 0); 1466 break; 1467 case ISCSI_OP_REJECT: 1468 case ISCSI_OP_NOOP_OUT: 1469 case ISCSI_OP_TEXT: 1470 spin_lock_bh(&conn->cmd_lock); 1471 if (!list_empty(&cmd->i_conn_node)) 1472 list_del(&cmd->i_conn_node); 1473 spin_unlock_bh(&conn->cmd_lock); 1474 1475 /* 1476 * Handle special case for REJECT when iscsi_add_reject*() has 1477 * overwritten the original iscsi_opcode assignment, and the 1478 * associated cmd->se_cmd needs to be released. 1479 */ 1480 if (cmd->se_cmd.se_tfo != NULL) { 1481 pr_debug("Calling transport_generic_free_cmd from" 1482 " isert_put_cmd for 0x%02x\n", 1483 cmd->iscsi_opcode); 1484 transport_generic_free_cmd(&cmd->se_cmd, 0); 1485 break; 1486 } 1487 /* 1488 * Fall-through 1489 */ 1490 default: 1491 iscsit_release_cmd(cmd); 1492 break; 1493 } 1494 } 1495 1496 static void 1497 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) 1498 { 1499 if (tx_desc->dma_addr != 0) { 1500 pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); 1501 ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, 1502 ISER_HEADERS_LEN, DMA_TO_DEVICE); 1503 tx_desc->dma_addr = 0; 1504 } 1505 } 1506 1507 static void 1508 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, 1509 struct ib_device *ib_dev) 1510 { 1511 if (isert_cmd->pdu_buf_dma != 0) { 1512 pr_debug("Calling ib_dma_unmap_single for isert_cmd->pdu_buf_dma\n"); 1513 ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma, 1514 isert_cmd->pdu_buf_len, DMA_TO_DEVICE); 1515 isert_cmd->pdu_buf_dma = 0; 1516 } 1517 1518 isert_unmap_tx_desc(tx_desc, ib_dev); 1519 isert_put_cmd(isert_cmd); 1520 } 1521 1522 static void 1523 isert_completion_rdma_read(struct iser_tx_desc *tx_desc, 1524 struct isert_cmd *isert_cmd) 1525 { 1526 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 1527 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1528 struct se_cmd *se_cmd = &cmd->se_cmd; 1529 struct isert_conn *isert_conn = isert_cmd->conn; 1530 struct isert_device *device = isert_conn->conn_device; 1531 1532 iscsit_stop_dataout_timer(cmd); 1533 device->unreg_rdma_mem(isert_cmd, isert_conn); 1534 cmd->write_data_done = wr->cur_rdma_length; 1535 1536 pr_debug("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd); 1537 spin_lock_bh(&cmd->istate_lock); 1538 cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; 1539 cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; 1540 spin_unlock_bh(&cmd->istate_lock); 1541 1542 target_execute_cmd(se_cmd); 1543 } 1544 1545 static void 1546 isert_do_control_comp(struct work_struct *work) 1547 { 1548 struct isert_cmd *isert_cmd = container_of(work, 1549 struct isert_cmd, comp_work); 1550 struct isert_conn *isert_conn = isert_cmd->conn; 1551 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1552 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1553 1554 switch (cmd->i_state) { 1555 case ISTATE_SEND_TASKMGTRSP: 1556 pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); 1557 1558 atomic_dec(&isert_conn->post_send_buf_count); 1559 iscsit_tmr_post_handler(cmd, cmd->conn); 1560 1561 cmd->i_state = ISTATE_SENT_STATUS; 1562 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1563 break; 1564 case ISTATE_SEND_REJECT: 1565 pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); 1566 atomic_dec(&isert_conn->post_send_buf_count); 1567 1568 cmd->i_state = ISTATE_SENT_STATUS; 1569 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1570 break; 1571 case ISTATE_SEND_LOGOUTRSP: 1572 pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); 1573 /* 1574 * Call atomic_dec(&isert_conn->post_send_buf_count) 1575 * from isert_free_conn() 1576 */ 1577 isert_conn->logout_posted = true; 1578 iscsit_logout_post_handler(cmd, cmd->conn); 1579 break; 1580 case ISTATE_SEND_TEXTRSP: 1581 atomic_dec(&isert_conn->post_send_buf_count); 1582 cmd->i_state = ISTATE_SENT_STATUS; 1583 isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); 1584 break; 1585 default: 1586 pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); 1587 dump_stack(); 1588 break; 1589 } 1590 } 1591 1592 static void 1593 isert_response_completion(struct iser_tx_desc *tx_desc, 1594 struct isert_cmd *isert_cmd, 1595 struct isert_conn *isert_conn, 1596 struct ib_device *ib_dev) 1597 { 1598 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1599 1600 if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || 1601 cmd->i_state == ISTATE_SEND_LOGOUTRSP || 1602 cmd->i_state == ISTATE_SEND_REJECT || 1603 cmd->i_state == ISTATE_SEND_TEXTRSP) { 1604 isert_unmap_tx_desc(tx_desc, ib_dev); 1605 1606 INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); 1607 queue_work(isert_comp_wq, &isert_cmd->comp_work); 1608 return; 1609 } 1610 atomic_dec(&isert_conn->post_send_buf_count); 1611 1612 cmd->i_state = ISTATE_SENT_STATUS; 1613 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1614 } 1615 1616 static void 1617 __isert_send_completion(struct iser_tx_desc *tx_desc, 1618 struct isert_conn *isert_conn) 1619 { 1620 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1621 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1622 struct isert_rdma_wr *wr; 1623 1624 if (!isert_cmd) { 1625 atomic_dec(&isert_conn->post_send_buf_count); 1626 isert_unmap_tx_desc(tx_desc, ib_dev); 1627 return; 1628 } 1629 wr = &isert_cmd->rdma_wr; 1630 1631 switch (wr->iser_ib_op) { 1632 case ISER_IB_RECV: 1633 pr_err("isert_send_completion: Got ISER_IB_RECV\n"); 1634 dump_stack(); 1635 break; 1636 case ISER_IB_SEND: 1637 pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); 1638 isert_response_completion(tx_desc, isert_cmd, 1639 isert_conn, ib_dev); 1640 break; 1641 case ISER_IB_RDMA_WRITE: 1642 pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); 1643 dump_stack(); 1644 break; 1645 case ISER_IB_RDMA_READ: 1646 pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); 1647 1648 atomic_dec(&isert_conn->post_send_buf_count); 1649 isert_completion_rdma_read(tx_desc, isert_cmd); 1650 break; 1651 default: 1652 pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); 1653 dump_stack(); 1654 break; 1655 } 1656 } 1657 1658 static void 1659 isert_send_completion(struct iser_tx_desc *tx_desc, 1660 struct isert_conn *isert_conn) 1661 { 1662 struct llist_node *llnode = tx_desc->comp_llnode_batch; 1663 struct iser_tx_desc *t; 1664 /* 1665 * Drain coalesced completion llist starting from comp_llnode_batch 1666 * setup in isert_init_send_wr(), and then complete trailing tx_desc. 1667 */ 1668 while (llnode) { 1669 t = llist_entry(llnode, struct iser_tx_desc, comp_llnode); 1670 llnode = llist_next(llnode); 1671 __isert_send_completion(t, isert_conn); 1672 } 1673 __isert_send_completion(tx_desc, isert_conn); 1674 } 1675 1676 static void 1677 isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) 1678 { 1679 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1680 1681 if (tx_desc) { 1682 struct isert_cmd *isert_cmd = tx_desc->isert_cmd; 1683 1684 if (!isert_cmd) 1685 isert_unmap_tx_desc(tx_desc, ib_dev); 1686 else 1687 isert_completion_put(tx_desc, isert_cmd, ib_dev); 1688 } 1689 1690 if (isert_conn->post_recv_buf_count == 0 && 1691 atomic_read(&isert_conn->post_send_buf_count) == 0) { 1692 pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); 1693 pr_debug("Calling wake_up from isert_cq_comp_err\n"); 1694 1695 mutex_lock(&isert_conn->conn_mutex); 1696 if (isert_conn->state != ISER_CONN_DOWN) 1697 isert_conn->state = ISER_CONN_TERMINATING; 1698 mutex_unlock(&isert_conn->conn_mutex); 1699 1700 wake_up(&isert_conn->conn_wait_comp_err); 1701 } 1702 } 1703 1704 static void 1705 isert_cq_tx_work(struct work_struct *work) 1706 { 1707 struct isert_cq_desc *cq_desc = container_of(work, 1708 struct isert_cq_desc, cq_tx_work); 1709 struct isert_device *device = cq_desc->device; 1710 int cq_index = cq_desc->cq_index; 1711 struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; 1712 struct isert_conn *isert_conn; 1713 struct iser_tx_desc *tx_desc; 1714 struct ib_wc wc; 1715 1716 while (ib_poll_cq(tx_cq, 1, &wc) == 1) { 1717 tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; 1718 isert_conn = wc.qp->qp_context; 1719 1720 if (wc.status == IB_WC_SUCCESS) { 1721 isert_send_completion(tx_desc, isert_conn); 1722 } else { 1723 pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1724 pr_debug("TX wc.status: 0x%08x\n", wc.status); 1725 pr_debug("TX wc.vendor_err: 0x%08x\n", wc.vendor_err); 1726 atomic_dec(&isert_conn->post_send_buf_count); 1727 isert_cq_comp_err(tx_desc, isert_conn); 1728 } 1729 } 1730 1731 ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); 1732 } 1733 1734 static void 1735 isert_cq_tx_callback(struct ib_cq *cq, void *context) 1736 { 1737 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1738 1739 queue_work(isert_comp_wq, &cq_desc->cq_tx_work); 1740 } 1741 1742 static void 1743 isert_cq_rx_work(struct work_struct *work) 1744 { 1745 struct isert_cq_desc *cq_desc = container_of(work, 1746 struct isert_cq_desc, cq_rx_work); 1747 struct isert_device *device = cq_desc->device; 1748 int cq_index = cq_desc->cq_index; 1749 struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; 1750 struct isert_conn *isert_conn; 1751 struct iser_rx_desc *rx_desc; 1752 struct ib_wc wc; 1753 unsigned long xfer_len; 1754 1755 while (ib_poll_cq(rx_cq, 1, &wc) == 1) { 1756 rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; 1757 isert_conn = wc.qp->qp_context; 1758 1759 if (wc.status == IB_WC_SUCCESS) { 1760 xfer_len = (unsigned long)wc.byte_len; 1761 isert_rx_completion(rx_desc, isert_conn, xfer_len); 1762 } else { 1763 pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); 1764 if (wc.status != IB_WC_WR_FLUSH_ERR) { 1765 pr_debug("RX wc.status: 0x%08x\n", wc.status); 1766 pr_debug("RX wc.vendor_err: 0x%08x\n", 1767 wc.vendor_err); 1768 } 1769 isert_conn->post_recv_buf_count--; 1770 isert_cq_comp_err(NULL, isert_conn); 1771 } 1772 } 1773 1774 ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); 1775 } 1776 1777 static void 1778 isert_cq_rx_callback(struct ib_cq *cq, void *context) 1779 { 1780 struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; 1781 1782 queue_work(isert_rx_wq, &cq_desc->cq_rx_work); 1783 } 1784 1785 static int 1786 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) 1787 { 1788 struct ib_send_wr *wr_failed; 1789 int ret; 1790 1791 atomic_inc(&isert_conn->post_send_buf_count); 1792 1793 ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, 1794 &wr_failed); 1795 if (ret) { 1796 pr_err("ib_post_send failed with %d\n", ret); 1797 atomic_dec(&isert_conn->post_send_buf_count); 1798 return ret; 1799 } 1800 return ret; 1801 } 1802 1803 static int 1804 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 1805 { 1806 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1807 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1808 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1809 struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) 1810 &isert_cmd->tx_desc.iscsi_header; 1811 1812 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1813 iscsit_build_rsp_pdu(cmd, conn, true, hdr); 1814 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1815 /* 1816 * Attach SENSE DATA payload to iSCSI Response PDU 1817 */ 1818 if (cmd->se_cmd.sense_buffer && 1819 ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || 1820 (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { 1821 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1822 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1823 u32 padding, pdu_len; 1824 1825 put_unaligned_be16(cmd->se_cmd.scsi_sense_length, 1826 cmd->sense_buffer); 1827 cmd->se_cmd.scsi_sense_length += sizeof(__be16); 1828 1829 padding = -(cmd->se_cmd.scsi_sense_length) & 3; 1830 hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); 1831 pdu_len = cmd->se_cmd.scsi_sense_length + padding; 1832 1833 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1834 (void *)cmd->sense_buffer, pdu_len, 1835 DMA_TO_DEVICE); 1836 1837 isert_cmd->pdu_buf_len = pdu_len; 1838 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1839 tx_dsg->length = pdu_len; 1840 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1841 isert_cmd->tx_desc.num_sge = 2; 1842 } 1843 1844 isert_init_send_wr(isert_conn, isert_cmd, send_wr, true); 1845 1846 pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1847 1848 return isert_post_response(isert_conn, isert_cmd); 1849 } 1850 1851 static int 1852 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, 1853 bool nopout_response) 1854 { 1855 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1856 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1857 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1858 1859 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1860 iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) 1861 &isert_cmd->tx_desc.iscsi_header, 1862 nopout_response); 1863 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1864 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1865 1866 pr_debug("Posting NOPIN Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1867 1868 return isert_post_response(isert_conn, isert_cmd); 1869 } 1870 1871 static int 1872 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1873 { 1874 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1875 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1876 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1877 1878 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1879 iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) 1880 &isert_cmd->tx_desc.iscsi_header); 1881 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1882 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1883 1884 pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1885 1886 return isert_post_response(isert_conn, isert_cmd); 1887 } 1888 1889 static int 1890 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1891 { 1892 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1893 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1894 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1895 1896 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1897 iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) 1898 &isert_cmd->tx_desc.iscsi_header); 1899 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1900 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1901 1902 pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1903 1904 return isert_post_response(isert_conn, isert_cmd); 1905 } 1906 1907 static int 1908 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1909 { 1910 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1911 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1912 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1913 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1914 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1915 struct iscsi_reject *hdr = 1916 (struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header; 1917 1918 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1919 iscsit_build_reject(cmd, conn, hdr); 1920 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1921 1922 hton24(hdr->dlength, ISCSI_HDR_LEN); 1923 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1924 (void *)cmd->buf_ptr, ISCSI_HDR_LEN, 1925 DMA_TO_DEVICE); 1926 isert_cmd->pdu_buf_len = ISCSI_HDR_LEN; 1927 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1928 tx_dsg->length = ISCSI_HDR_LEN; 1929 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1930 isert_cmd->tx_desc.num_sge = 2; 1931 1932 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1933 1934 pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1935 1936 return isert_post_response(isert_conn, isert_cmd); 1937 } 1938 1939 static int 1940 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) 1941 { 1942 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 1943 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 1944 struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; 1945 struct iscsi_text_rsp *hdr = 1946 (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header; 1947 u32 txt_rsp_len; 1948 int rc; 1949 1950 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 1951 rc = iscsit_build_text_rsp(cmd, conn, hdr); 1952 if (rc < 0) 1953 return rc; 1954 1955 txt_rsp_len = rc; 1956 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 1957 1958 if (txt_rsp_len) { 1959 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1960 struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; 1961 void *txt_rsp_buf = cmd->buf_ptr; 1962 1963 isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev, 1964 txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE); 1965 1966 isert_cmd->pdu_buf_len = txt_rsp_len; 1967 tx_dsg->addr = isert_cmd->pdu_buf_dma; 1968 tx_dsg->length = txt_rsp_len; 1969 tx_dsg->lkey = isert_conn->conn_mr->lkey; 1970 isert_cmd->tx_desc.num_sge = 2; 1971 } 1972 isert_init_send_wr(isert_conn, isert_cmd, send_wr, false); 1973 1974 pr_debug("Posting Text Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); 1975 1976 return isert_post_response(isert_conn, isert_cmd); 1977 } 1978 1979 static int 1980 isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, 1981 struct ib_sge *ib_sge, struct ib_send_wr *send_wr, 1982 u32 data_left, u32 offset) 1983 { 1984 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 1985 struct scatterlist *sg_start, *tmp_sg; 1986 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 1987 u32 sg_off, page_off; 1988 int i = 0, sg_nents; 1989 1990 sg_off = offset / PAGE_SIZE; 1991 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 1992 sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); 1993 page_off = offset % PAGE_SIZE; 1994 1995 send_wr->sg_list = ib_sge; 1996 send_wr->num_sge = sg_nents; 1997 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 1998 /* 1999 * Perform mapping of TCM scatterlist memory ib_sge dma_addr. 2000 */ 2001 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2002 pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", 2003 (unsigned long long)tmp_sg->dma_address, 2004 tmp_sg->length, page_off); 2005 2006 ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; 2007 ib_sge->length = min_t(u32, data_left, 2008 ib_sg_dma_len(ib_dev, tmp_sg) - page_off); 2009 ib_sge->lkey = isert_conn->conn_mr->lkey; 2010 2011 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2012 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2013 page_off = 0; 2014 data_left -= ib_sge->length; 2015 ib_sge++; 2016 pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); 2017 } 2018 2019 pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", 2020 send_wr->sg_list, send_wr->num_sge); 2021 2022 return sg_nents; 2023 } 2024 2025 static int 2026 isert_map_rdma(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2027 struct isert_rdma_wr *wr) 2028 { 2029 struct se_cmd *se_cmd = &cmd->se_cmd; 2030 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2031 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2032 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2033 struct ib_send_wr *send_wr; 2034 struct ib_sge *ib_sge; 2035 struct scatterlist *sg_start; 2036 u32 sg_off = 0, sg_nents; 2037 u32 offset = 0, data_len, data_left, rdma_write_max, va_offset = 0; 2038 int ret = 0, count, i, ib_sge_cnt; 2039 2040 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2041 data_left = se_cmd->data_length; 2042 } else { 2043 sg_off = cmd->write_data_done / PAGE_SIZE; 2044 data_left = se_cmd->data_length - cmd->write_data_done; 2045 offset = cmd->write_data_done; 2046 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2047 } 2048 2049 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2050 sg_nents = se_cmd->t_data_nents - sg_off; 2051 2052 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2053 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2054 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2055 if (unlikely(!count)) { 2056 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2057 return -EINVAL; 2058 } 2059 wr->sge = sg_start; 2060 wr->num_sge = sg_nents; 2061 wr->cur_rdma_length = data_left; 2062 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2063 isert_cmd, count, sg_start, sg_nents, data_left); 2064 2065 ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); 2066 if (!ib_sge) { 2067 pr_warn("Unable to allocate ib_sge\n"); 2068 ret = -ENOMEM; 2069 goto unmap_sg; 2070 } 2071 wr->ib_sge = ib_sge; 2072 2073 wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); 2074 wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, 2075 GFP_KERNEL); 2076 if (!wr->send_wr) { 2077 pr_debug("Unable to allocate wr->send_wr\n"); 2078 ret = -ENOMEM; 2079 goto unmap_sg; 2080 } 2081 2082 wr->isert_cmd = isert_cmd; 2083 rdma_write_max = isert_conn->max_sge * PAGE_SIZE; 2084 2085 for (i = 0; i < wr->send_wr_num; i++) { 2086 send_wr = &isert_cmd->rdma_wr.send_wr[i]; 2087 data_len = min(data_left, rdma_write_max); 2088 2089 send_wr->send_flags = 0; 2090 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2091 send_wr->opcode = IB_WR_RDMA_WRITE; 2092 send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; 2093 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2094 if (i + 1 == wr->send_wr_num) 2095 send_wr->next = &isert_cmd->tx_desc.send_wr; 2096 else 2097 send_wr->next = &wr->send_wr[i + 1]; 2098 } else { 2099 send_wr->opcode = IB_WR_RDMA_READ; 2100 send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; 2101 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2102 if (i + 1 == wr->send_wr_num) 2103 send_wr->send_flags = IB_SEND_SIGNALED; 2104 else 2105 send_wr->next = &wr->send_wr[i + 1]; 2106 } 2107 2108 ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, 2109 send_wr, data_len, offset); 2110 ib_sge += ib_sge_cnt; 2111 2112 offset += data_len; 2113 va_offset += data_len; 2114 data_left -= data_len; 2115 } 2116 2117 return 0; 2118 unmap_sg: 2119 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2120 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2121 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2122 return ret; 2123 } 2124 2125 static int 2126 isert_map_fr_pagelist(struct ib_device *ib_dev, 2127 struct scatterlist *sg_start, int sg_nents, u64 *fr_pl) 2128 { 2129 u64 start_addr, end_addr, page, chunk_start = 0; 2130 struct scatterlist *tmp_sg; 2131 int i = 0, new_chunk, last_ent, n_pages; 2132 2133 n_pages = 0; 2134 new_chunk = 1; 2135 last_ent = sg_nents - 1; 2136 for_each_sg(sg_start, tmp_sg, sg_nents, i) { 2137 start_addr = ib_sg_dma_address(ib_dev, tmp_sg); 2138 if (new_chunk) 2139 chunk_start = start_addr; 2140 end_addr = start_addr + ib_sg_dma_len(ib_dev, tmp_sg); 2141 2142 pr_debug("SGL[%d] dma_addr: 0x%16llx len: %u\n", 2143 i, (unsigned long long)tmp_sg->dma_address, 2144 tmp_sg->length); 2145 2146 if ((end_addr & ~PAGE_MASK) && i < last_ent) { 2147 new_chunk = 0; 2148 continue; 2149 } 2150 new_chunk = 1; 2151 2152 page = chunk_start & PAGE_MASK; 2153 do { 2154 fr_pl[n_pages++] = page; 2155 pr_debug("Mapped page_list[%d] page_addr: 0x%16llx\n", 2156 n_pages - 1, page); 2157 page += PAGE_SIZE; 2158 } while (page < end_addr); 2159 } 2160 2161 return n_pages; 2162 } 2163 2164 static int 2165 isert_fast_reg_mr(struct fast_reg_descriptor *fr_desc, 2166 struct isert_cmd *isert_cmd, struct isert_conn *isert_conn, 2167 struct ib_sge *ib_sge, u32 offset, unsigned int data_len) 2168 { 2169 struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd; 2170 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2171 struct scatterlist *sg_start; 2172 u32 sg_off, page_off; 2173 struct ib_send_wr fr_wr, inv_wr; 2174 struct ib_send_wr *bad_wr, *wr = NULL; 2175 u8 key; 2176 int ret, sg_nents, pagelist_len; 2177 2178 sg_off = offset / PAGE_SIZE; 2179 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2180 sg_nents = min_t(unsigned int, cmd->se_cmd.t_data_nents - sg_off, 2181 ISCSI_ISER_SG_TABLESIZE); 2182 page_off = offset % PAGE_SIZE; 2183 2184 pr_debug("Cmd: %p use fr_desc %p sg_nents %d sg_off %d offset %u\n", 2185 isert_cmd, fr_desc, sg_nents, sg_off, offset); 2186 2187 pagelist_len = isert_map_fr_pagelist(ib_dev, sg_start, sg_nents, 2188 &fr_desc->data_frpl->page_list[0]); 2189 2190 if (!fr_desc->valid) { 2191 memset(&inv_wr, 0, sizeof(inv_wr)); 2192 inv_wr.opcode = IB_WR_LOCAL_INV; 2193 inv_wr.ex.invalidate_rkey = fr_desc->data_mr->rkey; 2194 wr = &inv_wr; 2195 /* Bump the key */ 2196 key = (u8)(fr_desc->data_mr->rkey & 0x000000FF); 2197 ib_update_fast_reg_key(fr_desc->data_mr, ++key); 2198 } 2199 2200 /* Prepare FASTREG WR */ 2201 memset(&fr_wr, 0, sizeof(fr_wr)); 2202 fr_wr.opcode = IB_WR_FAST_REG_MR; 2203 fr_wr.wr.fast_reg.iova_start = 2204 fr_desc->data_frpl->page_list[0] + page_off; 2205 fr_wr.wr.fast_reg.page_list = fr_desc->data_frpl; 2206 fr_wr.wr.fast_reg.page_list_len = pagelist_len; 2207 fr_wr.wr.fast_reg.page_shift = PAGE_SHIFT; 2208 fr_wr.wr.fast_reg.length = data_len; 2209 fr_wr.wr.fast_reg.rkey = fr_desc->data_mr->rkey; 2210 fr_wr.wr.fast_reg.access_flags = IB_ACCESS_LOCAL_WRITE; 2211 2212 if (!wr) 2213 wr = &fr_wr; 2214 else 2215 wr->next = &fr_wr; 2216 2217 ret = ib_post_send(isert_conn->conn_qp, wr, &bad_wr); 2218 if (ret) { 2219 pr_err("fast registration failed, ret:%d\n", ret); 2220 return ret; 2221 } 2222 fr_desc->valid = false; 2223 2224 ib_sge->lkey = fr_desc->data_mr->lkey; 2225 ib_sge->addr = fr_desc->data_frpl->page_list[0] + page_off; 2226 ib_sge->length = data_len; 2227 2228 pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u lkey: %08x\n", 2229 ib_sge->addr, ib_sge->length, ib_sge->lkey); 2230 2231 return ret; 2232 } 2233 2234 static int 2235 isert_reg_rdma_frwr(struct iscsi_conn *conn, struct iscsi_cmd *cmd, 2236 struct isert_rdma_wr *wr) 2237 { 2238 struct se_cmd *se_cmd = &cmd->se_cmd; 2239 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2240 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2241 struct ib_device *ib_dev = isert_conn->conn_cm_id->device; 2242 struct ib_send_wr *send_wr; 2243 struct ib_sge *ib_sge; 2244 struct scatterlist *sg_start; 2245 struct fast_reg_descriptor *fr_desc; 2246 u32 sg_off = 0, sg_nents; 2247 u32 offset = 0, data_len, data_left, rdma_write_max; 2248 int ret = 0, count; 2249 unsigned long flags; 2250 2251 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2252 data_left = se_cmd->data_length; 2253 } else { 2254 sg_off = cmd->write_data_done / PAGE_SIZE; 2255 data_left = se_cmd->data_length - cmd->write_data_done; 2256 offset = cmd->write_data_done; 2257 isert_cmd->tx_desc.isert_cmd = isert_cmd; 2258 } 2259 2260 sg_start = &cmd->se_cmd.t_data_sg[sg_off]; 2261 sg_nents = se_cmd->t_data_nents - sg_off; 2262 2263 count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, 2264 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2265 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2266 if (unlikely(!count)) { 2267 pr_err("Cmd: %p unrable to map SGs\n", isert_cmd); 2268 return -EINVAL; 2269 } 2270 wr->sge = sg_start; 2271 wr->num_sge = sg_nents; 2272 pr_debug("Mapped cmd: %p count: %u sg: %p sg_nents: %u rdma_len %d\n", 2273 isert_cmd, count, sg_start, sg_nents, data_left); 2274 2275 memset(&wr->s_ib_sge, 0, sizeof(*ib_sge)); 2276 ib_sge = &wr->s_ib_sge; 2277 wr->ib_sge = ib_sge; 2278 2279 wr->send_wr_num = 1; 2280 memset(&wr->s_send_wr, 0, sizeof(*send_wr)); 2281 wr->send_wr = &wr->s_send_wr; 2282 2283 wr->isert_cmd = isert_cmd; 2284 rdma_write_max = ISCSI_ISER_SG_TABLESIZE * PAGE_SIZE; 2285 2286 send_wr = &isert_cmd->rdma_wr.s_send_wr; 2287 send_wr->sg_list = ib_sge; 2288 send_wr->num_sge = 1; 2289 send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; 2290 if (wr->iser_ib_op == ISER_IB_RDMA_WRITE) { 2291 send_wr->opcode = IB_WR_RDMA_WRITE; 2292 send_wr->wr.rdma.remote_addr = isert_cmd->read_va; 2293 send_wr->wr.rdma.rkey = isert_cmd->read_stag; 2294 send_wr->send_flags = 0; 2295 send_wr->next = &isert_cmd->tx_desc.send_wr; 2296 } else { 2297 send_wr->opcode = IB_WR_RDMA_READ; 2298 send_wr->wr.rdma.remote_addr = isert_cmd->write_va; 2299 send_wr->wr.rdma.rkey = isert_cmd->write_stag; 2300 send_wr->send_flags = IB_SEND_SIGNALED; 2301 } 2302 2303 data_len = min(data_left, rdma_write_max); 2304 wr->cur_rdma_length = data_len; 2305 2306 /* if there is a single dma entry, dma mr is sufficient */ 2307 if (count == 1) { 2308 ib_sge->addr = ib_sg_dma_address(ib_dev, &sg_start[0]); 2309 ib_sge->length = ib_sg_dma_len(ib_dev, &sg_start[0]); 2310 ib_sge->lkey = isert_conn->conn_mr->lkey; 2311 wr->fr_desc = NULL; 2312 } else { 2313 spin_lock_irqsave(&isert_conn->conn_lock, flags); 2314 fr_desc = list_first_entry(&isert_conn->conn_frwr_pool, 2315 struct fast_reg_descriptor, list); 2316 list_del(&fr_desc->list); 2317 spin_unlock_irqrestore(&isert_conn->conn_lock, flags); 2318 wr->fr_desc = fr_desc; 2319 2320 ret = isert_fast_reg_mr(fr_desc, isert_cmd, isert_conn, 2321 ib_sge, offset, data_len); 2322 if (ret) { 2323 list_add_tail(&fr_desc->list, &isert_conn->conn_frwr_pool); 2324 goto unmap_sg; 2325 } 2326 } 2327 2328 return 0; 2329 2330 unmap_sg: 2331 ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, 2332 (wr->iser_ib_op == ISER_IB_RDMA_WRITE) ? 2333 DMA_TO_DEVICE : DMA_FROM_DEVICE); 2334 return ret; 2335 } 2336 2337 static int 2338 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) 2339 { 2340 struct se_cmd *se_cmd = &cmd->se_cmd; 2341 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2342 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2343 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2344 struct isert_device *device = isert_conn->conn_device; 2345 struct ib_send_wr *wr_failed; 2346 int rc; 2347 2348 pr_debug("Cmd: %p RDMA_WRITE data_length: %u\n", 2349 isert_cmd, se_cmd->data_length); 2350 wr->iser_ib_op = ISER_IB_RDMA_WRITE; 2351 rc = device->reg_rdma_mem(conn, cmd, wr); 2352 if (rc) { 2353 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2354 return rc; 2355 } 2356 2357 /* 2358 * Build isert_conn->tx_desc for iSCSI response PDU and attach 2359 */ 2360 isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); 2361 iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *) 2362 &isert_cmd->tx_desc.iscsi_header); 2363 isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); 2364 isert_init_send_wr(isert_conn, isert_cmd, 2365 &isert_cmd->tx_desc.send_wr, true); 2366 2367 atomic_inc(&isert_conn->post_send_buf_count); 2368 2369 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2370 if (rc) { 2371 pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); 2372 atomic_dec(&isert_conn->post_send_buf_count); 2373 } 2374 pr_debug("Cmd: %p posted RDMA_WRITE + Response for iSER Data READ\n", 2375 isert_cmd); 2376 2377 return 1; 2378 } 2379 2380 static int 2381 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) 2382 { 2383 struct se_cmd *se_cmd = &cmd->se_cmd; 2384 struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd); 2385 struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; 2386 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2387 struct isert_device *device = isert_conn->conn_device; 2388 struct ib_send_wr *wr_failed; 2389 int rc; 2390 2391 pr_debug("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n", 2392 isert_cmd, se_cmd->data_length, cmd->write_data_done); 2393 wr->iser_ib_op = ISER_IB_RDMA_READ; 2394 rc = device->reg_rdma_mem(conn, cmd, wr); 2395 if (rc) { 2396 pr_err("Cmd: %p failed to prepare RDMA res\n", isert_cmd); 2397 return rc; 2398 } 2399 2400 atomic_inc(&isert_conn->post_send_buf_count); 2401 2402 rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); 2403 if (rc) { 2404 pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); 2405 atomic_dec(&isert_conn->post_send_buf_count); 2406 } 2407 pr_debug("Cmd: %p posted RDMA_READ memory for ISER Data WRITE\n", 2408 isert_cmd); 2409 2410 return 0; 2411 } 2412 2413 static int 2414 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2415 { 2416 int ret; 2417 2418 switch (state) { 2419 case ISTATE_SEND_NOPIN_WANT_RESPONSE: 2420 ret = isert_put_nopin(cmd, conn, false); 2421 break; 2422 default: 2423 pr_err("Unknown immediate state: 0x%02x\n", state); 2424 ret = -EINVAL; 2425 break; 2426 } 2427 2428 return ret; 2429 } 2430 2431 static int 2432 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) 2433 { 2434 int ret; 2435 2436 switch (state) { 2437 case ISTATE_SEND_LOGOUTRSP: 2438 ret = isert_put_logout_rsp(cmd, conn); 2439 if (!ret) { 2440 pr_debug("Returning iSER Logout -EAGAIN\n"); 2441 ret = -EAGAIN; 2442 } 2443 break; 2444 case ISTATE_SEND_NOPIN: 2445 ret = isert_put_nopin(cmd, conn, true); 2446 break; 2447 case ISTATE_SEND_TASKMGTRSP: 2448 ret = isert_put_tm_rsp(cmd, conn); 2449 break; 2450 case ISTATE_SEND_REJECT: 2451 ret = isert_put_reject(cmd, conn); 2452 break; 2453 case ISTATE_SEND_TEXTRSP: 2454 ret = isert_put_text_rsp(cmd, conn); 2455 break; 2456 case ISTATE_SEND_STATUS: 2457 /* 2458 * Special case for sending non GOOD SCSI status from TX thread 2459 * context during pre se_cmd excecution failure. 2460 */ 2461 ret = isert_put_response(conn, cmd); 2462 break; 2463 default: 2464 pr_err("Unknown response state: 0x%02x\n", state); 2465 ret = -EINVAL; 2466 break; 2467 } 2468 2469 return ret; 2470 } 2471 2472 static int 2473 isert_setup_np(struct iscsi_np *np, 2474 struct __kernel_sockaddr_storage *ksockaddr) 2475 { 2476 struct isert_np *isert_np; 2477 struct rdma_cm_id *isert_lid; 2478 struct sockaddr *sa; 2479 int ret; 2480 2481 isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); 2482 if (!isert_np) { 2483 pr_err("Unable to allocate struct isert_np\n"); 2484 return -ENOMEM; 2485 } 2486 init_waitqueue_head(&isert_np->np_accept_wq); 2487 mutex_init(&isert_np->np_accept_mutex); 2488 INIT_LIST_HEAD(&isert_np->np_accept_list); 2489 init_completion(&isert_np->np_login_comp); 2490 2491 sa = (struct sockaddr *)ksockaddr; 2492 pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); 2493 /* 2494 * Setup the np->np_sockaddr from the passed sockaddr setup 2495 * in iscsi_target_configfs.c code.. 2496 */ 2497 memcpy(&np->np_sockaddr, ksockaddr, 2498 sizeof(struct __kernel_sockaddr_storage)); 2499 2500 isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, 2501 IB_QPT_RC); 2502 if (IS_ERR(isert_lid)) { 2503 pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", 2504 PTR_ERR(isert_lid)); 2505 ret = PTR_ERR(isert_lid); 2506 goto out; 2507 } 2508 2509 ret = rdma_bind_addr(isert_lid, sa); 2510 if (ret) { 2511 pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); 2512 goto out_lid; 2513 } 2514 2515 ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); 2516 if (ret) { 2517 pr_err("rdma_listen() for isert_lid failed: %d\n", ret); 2518 goto out_lid; 2519 } 2520 2521 isert_np->np_cm_id = isert_lid; 2522 np->np_context = isert_np; 2523 pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); 2524 2525 return 0; 2526 2527 out_lid: 2528 rdma_destroy_id(isert_lid); 2529 out: 2530 kfree(isert_np); 2531 return ret; 2532 } 2533 2534 static int 2535 isert_check_accept_queue(struct isert_np *isert_np) 2536 { 2537 int empty; 2538 2539 mutex_lock(&isert_np->np_accept_mutex); 2540 empty = list_empty(&isert_np->np_accept_list); 2541 mutex_unlock(&isert_np->np_accept_mutex); 2542 2543 return empty; 2544 } 2545 2546 static int 2547 isert_rdma_accept(struct isert_conn *isert_conn) 2548 { 2549 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2550 struct rdma_conn_param cp; 2551 int ret; 2552 2553 memset(&cp, 0, sizeof(struct rdma_conn_param)); 2554 cp.responder_resources = isert_conn->responder_resources; 2555 cp.initiator_depth = isert_conn->initiator_depth; 2556 cp.retry_count = 7; 2557 cp.rnr_retry_count = 7; 2558 2559 pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); 2560 2561 ret = rdma_accept(cm_id, &cp); 2562 if (ret) { 2563 pr_err("rdma_accept() failed with: %d\n", ret); 2564 return ret; 2565 } 2566 2567 pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); 2568 2569 return 0; 2570 } 2571 2572 static int 2573 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) 2574 { 2575 struct isert_conn *isert_conn = (struct isert_conn *)conn->context; 2576 int ret; 2577 2578 pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); 2579 /* 2580 * For login requests after the first PDU, isert_rx_login_req() will 2581 * kick schedule_delayed_work(&conn->login_work) as the packet is 2582 * received, which turns this callback from iscsi_target_do_login_rx() 2583 * into a NOP. 2584 */ 2585 if (!login->first_request) 2586 return 0; 2587 2588 ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); 2589 if (ret) 2590 return ret; 2591 2592 pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); 2593 return 0; 2594 } 2595 2596 static void 2597 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, 2598 struct isert_conn *isert_conn) 2599 { 2600 struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; 2601 struct rdma_route *cm_route = &cm_id->route; 2602 struct sockaddr_in *sock_in; 2603 struct sockaddr_in6 *sock_in6; 2604 2605 conn->login_family = np->np_sockaddr.ss_family; 2606 2607 if (np->np_sockaddr.ss_family == AF_INET6) { 2608 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; 2609 snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", 2610 &sock_in6->sin6_addr.in6_u); 2611 conn->login_port = ntohs(sock_in6->sin6_port); 2612 2613 sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; 2614 snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", 2615 &sock_in6->sin6_addr.in6_u); 2616 conn->local_port = ntohs(sock_in6->sin6_port); 2617 } else { 2618 sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; 2619 sprintf(conn->login_ip, "%pI4", 2620 &sock_in->sin_addr.s_addr); 2621 conn->login_port = ntohs(sock_in->sin_port); 2622 2623 sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; 2624 sprintf(conn->local_ip, "%pI4", 2625 &sock_in->sin_addr.s_addr); 2626 conn->local_port = ntohs(sock_in->sin_port); 2627 } 2628 } 2629 2630 static int 2631 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) 2632 { 2633 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2634 struct isert_conn *isert_conn; 2635 int max_accept = 0, ret; 2636 2637 accept_wait: 2638 ret = wait_event_interruptible(isert_np->np_accept_wq, 2639 !isert_check_accept_queue(isert_np) || 2640 np->np_thread_state == ISCSI_NP_THREAD_RESET); 2641 if (max_accept > 5) 2642 return -ENODEV; 2643 2644 spin_lock_bh(&np->np_thread_lock); 2645 if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { 2646 spin_unlock_bh(&np->np_thread_lock); 2647 pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); 2648 return -ENODEV; 2649 } 2650 spin_unlock_bh(&np->np_thread_lock); 2651 2652 mutex_lock(&isert_np->np_accept_mutex); 2653 if (list_empty(&isert_np->np_accept_list)) { 2654 mutex_unlock(&isert_np->np_accept_mutex); 2655 max_accept++; 2656 goto accept_wait; 2657 } 2658 isert_conn = list_first_entry(&isert_np->np_accept_list, 2659 struct isert_conn, conn_accept_node); 2660 list_del_init(&isert_conn->conn_accept_node); 2661 mutex_unlock(&isert_np->np_accept_mutex); 2662 2663 conn->context = isert_conn; 2664 isert_conn->conn = conn; 2665 max_accept = 0; 2666 2667 ret = isert_rdma_post_recvl(isert_conn); 2668 if (ret) 2669 return ret; 2670 2671 ret = isert_rdma_accept(isert_conn); 2672 if (ret) 2673 return ret; 2674 2675 isert_set_conn_info(np, conn, isert_conn); 2676 2677 pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); 2678 return 0; 2679 } 2680 2681 static void 2682 isert_free_np(struct iscsi_np *np) 2683 { 2684 struct isert_np *isert_np = (struct isert_np *)np->np_context; 2685 2686 rdma_destroy_id(isert_np->np_cm_id); 2687 2688 np->np_context = NULL; 2689 kfree(isert_np); 2690 } 2691 2692 static int isert_check_state(struct isert_conn *isert_conn, int state) 2693 { 2694 int ret; 2695 2696 mutex_lock(&isert_conn->conn_mutex); 2697 ret = (isert_conn->state == state); 2698 mutex_unlock(&isert_conn->conn_mutex); 2699 2700 return ret; 2701 } 2702 2703 static void isert_free_conn(struct iscsi_conn *conn) 2704 { 2705 struct isert_conn *isert_conn = conn->context; 2706 2707 pr_debug("isert_free_conn: Starting \n"); 2708 /* 2709 * Decrement post_send_buf_count for special case when called 2710 * from isert_do_control_comp() -> iscsit_logout_post_handler() 2711 */ 2712 mutex_lock(&isert_conn->conn_mutex); 2713 if (isert_conn->logout_posted) 2714 atomic_dec(&isert_conn->post_send_buf_count); 2715 2716 if (isert_conn->conn_cm_id && isert_conn->state != ISER_CONN_DOWN) { 2717 pr_debug("Calling rdma_disconnect from isert_free_conn\n"); 2718 rdma_disconnect(isert_conn->conn_cm_id); 2719 } 2720 /* 2721 * Only wait for conn_wait_comp_err if the isert_conn made it 2722 * into full feature phase.. 2723 */ 2724 if (isert_conn->state == ISER_CONN_UP) { 2725 pr_debug("isert_free_conn: Before wait_event comp_err %d\n", 2726 isert_conn->state); 2727 mutex_unlock(&isert_conn->conn_mutex); 2728 2729 wait_event(isert_conn->conn_wait_comp_err, 2730 (isert_check_state(isert_conn, ISER_CONN_TERMINATING))); 2731 2732 wait_event(isert_conn->conn_wait, 2733 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2734 2735 isert_put_conn(isert_conn); 2736 return; 2737 } 2738 if (isert_conn->state == ISER_CONN_INIT) { 2739 mutex_unlock(&isert_conn->conn_mutex); 2740 isert_put_conn(isert_conn); 2741 return; 2742 } 2743 pr_debug("isert_free_conn: wait_event conn_wait %d\n", 2744 isert_conn->state); 2745 mutex_unlock(&isert_conn->conn_mutex); 2746 2747 wait_event(isert_conn->conn_wait, 2748 (isert_check_state(isert_conn, ISER_CONN_DOWN))); 2749 2750 isert_put_conn(isert_conn); 2751 } 2752 2753 static struct iscsit_transport iser_target_transport = { 2754 .name = "IB/iSER", 2755 .transport_type = ISCSI_INFINIBAND, 2756 .priv_size = sizeof(struct isert_cmd), 2757 .owner = THIS_MODULE, 2758 .iscsit_setup_np = isert_setup_np, 2759 .iscsit_accept_np = isert_accept_np, 2760 .iscsit_free_np = isert_free_np, 2761 .iscsit_free_conn = isert_free_conn, 2762 .iscsit_get_login_rx = isert_get_login_rx, 2763 .iscsit_put_login_tx = isert_put_login_tx, 2764 .iscsit_immediate_queue = isert_immediate_queue, 2765 .iscsit_response_queue = isert_response_queue, 2766 .iscsit_get_dataout = isert_get_dataout, 2767 .iscsit_queue_data_in = isert_put_datain, 2768 .iscsit_queue_status = isert_put_response, 2769 }; 2770 2771 static int __init isert_init(void) 2772 { 2773 int ret; 2774 2775 isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); 2776 if (!isert_rx_wq) { 2777 pr_err("Unable to allocate isert_rx_wq\n"); 2778 return -ENOMEM; 2779 } 2780 2781 isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); 2782 if (!isert_comp_wq) { 2783 pr_err("Unable to allocate isert_comp_wq\n"); 2784 ret = -ENOMEM; 2785 goto destroy_rx_wq; 2786 } 2787 2788 iscsit_register_transport(&iser_target_transport); 2789 pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); 2790 return 0; 2791 2792 destroy_rx_wq: 2793 destroy_workqueue(isert_rx_wq); 2794 return ret; 2795 } 2796 2797 static void __exit isert_exit(void) 2798 { 2799 destroy_workqueue(isert_comp_wq); 2800 destroy_workqueue(isert_rx_wq); 2801 iscsit_unregister_transport(&iser_target_transport); 2802 pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); 2803 } 2804 2805 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); 2806 MODULE_VERSION("0.1"); 2807 MODULE_AUTHOR("nab@Linux-iSCSI.org"); 2808 MODULE_LICENSE("GPL"); 2809 2810 module_init(isert_init); 2811 module_exit(isert_exit); 2812