1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/kernel.h> 34 #include <linux/slab.h> 35 #include <linux/mm.h> 36 #include <linux/scatterlist.h> 37 #include <linux/kfifo.h> 38 #include <scsi/scsi_cmnd.h> 39 #include <scsi/scsi_host.h> 40 41 #include "iscsi_iser.h" 42 43 /* Register user buffer memory and initialize passive rdma 44 * dto descriptor. Data size is stored in 45 * task->data[ISER_DIR_IN].data_len, Protection size 46 * os stored in task->prot[ISER_DIR_IN].data_len 47 */ 48 static int iser_prepare_read_cmd(struct iscsi_task *task) 49 50 { 51 struct iscsi_iser_task *iser_task = task->dd_data; 52 struct iser_mem_reg *mem_reg; 53 int err; 54 struct iser_ctrl *hdr = &iser_task->desc.iser_header; 55 56 err = iser_dma_map_task_data(iser_task, 57 ISER_DIR_IN, 58 DMA_FROM_DEVICE); 59 if (err) 60 return err; 61 62 err = iser_reg_mem_fastreg(iser_task, ISER_DIR_IN, false); 63 if (err) { 64 iser_err("Failed to set up Data-IN RDMA\n"); 65 goto out_err; 66 } 67 mem_reg = &iser_task->rdma_reg[ISER_DIR_IN]; 68 69 hdr->flags |= ISER_RSV; 70 hdr->read_stag = cpu_to_be32(mem_reg->rkey); 71 hdr->read_va = cpu_to_be64(mem_reg->sge.addr); 72 73 iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", 74 task->itt, mem_reg->rkey, 75 (unsigned long long)mem_reg->sge.addr); 76 77 return 0; 78 79 out_err: 80 iser_dma_unmap_task_data(iser_task, ISER_DIR_IN, DMA_FROM_DEVICE); 81 return err; 82 } 83 84 /* Register user buffer memory and initialize passive rdma 85 * dto descriptor. Data size is stored in 86 * task->data[ISER_DIR_OUT].data_len, Protection size 87 * is stored at task->prot[ISER_DIR_OUT].data_len 88 */ 89 static int iser_prepare_write_cmd(struct iscsi_task *task, unsigned int imm_sz, 90 unsigned int unsol_sz, unsigned int edtl) 91 { 92 struct iscsi_iser_task *iser_task = task->dd_data; 93 struct iser_mem_reg *mem_reg; 94 int err; 95 struct iser_ctrl *hdr = &iser_task->desc.iser_header; 96 struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; 97 struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; 98 99 err = iser_dma_map_task_data(iser_task, 100 ISER_DIR_OUT, 101 DMA_TO_DEVICE); 102 if (err) 103 return err; 104 105 err = iser_reg_mem_fastreg(iser_task, ISER_DIR_OUT, 106 buf_out->data_len == imm_sz); 107 if (err) { 108 iser_err("Failed to register write cmd RDMA mem\n"); 109 goto out_err; 110 } 111 112 mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT]; 113 114 if (unsol_sz < edtl) { 115 hdr->flags |= ISER_WSV; 116 if (buf_out->data_len > imm_sz) { 117 hdr->write_stag = cpu_to_be32(mem_reg->rkey); 118 hdr->write_va = cpu_to_be64(mem_reg->sge.addr + unsol_sz); 119 } 120 121 iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X VA:%#llX + unsol:%d\n", 122 task->itt, mem_reg->rkey, 123 (unsigned long long)mem_reg->sge.addr, unsol_sz); 124 } 125 126 if (imm_sz > 0) { 127 iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", 128 task->itt, imm_sz); 129 tx_dsg->addr = mem_reg->sge.addr; 130 tx_dsg->length = imm_sz; 131 tx_dsg->lkey = mem_reg->sge.lkey; 132 iser_task->desc.num_sge = 2; 133 } 134 135 return 0; 136 137 out_err: 138 iser_dma_unmap_task_data(iser_task, ISER_DIR_OUT, DMA_TO_DEVICE); 139 return err; 140 } 141 142 /* creates a new tx descriptor and adds header regd buffer */ 143 static void iser_create_send_desc(struct iser_conn *iser_conn, 144 struct iser_tx_desc *tx_desc) 145 { 146 struct iser_device *device = iser_conn->ib_conn.device; 147 148 ib_dma_sync_single_for_cpu(device->ib_device, 149 tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); 150 151 memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl)); 152 tx_desc->iser_header.flags = ISER_VER; 153 tx_desc->num_sge = 1; 154 } 155 156 static void iser_free_login_buf(struct iser_conn *iser_conn) 157 { 158 struct iser_device *device = iser_conn->ib_conn.device; 159 struct iser_login_desc *desc = &iser_conn->login_desc; 160 161 if (!desc->req) 162 return; 163 164 ib_dma_unmap_single(device->ib_device, desc->req_dma, 165 ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); 166 167 ib_dma_unmap_single(device->ib_device, desc->rsp_dma, 168 ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); 169 170 kfree(desc->req); 171 kfree(desc->rsp); 172 173 /* make sure we never redo any unmapping */ 174 desc->req = NULL; 175 desc->rsp = NULL; 176 } 177 178 static int iser_alloc_login_buf(struct iser_conn *iser_conn) 179 { 180 struct iser_device *device = iser_conn->ib_conn.device; 181 struct iser_login_desc *desc = &iser_conn->login_desc; 182 183 desc->req = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN, GFP_KERNEL); 184 if (!desc->req) 185 return -ENOMEM; 186 187 desc->req_dma = ib_dma_map_single(device->ib_device, desc->req, 188 ISCSI_DEF_MAX_RECV_SEG_LEN, 189 DMA_TO_DEVICE); 190 if (ib_dma_mapping_error(device->ib_device, 191 desc->req_dma)) 192 goto free_req; 193 194 desc->rsp = kmalloc(ISER_RX_LOGIN_SIZE, GFP_KERNEL); 195 if (!desc->rsp) 196 goto unmap_req; 197 198 desc->rsp_dma = ib_dma_map_single(device->ib_device, desc->rsp, 199 ISER_RX_LOGIN_SIZE, 200 DMA_FROM_DEVICE); 201 if (ib_dma_mapping_error(device->ib_device, 202 desc->rsp_dma)) 203 goto free_rsp; 204 205 return 0; 206 207 free_rsp: 208 kfree(desc->rsp); 209 unmap_req: 210 ib_dma_unmap_single(device->ib_device, desc->req_dma, 211 ISCSI_DEF_MAX_RECV_SEG_LEN, 212 DMA_TO_DEVICE); 213 free_req: 214 kfree(desc->req); 215 216 return -ENOMEM; 217 } 218 219 int iser_alloc_rx_descriptors(struct iser_conn *iser_conn, 220 struct iscsi_session *session) 221 { 222 int i, j; 223 u64 dma_addr; 224 struct iser_rx_desc *rx_desc; 225 struct ib_sge *rx_sg; 226 struct ib_conn *ib_conn = &iser_conn->ib_conn; 227 struct iser_device *device = ib_conn->device; 228 229 iser_conn->qp_max_recv_dtos = session->cmds_max; 230 231 if (iser_alloc_fastreg_pool(ib_conn, session->scsi_cmds_max, 232 iser_conn->pages_per_mr)) 233 goto create_rdma_reg_res_failed; 234 235 if (iser_alloc_login_buf(iser_conn)) 236 goto alloc_login_buf_fail; 237 238 iser_conn->num_rx_descs = session->cmds_max; 239 iser_conn->rx_descs = kmalloc_array(iser_conn->num_rx_descs, 240 sizeof(struct iser_rx_desc), 241 GFP_KERNEL); 242 if (!iser_conn->rx_descs) 243 goto rx_desc_alloc_fail; 244 245 rx_desc = iser_conn->rx_descs; 246 247 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) { 248 dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, 249 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 250 if (ib_dma_mapping_error(device->ib_device, dma_addr)) 251 goto rx_desc_dma_map_failed; 252 253 rx_desc->dma_addr = dma_addr; 254 rx_desc->cqe.done = iser_task_rsp; 255 rx_sg = &rx_desc->rx_sg; 256 rx_sg->addr = rx_desc->dma_addr; 257 rx_sg->length = ISER_RX_PAYLOAD_SIZE; 258 rx_sg->lkey = device->pd->local_dma_lkey; 259 } 260 261 return 0; 262 263 rx_desc_dma_map_failed: 264 rx_desc = iser_conn->rx_descs; 265 for (j = 0; j < i; j++, rx_desc++) 266 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 267 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 268 kfree(iser_conn->rx_descs); 269 iser_conn->rx_descs = NULL; 270 rx_desc_alloc_fail: 271 iser_free_login_buf(iser_conn); 272 alloc_login_buf_fail: 273 iser_free_fastreg_pool(ib_conn); 274 create_rdma_reg_res_failed: 275 iser_err("failed allocating rx descriptors / data buffers\n"); 276 return -ENOMEM; 277 } 278 279 void iser_free_rx_descriptors(struct iser_conn *iser_conn) 280 { 281 int i; 282 struct iser_rx_desc *rx_desc; 283 struct ib_conn *ib_conn = &iser_conn->ib_conn; 284 struct iser_device *device = ib_conn->device; 285 286 iser_free_fastreg_pool(ib_conn); 287 288 rx_desc = iser_conn->rx_descs; 289 for (i = 0; i < iser_conn->qp_max_recv_dtos; i++, rx_desc++) 290 ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, 291 ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); 292 kfree(iser_conn->rx_descs); 293 /* make sure we never redo any unmapping */ 294 iser_conn->rx_descs = NULL; 295 296 iser_free_login_buf(iser_conn); 297 } 298 299 static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) 300 { 301 struct iser_conn *iser_conn = conn->dd_data; 302 struct iscsi_session *session = conn->session; 303 int err = 0; 304 int i; 305 306 iser_dbg("req op %x flags %x\n", req->opcode, req->flags); 307 /* check if this is the last login - going to full feature phase */ 308 if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) 309 goto out; 310 311 if (session->discovery_sess) { 312 iser_info("Discovery session, re-using login RX buffer\n"); 313 goto out; 314 } 315 316 iser_info("Normal session, posting batch of RX %d buffers\n", 317 iser_conn->qp_max_recv_dtos - 1); 318 319 /* 320 * Initial post receive buffers. 321 * There is one already posted recv buffer (for the last login 322 * response). Therefore, the first recv buffer is skipped here. 323 */ 324 for (i = 1; i < iser_conn->qp_max_recv_dtos; i++) { 325 err = iser_post_recvm(iser_conn, &iser_conn->rx_descs[i]); 326 if (err) 327 goto out; 328 } 329 out: 330 return err; 331 } 332 333 /** 334 * iser_send_command - send command PDU 335 * @conn: link to matching iscsi connection 336 * @task: SCSI command task 337 */ 338 int iser_send_command(struct iscsi_conn *conn, struct iscsi_task *task) 339 { 340 struct iser_conn *iser_conn = conn->dd_data; 341 struct iscsi_iser_task *iser_task = task->dd_data; 342 unsigned long edtl; 343 int err; 344 struct iser_data_buf *data_buf, *prot_buf; 345 struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; 346 struct scsi_cmnd *sc = task->sc; 347 struct iser_tx_desc *tx_desc = &iser_task->desc; 348 349 edtl = ntohl(hdr->data_length); 350 351 /* build the tx desc regd header and add it to the tx desc dto */ 352 tx_desc->type = ISCSI_TX_SCSI_COMMAND; 353 tx_desc->cqe.done = iser_cmd_comp; 354 iser_create_send_desc(iser_conn, tx_desc); 355 356 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 357 data_buf = &iser_task->data[ISER_DIR_IN]; 358 prot_buf = &iser_task->prot[ISER_DIR_IN]; 359 } else { 360 data_buf = &iser_task->data[ISER_DIR_OUT]; 361 prot_buf = &iser_task->prot[ISER_DIR_OUT]; 362 } 363 364 if (scsi_sg_count(sc)) { /* using a scatter list */ 365 data_buf->sg = scsi_sglist(sc); 366 data_buf->size = scsi_sg_count(sc); 367 } 368 data_buf->data_len = scsi_bufflen(sc); 369 370 if (scsi_prot_sg_count(sc)) { 371 prot_buf->sg = scsi_prot_sglist(sc); 372 prot_buf->size = scsi_prot_sg_count(sc); 373 prot_buf->data_len = (data_buf->data_len >> 374 ilog2(sc->device->sector_size)) * 8; 375 } 376 377 if (hdr->flags & ISCSI_FLAG_CMD_READ) { 378 err = iser_prepare_read_cmd(task); 379 if (err) 380 goto send_command_error; 381 } 382 if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { 383 err = iser_prepare_write_cmd(task, 384 task->imm_count, 385 task->imm_count + 386 task->unsol_r2t.data_length, 387 edtl); 388 if (err) 389 goto send_command_error; 390 } 391 392 iser_task->status = ISER_TASK_STATUS_STARTED; 393 394 err = iser_post_send(&iser_conn->ib_conn, tx_desc); 395 if (!err) 396 return 0; 397 398 send_command_error: 399 iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); 400 return err; 401 } 402 403 /** 404 * iser_send_data_out - send data out PDU 405 * @conn: link to matching iscsi connection 406 * @task: SCSI command task 407 * @hdr: pointer to the LLD's iSCSI message header 408 */ 409 int iser_send_data_out(struct iscsi_conn *conn, struct iscsi_task *task, 410 struct iscsi_data *hdr) 411 { 412 struct iser_conn *iser_conn = conn->dd_data; 413 struct iscsi_iser_task *iser_task = task->dd_data; 414 struct iser_tx_desc *tx_desc; 415 struct iser_mem_reg *mem_reg; 416 unsigned long buf_offset; 417 unsigned long data_seg_len; 418 uint32_t itt; 419 int err; 420 struct ib_sge *tx_dsg; 421 422 itt = (__force uint32_t)hdr->itt; 423 data_seg_len = ntoh24(hdr->dlength); 424 buf_offset = ntohl(hdr->offset); 425 426 iser_dbg("%s itt %d dseg_len %d offset %d\n", 427 __func__,(int)itt,(int)data_seg_len,(int)buf_offset); 428 429 tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC); 430 if (!tx_desc) 431 return -ENOMEM; 432 433 tx_desc->type = ISCSI_TX_DATAOUT; 434 tx_desc->cqe.done = iser_dataout_comp; 435 tx_desc->iser_header.flags = ISER_VER; 436 memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); 437 438 /* build the tx desc */ 439 err = iser_initialize_task_headers(task, tx_desc); 440 if (err) 441 goto send_data_out_error; 442 443 mem_reg = &iser_task->rdma_reg[ISER_DIR_OUT]; 444 tx_dsg = &tx_desc->tx_sg[1]; 445 tx_dsg->addr = mem_reg->sge.addr + buf_offset; 446 tx_dsg->length = data_seg_len; 447 tx_dsg->lkey = mem_reg->sge.lkey; 448 tx_desc->num_sge = 2; 449 450 if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { 451 iser_err("Offset:%ld & DSL:%ld in Data-Out inconsistent with total len:%ld, itt:%d\n", 452 buf_offset, data_seg_len, 453 iser_task->data[ISER_DIR_OUT].data_len, itt); 454 err = -EINVAL; 455 goto send_data_out_error; 456 } 457 iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", 458 itt, buf_offset, data_seg_len); 459 460 461 err = iser_post_send(&iser_conn->ib_conn, tx_desc); 462 if (!err) 463 return 0; 464 465 send_data_out_error: 466 kmem_cache_free(ig.desc_cache, tx_desc); 467 iser_err("conn %p failed err %d\n", conn, err); 468 return err; 469 } 470 471 int iser_send_control(struct iscsi_conn *conn, struct iscsi_task *task) 472 { 473 struct iser_conn *iser_conn = conn->dd_data; 474 struct iscsi_iser_task *iser_task = task->dd_data; 475 struct iser_tx_desc *mdesc = &iser_task->desc; 476 unsigned long data_seg_len; 477 int err = 0; 478 struct iser_device *device; 479 480 /* build the tx desc regd header and add it to the tx desc dto */ 481 mdesc->type = ISCSI_TX_CONTROL; 482 mdesc->cqe.done = iser_ctrl_comp; 483 iser_create_send_desc(iser_conn, mdesc); 484 485 device = iser_conn->ib_conn.device; 486 487 data_seg_len = ntoh24(task->hdr->dlength); 488 489 if (data_seg_len > 0) { 490 struct iser_login_desc *desc = &iser_conn->login_desc; 491 struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; 492 493 if (task != conn->login_task) { 494 iser_err("data present on non login task!!!\n"); 495 goto send_control_error; 496 } 497 498 ib_dma_sync_single_for_cpu(device->ib_device, desc->req_dma, 499 task->data_count, DMA_TO_DEVICE); 500 501 memcpy(desc->req, task->data, task->data_count); 502 503 ib_dma_sync_single_for_device(device->ib_device, desc->req_dma, 504 task->data_count, DMA_TO_DEVICE); 505 506 tx_dsg->addr = desc->req_dma; 507 tx_dsg->length = task->data_count; 508 tx_dsg->lkey = device->pd->local_dma_lkey; 509 mdesc->num_sge = 2; 510 } 511 512 if (task == conn->login_task) { 513 iser_dbg("op %x dsl %lx, posting login rx buffer\n", 514 task->hdr->opcode, data_seg_len); 515 err = iser_post_recvl(iser_conn); 516 if (err) 517 goto send_control_error; 518 err = iser_post_rx_bufs(conn, task->hdr); 519 if (err) 520 goto send_control_error; 521 } 522 523 err = iser_post_send(&iser_conn->ib_conn, mdesc); 524 if (!err) 525 return 0; 526 527 send_control_error: 528 iser_err("conn %p failed err %d\n",conn, err); 529 return err; 530 } 531 532 void iser_login_rsp(struct ib_cq *cq, struct ib_wc *wc) 533 { 534 struct ib_conn *ib_conn = wc->qp->qp_context; 535 struct iser_conn *iser_conn = to_iser_conn(ib_conn); 536 struct iser_login_desc *desc = iser_login(wc->wr_cqe); 537 struct iscsi_hdr *hdr; 538 char *data; 539 int length; 540 bool full_feature_phase; 541 542 if (unlikely(wc->status != IB_WC_SUCCESS)) { 543 iser_err_comp(wc, "login_rsp"); 544 return; 545 } 546 547 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, 548 desc->rsp_dma, ISER_RX_LOGIN_SIZE, 549 DMA_FROM_DEVICE); 550 551 hdr = desc->rsp + sizeof(struct iser_ctrl); 552 data = desc->rsp + ISER_HEADERS_LEN; 553 length = wc->byte_len - ISER_HEADERS_LEN; 554 full_feature_phase = ((hdr->flags & ISCSI_FULL_FEATURE_PHASE) == 555 ISCSI_FULL_FEATURE_PHASE) && 556 (hdr->flags & ISCSI_FLAG_CMD_FINAL); 557 558 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 559 hdr->itt, length); 560 561 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, data, length); 562 563 ib_dma_sync_single_for_device(ib_conn->device->ib_device, 564 desc->rsp_dma, ISER_RX_LOGIN_SIZE, 565 DMA_FROM_DEVICE); 566 567 if (!full_feature_phase || 568 iser_conn->iscsi_conn->session->discovery_sess) 569 return; 570 571 /* Post the first RX buffer that is skipped in iser_post_rx_bufs() */ 572 iser_post_recvm(iser_conn, iser_conn->rx_descs); 573 } 574 575 static inline int iser_inv_desc(struct iser_fr_desc *desc, u32 rkey) 576 { 577 if (unlikely((!desc->sig_protected && rkey != desc->rsc.mr->rkey) || 578 (desc->sig_protected && rkey != desc->rsc.sig_mr->rkey))) { 579 iser_err("Bogus remote invalidation for rkey %#x\n", rkey); 580 return -EINVAL; 581 } 582 583 desc->rsc.mr_valid = 0; 584 585 return 0; 586 } 587 588 static int iser_check_remote_inv(struct iser_conn *iser_conn, struct ib_wc *wc, 589 struct iscsi_hdr *hdr) 590 { 591 if (wc->wc_flags & IB_WC_WITH_INVALIDATE) { 592 struct iscsi_task *task; 593 u32 rkey = wc->ex.invalidate_rkey; 594 595 iser_dbg("conn %p: remote invalidation for rkey %#x\n", 596 iser_conn, rkey); 597 598 if (unlikely(!iser_conn->snd_w_inv)) { 599 iser_err("conn %p: unexpected remote invalidation, terminating connection\n", 600 iser_conn); 601 return -EPROTO; 602 } 603 604 task = iscsi_itt_to_ctask(iser_conn->iscsi_conn, hdr->itt); 605 if (likely(task)) { 606 struct iscsi_iser_task *iser_task = task->dd_data; 607 struct iser_fr_desc *desc; 608 609 if (iser_task->dir[ISER_DIR_IN]) { 610 desc = iser_task->rdma_reg[ISER_DIR_IN].desc; 611 if (unlikely(iser_inv_desc(desc, rkey))) 612 return -EINVAL; 613 } 614 615 if (iser_task->dir[ISER_DIR_OUT]) { 616 desc = iser_task->rdma_reg[ISER_DIR_OUT].desc; 617 if (unlikely(iser_inv_desc(desc, rkey))) 618 return -EINVAL; 619 } 620 } else { 621 iser_err("failed to get task for itt=%d\n", hdr->itt); 622 return -EINVAL; 623 } 624 } 625 626 return 0; 627 } 628 629 630 void iser_task_rsp(struct ib_cq *cq, struct ib_wc *wc) 631 { 632 struct ib_conn *ib_conn = wc->qp->qp_context; 633 struct iser_conn *iser_conn = to_iser_conn(ib_conn); 634 struct iser_rx_desc *desc = iser_rx(wc->wr_cqe); 635 struct iscsi_hdr *hdr; 636 int length, err; 637 638 if (unlikely(wc->status != IB_WC_SUCCESS)) { 639 iser_err_comp(wc, "task_rsp"); 640 return; 641 } 642 643 ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, 644 desc->dma_addr, ISER_RX_PAYLOAD_SIZE, 645 DMA_FROM_DEVICE); 646 647 hdr = &desc->iscsi_header; 648 length = wc->byte_len - ISER_HEADERS_LEN; 649 650 iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, 651 hdr->itt, length); 652 653 if (iser_check_remote_inv(iser_conn, wc, hdr)) { 654 iscsi_conn_failure(iser_conn->iscsi_conn, 655 ISCSI_ERR_CONN_FAILED); 656 return; 657 } 658 659 iscsi_iser_recv(iser_conn->iscsi_conn, hdr, desc->data, length); 660 661 ib_dma_sync_single_for_device(ib_conn->device->ib_device, 662 desc->dma_addr, ISER_RX_PAYLOAD_SIZE, 663 DMA_FROM_DEVICE); 664 665 err = iser_post_recvm(iser_conn, desc); 666 if (err) 667 iser_err("posting rx buffer err %d\n", err); 668 } 669 670 void iser_cmd_comp(struct ib_cq *cq, struct ib_wc *wc) 671 { 672 if (unlikely(wc->status != IB_WC_SUCCESS)) 673 iser_err_comp(wc, "command"); 674 } 675 676 void iser_ctrl_comp(struct ib_cq *cq, struct ib_wc *wc) 677 { 678 struct iser_tx_desc *desc = iser_tx(wc->wr_cqe); 679 struct iscsi_task *task; 680 681 if (unlikely(wc->status != IB_WC_SUCCESS)) { 682 iser_err_comp(wc, "control"); 683 return; 684 } 685 686 /* this arithmetic is legal by libiscsi dd_data allocation */ 687 task = (void *)desc - sizeof(struct iscsi_task); 688 if (task->hdr->itt == RESERVED_ITT) 689 iscsi_put_task(task); 690 } 691 692 void iser_dataout_comp(struct ib_cq *cq, struct ib_wc *wc) 693 { 694 struct iser_tx_desc *desc = iser_tx(wc->wr_cqe); 695 struct ib_conn *ib_conn = wc->qp->qp_context; 696 struct iser_device *device = ib_conn->device; 697 698 if (unlikely(wc->status != IB_WC_SUCCESS)) 699 iser_err_comp(wc, "dataout"); 700 701 ib_dma_unmap_single(device->ib_device, desc->dma_addr, 702 ISER_HEADERS_LEN, DMA_TO_DEVICE); 703 kmem_cache_free(ig.desc_cache, desc); 704 } 705 706 void iser_task_rdma_init(struct iscsi_iser_task *iser_task) 707 708 { 709 iser_task->status = ISER_TASK_STATUS_INIT; 710 711 iser_task->dir[ISER_DIR_IN] = 0; 712 iser_task->dir[ISER_DIR_OUT] = 0; 713 714 iser_task->data[ISER_DIR_IN].data_len = 0; 715 iser_task->data[ISER_DIR_OUT].data_len = 0; 716 717 iser_task->prot[ISER_DIR_IN].data_len = 0; 718 iser_task->prot[ISER_DIR_OUT].data_len = 0; 719 720 iser_task->prot[ISER_DIR_IN].dma_nents = 0; 721 iser_task->prot[ISER_DIR_OUT].dma_nents = 0; 722 723 memset(&iser_task->rdma_reg[ISER_DIR_IN], 0, 724 sizeof(struct iser_mem_reg)); 725 memset(&iser_task->rdma_reg[ISER_DIR_OUT], 0, 726 sizeof(struct iser_mem_reg)); 727 } 728 729 void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) 730 { 731 732 if (iser_task->dir[ISER_DIR_IN]) { 733 iser_unreg_mem_fastreg(iser_task, ISER_DIR_IN); 734 iser_dma_unmap_task_data(iser_task, ISER_DIR_IN, 735 DMA_FROM_DEVICE); 736 } 737 738 if (iser_task->dir[ISER_DIR_OUT]) { 739 iser_unreg_mem_fastreg(iser_task, ISER_DIR_OUT); 740 iser_dma_unmap_task_data(iser_task, ISER_DIR_OUT, 741 DMA_TO_DEVICE); 742 } 743 } 744