1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/module.h> 34 #include <linux/kernel.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/highmem.h> 38 #include <linux/scatterlist.h> 39 40 #include "iscsi_iser.h" 41 static 42 int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task, 43 struct iser_data_buf *mem, 44 struct iser_reg_resources *rsc, 45 struct iser_mem_reg *mem_reg); 46 static 47 int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 48 struct iser_data_buf *mem, 49 struct iser_reg_resources *rsc, 50 struct iser_mem_reg *mem_reg); 51 52 static struct iser_reg_ops fastreg_ops = { 53 .alloc_reg_res = iser_alloc_fastreg_pool, 54 .free_reg_res = iser_free_fastreg_pool, 55 .reg_mem = iser_fast_reg_mr, 56 .unreg_mem = iser_unreg_mem_fastreg, 57 .reg_desc_get = iser_reg_desc_get_fr, 58 .reg_desc_put = iser_reg_desc_put_fr, 59 }; 60 61 static struct iser_reg_ops fmr_ops = { 62 .alloc_reg_res = iser_alloc_fmr_pool, 63 .free_reg_res = iser_free_fmr_pool, 64 .reg_mem = iser_fast_reg_fmr, 65 .unreg_mem = iser_unreg_mem_fmr, 66 .reg_desc_get = iser_reg_desc_get_fmr, 67 .reg_desc_put = iser_reg_desc_put_fmr, 68 }; 69 70 int iser_assign_reg_ops(struct iser_device *device) 71 { 72 struct ib_device_attr *dev_attr = &device->dev_attr; 73 74 /* Assign function handles - based on FMR support */ 75 if (device->ib_device->alloc_fmr && device->ib_device->dealloc_fmr && 76 device->ib_device->map_phys_fmr && device->ib_device->unmap_fmr) { 77 iser_info("FMR supported, using FMR for registration\n"); 78 device->reg_ops = &fmr_ops; 79 } else 80 if (dev_attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) { 81 iser_info("FastReg supported, using FastReg for registration\n"); 82 device->reg_ops = &fastreg_ops; 83 } else { 84 iser_err("IB device does not support FMRs nor FastRegs, can't register memory\n"); 85 return -1; 86 } 87 88 return 0; 89 } 90 91 struct iser_fr_desc * 92 iser_reg_desc_get_fr(struct ib_conn *ib_conn) 93 { 94 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 95 struct iser_fr_desc *desc; 96 unsigned long flags; 97 98 spin_lock_irqsave(&fr_pool->lock, flags); 99 desc = list_first_entry(&fr_pool->list, 100 struct iser_fr_desc, list); 101 list_del(&desc->list); 102 spin_unlock_irqrestore(&fr_pool->lock, flags); 103 104 return desc; 105 } 106 107 void 108 iser_reg_desc_put_fr(struct ib_conn *ib_conn, 109 struct iser_fr_desc *desc) 110 { 111 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 112 unsigned long flags; 113 114 spin_lock_irqsave(&fr_pool->lock, flags); 115 list_add(&desc->list, &fr_pool->list); 116 spin_unlock_irqrestore(&fr_pool->lock, flags); 117 } 118 119 struct iser_fr_desc * 120 iser_reg_desc_get_fmr(struct ib_conn *ib_conn) 121 { 122 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 123 124 return list_first_entry(&fr_pool->list, 125 struct iser_fr_desc, list); 126 } 127 128 void 129 iser_reg_desc_put_fmr(struct ib_conn *ib_conn, 130 struct iser_fr_desc *desc) 131 { 132 } 133 134 #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) 135 136 /** 137 * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses 138 * and returns the length of resulting physical address array (may be less than 139 * the original due to possible compaction). 140 * 141 * we build a "page vec" under the assumption that the SG meets the RDMA 142 * alignment requirements. Other then the first and last SG elements, all 143 * the "internal" elements can be compacted into a list whose elements are 144 * dma addresses of physical pages. The code supports also the weird case 145 * where --few fragments of the same page-- are present in the SG as 146 * consecutive elements. Also, it handles one entry SG. 147 */ 148 149 static int iser_sg_to_page_vec(struct iser_data_buf *data, 150 struct ib_device *ibdev, u64 *pages, 151 int *offset, int *data_size) 152 { 153 struct scatterlist *sg, *sgl = data->sg; 154 u64 start_addr, end_addr, page, chunk_start = 0; 155 unsigned long total_sz = 0; 156 unsigned int dma_len; 157 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; 158 159 /* compute the offset of first element */ 160 *offset = (u64) sgl[0].offset & ~MASK_4K; 161 162 new_chunk = 1; 163 cur_page = 0; 164 for_each_sg(sgl, sg, data->dma_nents, i) { 165 start_addr = ib_sg_dma_address(ibdev, sg); 166 if (new_chunk) 167 chunk_start = start_addr; 168 dma_len = ib_sg_dma_len(ibdev, sg); 169 end_addr = start_addr + dma_len; 170 total_sz += dma_len; 171 172 /* collect page fragments until aligned or end of SG list */ 173 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) { 174 new_chunk = 0; 175 continue; 176 } 177 new_chunk = 1; 178 179 /* address of the first page in the contiguous chunk; 180 masking relevant for the very first SG entry, 181 which might be unaligned */ 182 page = chunk_start & MASK_4K; 183 do { 184 pages[cur_page++] = page; 185 page += SIZE_4K; 186 } while (page < end_addr); 187 } 188 189 *data_size = total_sz; 190 iser_dbg("page_vec->data_size:%d cur_page %d\n", 191 *data_size, cur_page); 192 return cur_page; 193 } 194 195 static void iser_data_buf_dump(struct iser_data_buf *data, 196 struct ib_device *ibdev) 197 { 198 struct scatterlist *sg; 199 int i; 200 201 for_each_sg(data->sg, sg, data->dma_nents, i) 202 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p " 203 "off:0x%x sz:0x%x dma_len:0x%x\n", 204 i, (unsigned long)ib_sg_dma_address(ibdev, sg), 205 sg_page(sg), sg->offset, 206 sg->length, ib_sg_dma_len(ibdev, sg)); 207 } 208 209 static void iser_dump_page_vec(struct iser_page_vec *page_vec) 210 { 211 int i; 212 213 iser_err("page vec length %d data size %d\n", 214 page_vec->length, page_vec->data_size); 215 for (i = 0; i < page_vec->length; i++) 216 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]); 217 } 218 219 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 220 struct iser_data_buf *data, 221 enum iser_data_dir iser_dir, 222 enum dma_data_direction dma_dir) 223 { 224 struct ib_device *dev; 225 226 iser_task->dir[iser_dir] = 1; 227 dev = iser_task->iser_conn->ib_conn.device->ib_device; 228 229 data->dma_nents = ib_dma_map_sg(dev, data->sg, data->size, dma_dir); 230 if (data->dma_nents == 0) { 231 iser_err("dma_map_sg failed!!!\n"); 232 return -EINVAL; 233 } 234 return 0; 235 } 236 237 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 238 struct iser_data_buf *data, 239 enum dma_data_direction dir) 240 { 241 struct ib_device *dev; 242 243 dev = iser_task->iser_conn->ib_conn.device->ib_device; 244 ib_dma_unmap_sg(dev, data->sg, data->size, dir); 245 } 246 247 static int 248 iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem, 249 struct iser_mem_reg *reg) 250 { 251 struct scatterlist *sg = mem->sg; 252 253 reg->sge.lkey = device->pd->local_dma_lkey; 254 reg->rkey = device->mr->rkey; 255 reg->sge.addr = ib_sg_dma_address(device->ib_device, &sg[0]); 256 reg->sge.length = ib_sg_dma_len(device->ib_device, &sg[0]); 257 258 iser_dbg("Single DMA entry: lkey=0x%x, rkey=0x%x, addr=0x%llx," 259 " length=0x%x\n", reg->sge.lkey, reg->rkey, 260 reg->sge.addr, reg->sge.length); 261 262 return 0; 263 } 264 265 /** 266 * iser_reg_page_vec - Register physical memory 267 * 268 * returns: 0 on success, errno code on failure 269 */ 270 static 271 int iser_fast_reg_fmr(struct iscsi_iser_task *iser_task, 272 struct iser_data_buf *mem, 273 struct iser_reg_resources *rsc, 274 struct iser_mem_reg *reg) 275 { 276 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 277 struct iser_device *device = ib_conn->device; 278 struct iser_page_vec *page_vec = rsc->page_vec; 279 struct ib_fmr_pool *fmr_pool = rsc->fmr_pool; 280 struct ib_pool_fmr *fmr; 281 int ret, plen; 282 283 plen = iser_sg_to_page_vec(mem, device->ib_device, 284 page_vec->pages, 285 &page_vec->offset, 286 &page_vec->data_size); 287 page_vec->length = plen; 288 if (plen * SIZE_4K < page_vec->data_size) { 289 iser_err("page vec too short to hold this SG\n"); 290 iser_data_buf_dump(mem, device->ib_device); 291 iser_dump_page_vec(page_vec); 292 return -EINVAL; 293 } 294 295 fmr = ib_fmr_pool_map_phys(fmr_pool, 296 page_vec->pages, 297 page_vec->length, 298 page_vec->pages[0]); 299 if (IS_ERR(fmr)) { 300 ret = PTR_ERR(fmr); 301 iser_err("ib_fmr_pool_map_phys failed: %d\n", ret); 302 return ret; 303 } 304 305 reg->sge.lkey = fmr->fmr->lkey; 306 reg->rkey = fmr->fmr->rkey; 307 reg->sge.addr = page_vec->pages[0] + page_vec->offset; 308 reg->sge.length = page_vec->data_size; 309 reg->mem_h = fmr; 310 311 iser_dbg("fmr reg: lkey=0x%x, rkey=0x%x, addr=0x%llx," 312 " length=0x%x\n", reg->sge.lkey, reg->rkey, 313 reg->sge.addr, reg->sge.length); 314 315 return 0; 316 } 317 318 /** 319 * Unregister (previosuly registered using FMR) memory. 320 * If memory is non-FMR does nothing. 321 */ 322 void iser_unreg_mem_fmr(struct iscsi_iser_task *iser_task, 323 enum iser_data_dir cmd_dir) 324 { 325 struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir]; 326 int ret; 327 328 if (!reg->mem_h) 329 return; 330 331 iser_dbg("PHYSICAL Mem.Unregister mem_h %p\n", reg->mem_h); 332 333 ret = ib_fmr_pool_unmap((struct ib_pool_fmr *)reg->mem_h); 334 if (ret) 335 iser_err("ib_fmr_pool_unmap failed %d\n", ret); 336 337 reg->mem_h = NULL; 338 } 339 340 void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, 341 enum iser_data_dir cmd_dir) 342 { 343 struct iser_device *device = iser_task->iser_conn->ib_conn.device; 344 struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir]; 345 346 if (!reg->mem_h) 347 return; 348 349 device->reg_ops->reg_desc_put(&iser_task->iser_conn->ib_conn, 350 reg->mem_h); 351 reg->mem_h = NULL; 352 } 353 354 static void 355 iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, 356 struct ib_sig_domain *domain) 357 { 358 domain->sig_type = IB_SIG_TYPE_T10_DIF; 359 domain->sig.dif.pi_interval = scsi_prot_interval(sc); 360 domain->sig.dif.ref_tag = scsi_prot_ref_tag(sc); 361 /* 362 * At the moment we hard code those, but in the future 363 * we will take them from sc. 364 */ 365 domain->sig.dif.apptag_check_mask = 0xffff; 366 domain->sig.dif.app_escape = true; 367 domain->sig.dif.ref_escape = true; 368 if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) 369 domain->sig.dif.ref_remap = true; 370 }; 371 372 static int 373 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 374 { 375 switch (scsi_get_prot_op(sc)) { 376 case SCSI_PROT_WRITE_INSERT: 377 case SCSI_PROT_READ_STRIP: 378 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 379 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 380 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 381 break; 382 case SCSI_PROT_READ_INSERT: 383 case SCSI_PROT_WRITE_STRIP: 384 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 385 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 386 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 387 IB_T10DIF_CSUM : IB_T10DIF_CRC; 388 break; 389 case SCSI_PROT_READ_PASS: 390 case SCSI_PROT_WRITE_PASS: 391 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 392 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 393 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 394 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 395 IB_T10DIF_CSUM : IB_T10DIF_CRC; 396 break; 397 default: 398 iser_err("Unsupported PI operation %d\n", 399 scsi_get_prot_op(sc)); 400 return -EINVAL; 401 } 402 403 return 0; 404 } 405 406 static inline void 407 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 408 { 409 *mask = 0; 410 if (sc->prot_flags & SCSI_PROT_REF_CHECK) 411 *mask |= ISER_CHECK_REFTAG; 412 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) 413 *mask |= ISER_CHECK_GUARD; 414 } 415 416 static void 417 iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 418 { 419 u32 rkey; 420 421 inv_wr->opcode = IB_WR_LOCAL_INV; 422 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 423 inv_wr->ex.invalidate_rkey = mr->rkey; 424 inv_wr->send_flags = 0; 425 inv_wr->num_sge = 0; 426 427 rkey = ib_inc_rkey(mr->rkey); 428 ib_update_fast_reg_key(mr, rkey); 429 } 430 431 static int 432 iser_reg_sig_mr(struct iscsi_iser_task *iser_task, 433 struct iser_pi_context *pi_ctx, 434 struct iser_mem_reg *data_reg, 435 struct iser_mem_reg *prot_reg, 436 struct iser_mem_reg *sig_reg) 437 { 438 struct iser_tx_desc *tx_desc = &iser_task->desc; 439 struct ib_sig_attrs *sig_attrs = &tx_desc->sig_attrs; 440 struct ib_sig_handover_wr *wr; 441 int ret; 442 443 memset(sig_attrs, 0, sizeof(*sig_attrs)); 444 ret = iser_set_sig_attrs(iser_task->sc, sig_attrs); 445 if (ret) 446 goto err; 447 448 iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask); 449 450 if (!pi_ctx->sig_mr_valid) 451 iser_inv_rkey(iser_tx_next_wr(tx_desc), pi_ctx->sig_mr); 452 453 wr = sig_handover_wr(iser_tx_next_wr(tx_desc)); 454 wr->wr.opcode = IB_WR_REG_SIG_MR; 455 wr->wr.wr_id = ISER_FASTREG_LI_WRID; 456 wr->wr.sg_list = &data_reg->sge; 457 wr->wr.num_sge = 1; 458 wr->wr.send_flags = 0; 459 wr->sig_attrs = sig_attrs; 460 wr->sig_mr = pi_ctx->sig_mr; 461 if (scsi_prot_sg_count(iser_task->sc)) 462 wr->prot = &prot_reg->sge; 463 else 464 wr->prot = NULL; 465 wr->access_flags = IB_ACCESS_LOCAL_WRITE | 466 IB_ACCESS_REMOTE_READ | 467 IB_ACCESS_REMOTE_WRITE; 468 pi_ctx->sig_mr_valid = 0; 469 470 sig_reg->sge.lkey = pi_ctx->sig_mr->lkey; 471 sig_reg->rkey = pi_ctx->sig_mr->rkey; 472 sig_reg->sge.addr = 0; 473 sig_reg->sge.length = scsi_transfer_length(iser_task->sc); 474 475 iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=%u\n", 476 sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr, 477 sig_reg->sge.length); 478 err: 479 return ret; 480 } 481 482 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 483 struct iser_data_buf *mem, 484 struct iser_reg_resources *rsc, 485 struct iser_mem_reg *reg) 486 { 487 struct iser_tx_desc *tx_desc = &iser_task->desc; 488 struct ib_mr *mr = rsc->mr; 489 struct ib_reg_wr *wr; 490 int n; 491 492 if (!rsc->mr_valid) 493 iser_inv_rkey(iser_tx_next_wr(tx_desc), mr); 494 495 n = ib_map_mr_sg(mr, mem->sg, mem->size, SIZE_4K); 496 if (unlikely(n != mem->size)) { 497 iser_err("failed to map sg (%d/%d)\n", 498 n, mem->size); 499 return n < 0 ? n : -EINVAL; 500 } 501 502 wr = reg_wr(iser_tx_next_wr(tx_desc)); 503 wr->wr.opcode = IB_WR_REG_MR; 504 wr->wr.wr_id = ISER_FASTREG_LI_WRID; 505 wr->wr.send_flags = 0; 506 wr->wr.num_sge = 0; 507 wr->mr = mr; 508 wr->key = mr->rkey; 509 wr->access = IB_ACCESS_LOCAL_WRITE | 510 IB_ACCESS_REMOTE_WRITE | 511 IB_ACCESS_REMOTE_READ; 512 513 rsc->mr_valid = 0; 514 515 reg->sge.lkey = mr->lkey; 516 reg->rkey = mr->rkey; 517 reg->sge.addr = mr->iova; 518 reg->sge.length = mr->length; 519 520 iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=0x%x\n", 521 reg->sge.lkey, reg->rkey, reg->sge.addr, reg->sge.length); 522 523 return 0; 524 } 525 526 static int 527 iser_reg_prot_sg(struct iscsi_iser_task *task, 528 struct iser_data_buf *mem, 529 struct iser_fr_desc *desc, 530 bool use_dma_key, 531 struct iser_mem_reg *reg) 532 { 533 struct iser_device *device = task->iser_conn->ib_conn.device; 534 535 if (use_dma_key) 536 return iser_reg_dma(device, mem, reg); 537 538 return device->reg_ops->reg_mem(task, mem, &desc->pi_ctx->rsc, reg); 539 } 540 541 static int 542 iser_reg_data_sg(struct iscsi_iser_task *task, 543 struct iser_data_buf *mem, 544 struct iser_fr_desc *desc, 545 bool use_dma_key, 546 struct iser_mem_reg *reg) 547 { 548 struct iser_device *device = task->iser_conn->ib_conn.device; 549 550 if (use_dma_key) 551 return iser_reg_dma(device, mem, reg); 552 553 return device->reg_ops->reg_mem(task, mem, &desc->rsc, reg); 554 } 555 556 int iser_reg_rdma_mem(struct iscsi_iser_task *task, 557 enum iser_data_dir dir) 558 { 559 struct ib_conn *ib_conn = &task->iser_conn->ib_conn; 560 struct iser_device *device = ib_conn->device; 561 struct iser_data_buf *mem = &task->data[dir]; 562 struct iser_mem_reg *reg = &task->rdma_reg[dir]; 563 struct iser_mem_reg *data_reg; 564 struct iser_fr_desc *desc = NULL; 565 bool use_dma_key; 566 int err; 567 568 use_dma_key = (mem->dma_nents == 1 && !iser_always_reg && 569 scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL); 570 571 if (!use_dma_key) { 572 desc = device->reg_ops->reg_desc_get(ib_conn); 573 reg->mem_h = desc; 574 } 575 576 if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL) 577 data_reg = reg; 578 else 579 data_reg = &task->desc.data_reg; 580 581 err = iser_reg_data_sg(task, mem, desc, use_dma_key, data_reg); 582 if (unlikely(err)) 583 goto err_reg; 584 585 if (scsi_get_prot_op(task->sc) != SCSI_PROT_NORMAL) { 586 struct iser_mem_reg *prot_reg = &task->desc.prot_reg; 587 588 if (scsi_prot_sg_count(task->sc)) { 589 mem = &task->prot[dir]; 590 err = iser_reg_prot_sg(task, mem, desc, 591 use_dma_key, prot_reg); 592 if (unlikely(err)) 593 goto err_reg; 594 } 595 596 err = iser_reg_sig_mr(task, desc->pi_ctx, data_reg, 597 prot_reg, reg); 598 if (unlikely(err)) 599 goto err_reg; 600 601 desc->pi_ctx->sig_protected = 1; 602 } 603 604 return 0; 605 606 err_reg: 607 if (desc) 608 device->reg_ops->reg_desc_put(ib_conn, desc); 609 610 return err; 611 } 612 613 void iser_unreg_rdma_mem(struct iscsi_iser_task *task, 614 enum iser_data_dir dir) 615 { 616 struct iser_device *device = task->iser_conn->ib_conn.device; 617 618 device->reg_ops->unreg_mem(task, dir); 619 } 620