1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/module.h> 34 #include <linux/kernel.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/highmem.h> 38 #include <linux/scatterlist.h> 39 40 #include "iscsi_iser.h" 41 42 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ 43 44 /** 45 * iser_start_rdma_unaligned_sg 46 */ 47 static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 48 struct iser_data_buf *data, 49 struct iser_data_buf *data_copy, 50 enum iser_data_dir cmd_dir) 51 { 52 struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device; 53 struct scatterlist *sgl = (struct scatterlist *)data->buf; 54 struct scatterlist *sg; 55 char *mem = NULL; 56 unsigned long cmd_data_len = 0; 57 int dma_nents, i; 58 59 for_each_sg(sgl, sg, data->size, i) 60 cmd_data_len += ib_sg_dma_len(dev, sg); 61 62 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 63 mem = (void *)__get_free_pages(GFP_ATOMIC, 64 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); 65 else 66 mem = kmalloc(cmd_data_len, GFP_ATOMIC); 67 68 if (mem == NULL) { 69 iser_err("Failed to allocate mem size %d %d for copying sglist\n", 70 data->size, (int)cmd_data_len); 71 return -ENOMEM; 72 } 73 74 if (cmd_dir == ISER_DIR_OUT) { 75 /* copy the unaligned sg the buffer which is used for RDMA */ 76 char *p, *from; 77 78 sgl = (struct scatterlist *)data->buf; 79 p = mem; 80 for_each_sg(sgl, sg, data->size, i) { 81 from = kmap_atomic(sg_page(sg)); 82 memcpy(p, 83 from + sg->offset, 84 sg->length); 85 kunmap_atomic(from); 86 p += sg->length; 87 } 88 } 89 90 sg_init_one(&data_copy->sg_single, mem, cmd_data_len); 91 data_copy->buf = &data_copy->sg_single; 92 data_copy->size = 1; 93 data_copy->copy_buf = mem; 94 95 dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1, 96 (cmd_dir == ISER_DIR_OUT) ? 97 DMA_TO_DEVICE : DMA_FROM_DEVICE); 98 BUG_ON(dma_nents == 0); 99 100 data_copy->dma_nents = dma_nents; 101 data_copy->data_len = cmd_data_len; 102 103 return 0; 104 } 105 106 /** 107 * iser_finalize_rdma_unaligned_sg 108 */ 109 110 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 111 struct iser_data_buf *data, 112 struct iser_data_buf *data_copy, 113 enum iser_data_dir cmd_dir) 114 { 115 struct ib_device *dev; 116 unsigned long cmd_data_len; 117 118 dev = iser_task->iser_conn->ib_conn.device->ib_device; 119 120 ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, 121 (cmd_dir == ISER_DIR_OUT) ? 122 DMA_TO_DEVICE : DMA_FROM_DEVICE); 123 124 if (cmd_dir == ISER_DIR_IN) { 125 char *mem; 126 struct scatterlist *sgl, *sg; 127 unsigned char *p, *to; 128 unsigned int sg_size; 129 int i; 130 131 /* copy back read RDMA to unaligned sg */ 132 mem = data_copy->copy_buf; 133 134 sgl = (struct scatterlist *)data->buf; 135 sg_size = data->size; 136 137 p = mem; 138 for_each_sg(sgl, sg, sg_size, i) { 139 to = kmap_atomic(sg_page(sg)); 140 memcpy(to + sg->offset, 141 p, 142 sg->length); 143 kunmap_atomic(to); 144 p += sg->length; 145 } 146 } 147 148 cmd_data_len = data->data_len; 149 150 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 151 free_pages((unsigned long)data_copy->copy_buf, 152 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); 153 else 154 kfree(data_copy->copy_buf); 155 156 data_copy->copy_buf = NULL; 157 } 158 159 #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) 160 161 /** 162 * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses 163 * and returns the length of resulting physical address array (may be less than 164 * the original due to possible compaction). 165 * 166 * we build a "page vec" under the assumption that the SG meets the RDMA 167 * alignment requirements. Other then the first and last SG elements, all 168 * the "internal" elements can be compacted into a list whose elements are 169 * dma addresses of physical pages. The code supports also the weird case 170 * where --few fragments of the same page-- are present in the SG as 171 * consecutive elements. Also, it handles one entry SG. 172 */ 173 174 static int iser_sg_to_page_vec(struct iser_data_buf *data, 175 struct ib_device *ibdev, u64 *pages, 176 int *offset, int *data_size) 177 { 178 struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf; 179 u64 start_addr, end_addr, page, chunk_start = 0; 180 unsigned long total_sz = 0; 181 unsigned int dma_len; 182 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; 183 184 /* compute the offset of first element */ 185 *offset = (u64) sgl[0].offset & ~MASK_4K; 186 187 new_chunk = 1; 188 cur_page = 0; 189 for_each_sg(sgl, sg, data->dma_nents, i) { 190 start_addr = ib_sg_dma_address(ibdev, sg); 191 if (new_chunk) 192 chunk_start = start_addr; 193 dma_len = ib_sg_dma_len(ibdev, sg); 194 end_addr = start_addr + dma_len; 195 total_sz += dma_len; 196 197 /* collect page fragments until aligned or end of SG list */ 198 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) { 199 new_chunk = 0; 200 continue; 201 } 202 new_chunk = 1; 203 204 /* address of the first page in the contiguous chunk; 205 masking relevant for the very first SG entry, 206 which might be unaligned */ 207 page = chunk_start & MASK_4K; 208 do { 209 pages[cur_page++] = page; 210 page += SIZE_4K; 211 } while (page < end_addr); 212 } 213 214 *data_size = total_sz; 215 iser_dbg("page_vec->data_size:%d cur_page %d\n", 216 *data_size, cur_page); 217 return cur_page; 218 } 219 220 221 /** 222 * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned 223 * for RDMA sub-list of a scatter-gather list of memory buffers, and returns 224 * the number of entries which are aligned correctly. Supports the case where 225 * consecutive SG elements are actually fragments of the same physcial page. 226 */ 227 static int iser_data_buf_aligned_len(struct iser_data_buf *data, 228 struct ib_device *ibdev) 229 { 230 struct scatterlist *sgl, *sg, *next_sg = NULL; 231 u64 start_addr, end_addr; 232 int i, ret_len, start_check = 0; 233 234 if (data->dma_nents == 1) 235 return 1; 236 237 sgl = (struct scatterlist *)data->buf; 238 start_addr = ib_sg_dma_address(ibdev, sgl); 239 240 for_each_sg(sgl, sg, data->dma_nents, i) { 241 if (start_check && !IS_4K_ALIGNED(start_addr)) 242 break; 243 244 next_sg = sg_next(sg); 245 if (!next_sg) 246 break; 247 248 end_addr = start_addr + ib_sg_dma_len(ibdev, sg); 249 start_addr = ib_sg_dma_address(ibdev, next_sg); 250 251 if (end_addr == start_addr) { 252 start_check = 0; 253 continue; 254 } else 255 start_check = 1; 256 257 if (!IS_4K_ALIGNED(end_addr)) 258 break; 259 } 260 ret_len = (next_sg) ? i : i+1; 261 iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n", 262 ret_len, data->dma_nents, data); 263 return ret_len; 264 } 265 266 static void iser_data_buf_dump(struct iser_data_buf *data, 267 struct ib_device *ibdev) 268 { 269 struct scatterlist *sgl = (struct scatterlist *)data->buf; 270 struct scatterlist *sg; 271 int i; 272 273 for_each_sg(sgl, sg, data->dma_nents, i) 274 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p " 275 "off:0x%x sz:0x%x dma_len:0x%x\n", 276 i, (unsigned long)ib_sg_dma_address(ibdev, sg), 277 sg_page(sg), sg->offset, 278 sg->length, ib_sg_dma_len(ibdev, sg)); 279 } 280 281 static void iser_dump_page_vec(struct iser_page_vec *page_vec) 282 { 283 int i; 284 285 iser_err("page vec length %d data size %d\n", 286 page_vec->length, page_vec->data_size); 287 for (i = 0; i < page_vec->length; i++) 288 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]); 289 } 290 291 static void iser_page_vec_build(struct iser_data_buf *data, 292 struct iser_page_vec *page_vec, 293 struct ib_device *ibdev) 294 { 295 int page_vec_len = 0; 296 297 page_vec->length = 0; 298 page_vec->offset = 0; 299 300 iser_dbg("Translating sg sz: %d\n", data->dma_nents); 301 page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages, 302 &page_vec->offset, 303 &page_vec->data_size); 304 iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len); 305 306 page_vec->length = page_vec_len; 307 308 if (page_vec_len * SIZE_4K < page_vec->data_size) { 309 iser_err("page_vec too short to hold this SG\n"); 310 iser_data_buf_dump(data, ibdev); 311 iser_dump_page_vec(page_vec); 312 BUG(); 313 } 314 } 315 316 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 317 struct iser_data_buf *data, 318 enum iser_data_dir iser_dir, 319 enum dma_data_direction dma_dir) 320 { 321 struct ib_device *dev; 322 323 iser_task->dir[iser_dir] = 1; 324 dev = iser_task->iser_conn->ib_conn.device->ib_device; 325 326 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); 327 if (data->dma_nents == 0) { 328 iser_err("dma_map_sg failed!!!\n"); 329 return -EINVAL; 330 } 331 return 0; 332 } 333 334 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 335 struct iser_data_buf *data) 336 { 337 struct ib_device *dev; 338 339 dev = iser_task->iser_conn->ib_conn.device->ib_device; 340 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); 341 } 342 343 static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task, 344 struct ib_device *ibdev, 345 struct iser_data_buf *mem, 346 struct iser_data_buf *mem_copy, 347 enum iser_data_dir cmd_dir, 348 int aligned_len) 349 { 350 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn; 351 352 iscsi_conn->fmr_unalign_cnt++; 353 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", 354 aligned_len, mem->size); 355 356 if (iser_debug_level > 0) 357 iser_data_buf_dump(mem, ibdev); 358 359 /* unmap the command data before accessing it */ 360 iser_dma_unmap_task_data(iser_task, mem); 361 362 /* allocate copy buf, if we are writing, copy the */ 363 /* unaligned scatterlist, dma map the copy */ 364 if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0) 365 return -ENOMEM; 366 367 return 0; 368 } 369 370 /** 371 * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA, 372 * using FMR (if possible) obtaining rkey and va 373 * 374 * returns 0 on success, errno code on failure 375 */ 376 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, 377 enum iser_data_dir cmd_dir) 378 { 379 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 380 struct iser_device *device = ib_conn->device; 381 struct ib_device *ibdev = device->ib_device; 382 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 383 struct iser_regd_buf *regd_buf; 384 int aligned_len; 385 int err; 386 int i; 387 struct scatterlist *sg; 388 389 regd_buf = &iser_task->rdma_regd[cmd_dir]; 390 391 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 392 if (aligned_len != mem->dma_nents) { 393 err = fall_to_bounce_buf(iser_task, ibdev, mem, 394 &iser_task->data_copy[cmd_dir], 395 cmd_dir, aligned_len); 396 if (err) { 397 iser_err("failed to allocate bounce buffer\n"); 398 return err; 399 } 400 mem = &iser_task->data_copy[cmd_dir]; 401 } 402 403 /* if there a single dma entry, FMR is not needed */ 404 if (mem->dma_nents == 1) { 405 sg = (struct scatterlist *)mem->buf; 406 407 regd_buf->reg.lkey = device->mr->lkey; 408 regd_buf->reg.rkey = device->mr->rkey; 409 regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]); 410 regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]); 411 412 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " 413 "va: 0x%08lX sz: %ld]\n", 414 (unsigned int)regd_buf->reg.lkey, 415 (unsigned int)regd_buf->reg.rkey, 416 (unsigned long)regd_buf->reg.va, 417 (unsigned long)regd_buf->reg.len); 418 } else { /* use FMR for multiple dma entries */ 419 iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev); 420 err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec, 421 ®d_buf->reg); 422 if (err && err != -EAGAIN) { 423 iser_data_buf_dump(mem, ibdev); 424 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", 425 mem->dma_nents, 426 ntoh24(iser_task->desc.iscsi_header.dlength)); 427 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", 428 ib_conn->fmr.page_vec->data_size, 429 ib_conn->fmr.page_vec->length, 430 ib_conn->fmr.page_vec->offset); 431 for (i = 0; i < ib_conn->fmr.page_vec->length; i++) 432 iser_err("page_vec[%d] = 0x%llx\n", i, 433 (unsigned long long)ib_conn->fmr.page_vec->pages[i]); 434 } 435 if (err) 436 return err; 437 } 438 return 0; 439 } 440 441 static void 442 iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, 443 struct ib_sig_domain *domain) 444 { 445 domain->sig_type = IB_SIG_TYPE_T10_DIF; 446 domain->sig.dif.pi_interval = scsi_prot_interval(sc); 447 domain->sig.dif.ref_tag = scsi_prot_ref_tag(sc); 448 /* 449 * At the moment we hard code those, but in the future 450 * we will take them from sc. 451 */ 452 domain->sig.dif.apptag_check_mask = 0xffff; 453 domain->sig.dif.app_escape = true; 454 domain->sig.dif.ref_escape = true; 455 if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) 456 domain->sig.dif.ref_remap = true; 457 }; 458 459 static int 460 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 461 { 462 switch (scsi_get_prot_op(sc)) { 463 case SCSI_PROT_WRITE_INSERT: 464 case SCSI_PROT_READ_STRIP: 465 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 466 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 467 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 468 break; 469 case SCSI_PROT_READ_INSERT: 470 case SCSI_PROT_WRITE_STRIP: 471 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 472 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 473 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 474 IB_T10DIF_CSUM : IB_T10DIF_CRC; 475 break; 476 case SCSI_PROT_READ_PASS: 477 case SCSI_PROT_WRITE_PASS: 478 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 479 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 480 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 481 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 482 IB_T10DIF_CSUM : IB_T10DIF_CRC; 483 break; 484 default: 485 iser_err("Unsupported PI operation %d\n", 486 scsi_get_prot_op(sc)); 487 return -EINVAL; 488 } 489 490 return 0; 491 } 492 493 static inline void 494 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 495 { 496 *mask = 0; 497 if (sc->prot_flags & SCSI_PROT_REF_CHECK) 498 *mask |= ISER_CHECK_REFTAG; 499 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) 500 *mask |= ISER_CHECK_GUARD; 501 } 502 503 static void 504 iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 505 { 506 u32 rkey; 507 508 memset(inv_wr, 0, sizeof(*inv_wr)); 509 inv_wr->opcode = IB_WR_LOCAL_INV; 510 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 511 inv_wr->ex.invalidate_rkey = mr->rkey; 512 513 rkey = ib_inc_rkey(mr->rkey); 514 ib_update_fast_reg_key(mr, rkey); 515 } 516 517 static int 518 iser_reg_sig_mr(struct iscsi_iser_task *iser_task, 519 struct fast_reg_descriptor *desc, struct ib_sge *data_sge, 520 struct ib_sge *prot_sge, struct ib_sge *sig_sge) 521 { 522 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 523 struct iser_pi_context *pi_ctx = desc->pi_ctx; 524 struct ib_send_wr sig_wr, inv_wr; 525 struct ib_send_wr *bad_wr, *wr = NULL; 526 struct ib_sig_attrs sig_attrs; 527 int ret; 528 529 memset(&sig_attrs, 0, sizeof(sig_attrs)); 530 ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs); 531 if (ret) 532 goto err; 533 534 iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask); 535 536 if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) { 537 iser_inv_rkey(&inv_wr, pi_ctx->sig_mr); 538 wr = &inv_wr; 539 } 540 541 memset(&sig_wr, 0, sizeof(sig_wr)); 542 sig_wr.opcode = IB_WR_REG_SIG_MR; 543 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 544 sig_wr.sg_list = data_sge; 545 sig_wr.num_sge = 1; 546 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 547 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 548 if (scsi_prot_sg_count(iser_task->sc)) 549 sig_wr.wr.sig_handover.prot = prot_sge; 550 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE | 551 IB_ACCESS_REMOTE_READ | 552 IB_ACCESS_REMOTE_WRITE; 553 554 if (!wr) 555 wr = &sig_wr; 556 else 557 wr->next = &sig_wr; 558 559 ret = ib_post_send(ib_conn->qp, wr, &bad_wr); 560 if (ret) { 561 iser_err("reg_sig_mr failed, ret:%d\n", ret); 562 goto err; 563 } 564 desc->reg_indicators &= ~ISER_SIG_KEY_VALID; 565 566 sig_sge->lkey = pi_ctx->sig_mr->lkey; 567 sig_sge->addr = 0; 568 sig_sge->length = scsi_transfer_length(iser_task->sc); 569 570 iser_dbg("sig_sge: addr: 0x%llx length: %u lkey: 0x%x\n", 571 sig_sge->addr, sig_sge->length, 572 sig_sge->lkey); 573 err: 574 return ret; 575 } 576 577 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 578 struct iser_regd_buf *regd_buf, 579 struct iser_data_buf *mem, 580 enum iser_reg_indicator ind, 581 struct ib_sge *sge) 582 { 583 struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; 584 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 585 struct iser_device *device = ib_conn->device; 586 struct ib_device *ibdev = device->ib_device; 587 struct ib_mr *mr; 588 struct ib_fast_reg_page_list *frpl; 589 struct ib_send_wr fastreg_wr, inv_wr; 590 struct ib_send_wr *bad_wr, *wr = NULL; 591 int ret, offset, size, plen; 592 593 /* if there a single dma entry, dma mr suffices */ 594 if (mem->dma_nents == 1) { 595 struct scatterlist *sg = (struct scatterlist *)mem->buf; 596 597 sge->lkey = device->mr->lkey; 598 sge->addr = ib_sg_dma_address(ibdev, &sg[0]); 599 sge->length = ib_sg_dma_len(ibdev, &sg[0]); 600 601 iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n", 602 sge->lkey, sge->addr, sge->length); 603 return 0; 604 } 605 606 if (ind == ISER_DATA_KEY_VALID) { 607 mr = desc->data_mr; 608 frpl = desc->data_frpl; 609 } else { 610 mr = desc->pi_ctx->prot_mr; 611 frpl = desc->pi_ctx->prot_frpl; 612 } 613 614 plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list, 615 &offset, &size); 616 if (plen * SIZE_4K < size) { 617 iser_err("fast reg page_list too short to hold this SG\n"); 618 return -EINVAL; 619 } 620 621 if (!(desc->reg_indicators & ind)) { 622 iser_inv_rkey(&inv_wr, mr); 623 wr = &inv_wr; 624 } 625 626 /* Prepare FASTREG WR */ 627 memset(&fastreg_wr, 0, sizeof(fastreg_wr)); 628 fastreg_wr.wr_id = ISER_FASTREG_LI_WRID; 629 fastreg_wr.opcode = IB_WR_FAST_REG_MR; 630 fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset; 631 fastreg_wr.wr.fast_reg.page_list = frpl; 632 fastreg_wr.wr.fast_reg.page_list_len = plen; 633 fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K; 634 fastreg_wr.wr.fast_reg.length = size; 635 fastreg_wr.wr.fast_reg.rkey = mr->rkey; 636 fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE | 637 IB_ACCESS_REMOTE_WRITE | 638 IB_ACCESS_REMOTE_READ); 639 640 if (!wr) 641 wr = &fastreg_wr; 642 else 643 wr->next = &fastreg_wr; 644 645 ret = ib_post_send(ib_conn->qp, wr, &bad_wr); 646 if (ret) { 647 iser_err("fast registration failed, ret:%d\n", ret); 648 return ret; 649 } 650 desc->reg_indicators &= ~ind; 651 652 sge->lkey = mr->lkey; 653 sge->addr = frpl->page_list[0] + offset; 654 sge->length = size; 655 656 return ret; 657 } 658 659 /** 660 * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA, 661 * using Fast Registration WR (if possible) obtaining rkey and va 662 * 663 * returns 0 on success, errno code on failure 664 */ 665 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, 666 enum iser_data_dir cmd_dir) 667 { 668 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 669 struct iser_device *device = ib_conn->device; 670 struct ib_device *ibdev = device->ib_device; 671 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 672 struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir]; 673 struct fast_reg_descriptor *desc = NULL; 674 struct ib_sge data_sge; 675 int err, aligned_len; 676 unsigned long flags; 677 678 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 679 if (aligned_len != mem->dma_nents) { 680 err = fall_to_bounce_buf(iser_task, ibdev, mem, 681 &iser_task->data_copy[cmd_dir], 682 cmd_dir, aligned_len); 683 if (err) { 684 iser_err("failed to allocate bounce buffer\n"); 685 return err; 686 } 687 mem = &iser_task->data_copy[cmd_dir]; 688 } 689 690 if (mem->dma_nents != 1 || 691 scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { 692 spin_lock_irqsave(&ib_conn->lock, flags); 693 desc = list_first_entry(&ib_conn->fastreg.pool, 694 struct fast_reg_descriptor, list); 695 list_del(&desc->list); 696 spin_unlock_irqrestore(&ib_conn->lock, flags); 697 regd_buf->reg.mem_h = desc; 698 } 699 700 err = iser_fast_reg_mr(iser_task, regd_buf, mem, 701 ISER_DATA_KEY_VALID, &data_sge); 702 if (err) 703 goto err_reg; 704 705 if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { 706 struct ib_sge prot_sge, sig_sge; 707 708 memset(&prot_sge, 0, sizeof(prot_sge)); 709 if (scsi_prot_sg_count(iser_task->sc)) { 710 mem = &iser_task->prot[cmd_dir]; 711 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 712 if (aligned_len != mem->dma_nents) { 713 err = fall_to_bounce_buf(iser_task, ibdev, mem, 714 &iser_task->prot_copy[cmd_dir], 715 cmd_dir, aligned_len); 716 if (err) { 717 iser_err("failed to allocate bounce buffer\n"); 718 return err; 719 } 720 mem = &iser_task->prot_copy[cmd_dir]; 721 } 722 723 err = iser_fast_reg_mr(iser_task, regd_buf, mem, 724 ISER_PROT_KEY_VALID, &prot_sge); 725 if (err) 726 goto err_reg; 727 } 728 729 err = iser_reg_sig_mr(iser_task, desc, &data_sge, 730 &prot_sge, &sig_sge); 731 if (err) { 732 iser_err("Failed to register signature mr\n"); 733 return err; 734 } 735 desc->reg_indicators |= ISER_FASTREG_PROTECTED; 736 737 regd_buf->reg.lkey = sig_sge.lkey; 738 regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey; 739 regd_buf->reg.va = sig_sge.addr; 740 regd_buf->reg.len = sig_sge.length; 741 } else { 742 if (desc) 743 regd_buf->reg.rkey = desc->data_mr->rkey; 744 else 745 regd_buf->reg.rkey = device->mr->rkey; 746 747 regd_buf->reg.lkey = data_sge.lkey; 748 regd_buf->reg.va = data_sge.addr; 749 regd_buf->reg.len = data_sge.length; 750 } 751 752 return 0; 753 err_reg: 754 if (desc) { 755 spin_lock_irqsave(&ib_conn->lock, flags); 756 list_add_tail(&desc->list, &ib_conn->fastreg.pool); 757 spin_unlock_irqrestore(&ib_conn->lock, flags); 758 } 759 760 return err; 761 } 762