1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/module.h> 34 #include <linux/kernel.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/highmem.h> 38 #include <linux/scatterlist.h> 39 40 #include "iscsi_iser.h" 41 42 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ 43 44 /** 45 * iser_start_rdma_unaligned_sg 46 */ 47 static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 48 struct iser_data_buf *data, 49 struct iser_data_buf *data_copy, 50 enum iser_data_dir cmd_dir) 51 { 52 struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device; 53 struct scatterlist *sgl = (struct scatterlist *)data->buf; 54 struct scatterlist *sg; 55 char *mem = NULL; 56 unsigned long cmd_data_len = 0; 57 int dma_nents, i; 58 59 for_each_sg(sgl, sg, data->size, i) 60 cmd_data_len += ib_sg_dma_len(dev, sg); 61 62 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 63 mem = (void *)__get_free_pages(GFP_ATOMIC, 64 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); 65 else 66 mem = kmalloc(cmd_data_len, GFP_ATOMIC); 67 68 if (mem == NULL) { 69 iser_err("Failed to allocate mem size %d %d for copying sglist\n", 70 data->size, (int)cmd_data_len); 71 return -ENOMEM; 72 } 73 74 if (cmd_dir == ISER_DIR_OUT) { 75 /* copy the unaligned sg the buffer which is used for RDMA */ 76 char *p, *from; 77 78 sgl = (struct scatterlist *)data->buf; 79 p = mem; 80 for_each_sg(sgl, sg, data->size, i) { 81 from = kmap_atomic(sg_page(sg)); 82 memcpy(p, 83 from + sg->offset, 84 sg->length); 85 kunmap_atomic(from); 86 p += sg->length; 87 } 88 } 89 90 sg_init_one(&data_copy->sg_single, mem, cmd_data_len); 91 data_copy->buf = &data_copy->sg_single; 92 data_copy->size = 1; 93 data_copy->copy_buf = mem; 94 95 dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1, 96 (cmd_dir == ISER_DIR_OUT) ? 97 DMA_TO_DEVICE : DMA_FROM_DEVICE); 98 BUG_ON(dma_nents == 0); 99 100 data_copy->dma_nents = dma_nents; 101 data_copy->data_len = cmd_data_len; 102 103 return 0; 104 } 105 106 /** 107 * iser_finalize_rdma_unaligned_sg 108 */ 109 110 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 111 struct iser_data_buf *data, 112 struct iser_data_buf *data_copy, 113 enum iser_data_dir cmd_dir) 114 { 115 struct ib_device *dev; 116 unsigned long cmd_data_len; 117 118 dev = iser_task->iser_conn->ib_conn.device->ib_device; 119 120 ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, 121 (cmd_dir == ISER_DIR_OUT) ? 122 DMA_TO_DEVICE : DMA_FROM_DEVICE); 123 124 if (cmd_dir == ISER_DIR_IN) { 125 char *mem; 126 struct scatterlist *sgl, *sg; 127 unsigned char *p, *to; 128 unsigned int sg_size; 129 int i; 130 131 /* copy back read RDMA to unaligned sg */ 132 mem = data_copy->copy_buf; 133 134 sgl = (struct scatterlist *)data->buf; 135 sg_size = data->size; 136 137 p = mem; 138 for_each_sg(sgl, sg, sg_size, i) { 139 to = kmap_atomic(sg_page(sg)); 140 memcpy(to + sg->offset, 141 p, 142 sg->length); 143 kunmap_atomic(to); 144 p += sg->length; 145 } 146 } 147 148 cmd_data_len = data->data_len; 149 150 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 151 free_pages((unsigned long)data_copy->copy_buf, 152 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); 153 else 154 kfree(data_copy->copy_buf); 155 156 data_copy->copy_buf = NULL; 157 } 158 159 #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) 160 161 /** 162 * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses 163 * and returns the length of resulting physical address array (may be less than 164 * the original due to possible compaction). 165 * 166 * we build a "page vec" under the assumption that the SG meets the RDMA 167 * alignment requirements. Other then the first and last SG elements, all 168 * the "internal" elements can be compacted into a list whose elements are 169 * dma addresses of physical pages. The code supports also the weird case 170 * where --few fragments of the same page-- are present in the SG as 171 * consecutive elements. Also, it handles one entry SG. 172 */ 173 174 static int iser_sg_to_page_vec(struct iser_data_buf *data, 175 struct ib_device *ibdev, u64 *pages, 176 int *offset, int *data_size) 177 { 178 struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf; 179 u64 start_addr, end_addr, page, chunk_start = 0; 180 unsigned long total_sz = 0; 181 unsigned int dma_len; 182 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; 183 184 /* compute the offset of first element */ 185 *offset = (u64) sgl[0].offset & ~MASK_4K; 186 187 new_chunk = 1; 188 cur_page = 0; 189 for_each_sg(sgl, sg, data->dma_nents, i) { 190 start_addr = ib_sg_dma_address(ibdev, sg); 191 if (new_chunk) 192 chunk_start = start_addr; 193 dma_len = ib_sg_dma_len(ibdev, sg); 194 end_addr = start_addr + dma_len; 195 total_sz += dma_len; 196 197 /* collect page fragments until aligned or end of SG list */ 198 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) { 199 new_chunk = 0; 200 continue; 201 } 202 new_chunk = 1; 203 204 /* address of the first page in the contiguous chunk; 205 masking relevant for the very first SG entry, 206 which might be unaligned */ 207 page = chunk_start & MASK_4K; 208 do { 209 pages[cur_page++] = page; 210 page += SIZE_4K; 211 } while (page < end_addr); 212 } 213 214 *data_size = total_sz; 215 iser_dbg("page_vec->data_size:%d cur_page %d\n", 216 *data_size, cur_page); 217 return cur_page; 218 } 219 220 221 /** 222 * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned 223 * for RDMA sub-list of a scatter-gather list of memory buffers, and returns 224 * the number of entries which are aligned correctly. Supports the case where 225 * consecutive SG elements are actually fragments of the same physcial page. 226 */ 227 static int iser_data_buf_aligned_len(struct iser_data_buf *data, 228 struct ib_device *ibdev) 229 { 230 struct scatterlist *sgl, *sg, *next_sg = NULL; 231 u64 start_addr, end_addr; 232 int i, ret_len, start_check = 0; 233 234 if (data->dma_nents == 1) 235 return 1; 236 237 sgl = (struct scatterlist *)data->buf; 238 start_addr = ib_sg_dma_address(ibdev, sgl); 239 240 for_each_sg(sgl, sg, data->dma_nents, i) { 241 if (start_check && !IS_4K_ALIGNED(start_addr)) 242 break; 243 244 next_sg = sg_next(sg); 245 if (!next_sg) 246 break; 247 248 end_addr = start_addr + ib_sg_dma_len(ibdev, sg); 249 start_addr = ib_sg_dma_address(ibdev, next_sg); 250 251 if (end_addr == start_addr) { 252 start_check = 0; 253 continue; 254 } else 255 start_check = 1; 256 257 if (!IS_4K_ALIGNED(end_addr)) 258 break; 259 } 260 ret_len = (next_sg) ? i : i+1; 261 iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n", 262 ret_len, data->dma_nents, data); 263 return ret_len; 264 } 265 266 static void iser_data_buf_dump(struct iser_data_buf *data, 267 struct ib_device *ibdev) 268 { 269 struct scatterlist *sgl = (struct scatterlist *)data->buf; 270 struct scatterlist *sg; 271 int i; 272 273 for_each_sg(sgl, sg, data->dma_nents, i) 274 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p " 275 "off:0x%x sz:0x%x dma_len:0x%x\n", 276 i, (unsigned long)ib_sg_dma_address(ibdev, sg), 277 sg_page(sg), sg->offset, 278 sg->length, ib_sg_dma_len(ibdev, sg)); 279 } 280 281 static void iser_dump_page_vec(struct iser_page_vec *page_vec) 282 { 283 int i; 284 285 iser_err("page vec length %d data size %d\n", 286 page_vec->length, page_vec->data_size); 287 for (i = 0; i < page_vec->length; i++) 288 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]); 289 } 290 291 static void iser_page_vec_build(struct iser_data_buf *data, 292 struct iser_page_vec *page_vec, 293 struct ib_device *ibdev) 294 { 295 int page_vec_len = 0; 296 297 page_vec->length = 0; 298 page_vec->offset = 0; 299 300 iser_dbg("Translating sg sz: %d\n", data->dma_nents); 301 page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages, 302 &page_vec->offset, 303 &page_vec->data_size); 304 iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len); 305 306 page_vec->length = page_vec_len; 307 308 if (page_vec_len * SIZE_4K < page_vec->data_size) { 309 iser_err("page_vec too short to hold this SG\n"); 310 iser_data_buf_dump(data, ibdev); 311 iser_dump_page_vec(page_vec); 312 BUG(); 313 } 314 } 315 316 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 317 struct iser_data_buf *data, 318 enum iser_data_dir iser_dir, 319 enum dma_data_direction dma_dir) 320 { 321 struct ib_device *dev; 322 323 iser_task->dir[iser_dir] = 1; 324 dev = iser_task->iser_conn->ib_conn.device->ib_device; 325 326 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); 327 if (data->dma_nents == 0) { 328 iser_err("dma_map_sg failed!!!\n"); 329 return -EINVAL; 330 } 331 return 0; 332 } 333 334 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 335 struct iser_data_buf *data, 336 enum dma_data_direction dir) 337 { 338 struct ib_device *dev; 339 340 dev = iser_task->iser_conn->ib_conn.device->ib_device; 341 ib_dma_unmap_sg(dev, data->buf, data->size, dir); 342 } 343 344 static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task, 345 struct ib_device *ibdev, 346 struct iser_data_buf *mem, 347 struct iser_data_buf *mem_copy, 348 enum iser_data_dir cmd_dir, 349 int aligned_len) 350 { 351 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn; 352 353 iscsi_conn->fmr_unalign_cnt++; 354 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", 355 aligned_len, mem->size); 356 357 if (iser_debug_level > 0) 358 iser_data_buf_dump(mem, ibdev); 359 360 /* unmap the command data before accessing it */ 361 iser_dma_unmap_task_data(iser_task, mem, 362 (cmd_dir == ISER_DIR_OUT) ? 363 DMA_TO_DEVICE : DMA_FROM_DEVICE); 364 365 /* allocate copy buf, if we are writing, copy the */ 366 /* unaligned scatterlist, dma map the copy */ 367 if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0) 368 return -ENOMEM; 369 370 return 0; 371 } 372 373 /** 374 * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA, 375 * using FMR (if possible) obtaining rkey and va 376 * 377 * returns 0 on success, errno code on failure 378 */ 379 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, 380 enum iser_data_dir cmd_dir) 381 { 382 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 383 struct iser_device *device = ib_conn->device; 384 struct ib_device *ibdev = device->ib_device; 385 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 386 struct iser_regd_buf *regd_buf; 387 int aligned_len; 388 int err; 389 int i; 390 struct scatterlist *sg; 391 392 regd_buf = &iser_task->rdma_regd[cmd_dir]; 393 394 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 395 if (aligned_len != mem->dma_nents) { 396 err = fall_to_bounce_buf(iser_task, ibdev, mem, 397 &iser_task->data_copy[cmd_dir], 398 cmd_dir, aligned_len); 399 if (err) { 400 iser_err("failed to allocate bounce buffer\n"); 401 return err; 402 } 403 mem = &iser_task->data_copy[cmd_dir]; 404 } 405 406 /* if there a single dma entry, FMR is not needed */ 407 if (mem->dma_nents == 1) { 408 sg = (struct scatterlist *)mem->buf; 409 410 regd_buf->reg.lkey = device->mr->lkey; 411 regd_buf->reg.rkey = device->mr->rkey; 412 regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]); 413 regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]); 414 415 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " 416 "va: 0x%08lX sz: %ld]\n", 417 (unsigned int)regd_buf->reg.lkey, 418 (unsigned int)regd_buf->reg.rkey, 419 (unsigned long)regd_buf->reg.va, 420 (unsigned long)regd_buf->reg.len); 421 } else { /* use FMR for multiple dma entries */ 422 iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev); 423 err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec, 424 ®d_buf->reg); 425 if (err && err != -EAGAIN) { 426 iser_data_buf_dump(mem, ibdev); 427 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", 428 mem->dma_nents, 429 ntoh24(iser_task->desc.iscsi_header.dlength)); 430 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", 431 ib_conn->fmr.page_vec->data_size, 432 ib_conn->fmr.page_vec->length, 433 ib_conn->fmr.page_vec->offset); 434 for (i = 0; i < ib_conn->fmr.page_vec->length; i++) 435 iser_err("page_vec[%d] = 0x%llx\n", i, 436 (unsigned long long)ib_conn->fmr.page_vec->pages[i]); 437 } 438 if (err) 439 return err; 440 } 441 return 0; 442 } 443 444 static void 445 iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, 446 struct ib_sig_domain *domain) 447 { 448 domain->sig_type = IB_SIG_TYPE_T10_DIF; 449 domain->sig.dif.pi_interval = scsi_prot_interval(sc); 450 domain->sig.dif.ref_tag = scsi_prot_ref_tag(sc); 451 /* 452 * At the moment we hard code those, but in the future 453 * we will take them from sc. 454 */ 455 domain->sig.dif.apptag_check_mask = 0xffff; 456 domain->sig.dif.app_escape = true; 457 domain->sig.dif.ref_escape = true; 458 if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) 459 domain->sig.dif.ref_remap = true; 460 }; 461 462 static int 463 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 464 { 465 switch (scsi_get_prot_op(sc)) { 466 case SCSI_PROT_WRITE_INSERT: 467 case SCSI_PROT_READ_STRIP: 468 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 469 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 470 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 471 break; 472 case SCSI_PROT_READ_INSERT: 473 case SCSI_PROT_WRITE_STRIP: 474 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 475 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 476 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 477 IB_T10DIF_CSUM : IB_T10DIF_CRC; 478 break; 479 case SCSI_PROT_READ_PASS: 480 case SCSI_PROT_WRITE_PASS: 481 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 482 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 483 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 484 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 485 IB_T10DIF_CSUM : IB_T10DIF_CRC; 486 break; 487 default: 488 iser_err("Unsupported PI operation %d\n", 489 scsi_get_prot_op(sc)); 490 return -EINVAL; 491 } 492 493 return 0; 494 } 495 496 static inline void 497 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 498 { 499 *mask = 0; 500 if (sc->prot_flags & SCSI_PROT_REF_CHECK) 501 *mask |= ISER_CHECK_REFTAG; 502 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) 503 *mask |= ISER_CHECK_GUARD; 504 } 505 506 static void 507 iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) 508 { 509 u32 rkey; 510 511 memset(inv_wr, 0, sizeof(*inv_wr)); 512 inv_wr->opcode = IB_WR_LOCAL_INV; 513 inv_wr->wr_id = ISER_FASTREG_LI_WRID; 514 inv_wr->ex.invalidate_rkey = mr->rkey; 515 516 rkey = ib_inc_rkey(mr->rkey); 517 ib_update_fast_reg_key(mr, rkey); 518 } 519 520 static int 521 iser_reg_sig_mr(struct iscsi_iser_task *iser_task, 522 struct fast_reg_descriptor *desc, struct ib_sge *data_sge, 523 struct ib_sge *prot_sge, struct ib_sge *sig_sge) 524 { 525 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 526 struct iser_pi_context *pi_ctx = desc->pi_ctx; 527 struct ib_send_wr sig_wr, inv_wr; 528 struct ib_send_wr *bad_wr, *wr = NULL; 529 struct ib_sig_attrs sig_attrs; 530 int ret; 531 532 memset(&sig_attrs, 0, sizeof(sig_attrs)); 533 ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs); 534 if (ret) 535 goto err; 536 537 iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask); 538 539 if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) { 540 iser_inv_rkey(&inv_wr, pi_ctx->sig_mr); 541 wr = &inv_wr; 542 } 543 544 memset(&sig_wr, 0, sizeof(sig_wr)); 545 sig_wr.opcode = IB_WR_REG_SIG_MR; 546 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 547 sig_wr.sg_list = data_sge; 548 sig_wr.num_sge = 1; 549 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 550 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 551 if (scsi_prot_sg_count(iser_task->sc)) 552 sig_wr.wr.sig_handover.prot = prot_sge; 553 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE | 554 IB_ACCESS_REMOTE_READ | 555 IB_ACCESS_REMOTE_WRITE; 556 557 if (!wr) 558 wr = &sig_wr; 559 else 560 wr->next = &sig_wr; 561 562 ret = ib_post_send(ib_conn->qp, wr, &bad_wr); 563 if (ret) { 564 iser_err("reg_sig_mr failed, ret:%d\n", ret); 565 goto err; 566 } 567 desc->reg_indicators &= ~ISER_SIG_KEY_VALID; 568 569 sig_sge->lkey = pi_ctx->sig_mr->lkey; 570 sig_sge->addr = 0; 571 sig_sge->length = scsi_transfer_length(iser_task->sc); 572 573 iser_dbg("sig_sge: addr: 0x%llx length: %u lkey: 0x%x\n", 574 sig_sge->addr, sig_sge->length, 575 sig_sge->lkey); 576 err: 577 return ret; 578 } 579 580 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 581 struct iser_regd_buf *regd_buf, 582 struct iser_data_buf *mem, 583 enum iser_reg_indicator ind, 584 struct ib_sge *sge) 585 { 586 struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; 587 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 588 struct iser_device *device = ib_conn->device; 589 struct ib_device *ibdev = device->ib_device; 590 struct ib_mr *mr; 591 struct ib_fast_reg_page_list *frpl; 592 struct ib_send_wr fastreg_wr, inv_wr; 593 struct ib_send_wr *bad_wr, *wr = NULL; 594 int ret, offset, size, plen; 595 596 /* if there a single dma entry, dma mr suffices */ 597 if (mem->dma_nents == 1) { 598 struct scatterlist *sg = (struct scatterlist *)mem->buf; 599 600 sge->lkey = device->mr->lkey; 601 sge->addr = ib_sg_dma_address(ibdev, &sg[0]); 602 sge->length = ib_sg_dma_len(ibdev, &sg[0]); 603 604 iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n", 605 sge->lkey, sge->addr, sge->length); 606 return 0; 607 } 608 609 if (ind == ISER_DATA_KEY_VALID) { 610 mr = desc->data_mr; 611 frpl = desc->data_frpl; 612 } else { 613 mr = desc->pi_ctx->prot_mr; 614 frpl = desc->pi_ctx->prot_frpl; 615 } 616 617 plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list, 618 &offset, &size); 619 if (plen * SIZE_4K < size) { 620 iser_err("fast reg page_list too short to hold this SG\n"); 621 return -EINVAL; 622 } 623 624 if (!(desc->reg_indicators & ind)) { 625 iser_inv_rkey(&inv_wr, mr); 626 wr = &inv_wr; 627 } 628 629 /* Prepare FASTREG WR */ 630 memset(&fastreg_wr, 0, sizeof(fastreg_wr)); 631 fastreg_wr.wr_id = ISER_FASTREG_LI_WRID; 632 fastreg_wr.opcode = IB_WR_FAST_REG_MR; 633 fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset; 634 fastreg_wr.wr.fast_reg.page_list = frpl; 635 fastreg_wr.wr.fast_reg.page_list_len = plen; 636 fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K; 637 fastreg_wr.wr.fast_reg.length = size; 638 fastreg_wr.wr.fast_reg.rkey = mr->rkey; 639 fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE | 640 IB_ACCESS_REMOTE_WRITE | 641 IB_ACCESS_REMOTE_READ); 642 643 if (!wr) 644 wr = &fastreg_wr; 645 else 646 wr->next = &fastreg_wr; 647 648 ret = ib_post_send(ib_conn->qp, wr, &bad_wr); 649 if (ret) { 650 iser_err("fast registration failed, ret:%d\n", ret); 651 return ret; 652 } 653 desc->reg_indicators &= ~ind; 654 655 sge->lkey = mr->lkey; 656 sge->addr = frpl->page_list[0] + offset; 657 sge->length = size; 658 659 return ret; 660 } 661 662 /** 663 * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA, 664 * using Fast Registration WR (if possible) obtaining rkey and va 665 * 666 * returns 0 on success, errno code on failure 667 */ 668 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, 669 enum iser_data_dir cmd_dir) 670 { 671 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 672 struct iser_device *device = ib_conn->device; 673 struct ib_device *ibdev = device->ib_device; 674 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 675 struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir]; 676 struct fast_reg_descriptor *desc = NULL; 677 struct ib_sge data_sge; 678 int err, aligned_len; 679 unsigned long flags; 680 681 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 682 if (aligned_len != mem->dma_nents) { 683 err = fall_to_bounce_buf(iser_task, ibdev, mem, 684 &iser_task->data_copy[cmd_dir], 685 cmd_dir, aligned_len); 686 if (err) { 687 iser_err("failed to allocate bounce buffer\n"); 688 return err; 689 } 690 mem = &iser_task->data_copy[cmd_dir]; 691 } 692 693 if (mem->dma_nents != 1 || 694 scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { 695 spin_lock_irqsave(&ib_conn->lock, flags); 696 desc = list_first_entry(&ib_conn->fastreg.pool, 697 struct fast_reg_descriptor, list); 698 list_del(&desc->list); 699 spin_unlock_irqrestore(&ib_conn->lock, flags); 700 regd_buf->reg.mem_h = desc; 701 } 702 703 err = iser_fast_reg_mr(iser_task, regd_buf, mem, 704 ISER_DATA_KEY_VALID, &data_sge); 705 if (err) 706 goto err_reg; 707 708 if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { 709 struct ib_sge prot_sge, sig_sge; 710 711 memset(&prot_sge, 0, sizeof(prot_sge)); 712 if (scsi_prot_sg_count(iser_task->sc)) { 713 mem = &iser_task->prot[cmd_dir]; 714 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 715 if (aligned_len != mem->dma_nents) { 716 err = fall_to_bounce_buf(iser_task, ibdev, mem, 717 &iser_task->prot_copy[cmd_dir], 718 cmd_dir, aligned_len); 719 if (err) { 720 iser_err("failed to allocate bounce buffer\n"); 721 return err; 722 } 723 mem = &iser_task->prot_copy[cmd_dir]; 724 } 725 726 err = iser_fast_reg_mr(iser_task, regd_buf, mem, 727 ISER_PROT_KEY_VALID, &prot_sge); 728 if (err) 729 goto err_reg; 730 } 731 732 err = iser_reg_sig_mr(iser_task, desc, &data_sge, 733 &prot_sge, &sig_sge); 734 if (err) { 735 iser_err("Failed to register signature mr\n"); 736 return err; 737 } 738 desc->reg_indicators |= ISER_FASTREG_PROTECTED; 739 740 regd_buf->reg.lkey = sig_sge.lkey; 741 regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey; 742 regd_buf->reg.va = sig_sge.addr; 743 regd_buf->reg.len = sig_sge.length; 744 } else { 745 if (desc) 746 regd_buf->reg.rkey = desc->data_mr->rkey; 747 else 748 regd_buf->reg.rkey = device->mr->rkey; 749 750 regd_buf->reg.lkey = data_sge.lkey; 751 regd_buf->reg.va = data_sge.addr; 752 regd_buf->reg.len = data_sge.length; 753 } 754 755 return 0; 756 err_reg: 757 if (desc) { 758 spin_lock_irqsave(&ib_conn->lock, flags); 759 list_add_tail(&desc->list, &ib_conn->fastreg.pool); 760 spin_unlock_irqrestore(&ib_conn->lock, flags); 761 } 762 763 return err; 764 } 765