1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/module.h> 34 #include <linux/kernel.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/highmem.h> 38 #include <linux/scatterlist.h> 39 40 #include "iscsi_iser.h" 41 42 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ 43 44 /** 45 * iser_start_rdma_unaligned_sg 46 */ 47 static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 48 struct iser_data_buf *data, 49 struct iser_data_buf *data_copy, 50 enum iser_data_dir cmd_dir) 51 { 52 struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device; 53 struct scatterlist *sgl = (struct scatterlist *)data->buf; 54 struct scatterlist *sg; 55 char *mem = NULL; 56 unsigned long cmd_data_len = 0; 57 int dma_nents, i; 58 59 for_each_sg(sgl, sg, data->size, i) 60 cmd_data_len += ib_sg_dma_len(dev, sg); 61 62 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 63 mem = (void *)__get_free_pages(GFP_ATOMIC, 64 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); 65 else 66 mem = kmalloc(cmd_data_len, GFP_ATOMIC); 67 68 if (mem == NULL) { 69 iser_err("Failed to allocate mem size %d %d for copying sglist\n", 70 data->size, (int)cmd_data_len); 71 return -ENOMEM; 72 } 73 74 if (cmd_dir == ISER_DIR_OUT) { 75 /* copy the unaligned sg the buffer which is used for RDMA */ 76 int i; 77 char *p, *from; 78 79 sgl = (struct scatterlist *)data->buf; 80 p = mem; 81 for_each_sg(sgl, sg, data->size, i) { 82 from = kmap_atomic(sg_page(sg)); 83 memcpy(p, 84 from + sg->offset, 85 sg->length); 86 kunmap_atomic(from); 87 p += sg->length; 88 } 89 } 90 91 sg_init_one(&data_copy->sg_single, mem, cmd_data_len); 92 data_copy->buf = &data_copy->sg_single; 93 data_copy->size = 1; 94 data_copy->copy_buf = mem; 95 96 dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1, 97 (cmd_dir == ISER_DIR_OUT) ? 98 DMA_TO_DEVICE : DMA_FROM_DEVICE); 99 BUG_ON(dma_nents == 0); 100 101 data_copy->dma_nents = dma_nents; 102 data_copy->data_len = cmd_data_len; 103 104 return 0; 105 } 106 107 /** 108 * iser_finalize_rdma_unaligned_sg 109 */ 110 111 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, 112 struct iser_data_buf *data, 113 struct iser_data_buf *data_copy, 114 enum iser_data_dir cmd_dir) 115 { 116 struct ib_device *dev; 117 unsigned long cmd_data_len; 118 119 dev = iser_task->iser_conn->ib_conn.device->ib_device; 120 121 ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, 122 (cmd_dir == ISER_DIR_OUT) ? 123 DMA_TO_DEVICE : DMA_FROM_DEVICE); 124 125 if (cmd_dir == ISER_DIR_IN) { 126 char *mem; 127 struct scatterlist *sgl, *sg; 128 unsigned char *p, *to; 129 unsigned int sg_size; 130 int i; 131 132 /* copy back read RDMA to unaligned sg */ 133 mem = data_copy->copy_buf; 134 135 sgl = (struct scatterlist *)data->buf; 136 sg_size = data->size; 137 138 p = mem; 139 for_each_sg(sgl, sg, sg_size, i) { 140 to = kmap_atomic(sg_page(sg)); 141 memcpy(to + sg->offset, 142 p, 143 sg->length); 144 kunmap_atomic(to); 145 p += sg->length; 146 } 147 } 148 149 cmd_data_len = data->data_len; 150 151 if (cmd_data_len > ISER_KMALLOC_THRESHOLD) 152 free_pages((unsigned long)data_copy->copy_buf, 153 ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); 154 else 155 kfree(data_copy->copy_buf); 156 157 data_copy->copy_buf = NULL; 158 } 159 160 #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) 161 162 /** 163 * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses 164 * and returns the length of resulting physical address array (may be less than 165 * the original due to possible compaction). 166 * 167 * we build a "page vec" under the assumption that the SG meets the RDMA 168 * alignment requirements. Other then the first and last SG elements, all 169 * the "internal" elements can be compacted into a list whose elements are 170 * dma addresses of physical pages. The code supports also the weird case 171 * where --few fragments of the same page-- are present in the SG as 172 * consecutive elements. Also, it handles one entry SG. 173 */ 174 175 static int iser_sg_to_page_vec(struct iser_data_buf *data, 176 struct ib_device *ibdev, u64 *pages, 177 int *offset, int *data_size) 178 { 179 struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf; 180 u64 start_addr, end_addr, page, chunk_start = 0; 181 unsigned long total_sz = 0; 182 unsigned int dma_len; 183 int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; 184 185 /* compute the offset of first element */ 186 *offset = (u64) sgl[0].offset & ~MASK_4K; 187 188 new_chunk = 1; 189 cur_page = 0; 190 for_each_sg(sgl, sg, data->dma_nents, i) { 191 start_addr = ib_sg_dma_address(ibdev, sg); 192 if (new_chunk) 193 chunk_start = start_addr; 194 dma_len = ib_sg_dma_len(ibdev, sg); 195 end_addr = start_addr + dma_len; 196 total_sz += dma_len; 197 198 /* collect page fragments until aligned or end of SG list */ 199 if (!IS_4K_ALIGNED(end_addr) && i < last_ent) { 200 new_chunk = 0; 201 continue; 202 } 203 new_chunk = 1; 204 205 /* address of the first page in the contiguous chunk; 206 masking relevant for the very first SG entry, 207 which might be unaligned */ 208 page = chunk_start & MASK_4K; 209 do { 210 pages[cur_page++] = page; 211 page += SIZE_4K; 212 } while (page < end_addr); 213 } 214 215 *data_size = total_sz; 216 iser_dbg("page_vec->data_size:%d cur_page %d\n", 217 *data_size, cur_page); 218 return cur_page; 219 } 220 221 222 /** 223 * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned 224 * for RDMA sub-list of a scatter-gather list of memory buffers, and returns 225 * the number of entries which are aligned correctly. Supports the case where 226 * consecutive SG elements are actually fragments of the same physcial page. 227 */ 228 static int iser_data_buf_aligned_len(struct iser_data_buf *data, 229 struct ib_device *ibdev) 230 { 231 struct scatterlist *sgl, *sg, *next_sg = NULL; 232 u64 start_addr, end_addr; 233 int i, ret_len, start_check = 0; 234 235 if (data->dma_nents == 1) 236 return 1; 237 238 sgl = (struct scatterlist *)data->buf; 239 start_addr = ib_sg_dma_address(ibdev, sgl); 240 241 for_each_sg(sgl, sg, data->dma_nents, i) { 242 if (start_check && !IS_4K_ALIGNED(start_addr)) 243 break; 244 245 next_sg = sg_next(sg); 246 if (!next_sg) 247 break; 248 249 end_addr = start_addr + ib_sg_dma_len(ibdev, sg); 250 start_addr = ib_sg_dma_address(ibdev, next_sg); 251 252 if (end_addr == start_addr) { 253 start_check = 0; 254 continue; 255 } else 256 start_check = 1; 257 258 if (!IS_4K_ALIGNED(end_addr)) 259 break; 260 } 261 ret_len = (next_sg) ? i : i+1; 262 iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n", 263 ret_len, data->dma_nents, data); 264 return ret_len; 265 } 266 267 static void iser_data_buf_dump(struct iser_data_buf *data, 268 struct ib_device *ibdev) 269 { 270 struct scatterlist *sgl = (struct scatterlist *)data->buf; 271 struct scatterlist *sg; 272 int i; 273 274 for_each_sg(sgl, sg, data->dma_nents, i) 275 iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p " 276 "off:0x%x sz:0x%x dma_len:0x%x\n", 277 i, (unsigned long)ib_sg_dma_address(ibdev, sg), 278 sg_page(sg), sg->offset, 279 sg->length, ib_sg_dma_len(ibdev, sg)); 280 } 281 282 static void iser_dump_page_vec(struct iser_page_vec *page_vec) 283 { 284 int i; 285 286 iser_err("page vec length %d data size %d\n", 287 page_vec->length, page_vec->data_size); 288 for (i = 0; i < page_vec->length; i++) 289 iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]); 290 } 291 292 static void iser_page_vec_build(struct iser_data_buf *data, 293 struct iser_page_vec *page_vec, 294 struct ib_device *ibdev) 295 { 296 int page_vec_len = 0; 297 298 page_vec->length = 0; 299 page_vec->offset = 0; 300 301 iser_dbg("Translating sg sz: %d\n", data->dma_nents); 302 page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages, 303 &page_vec->offset, 304 &page_vec->data_size); 305 iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len); 306 307 page_vec->length = page_vec_len; 308 309 if (page_vec_len * SIZE_4K < page_vec->data_size) { 310 iser_err("page_vec too short to hold this SG\n"); 311 iser_data_buf_dump(data, ibdev); 312 iser_dump_page_vec(page_vec); 313 BUG(); 314 } 315 } 316 317 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 318 struct iser_data_buf *data, 319 enum iser_data_dir iser_dir, 320 enum dma_data_direction dma_dir) 321 { 322 struct ib_device *dev; 323 324 iser_task->dir[iser_dir] = 1; 325 dev = iser_task->iser_conn->ib_conn.device->ib_device; 326 327 data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); 328 if (data->dma_nents == 0) { 329 iser_err("dma_map_sg failed!!!\n"); 330 return -EINVAL; 331 } 332 return 0; 333 } 334 335 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 336 struct iser_data_buf *data) 337 { 338 struct ib_device *dev; 339 340 dev = iser_task->iser_conn->ib_conn.device->ib_device; 341 ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); 342 } 343 344 static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task, 345 struct ib_device *ibdev, 346 struct iser_data_buf *mem, 347 struct iser_data_buf *mem_copy, 348 enum iser_data_dir cmd_dir, 349 int aligned_len) 350 { 351 struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn; 352 353 iscsi_conn->fmr_unalign_cnt++; 354 iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", 355 aligned_len, mem->size); 356 357 if (iser_debug_level > 0) 358 iser_data_buf_dump(mem, ibdev); 359 360 /* unmap the command data before accessing it */ 361 iser_dma_unmap_task_data(iser_task, mem); 362 363 /* allocate copy buf, if we are writing, copy the */ 364 /* unaligned scatterlist, dma map the copy */ 365 if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0) 366 return -ENOMEM; 367 368 return 0; 369 } 370 371 /** 372 * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA, 373 * using FMR (if possible) obtaining rkey and va 374 * 375 * returns 0 on success, errno code on failure 376 */ 377 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, 378 enum iser_data_dir cmd_dir) 379 { 380 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 381 struct iser_device *device = ib_conn->device; 382 struct ib_device *ibdev = device->ib_device; 383 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 384 struct iser_regd_buf *regd_buf; 385 int aligned_len; 386 int err; 387 int i; 388 struct scatterlist *sg; 389 390 regd_buf = &iser_task->rdma_regd[cmd_dir]; 391 392 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 393 if (aligned_len != mem->dma_nents) { 394 err = fall_to_bounce_buf(iser_task, ibdev, mem, 395 &iser_task->data_copy[cmd_dir], 396 cmd_dir, aligned_len); 397 if (err) { 398 iser_err("failed to allocate bounce buffer\n"); 399 return err; 400 } 401 mem = &iser_task->data_copy[cmd_dir]; 402 } 403 404 /* if there a single dma entry, FMR is not needed */ 405 if (mem->dma_nents == 1) { 406 sg = (struct scatterlist *)mem->buf; 407 408 regd_buf->reg.lkey = device->mr->lkey; 409 regd_buf->reg.rkey = device->mr->rkey; 410 regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]); 411 regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]); 412 regd_buf->reg.is_mr = 0; 413 414 iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " 415 "va: 0x%08lX sz: %ld]\n", 416 (unsigned int)regd_buf->reg.lkey, 417 (unsigned int)regd_buf->reg.rkey, 418 (unsigned long)regd_buf->reg.va, 419 (unsigned long)regd_buf->reg.len); 420 } else { /* use FMR for multiple dma entries */ 421 iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev); 422 err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec, 423 ®d_buf->reg); 424 if (err && err != -EAGAIN) { 425 iser_data_buf_dump(mem, ibdev); 426 iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", 427 mem->dma_nents, 428 ntoh24(iser_task->desc.iscsi_header.dlength)); 429 iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", 430 ib_conn->fmr.page_vec->data_size, 431 ib_conn->fmr.page_vec->length, 432 ib_conn->fmr.page_vec->offset); 433 for (i = 0; i < ib_conn->fmr.page_vec->length; i++) 434 iser_err("page_vec[%d] = 0x%llx\n", i, 435 (unsigned long long)ib_conn->fmr.page_vec->pages[i]); 436 } 437 if (err) 438 return err; 439 } 440 return 0; 441 } 442 443 static inline void 444 iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, 445 struct ib_sig_domain *domain) 446 { 447 domain->sig_type = IB_SIG_TYPE_T10_DIF; 448 domain->sig.dif.pi_interval = sc->device->sector_size; 449 domain->sig.dif.ref_tag = scsi_get_lba(sc) & 0xffffffff; 450 /* 451 * At the moment we hard code those, but in the future 452 * we will take them from sc. 453 */ 454 domain->sig.dif.apptag_check_mask = 0xffff; 455 domain->sig.dif.app_escape = true; 456 domain->sig.dif.ref_escape = true; 457 if (scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE1 || 458 scsi_get_prot_type(sc) == SCSI_PROT_DIF_TYPE2) 459 domain->sig.dif.ref_remap = true; 460 }; 461 462 static int 463 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 464 { 465 switch (scsi_get_prot_op(sc)) { 466 case SCSI_PROT_WRITE_INSERT: 467 case SCSI_PROT_READ_STRIP: 468 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 469 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 470 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 471 break; 472 case SCSI_PROT_READ_INSERT: 473 case SCSI_PROT_WRITE_STRIP: 474 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 475 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 476 /* 477 * At the moment we use this modparam to tell what is 478 * the memory bg_type, in the future we will take it 479 * from sc. 480 */ 481 sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM : 482 IB_T10DIF_CRC; 483 break; 484 case SCSI_PROT_READ_PASS: 485 case SCSI_PROT_WRITE_PASS: 486 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); 487 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 488 iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); 489 /* 490 * At the moment we use this modparam to tell what is 491 * the memory bg_type, in the future we will take it 492 * from sc. 493 */ 494 sig_attrs->mem.sig.dif.bg_type = iser_pi_guard ? IB_T10DIF_CSUM : 495 IB_T10DIF_CRC; 496 break; 497 default: 498 iser_err("Unsupported PI operation %d\n", 499 scsi_get_prot_op(sc)); 500 return -EINVAL; 501 } 502 503 return 0; 504 } 505 506 static int 507 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 508 { 509 switch (scsi_get_prot_type(sc)) { 510 case SCSI_PROT_DIF_TYPE0: 511 break; 512 case SCSI_PROT_DIF_TYPE1: 513 case SCSI_PROT_DIF_TYPE2: 514 *mask = ISER_CHECK_GUARD | ISER_CHECK_REFTAG; 515 break; 516 case SCSI_PROT_DIF_TYPE3: 517 *mask = ISER_CHECK_GUARD; 518 break; 519 default: 520 iser_err("Unsupported protection type %d\n", 521 scsi_get_prot_type(sc)); 522 return -EINVAL; 523 } 524 525 return 0; 526 } 527 528 static int 529 iser_reg_sig_mr(struct iscsi_iser_task *iser_task, 530 struct fast_reg_descriptor *desc, struct ib_sge *data_sge, 531 struct ib_sge *prot_sge, struct ib_sge *sig_sge) 532 { 533 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 534 struct iser_pi_context *pi_ctx = desc->pi_ctx; 535 struct ib_send_wr sig_wr, inv_wr; 536 struct ib_send_wr *bad_wr, *wr = NULL; 537 struct ib_sig_attrs sig_attrs; 538 int ret; 539 u32 key; 540 541 memset(&sig_attrs, 0, sizeof(sig_attrs)); 542 ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs); 543 if (ret) 544 goto err; 545 546 ret = iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask); 547 if (ret) 548 goto err; 549 550 if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) { 551 memset(&inv_wr, 0, sizeof(inv_wr)); 552 inv_wr.opcode = IB_WR_LOCAL_INV; 553 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 554 inv_wr.ex.invalidate_rkey = pi_ctx->sig_mr->rkey; 555 wr = &inv_wr; 556 /* Bump the key */ 557 key = (u8)(pi_ctx->sig_mr->rkey & 0x000000FF); 558 ib_update_fast_reg_key(pi_ctx->sig_mr, ++key); 559 } 560 561 memset(&sig_wr, 0, sizeof(sig_wr)); 562 sig_wr.opcode = IB_WR_REG_SIG_MR; 563 sig_wr.wr_id = ISER_FASTREG_LI_WRID; 564 sig_wr.sg_list = data_sge; 565 sig_wr.num_sge = 1; 566 sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; 567 sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; 568 if (scsi_prot_sg_count(iser_task->sc)) 569 sig_wr.wr.sig_handover.prot = prot_sge; 570 sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE | 571 IB_ACCESS_REMOTE_READ | 572 IB_ACCESS_REMOTE_WRITE; 573 574 if (!wr) 575 wr = &sig_wr; 576 else 577 wr->next = &sig_wr; 578 579 ret = ib_post_send(ib_conn->qp, wr, &bad_wr); 580 if (ret) { 581 iser_err("reg_sig_mr failed, ret:%d\n", ret); 582 goto err; 583 } 584 desc->reg_indicators &= ~ISER_SIG_KEY_VALID; 585 586 sig_sge->lkey = pi_ctx->sig_mr->lkey; 587 sig_sge->addr = 0; 588 sig_sge->length = data_sge->length + prot_sge->length; 589 if (scsi_get_prot_op(iser_task->sc) == SCSI_PROT_WRITE_INSERT || 590 scsi_get_prot_op(iser_task->sc) == SCSI_PROT_READ_STRIP) { 591 sig_sge->length += (data_sge->length / 592 iser_task->sc->device->sector_size) * 8; 593 } 594 595 iser_dbg("sig_sge: addr: 0x%llx length: %u lkey: 0x%x\n", 596 sig_sge->addr, sig_sge->length, 597 sig_sge->lkey); 598 err: 599 return ret; 600 } 601 602 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 603 struct iser_regd_buf *regd_buf, 604 struct iser_data_buf *mem, 605 enum iser_reg_indicator ind, 606 struct ib_sge *sge) 607 { 608 struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; 609 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 610 struct iser_device *device = ib_conn->device; 611 struct ib_device *ibdev = device->ib_device; 612 struct ib_mr *mr; 613 struct ib_fast_reg_page_list *frpl; 614 struct ib_send_wr fastreg_wr, inv_wr; 615 struct ib_send_wr *bad_wr, *wr = NULL; 616 u8 key; 617 int ret, offset, size, plen; 618 619 /* if there a single dma entry, dma mr suffices */ 620 if (mem->dma_nents == 1) { 621 struct scatterlist *sg = (struct scatterlist *)mem->buf; 622 623 sge->lkey = device->mr->lkey; 624 sge->addr = ib_sg_dma_address(ibdev, &sg[0]); 625 sge->length = ib_sg_dma_len(ibdev, &sg[0]); 626 627 iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n", 628 sge->lkey, sge->addr, sge->length); 629 return 0; 630 } 631 632 if (ind == ISER_DATA_KEY_VALID) { 633 mr = desc->data_mr; 634 frpl = desc->data_frpl; 635 } else { 636 mr = desc->pi_ctx->prot_mr; 637 frpl = desc->pi_ctx->prot_frpl; 638 } 639 640 plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list, 641 &offset, &size); 642 if (plen * SIZE_4K < size) { 643 iser_err("fast reg page_list too short to hold this SG\n"); 644 return -EINVAL; 645 } 646 647 if (!(desc->reg_indicators & ind)) { 648 memset(&inv_wr, 0, sizeof(inv_wr)); 649 inv_wr.wr_id = ISER_FASTREG_LI_WRID; 650 inv_wr.opcode = IB_WR_LOCAL_INV; 651 inv_wr.ex.invalidate_rkey = mr->rkey; 652 wr = &inv_wr; 653 /* Bump the key */ 654 key = (u8)(mr->rkey & 0x000000FF); 655 ib_update_fast_reg_key(mr, ++key); 656 } 657 658 /* Prepare FASTREG WR */ 659 memset(&fastreg_wr, 0, sizeof(fastreg_wr)); 660 fastreg_wr.wr_id = ISER_FASTREG_LI_WRID; 661 fastreg_wr.opcode = IB_WR_FAST_REG_MR; 662 fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset; 663 fastreg_wr.wr.fast_reg.page_list = frpl; 664 fastreg_wr.wr.fast_reg.page_list_len = plen; 665 fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K; 666 fastreg_wr.wr.fast_reg.length = size; 667 fastreg_wr.wr.fast_reg.rkey = mr->rkey; 668 fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE | 669 IB_ACCESS_REMOTE_WRITE | 670 IB_ACCESS_REMOTE_READ); 671 672 if (!wr) 673 wr = &fastreg_wr; 674 else 675 wr->next = &fastreg_wr; 676 677 ret = ib_post_send(ib_conn->qp, wr, &bad_wr); 678 if (ret) { 679 iser_err("fast registration failed, ret:%d\n", ret); 680 return ret; 681 } 682 desc->reg_indicators &= ~ind; 683 684 sge->lkey = mr->lkey; 685 sge->addr = frpl->page_list[0] + offset; 686 sge->length = size; 687 688 return ret; 689 } 690 691 /** 692 * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA, 693 * using Fast Registration WR (if possible) obtaining rkey and va 694 * 695 * returns 0 on success, errno code on failure 696 */ 697 int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, 698 enum iser_data_dir cmd_dir) 699 { 700 struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; 701 struct iser_device *device = ib_conn->device; 702 struct ib_device *ibdev = device->ib_device; 703 struct iser_data_buf *mem = &iser_task->data[cmd_dir]; 704 struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir]; 705 struct fast_reg_descriptor *desc = NULL; 706 struct ib_sge data_sge; 707 int err, aligned_len; 708 unsigned long flags; 709 710 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 711 if (aligned_len != mem->dma_nents) { 712 err = fall_to_bounce_buf(iser_task, ibdev, mem, 713 &iser_task->data_copy[cmd_dir], 714 cmd_dir, aligned_len); 715 if (err) { 716 iser_err("failed to allocate bounce buffer\n"); 717 return err; 718 } 719 mem = &iser_task->data_copy[cmd_dir]; 720 } 721 722 if (mem->dma_nents != 1 || 723 scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { 724 spin_lock_irqsave(&ib_conn->lock, flags); 725 desc = list_first_entry(&ib_conn->fastreg.pool, 726 struct fast_reg_descriptor, list); 727 list_del(&desc->list); 728 spin_unlock_irqrestore(&ib_conn->lock, flags); 729 regd_buf->reg.mem_h = desc; 730 } 731 732 err = iser_fast_reg_mr(iser_task, regd_buf, mem, 733 ISER_DATA_KEY_VALID, &data_sge); 734 if (err) 735 goto err_reg; 736 737 if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { 738 struct ib_sge prot_sge, sig_sge; 739 740 memset(&prot_sge, 0, sizeof(prot_sge)); 741 if (scsi_prot_sg_count(iser_task->sc)) { 742 mem = &iser_task->prot[cmd_dir]; 743 aligned_len = iser_data_buf_aligned_len(mem, ibdev); 744 if (aligned_len != mem->dma_nents) { 745 err = fall_to_bounce_buf(iser_task, ibdev, mem, 746 &iser_task->prot_copy[cmd_dir], 747 cmd_dir, aligned_len); 748 if (err) { 749 iser_err("failed to allocate bounce buffer\n"); 750 return err; 751 } 752 mem = &iser_task->prot_copy[cmd_dir]; 753 } 754 755 err = iser_fast_reg_mr(iser_task, regd_buf, mem, 756 ISER_PROT_KEY_VALID, &prot_sge); 757 if (err) 758 goto err_reg; 759 } 760 761 err = iser_reg_sig_mr(iser_task, desc, &data_sge, 762 &prot_sge, &sig_sge); 763 if (err) { 764 iser_err("Failed to register signature mr\n"); 765 return err; 766 } 767 desc->reg_indicators |= ISER_FASTREG_PROTECTED; 768 769 regd_buf->reg.lkey = sig_sge.lkey; 770 regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey; 771 regd_buf->reg.va = sig_sge.addr; 772 regd_buf->reg.len = sig_sge.length; 773 regd_buf->reg.is_mr = 1; 774 } else { 775 if (desc) { 776 regd_buf->reg.rkey = desc->data_mr->rkey; 777 regd_buf->reg.is_mr = 1; 778 } else { 779 regd_buf->reg.rkey = device->mr->rkey; 780 regd_buf->reg.is_mr = 0; 781 } 782 783 regd_buf->reg.lkey = data_sge.lkey; 784 regd_buf->reg.va = data_sge.addr; 785 regd_buf->reg.len = data_sge.length; 786 } 787 788 return 0; 789 err_reg: 790 if (desc) { 791 spin_lock_irqsave(&ib_conn->lock, flags); 792 list_add_tail(&desc->list, &ib_conn->fastreg.pool); 793 spin_unlock_irqrestore(&ib_conn->lock, flags); 794 } 795 796 return err; 797 } 798