1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/module.h> 34 #include <linux/kernel.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/highmem.h> 38 #include <linux/scatterlist.h> 39 40 #include "iscsi_iser.h" 41 42 void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc) 43 { 44 iser_err_comp(wc, "memreg"); 45 } 46 47 static struct iser_fr_desc * 48 iser_reg_desc_get_fr(struct ib_conn *ib_conn) 49 { 50 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 51 struct iser_fr_desc *desc; 52 unsigned long flags; 53 54 spin_lock_irqsave(&fr_pool->lock, flags); 55 desc = list_first_entry(&fr_pool->list, 56 struct iser_fr_desc, list); 57 list_del(&desc->list); 58 spin_unlock_irqrestore(&fr_pool->lock, flags); 59 60 return desc; 61 } 62 63 static void 64 iser_reg_desc_put_fr(struct ib_conn *ib_conn, 65 struct iser_fr_desc *desc) 66 { 67 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 68 unsigned long flags; 69 70 spin_lock_irqsave(&fr_pool->lock, flags); 71 list_add(&desc->list, &fr_pool->list); 72 spin_unlock_irqrestore(&fr_pool->lock, flags); 73 } 74 75 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 76 struct iser_data_buf *data, 77 enum iser_data_dir iser_dir, 78 enum dma_data_direction dma_dir) 79 { 80 struct ib_device *dev; 81 82 iser_task->dir[iser_dir] = 1; 83 dev = iser_task->iser_conn->ib_conn.device->ib_device; 84 85 data->dma_nents = ib_dma_map_sg(dev, data->sg, data->size, dma_dir); 86 if (unlikely(data->dma_nents == 0)) { 87 iser_err("dma_map_sg failed!!!\n"); 88 return -EINVAL; 89 } 90 return 0; 91 } 92 93 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 94 struct iser_data_buf *data, 95 enum dma_data_direction dir) 96 { 97 struct ib_device *dev; 98 99 dev = iser_task->iser_conn->ib_conn.device->ib_device; 100 ib_dma_unmap_sg(dev, data->sg, data->size, dir); 101 } 102 103 static int 104 iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem, 105 struct iser_mem_reg *reg) 106 { 107 struct scatterlist *sg = mem->sg; 108 109 reg->sge.lkey = device->pd->local_dma_lkey; 110 /* 111 * FIXME: rework the registration code path to differentiate 112 * rkey/lkey use cases 113 */ 114 115 if (device->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) 116 reg->rkey = device->pd->unsafe_global_rkey; 117 else 118 reg->rkey = 0; 119 reg->sge.addr = sg_dma_address(&sg[0]); 120 reg->sge.length = sg_dma_len(&sg[0]); 121 122 iser_dbg("Single DMA entry: lkey=0x%x, rkey=0x%x, addr=0x%llx," 123 " length=0x%x\n", reg->sge.lkey, reg->rkey, 124 reg->sge.addr, reg->sge.length); 125 126 return 0; 127 } 128 129 void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, 130 enum iser_data_dir cmd_dir) 131 { 132 struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir]; 133 struct iser_fr_desc *desc; 134 struct ib_mr_status mr_status; 135 136 desc = reg->mem_h; 137 if (!desc) 138 return; 139 140 /* 141 * The signature MR cannot be invalidated and reused without checking. 142 * libiscsi calls the check_protection transport handler only if 143 * SCSI-Response is received. And the signature MR is not checked if 144 * the task is completed for some other reason like a timeout or error 145 * handling. That's why we must check the signature MR here before 146 * putting it to the free pool. 147 */ 148 if (unlikely(desc->sig_protected)) { 149 desc->sig_protected = false; 150 ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS, 151 &mr_status); 152 } 153 iser_reg_desc_put_fr(&iser_task->iser_conn->ib_conn, reg->mem_h); 154 reg->mem_h = NULL; 155 } 156 157 static void 158 iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_domain *domain) 159 { 160 domain->sig_type = IB_SIG_TYPE_T10_DIF; 161 domain->sig.dif.pi_interval = scsi_prot_interval(sc); 162 domain->sig.dif.ref_tag = t10_pi_ref_tag(sc->request); 163 /* 164 * At the moment we hard code those, but in the future 165 * we will take them from sc. 166 */ 167 domain->sig.dif.apptag_check_mask = 0xffff; 168 domain->sig.dif.app_escape = true; 169 domain->sig.dif.ref_escape = true; 170 if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) 171 domain->sig.dif.ref_remap = true; 172 } 173 174 static int 175 iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) 176 { 177 switch (scsi_get_prot_op(sc)) { 178 case SCSI_PROT_WRITE_INSERT: 179 case SCSI_PROT_READ_STRIP: 180 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 181 iser_set_dif_domain(sc, &sig_attrs->wire); 182 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 183 break; 184 case SCSI_PROT_READ_INSERT: 185 case SCSI_PROT_WRITE_STRIP: 186 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 187 iser_set_dif_domain(sc, &sig_attrs->mem); 188 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 189 IB_T10DIF_CSUM : IB_T10DIF_CRC; 190 break; 191 case SCSI_PROT_READ_PASS: 192 case SCSI_PROT_WRITE_PASS: 193 iser_set_dif_domain(sc, &sig_attrs->wire); 194 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 195 iser_set_dif_domain(sc, &sig_attrs->mem); 196 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 197 IB_T10DIF_CSUM : IB_T10DIF_CRC; 198 break; 199 default: 200 iser_err("Unsupported PI operation %d\n", 201 scsi_get_prot_op(sc)); 202 return -EINVAL; 203 } 204 205 return 0; 206 } 207 208 static inline void 209 iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 210 { 211 *mask = 0; 212 if (sc->prot_flags & SCSI_PROT_REF_CHECK) 213 *mask |= IB_SIG_CHECK_REFTAG; 214 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) 215 *mask |= IB_SIG_CHECK_GUARD; 216 } 217 218 static inline void 219 iser_inv_rkey(struct ib_send_wr *inv_wr, 220 struct ib_mr *mr, 221 struct ib_cqe *cqe, 222 struct ib_send_wr *next_wr) 223 { 224 inv_wr->opcode = IB_WR_LOCAL_INV; 225 inv_wr->wr_cqe = cqe; 226 inv_wr->ex.invalidate_rkey = mr->rkey; 227 inv_wr->send_flags = 0; 228 inv_wr->num_sge = 0; 229 inv_wr->next = next_wr; 230 } 231 232 static int 233 iser_reg_sig_mr(struct iscsi_iser_task *iser_task, 234 struct iser_data_buf *mem, 235 struct iser_data_buf *sig_mem, 236 struct iser_reg_resources *rsc, 237 struct iser_mem_reg *sig_reg) 238 { 239 struct iser_tx_desc *tx_desc = &iser_task->desc; 240 struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; 241 struct ib_mr *mr = rsc->sig_mr; 242 struct ib_sig_attrs *sig_attrs = mr->sig_attrs; 243 struct ib_reg_wr *wr = &tx_desc->reg_wr; 244 int ret; 245 246 memset(sig_attrs, 0, sizeof(*sig_attrs)); 247 ret = iser_set_sig_attrs(iser_task->sc, sig_attrs); 248 if (ret) 249 goto err; 250 251 iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask); 252 253 if (rsc->mr_valid) 254 iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); 255 256 ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 257 258 ret = ib_map_mr_sg_pi(mr, mem->sg, mem->dma_nents, NULL, 259 sig_mem->sg, sig_mem->dma_nents, NULL, SZ_4K); 260 if (unlikely(ret)) { 261 iser_err("failed to map PI sg (%d)\n", 262 mem->dma_nents + sig_mem->dma_nents); 263 goto err; 264 } 265 266 memset(wr, 0, sizeof(*wr)); 267 wr->wr.next = &tx_desc->send_wr; 268 wr->wr.opcode = IB_WR_REG_MR_INTEGRITY; 269 wr->wr.wr_cqe = cqe; 270 wr->wr.num_sge = 0; 271 wr->wr.send_flags = 0; 272 wr->mr = mr; 273 wr->key = mr->rkey; 274 wr->access = IB_ACCESS_LOCAL_WRITE | 275 IB_ACCESS_REMOTE_READ | 276 IB_ACCESS_REMOTE_WRITE; 277 rsc->mr_valid = 1; 278 279 sig_reg->sge.lkey = mr->lkey; 280 sig_reg->rkey = mr->rkey; 281 sig_reg->sge.addr = mr->iova; 282 sig_reg->sge.length = mr->length; 283 284 iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=%u\n", 285 sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr, 286 sig_reg->sge.length); 287 err: 288 return ret; 289 } 290 291 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 292 struct iser_data_buf *mem, 293 struct iser_reg_resources *rsc, 294 struct iser_mem_reg *reg) 295 { 296 struct iser_tx_desc *tx_desc = &iser_task->desc; 297 struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; 298 struct ib_mr *mr = rsc->mr; 299 struct ib_reg_wr *wr = &tx_desc->reg_wr; 300 int n; 301 302 if (rsc->mr_valid) 303 iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); 304 305 ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 306 307 n = ib_map_mr_sg(mr, mem->sg, mem->dma_nents, NULL, SZ_4K); 308 if (unlikely(n != mem->dma_nents)) { 309 iser_err("failed to map sg (%d/%d)\n", 310 n, mem->dma_nents); 311 return n < 0 ? n : -EINVAL; 312 } 313 314 wr->wr.next = &tx_desc->send_wr; 315 wr->wr.opcode = IB_WR_REG_MR; 316 wr->wr.wr_cqe = cqe; 317 wr->wr.send_flags = 0; 318 wr->wr.num_sge = 0; 319 wr->mr = mr; 320 wr->key = mr->rkey; 321 wr->access = IB_ACCESS_LOCAL_WRITE | 322 IB_ACCESS_REMOTE_WRITE | 323 IB_ACCESS_REMOTE_READ; 324 325 rsc->mr_valid = 1; 326 327 reg->sge.lkey = mr->lkey; 328 reg->rkey = mr->rkey; 329 reg->sge.addr = mr->iova; 330 reg->sge.length = mr->length; 331 332 iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=0x%x\n", 333 reg->sge.lkey, reg->rkey, reg->sge.addr, reg->sge.length); 334 335 return 0; 336 } 337 338 static int 339 iser_reg_data_sg(struct iscsi_iser_task *task, 340 struct iser_data_buf *mem, 341 struct iser_fr_desc *desc, 342 bool use_dma_key, 343 struct iser_mem_reg *reg) 344 { 345 struct iser_device *device = task->iser_conn->ib_conn.device; 346 347 if (use_dma_key) 348 return iser_reg_dma(device, mem, reg); 349 350 return iser_fast_reg_mr(task, mem, &desc->rsc, reg); 351 } 352 353 int iser_reg_mem_fastreg(struct iscsi_iser_task *task, 354 enum iser_data_dir dir, 355 bool all_imm) 356 { 357 struct ib_conn *ib_conn = &task->iser_conn->ib_conn; 358 struct iser_data_buf *mem = &task->data[dir]; 359 struct iser_mem_reg *reg = &task->rdma_reg[dir]; 360 struct iser_fr_desc *desc = NULL; 361 bool use_dma_key; 362 int err; 363 364 use_dma_key = mem->dma_nents == 1 && (all_imm || !iser_always_reg) && 365 scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL; 366 367 if (!use_dma_key) { 368 desc = iser_reg_desc_get_fr(ib_conn); 369 reg->mem_h = desc; 370 } 371 372 if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL) { 373 err = iser_reg_data_sg(task, mem, desc, use_dma_key, reg); 374 if (unlikely(err)) 375 goto err_reg; 376 } else { 377 err = iser_reg_sig_mr(task, mem, &task->prot[dir], 378 &desc->rsc, reg); 379 if (unlikely(err)) 380 goto err_reg; 381 382 desc->sig_protected = true; 383 } 384 385 return 0; 386 387 err_reg: 388 if (desc) 389 iser_reg_desc_put_fr(ib_conn, desc); 390 391 return err; 392 } 393