1 /* 2 * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. 3 * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 #include <linux/module.h> 34 #include <linux/kernel.h> 35 #include <linux/slab.h> 36 #include <linux/mm.h> 37 #include <linux/highmem.h> 38 #include <linux/scatterlist.h> 39 40 #include "iscsi_iser.h" 41 42 void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc) 43 { 44 iser_err_comp(wc, "memreg"); 45 } 46 47 static struct iser_fr_desc *iser_reg_desc_get_fr(struct ib_conn *ib_conn) 48 { 49 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 50 struct iser_fr_desc *desc; 51 unsigned long flags; 52 53 spin_lock_irqsave(&fr_pool->lock, flags); 54 desc = list_first_entry(&fr_pool->list, 55 struct iser_fr_desc, list); 56 list_del(&desc->list); 57 spin_unlock_irqrestore(&fr_pool->lock, flags); 58 59 return desc; 60 } 61 62 static void iser_reg_desc_put_fr(struct ib_conn *ib_conn, 63 struct iser_fr_desc *desc) 64 { 65 struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; 66 unsigned long flags; 67 68 spin_lock_irqsave(&fr_pool->lock, flags); 69 list_add(&desc->list, &fr_pool->list); 70 spin_unlock_irqrestore(&fr_pool->lock, flags); 71 } 72 73 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, 74 struct iser_data_buf *data, 75 enum iser_data_dir iser_dir, 76 enum dma_data_direction dma_dir) 77 { 78 struct ib_device *dev; 79 80 iser_task->dir[iser_dir] = 1; 81 dev = iser_task->iser_conn->ib_conn.device->ib_device; 82 83 data->dma_nents = ib_dma_map_sg(dev, data->sg, data->size, dma_dir); 84 if (unlikely(data->dma_nents == 0)) { 85 iser_err("dma_map_sg failed!!!\n"); 86 return -EINVAL; 87 } 88 return 0; 89 } 90 91 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, 92 struct iser_data_buf *data, 93 enum dma_data_direction dir) 94 { 95 struct ib_device *dev; 96 97 dev = iser_task->iser_conn->ib_conn.device->ib_device; 98 ib_dma_unmap_sg(dev, data->sg, data->size, dir); 99 } 100 101 static int iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem, 102 struct iser_mem_reg *reg) 103 { 104 struct scatterlist *sg = mem->sg; 105 106 reg->sge.lkey = device->pd->local_dma_lkey; 107 /* 108 * FIXME: rework the registration code path to differentiate 109 * rkey/lkey use cases 110 */ 111 112 if (device->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) 113 reg->rkey = device->pd->unsafe_global_rkey; 114 else 115 reg->rkey = 0; 116 reg->sge.addr = sg_dma_address(&sg[0]); 117 reg->sge.length = sg_dma_len(&sg[0]); 118 119 iser_dbg("Single DMA entry: lkey=0x%x, rkey=0x%x, addr=0x%llx," 120 " length=0x%x\n", reg->sge.lkey, reg->rkey, 121 reg->sge.addr, reg->sge.length); 122 123 return 0; 124 } 125 126 void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, 127 enum iser_data_dir cmd_dir) 128 { 129 struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir]; 130 struct iser_fr_desc *desc; 131 struct ib_mr_status mr_status; 132 133 desc = reg->mem_h; 134 if (!desc) 135 return; 136 137 /* 138 * The signature MR cannot be invalidated and reused without checking. 139 * libiscsi calls the check_protection transport handler only if 140 * SCSI-Response is received. And the signature MR is not checked if 141 * the task is completed for some other reason like a timeout or error 142 * handling. That's why we must check the signature MR here before 143 * putting it to the free pool. 144 */ 145 if (unlikely(desc->sig_protected)) { 146 desc->sig_protected = false; 147 ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS, 148 &mr_status); 149 } 150 iser_reg_desc_put_fr(&iser_task->iser_conn->ib_conn, reg->mem_h); 151 reg->mem_h = NULL; 152 } 153 154 static void iser_set_dif_domain(struct scsi_cmnd *sc, 155 struct ib_sig_domain *domain) 156 { 157 domain->sig_type = IB_SIG_TYPE_T10_DIF; 158 domain->sig.dif.pi_interval = scsi_prot_interval(sc); 159 domain->sig.dif.ref_tag = t10_pi_ref_tag(scsi_cmd_to_rq(sc)); 160 /* 161 * At the moment we hard code those, but in the future 162 * we will take them from sc. 163 */ 164 domain->sig.dif.apptag_check_mask = 0xffff; 165 domain->sig.dif.app_escape = true; 166 domain->sig.dif.ref_escape = true; 167 if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) 168 domain->sig.dif.ref_remap = true; 169 } 170 171 static int iser_set_sig_attrs(struct scsi_cmnd *sc, 172 struct ib_sig_attrs *sig_attrs) 173 { 174 switch (scsi_get_prot_op(sc)) { 175 case SCSI_PROT_WRITE_INSERT: 176 case SCSI_PROT_READ_STRIP: 177 sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; 178 iser_set_dif_domain(sc, &sig_attrs->wire); 179 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 180 break; 181 case SCSI_PROT_READ_INSERT: 182 case SCSI_PROT_WRITE_STRIP: 183 sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; 184 iser_set_dif_domain(sc, &sig_attrs->mem); 185 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 186 IB_T10DIF_CSUM : IB_T10DIF_CRC; 187 break; 188 case SCSI_PROT_READ_PASS: 189 case SCSI_PROT_WRITE_PASS: 190 iser_set_dif_domain(sc, &sig_attrs->wire); 191 sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; 192 iser_set_dif_domain(sc, &sig_attrs->mem); 193 sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? 194 IB_T10DIF_CSUM : IB_T10DIF_CRC; 195 break; 196 default: 197 iser_err("Unsupported PI operation %d\n", 198 scsi_get_prot_op(sc)); 199 return -EINVAL; 200 } 201 202 return 0; 203 } 204 205 static inline void iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) 206 { 207 *mask = 0; 208 if (sc->prot_flags & SCSI_PROT_REF_CHECK) 209 *mask |= IB_SIG_CHECK_REFTAG; 210 if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) 211 *mask |= IB_SIG_CHECK_GUARD; 212 } 213 214 static inline void iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr, 215 struct ib_cqe *cqe, struct ib_send_wr *next_wr) 216 { 217 inv_wr->opcode = IB_WR_LOCAL_INV; 218 inv_wr->wr_cqe = cqe; 219 inv_wr->ex.invalidate_rkey = mr->rkey; 220 inv_wr->send_flags = 0; 221 inv_wr->num_sge = 0; 222 inv_wr->next = next_wr; 223 } 224 225 static int iser_reg_sig_mr(struct iscsi_iser_task *iser_task, 226 struct iser_data_buf *mem, 227 struct iser_data_buf *sig_mem, 228 struct iser_reg_resources *rsc, 229 struct iser_mem_reg *sig_reg) 230 { 231 struct iser_tx_desc *tx_desc = &iser_task->desc; 232 struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; 233 struct ib_mr *mr = rsc->sig_mr; 234 struct ib_sig_attrs *sig_attrs = mr->sig_attrs; 235 struct ib_reg_wr *wr = &tx_desc->reg_wr; 236 int ret; 237 238 memset(sig_attrs, 0, sizeof(*sig_attrs)); 239 ret = iser_set_sig_attrs(iser_task->sc, sig_attrs); 240 if (ret) 241 goto err; 242 243 iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask); 244 245 if (rsc->mr_valid) 246 iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); 247 248 ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 249 250 ret = ib_map_mr_sg_pi(mr, mem->sg, mem->dma_nents, NULL, 251 sig_mem->sg, sig_mem->dma_nents, NULL, SZ_4K); 252 if (unlikely(ret)) { 253 iser_err("failed to map PI sg (%d)\n", 254 mem->dma_nents + sig_mem->dma_nents); 255 goto err; 256 } 257 258 memset(wr, 0, sizeof(*wr)); 259 wr->wr.next = &tx_desc->send_wr; 260 wr->wr.opcode = IB_WR_REG_MR_INTEGRITY; 261 wr->wr.wr_cqe = cqe; 262 wr->wr.num_sge = 0; 263 wr->wr.send_flags = 0; 264 wr->mr = mr; 265 wr->key = mr->rkey; 266 wr->access = IB_ACCESS_LOCAL_WRITE | 267 IB_ACCESS_REMOTE_READ | 268 IB_ACCESS_REMOTE_WRITE; 269 rsc->mr_valid = 1; 270 271 sig_reg->sge.lkey = mr->lkey; 272 sig_reg->rkey = mr->rkey; 273 sig_reg->sge.addr = mr->iova; 274 sig_reg->sge.length = mr->length; 275 276 iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=%u\n", 277 sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr, 278 sig_reg->sge.length); 279 err: 280 return ret; 281 } 282 283 static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, 284 struct iser_data_buf *mem, 285 struct iser_reg_resources *rsc, 286 struct iser_mem_reg *reg) 287 { 288 struct iser_tx_desc *tx_desc = &iser_task->desc; 289 struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; 290 struct ib_mr *mr = rsc->mr; 291 struct ib_reg_wr *wr = &tx_desc->reg_wr; 292 int n; 293 294 if (rsc->mr_valid) 295 iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); 296 297 ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); 298 299 n = ib_map_mr_sg(mr, mem->sg, mem->dma_nents, NULL, SZ_4K); 300 if (unlikely(n != mem->dma_nents)) { 301 iser_err("failed to map sg (%d/%d)\n", 302 n, mem->dma_nents); 303 return n < 0 ? n : -EINVAL; 304 } 305 306 wr->wr.next = &tx_desc->send_wr; 307 wr->wr.opcode = IB_WR_REG_MR; 308 wr->wr.wr_cqe = cqe; 309 wr->wr.send_flags = 0; 310 wr->wr.num_sge = 0; 311 wr->mr = mr; 312 wr->key = mr->rkey; 313 wr->access = IB_ACCESS_LOCAL_WRITE | 314 IB_ACCESS_REMOTE_WRITE | 315 IB_ACCESS_REMOTE_READ; 316 317 rsc->mr_valid = 1; 318 319 reg->sge.lkey = mr->lkey; 320 reg->rkey = mr->rkey; 321 reg->sge.addr = mr->iova; 322 reg->sge.length = mr->length; 323 324 iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=0x%x\n", 325 reg->sge.lkey, reg->rkey, reg->sge.addr, reg->sge.length); 326 327 return 0; 328 } 329 330 static int iser_reg_data_sg(struct iscsi_iser_task *task, 331 struct iser_data_buf *mem, 332 struct iser_fr_desc *desc, bool use_dma_key, 333 struct iser_mem_reg *reg) 334 { 335 struct iser_device *device = task->iser_conn->ib_conn.device; 336 337 if (use_dma_key) 338 return iser_reg_dma(device, mem, reg); 339 340 return iser_fast_reg_mr(task, mem, &desc->rsc, reg); 341 } 342 343 int iser_reg_mem_fastreg(struct iscsi_iser_task *task, 344 enum iser_data_dir dir, 345 bool all_imm) 346 { 347 struct ib_conn *ib_conn = &task->iser_conn->ib_conn; 348 struct iser_data_buf *mem = &task->data[dir]; 349 struct iser_mem_reg *reg = &task->rdma_reg[dir]; 350 struct iser_fr_desc *desc = NULL; 351 bool use_dma_key; 352 int err; 353 354 use_dma_key = mem->dma_nents == 1 && (all_imm || !iser_always_reg) && 355 scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL; 356 357 if (!use_dma_key) { 358 desc = iser_reg_desc_get_fr(ib_conn); 359 reg->mem_h = desc; 360 } 361 362 if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL) { 363 err = iser_reg_data_sg(task, mem, desc, use_dma_key, reg); 364 if (unlikely(err)) 365 goto err_reg; 366 } else { 367 err = iser_reg_sig_mr(task, mem, &task->prot[dir], 368 &desc->rsc, reg); 369 if (unlikely(err)) 370 goto err_reg; 371 372 desc->sig_protected = true; 373 } 374 375 return 0; 376 377 err_reg: 378 if (desc) 379 iser_reg_desc_put_fr(ib_conn, desc); 380 381 return err; 382 } 383