1 /* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/module.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/err.h> 39 #include <linux/string.h> 40 #include <linux/parser.h> 41 #include <linux/random.h> 42 #include <linux/jiffies.h> 43 #include <linux/lockdep.h> 44 #include <linux/inet.h> 45 #include <rdma/ib_cache.h> 46 47 #include <linux/atomic.h> 48 49 #include <scsi/scsi.h> 50 #include <scsi/scsi_device.h> 51 #include <scsi/scsi_dbg.h> 52 #include <scsi/scsi_tcq.h> 53 #include <scsi/srp.h> 54 #include <scsi/scsi_transport_srp.h> 55 56 #include "ib_srp.h" 57 58 #define DRV_NAME "ib_srp" 59 #define PFX DRV_NAME ": " 60 61 MODULE_AUTHOR("Roland Dreier"); 62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); 63 MODULE_LICENSE("Dual BSD/GPL"); 64 65 #if !defined(CONFIG_DYNAMIC_DEBUG) 66 #define DEFINE_DYNAMIC_DEBUG_METADATA(name, fmt) 67 #define DYNAMIC_DEBUG_BRANCH(descriptor) false 68 #endif 69 70 static unsigned int srp_sg_tablesize; 71 static unsigned int cmd_sg_entries; 72 static unsigned int indirect_sg_entries; 73 static bool allow_ext_sg; 74 static bool register_always = true; 75 static bool never_register; 76 static int topspin_workarounds = 1; 77 78 module_param(srp_sg_tablesize, uint, 0444); 79 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); 80 81 module_param(cmd_sg_entries, uint, 0444); 82 MODULE_PARM_DESC(cmd_sg_entries, 83 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); 84 85 module_param(indirect_sg_entries, uint, 0444); 86 MODULE_PARM_DESC(indirect_sg_entries, 87 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")"); 88 89 module_param(allow_ext_sg, bool, 0444); 90 MODULE_PARM_DESC(allow_ext_sg, 91 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)"); 92 93 module_param(topspin_workarounds, int, 0444); 94 MODULE_PARM_DESC(topspin_workarounds, 95 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0"); 96 97 module_param(register_always, bool, 0444); 98 MODULE_PARM_DESC(register_always, 99 "Use memory registration even for contiguous memory regions"); 100 101 module_param(never_register, bool, 0444); 102 MODULE_PARM_DESC(never_register, "Never register memory"); 103 104 static const struct kernel_param_ops srp_tmo_ops; 105 106 static int srp_reconnect_delay = 10; 107 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 108 S_IRUGO | S_IWUSR); 109 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts"); 110 111 static int srp_fast_io_fail_tmo = 15; 112 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo, 113 S_IRUGO | S_IWUSR); 114 MODULE_PARM_DESC(fast_io_fail_tmo, 115 "Number of seconds between the observation of a transport" 116 " layer error and failing all I/O. \"off\" means that this" 117 " functionality is disabled."); 118 119 static int srp_dev_loss_tmo = 600; 120 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo, 121 S_IRUGO | S_IWUSR); 122 MODULE_PARM_DESC(dev_loss_tmo, 123 "Maximum number of seconds that the SRP transport should" 124 " insulate transport layer errors. After this time has been" 125 " exceeded the SCSI host is removed. Should be" 126 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT) 127 " if fast_io_fail_tmo has not been set. \"off\" means that" 128 " this functionality is disabled."); 129 130 static bool srp_use_imm_data = true; 131 module_param_named(use_imm_data, srp_use_imm_data, bool, 0644); 132 MODULE_PARM_DESC(use_imm_data, 133 "Whether or not to request permission to use immediate data during SRP login."); 134 135 static unsigned int srp_max_imm_data = 8 * 1024; 136 module_param_named(max_imm_data, srp_max_imm_data, uint, 0644); 137 MODULE_PARM_DESC(max_imm_data, "Maximum immediate data size."); 138 139 static unsigned ch_count; 140 module_param(ch_count, uint, 0444); 141 MODULE_PARM_DESC(ch_count, 142 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA."); 143 144 static int srp_add_one(struct ib_device *device); 145 static void srp_remove_one(struct ib_device *device, void *client_data); 146 static void srp_rename_dev(struct ib_device *device, void *client_data); 147 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc); 148 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 149 const char *opname); 150 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 151 const struct ib_cm_event *event); 152 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 153 struct rdma_cm_event *event); 154 155 static struct scsi_transport_template *ib_srp_transport_template; 156 static struct workqueue_struct *srp_remove_wq; 157 158 static struct ib_client srp_client = { 159 .name = "srp", 160 .add = srp_add_one, 161 .remove = srp_remove_one, 162 .rename = srp_rename_dev 163 }; 164 165 static struct ib_sa_client srp_sa_client; 166 167 static int srp_tmo_get(char *buffer, const struct kernel_param *kp) 168 { 169 int tmo = *(int *)kp->arg; 170 171 if (tmo >= 0) 172 return sysfs_emit(buffer, "%d\n", tmo); 173 else 174 return sysfs_emit(buffer, "off\n"); 175 } 176 177 static int srp_tmo_set(const char *val, const struct kernel_param *kp) 178 { 179 int tmo, res; 180 181 res = srp_parse_tmo(&tmo, val); 182 if (res) 183 goto out; 184 185 if (kp->arg == &srp_reconnect_delay) 186 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo, 187 srp_dev_loss_tmo); 188 else if (kp->arg == &srp_fast_io_fail_tmo) 189 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo); 190 else 191 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo, 192 tmo); 193 if (res) 194 goto out; 195 *(int *)kp->arg = tmo; 196 197 out: 198 return res; 199 } 200 201 static const struct kernel_param_ops srp_tmo_ops = { 202 .get = srp_tmo_get, 203 .set = srp_tmo_set, 204 }; 205 206 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) 207 { 208 return (struct srp_target_port *) host->hostdata; 209 } 210 211 static const char *srp_target_info(struct Scsi_Host *host) 212 { 213 return host_to_target(host)->target_name; 214 } 215 216 static int srp_target_is_topspin(struct srp_target_port *target) 217 { 218 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad }; 219 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d }; 220 221 return topspin_workarounds && 222 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) || 223 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui)); 224 } 225 226 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size, 227 gfp_t gfp_mask, 228 enum dma_data_direction direction) 229 { 230 struct srp_iu *iu; 231 232 iu = kmalloc(sizeof *iu, gfp_mask); 233 if (!iu) 234 goto out; 235 236 iu->buf = kzalloc(size, gfp_mask); 237 if (!iu->buf) 238 goto out_free_iu; 239 240 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size, 241 direction); 242 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma)) 243 goto out_free_buf; 244 245 iu->size = size; 246 iu->direction = direction; 247 248 return iu; 249 250 out_free_buf: 251 kfree(iu->buf); 252 out_free_iu: 253 kfree(iu); 254 out: 255 return NULL; 256 } 257 258 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu) 259 { 260 if (!iu) 261 return; 262 263 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size, 264 iu->direction); 265 kfree(iu->buf); 266 kfree(iu); 267 } 268 269 static void srp_qp_event(struct ib_event *event, void *context) 270 { 271 pr_debug("QP event %s (%d)\n", 272 ib_event_msg(event->event), event->event); 273 } 274 275 static int srp_init_ib_qp(struct srp_target_port *target, 276 struct ib_qp *qp) 277 { 278 struct ib_qp_attr *attr; 279 int ret; 280 281 attr = kmalloc(sizeof *attr, GFP_KERNEL); 282 if (!attr) 283 return -ENOMEM; 284 285 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 286 target->srp_host->port, 287 be16_to_cpu(target->ib_cm.pkey), 288 &attr->pkey_index); 289 if (ret) 290 goto out; 291 292 attr->qp_state = IB_QPS_INIT; 293 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ | 294 IB_ACCESS_REMOTE_WRITE); 295 attr->port_num = target->srp_host->port; 296 297 ret = ib_modify_qp(qp, attr, 298 IB_QP_STATE | 299 IB_QP_PKEY_INDEX | 300 IB_QP_ACCESS_FLAGS | 301 IB_QP_PORT); 302 303 out: 304 kfree(attr); 305 return ret; 306 } 307 308 static int srp_new_ib_cm_id(struct srp_rdma_ch *ch) 309 { 310 struct srp_target_port *target = ch->target; 311 struct ib_cm_id *new_cm_id; 312 313 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, 314 srp_ib_cm_handler, ch); 315 if (IS_ERR(new_cm_id)) 316 return PTR_ERR(new_cm_id); 317 318 if (ch->ib_cm.cm_id) 319 ib_destroy_cm_id(ch->ib_cm.cm_id); 320 ch->ib_cm.cm_id = new_cm_id; 321 if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev, 322 target->srp_host->port)) 323 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA; 324 else 325 ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB; 326 ch->ib_cm.path.sgid = target->sgid; 327 ch->ib_cm.path.dgid = target->ib_cm.orig_dgid; 328 ch->ib_cm.path.pkey = target->ib_cm.pkey; 329 ch->ib_cm.path.service_id = target->ib_cm.service_id; 330 331 return 0; 332 } 333 334 static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch) 335 { 336 struct srp_target_port *target = ch->target; 337 struct rdma_cm_id *new_cm_id; 338 int ret; 339 340 new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch, 341 RDMA_PS_TCP, IB_QPT_RC); 342 if (IS_ERR(new_cm_id)) { 343 ret = PTR_ERR(new_cm_id); 344 new_cm_id = NULL; 345 goto out; 346 } 347 348 init_completion(&ch->done); 349 ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ? 350 &target->rdma_cm.src.sa : NULL, 351 &target->rdma_cm.dst.sa, 352 SRP_PATH_REC_TIMEOUT_MS); 353 if (ret) { 354 pr_err("No route available from %pISpsc to %pISpsc (%d)\n", 355 &target->rdma_cm.src, &target->rdma_cm.dst, ret); 356 goto out; 357 } 358 ret = wait_for_completion_interruptible(&ch->done); 359 if (ret < 0) 360 goto out; 361 362 ret = ch->status; 363 if (ret) { 364 pr_err("Resolving address %pISpsc failed (%d)\n", 365 &target->rdma_cm.dst, ret); 366 goto out; 367 } 368 369 swap(ch->rdma_cm.cm_id, new_cm_id); 370 371 out: 372 if (new_cm_id) 373 rdma_destroy_id(new_cm_id); 374 375 return ret; 376 } 377 378 static int srp_new_cm_id(struct srp_rdma_ch *ch) 379 { 380 struct srp_target_port *target = ch->target; 381 382 return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) : 383 srp_new_ib_cm_id(ch); 384 } 385 386 /** 387 * srp_destroy_fr_pool() - free the resources owned by a pool 388 * @pool: Fast registration pool to be destroyed. 389 */ 390 static void srp_destroy_fr_pool(struct srp_fr_pool *pool) 391 { 392 int i; 393 struct srp_fr_desc *d; 394 395 if (!pool) 396 return; 397 398 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 399 if (d->mr) 400 ib_dereg_mr(d->mr); 401 } 402 kfree(pool); 403 } 404 405 /** 406 * srp_create_fr_pool() - allocate and initialize a pool for fast registration 407 * @device: IB device to allocate fast registration descriptors for. 408 * @pd: Protection domain associated with the FR descriptors. 409 * @pool_size: Number of descriptors to allocate. 410 * @max_page_list_len: Maximum fast registration work request page list length. 411 */ 412 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, 413 struct ib_pd *pd, int pool_size, 414 int max_page_list_len) 415 { 416 struct srp_fr_pool *pool; 417 struct srp_fr_desc *d; 418 struct ib_mr *mr; 419 int i, ret = -EINVAL; 420 enum ib_mr_type mr_type; 421 422 if (pool_size <= 0) 423 goto err; 424 ret = -ENOMEM; 425 pool = kzalloc(struct_size(pool, desc, pool_size), GFP_KERNEL); 426 if (!pool) 427 goto err; 428 pool->size = pool_size; 429 pool->max_page_list_len = max_page_list_len; 430 spin_lock_init(&pool->lock); 431 INIT_LIST_HEAD(&pool->free_list); 432 433 if (device->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG) 434 mr_type = IB_MR_TYPE_SG_GAPS; 435 else 436 mr_type = IB_MR_TYPE_MEM_REG; 437 438 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 439 mr = ib_alloc_mr(pd, mr_type, max_page_list_len); 440 if (IS_ERR(mr)) { 441 ret = PTR_ERR(mr); 442 if (ret == -ENOMEM) 443 pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n", 444 dev_name(&device->dev)); 445 goto destroy_pool; 446 } 447 d->mr = mr; 448 list_add_tail(&d->entry, &pool->free_list); 449 } 450 451 out: 452 return pool; 453 454 destroy_pool: 455 srp_destroy_fr_pool(pool); 456 457 err: 458 pool = ERR_PTR(ret); 459 goto out; 460 } 461 462 /** 463 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration 464 * @pool: Pool to obtain descriptor from. 465 */ 466 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool) 467 { 468 struct srp_fr_desc *d = NULL; 469 unsigned long flags; 470 471 spin_lock_irqsave(&pool->lock, flags); 472 if (!list_empty(&pool->free_list)) { 473 d = list_first_entry(&pool->free_list, typeof(*d), entry); 474 list_del(&d->entry); 475 } 476 spin_unlock_irqrestore(&pool->lock, flags); 477 478 return d; 479 } 480 481 /** 482 * srp_fr_pool_put() - put an FR descriptor back in the free list 483 * @pool: Pool the descriptor was allocated from. 484 * @desc: Pointer to an array of fast registration descriptor pointers. 485 * @n: Number of descriptors to put back. 486 * 487 * Note: The caller must already have queued an invalidation request for 488 * desc->mr->rkey before calling this function. 489 */ 490 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc, 491 int n) 492 { 493 unsigned long flags; 494 int i; 495 496 spin_lock_irqsave(&pool->lock, flags); 497 for (i = 0; i < n; i++) 498 list_add(&desc[i]->entry, &pool->free_list); 499 spin_unlock_irqrestore(&pool->lock, flags); 500 } 501 502 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) 503 { 504 struct srp_device *dev = target->srp_host->srp_dev; 505 506 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size, 507 dev->max_pages_per_mr); 508 } 509 510 /** 511 * srp_destroy_qp() - destroy an RDMA queue pair 512 * @ch: SRP RDMA channel. 513 * 514 * Drain the qp before destroying it. This avoids that the receive 515 * completion handler can access the queue pair while it is 516 * being destroyed. 517 */ 518 static void srp_destroy_qp(struct srp_rdma_ch *ch) 519 { 520 spin_lock_irq(&ch->lock); 521 ib_process_cq_direct(ch->send_cq, -1); 522 spin_unlock_irq(&ch->lock); 523 524 ib_drain_qp(ch->qp); 525 ib_destroy_qp(ch->qp); 526 } 527 528 static int srp_create_ch_ib(struct srp_rdma_ch *ch) 529 { 530 struct srp_target_port *target = ch->target; 531 struct srp_device *dev = target->srp_host->srp_dev; 532 const struct ib_device_attr *attr = &dev->dev->attrs; 533 struct ib_qp_init_attr *init_attr; 534 struct ib_cq *recv_cq, *send_cq; 535 struct ib_qp *qp; 536 struct srp_fr_pool *fr_pool = NULL; 537 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2; 538 int ret; 539 540 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL); 541 if (!init_attr) 542 return -ENOMEM; 543 544 /* queue_size + 1 for ib_drain_rq() */ 545 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1, 546 ch->comp_vector, IB_POLL_SOFTIRQ); 547 if (IS_ERR(recv_cq)) { 548 ret = PTR_ERR(recv_cq); 549 goto err; 550 } 551 552 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size, 553 ch->comp_vector, IB_POLL_DIRECT); 554 if (IS_ERR(send_cq)) { 555 ret = PTR_ERR(send_cq); 556 goto err_recv_cq; 557 } 558 559 init_attr->event_handler = srp_qp_event; 560 init_attr->cap.max_send_wr = m * target->queue_size; 561 init_attr->cap.max_recv_wr = target->queue_size + 1; 562 init_attr->cap.max_recv_sge = 1; 563 init_attr->cap.max_send_sge = min(SRP_MAX_SGE, attr->max_send_sge); 564 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 565 init_attr->qp_type = IB_QPT_RC; 566 init_attr->send_cq = send_cq; 567 init_attr->recv_cq = recv_cq; 568 569 ch->max_imm_sge = min(init_attr->cap.max_send_sge - 1U, 255U); 570 571 if (target->using_rdma_cm) { 572 ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr); 573 qp = ch->rdma_cm.cm_id->qp; 574 } else { 575 qp = ib_create_qp(dev->pd, init_attr); 576 if (!IS_ERR(qp)) { 577 ret = srp_init_ib_qp(target, qp); 578 if (ret) 579 ib_destroy_qp(qp); 580 } else { 581 ret = PTR_ERR(qp); 582 } 583 } 584 if (ret) { 585 pr_err("QP creation failed for dev %s: %d\n", 586 dev_name(&dev->dev->dev), ret); 587 goto err_send_cq; 588 } 589 590 if (dev->use_fast_reg) { 591 fr_pool = srp_alloc_fr_pool(target); 592 if (IS_ERR(fr_pool)) { 593 ret = PTR_ERR(fr_pool); 594 shost_printk(KERN_WARNING, target->scsi_host, PFX 595 "FR pool allocation failed (%d)\n", ret); 596 goto err_qp; 597 } 598 } 599 600 if (ch->qp) 601 srp_destroy_qp(ch); 602 if (ch->recv_cq) 603 ib_free_cq(ch->recv_cq); 604 if (ch->send_cq) 605 ib_free_cq(ch->send_cq); 606 607 ch->qp = qp; 608 ch->recv_cq = recv_cq; 609 ch->send_cq = send_cq; 610 611 if (dev->use_fast_reg) { 612 if (ch->fr_pool) 613 srp_destroy_fr_pool(ch->fr_pool); 614 ch->fr_pool = fr_pool; 615 } 616 617 kfree(init_attr); 618 return 0; 619 620 err_qp: 621 if (target->using_rdma_cm) 622 rdma_destroy_qp(ch->rdma_cm.cm_id); 623 else 624 ib_destroy_qp(qp); 625 626 err_send_cq: 627 ib_free_cq(send_cq); 628 629 err_recv_cq: 630 ib_free_cq(recv_cq); 631 632 err: 633 kfree(init_attr); 634 return ret; 635 } 636 637 /* 638 * Note: this function may be called without srp_alloc_iu_bufs() having been 639 * invoked. Hence the ch->[rt]x_ring checks. 640 */ 641 static void srp_free_ch_ib(struct srp_target_port *target, 642 struct srp_rdma_ch *ch) 643 { 644 struct srp_device *dev = target->srp_host->srp_dev; 645 int i; 646 647 if (!ch->target) 648 return; 649 650 if (target->using_rdma_cm) { 651 if (ch->rdma_cm.cm_id) { 652 rdma_destroy_id(ch->rdma_cm.cm_id); 653 ch->rdma_cm.cm_id = NULL; 654 } 655 } else { 656 if (ch->ib_cm.cm_id) { 657 ib_destroy_cm_id(ch->ib_cm.cm_id); 658 ch->ib_cm.cm_id = NULL; 659 } 660 } 661 662 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */ 663 if (!ch->qp) 664 return; 665 666 if (dev->use_fast_reg) { 667 if (ch->fr_pool) 668 srp_destroy_fr_pool(ch->fr_pool); 669 } 670 671 srp_destroy_qp(ch); 672 ib_free_cq(ch->send_cq); 673 ib_free_cq(ch->recv_cq); 674 675 /* 676 * Avoid that the SCSI error handler tries to use this channel after 677 * it has been freed. The SCSI error handler can namely continue 678 * trying to perform recovery actions after scsi_remove_host() 679 * returned. 680 */ 681 ch->target = NULL; 682 683 ch->qp = NULL; 684 ch->send_cq = ch->recv_cq = NULL; 685 686 if (ch->rx_ring) { 687 for (i = 0; i < target->queue_size; ++i) 688 srp_free_iu(target->srp_host, ch->rx_ring[i]); 689 kfree(ch->rx_ring); 690 ch->rx_ring = NULL; 691 } 692 if (ch->tx_ring) { 693 for (i = 0; i < target->queue_size; ++i) 694 srp_free_iu(target->srp_host, ch->tx_ring[i]); 695 kfree(ch->tx_ring); 696 ch->tx_ring = NULL; 697 } 698 } 699 700 static void srp_path_rec_completion(int status, 701 struct sa_path_rec *pathrec, 702 void *ch_ptr) 703 { 704 struct srp_rdma_ch *ch = ch_ptr; 705 struct srp_target_port *target = ch->target; 706 707 ch->status = status; 708 if (status) 709 shost_printk(KERN_ERR, target->scsi_host, 710 PFX "Got failed path rec status %d\n", status); 711 else 712 ch->ib_cm.path = *pathrec; 713 complete(&ch->done); 714 } 715 716 static int srp_ib_lookup_path(struct srp_rdma_ch *ch) 717 { 718 struct srp_target_port *target = ch->target; 719 int ret; 720 721 ch->ib_cm.path.numb_path = 1; 722 723 init_completion(&ch->done); 724 725 ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client, 726 target->srp_host->srp_dev->dev, 727 target->srp_host->port, 728 &ch->ib_cm.path, 729 IB_SA_PATH_REC_SERVICE_ID | 730 IB_SA_PATH_REC_DGID | 731 IB_SA_PATH_REC_SGID | 732 IB_SA_PATH_REC_NUMB_PATH | 733 IB_SA_PATH_REC_PKEY, 734 SRP_PATH_REC_TIMEOUT_MS, 735 GFP_KERNEL, 736 srp_path_rec_completion, 737 ch, &ch->ib_cm.path_query); 738 if (ch->ib_cm.path_query_id < 0) 739 return ch->ib_cm.path_query_id; 740 741 ret = wait_for_completion_interruptible(&ch->done); 742 if (ret < 0) 743 return ret; 744 745 if (ch->status < 0) 746 shost_printk(KERN_WARNING, target->scsi_host, 747 PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n", 748 ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw, 749 be16_to_cpu(target->ib_cm.pkey), 750 be64_to_cpu(target->ib_cm.service_id)); 751 752 return ch->status; 753 } 754 755 static int srp_rdma_lookup_path(struct srp_rdma_ch *ch) 756 { 757 struct srp_target_port *target = ch->target; 758 int ret; 759 760 init_completion(&ch->done); 761 762 ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS); 763 if (ret) 764 return ret; 765 766 wait_for_completion_interruptible(&ch->done); 767 768 if (ch->status != 0) 769 shost_printk(KERN_WARNING, target->scsi_host, 770 PFX "Path resolution failed\n"); 771 772 return ch->status; 773 } 774 775 static int srp_lookup_path(struct srp_rdma_ch *ch) 776 { 777 struct srp_target_port *target = ch->target; 778 779 return target->using_rdma_cm ? srp_rdma_lookup_path(ch) : 780 srp_ib_lookup_path(ch); 781 } 782 783 static u8 srp_get_subnet_timeout(struct srp_host *host) 784 { 785 struct ib_port_attr attr; 786 int ret; 787 u8 subnet_timeout = 18; 788 789 ret = ib_query_port(host->srp_dev->dev, host->port, &attr); 790 if (ret == 0) 791 subnet_timeout = attr.subnet_timeout; 792 793 if (unlikely(subnet_timeout < 15)) 794 pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n", 795 dev_name(&host->srp_dev->dev->dev), subnet_timeout); 796 797 return subnet_timeout; 798 } 799 800 static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len, 801 bool multich) 802 { 803 struct srp_target_port *target = ch->target; 804 struct { 805 struct rdma_conn_param rdma_param; 806 struct srp_login_req_rdma rdma_req; 807 struct ib_cm_req_param ib_param; 808 struct srp_login_req ib_req; 809 } *req = NULL; 810 char *ipi, *tpi; 811 int status; 812 813 req = kzalloc(sizeof *req, GFP_KERNEL); 814 if (!req) 815 return -ENOMEM; 816 817 req->ib_param.flow_control = 1; 818 req->ib_param.retry_count = target->tl_retry_count; 819 820 /* 821 * Pick some arbitrary defaults here; we could make these 822 * module parameters if anyone cared about setting them. 823 */ 824 req->ib_param.responder_resources = 4; 825 req->ib_param.rnr_retry_count = 7; 826 req->ib_param.max_cm_retries = 15; 827 828 req->ib_req.opcode = SRP_LOGIN_REQ; 829 req->ib_req.tag = 0; 830 req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len); 831 req->ib_req.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | 832 SRP_BUF_FORMAT_INDIRECT); 833 req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI : 834 SRP_MULTICHAN_SINGLE); 835 if (srp_use_imm_data) { 836 req->ib_req.req_flags |= SRP_IMMED_REQUESTED; 837 req->ib_req.imm_data_offset = cpu_to_be16(SRP_IMM_DATA_OFFSET); 838 } 839 840 if (target->using_rdma_cm) { 841 req->rdma_param.flow_control = req->ib_param.flow_control; 842 req->rdma_param.responder_resources = 843 req->ib_param.responder_resources; 844 req->rdma_param.initiator_depth = req->ib_param.initiator_depth; 845 req->rdma_param.retry_count = req->ib_param.retry_count; 846 req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count; 847 req->rdma_param.private_data = &req->rdma_req; 848 req->rdma_param.private_data_len = sizeof(req->rdma_req); 849 850 req->rdma_req.opcode = req->ib_req.opcode; 851 req->rdma_req.tag = req->ib_req.tag; 852 req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len; 853 req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt; 854 req->rdma_req.req_flags = req->ib_req.req_flags; 855 req->rdma_req.imm_data_offset = req->ib_req.imm_data_offset; 856 857 ipi = req->rdma_req.initiator_port_id; 858 tpi = req->rdma_req.target_port_id; 859 } else { 860 u8 subnet_timeout; 861 862 subnet_timeout = srp_get_subnet_timeout(target->srp_host); 863 864 req->ib_param.primary_path = &ch->ib_cm.path; 865 req->ib_param.alternate_path = NULL; 866 req->ib_param.service_id = target->ib_cm.service_id; 867 get_random_bytes(&req->ib_param.starting_psn, 4); 868 req->ib_param.starting_psn &= 0xffffff; 869 req->ib_param.qp_num = ch->qp->qp_num; 870 req->ib_param.qp_type = ch->qp->qp_type; 871 req->ib_param.local_cm_response_timeout = subnet_timeout + 2; 872 req->ib_param.remote_cm_response_timeout = subnet_timeout + 2; 873 req->ib_param.private_data = &req->ib_req; 874 req->ib_param.private_data_len = sizeof(req->ib_req); 875 876 ipi = req->ib_req.initiator_port_id; 877 tpi = req->ib_req.target_port_id; 878 } 879 880 /* 881 * In the published SRP specification (draft rev. 16a), the 882 * port identifier format is 8 bytes of ID extension followed 883 * by 8 bytes of GUID. Older drafts put the two halves in the 884 * opposite order, so that the GUID comes first. 885 * 886 * Targets conforming to these obsolete drafts can be 887 * recognized by the I/O Class they report. 888 */ 889 if (target->io_class == SRP_REV10_IB_IO_CLASS) { 890 memcpy(ipi, &target->sgid.global.interface_id, 8); 891 memcpy(ipi + 8, &target->initiator_ext, 8); 892 memcpy(tpi, &target->ioc_guid, 8); 893 memcpy(tpi + 8, &target->id_ext, 8); 894 } else { 895 memcpy(ipi, &target->initiator_ext, 8); 896 memcpy(ipi + 8, &target->sgid.global.interface_id, 8); 897 memcpy(tpi, &target->id_ext, 8); 898 memcpy(tpi + 8, &target->ioc_guid, 8); 899 } 900 901 /* 902 * Topspin/Cisco SRP targets will reject our login unless we 903 * zero out the first 8 bytes of our initiator port ID and set 904 * the second 8 bytes to the local node GUID. 905 */ 906 if (srp_target_is_topspin(target)) { 907 shost_printk(KERN_DEBUG, target->scsi_host, 908 PFX "Topspin/Cisco initiator port ID workaround " 909 "activated for target GUID %016llx\n", 910 be64_to_cpu(target->ioc_guid)); 911 memset(ipi, 0, 8); 912 memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8); 913 } 914 915 if (target->using_rdma_cm) 916 status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param); 917 else 918 status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param); 919 920 kfree(req); 921 922 return status; 923 } 924 925 static bool srp_queue_remove_work(struct srp_target_port *target) 926 { 927 bool changed = false; 928 929 spin_lock_irq(&target->lock); 930 if (target->state != SRP_TARGET_REMOVED) { 931 target->state = SRP_TARGET_REMOVED; 932 changed = true; 933 } 934 spin_unlock_irq(&target->lock); 935 936 if (changed) 937 queue_work(srp_remove_wq, &target->remove_work); 938 939 return changed; 940 } 941 942 static void srp_disconnect_target(struct srp_target_port *target) 943 { 944 struct srp_rdma_ch *ch; 945 int i, ret; 946 947 /* XXX should send SRP_I_LOGOUT request */ 948 949 for (i = 0; i < target->ch_count; i++) { 950 ch = &target->ch[i]; 951 ch->connected = false; 952 ret = 0; 953 if (target->using_rdma_cm) { 954 if (ch->rdma_cm.cm_id) 955 rdma_disconnect(ch->rdma_cm.cm_id); 956 } else { 957 if (ch->ib_cm.cm_id) 958 ret = ib_send_cm_dreq(ch->ib_cm.cm_id, 959 NULL, 0); 960 } 961 if (ret < 0) { 962 shost_printk(KERN_DEBUG, target->scsi_host, 963 PFX "Sending CM DREQ failed\n"); 964 } 965 } 966 } 967 968 static int srp_exit_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 969 { 970 struct srp_target_port *target = host_to_target(shost); 971 struct srp_device *dev = target->srp_host->srp_dev; 972 struct ib_device *ibdev = dev->dev; 973 struct srp_request *req = scsi_cmd_priv(cmd); 974 975 kfree(req->fr_list); 976 if (req->indirect_dma_addr) { 977 ib_dma_unmap_single(ibdev, req->indirect_dma_addr, 978 target->indirect_size, 979 DMA_TO_DEVICE); 980 } 981 kfree(req->indirect_desc); 982 983 return 0; 984 } 985 986 static int srp_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd) 987 { 988 struct srp_target_port *target = host_to_target(shost); 989 struct srp_device *srp_dev = target->srp_host->srp_dev; 990 struct ib_device *ibdev = srp_dev->dev; 991 struct srp_request *req = scsi_cmd_priv(cmd); 992 dma_addr_t dma_addr; 993 int ret = -ENOMEM; 994 995 if (srp_dev->use_fast_reg) { 996 req->fr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *), 997 GFP_KERNEL); 998 if (!req->fr_list) 999 goto out; 1000 } 1001 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL); 1002 if (!req->indirect_desc) 1003 goto out; 1004 1005 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc, 1006 target->indirect_size, 1007 DMA_TO_DEVICE); 1008 if (ib_dma_mapping_error(ibdev, dma_addr)) { 1009 srp_exit_cmd_priv(shost, cmd); 1010 goto out; 1011 } 1012 1013 req->indirect_dma_addr = dma_addr; 1014 ret = 0; 1015 1016 out: 1017 return ret; 1018 } 1019 1020 /** 1021 * srp_del_scsi_host_attr() - Remove attributes defined in the host template. 1022 * @shost: SCSI host whose attributes to remove from sysfs. 1023 * 1024 * Note: Any attributes defined in the host template and that did not exist 1025 * before invocation of this function will be ignored. 1026 */ 1027 static void srp_del_scsi_host_attr(struct Scsi_Host *shost) 1028 { 1029 const struct attribute_group **g; 1030 struct attribute **attr; 1031 1032 for (g = shost->hostt->shost_groups; *g; ++g) { 1033 for (attr = (*g)->attrs; *attr; ++attr) { 1034 struct device_attribute *dev_attr = 1035 container_of(*attr, typeof(*dev_attr), attr); 1036 1037 device_remove_file(&shost->shost_dev, dev_attr); 1038 } 1039 } 1040 } 1041 1042 static void srp_remove_target(struct srp_target_port *target) 1043 { 1044 struct srp_rdma_ch *ch; 1045 int i; 1046 1047 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1048 1049 srp_del_scsi_host_attr(target->scsi_host); 1050 srp_rport_get(target->rport); 1051 srp_remove_host(target->scsi_host); 1052 scsi_remove_host(target->scsi_host); 1053 srp_stop_rport_timers(target->rport); 1054 srp_disconnect_target(target); 1055 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 1056 for (i = 0; i < target->ch_count; i++) { 1057 ch = &target->ch[i]; 1058 srp_free_ch_ib(target, ch); 1059 } 1060 cancel_work_sync(&target->tl_err_work); 1061 srp_rport_put(target->rport); 1062 kfree(target->ch); 1063 target->ch = NULL; 1064 1065 spin_lock(&target->srp_host->target_lock); 1066 list_del(&target->list); 1067 spin_unlock(&target->srp_host->target_lock); 1068 1069 scsi_host_put(target->scsi_host); 1070 } 1071 1072 static void srp_remove_work(struct work_struct *work) 1073 { 1074 struct srp_target_port *target = 1075 container_of(work, struct srp_target_port, remove_work); 1076 1077 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 1078 1079 srp_remove_target(target); 1080 } 1081 1082 static void srp_rport_delete(struct srp_rport *rport) 1083 { 1084 struct srp_target_port *target = rport->lld_data; 1085 1086 srp_queue_remove_work(target); 1087 } 1088 1089 /** 1090 * srp_connected_ch() - number of connected channels 1091 * @target: SRP target port. 1092 */ 1093 static int srp_connected_ch(struct srp_target_port *target) 1094 { 1095 int i, c = 0; 1096 1097 for (i = 0; i < target->ch_count; i++) 1098 c += target->ch[i].connected; 1099 1100 return c; 1101 } 1102 1103 static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len, 1104 bool multich) 1105 { 1106 struct srp_target_port *target = ch->target; 1107 int ret; 1108 1109 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0); 1110 1111 ret = srp_lookup_path(ch); 1112 if (ret) 1113 goto out; 1114 1115 while (1) { 1116 init_completion(&ch->done); 1117 ret = srp_send_req(ch, max_iu_len, multich); 1118 if (ret) 1119 goto out; 1120 ret = wait_for_completion_interruptible(&ch->done); 1121 if (ret < 0) 1122 goto out; 1123 1124 /* 1125 * The CM event handling code will set status to 1126 * SRP_PORT_REDIRECT if we get a port redirect REJ 1127 * back, or SRP_DLID_REDIRECT if we get a lid/qp 1128 * redirect REJ back. 1129 */ 1130 ret = ch->status; 1131 switch (ret) { 1132 case 0: 1133 ch->connected = true; 1134 goto out; 1135 1136 case SRP_PORT_REDIRECT: 1137 ret = srp_lookup_path(ch); 1138 if (ret) 1139 goto out; 1140 break; 1141 1142 case SRP_DLID_REDIRECT: 1143 break; 1144 1145 case SRP_STALE_CONN: 1146 shost_printk(KERN_ERR, target->scsi_host, PFX 1147 "giving up on stale connection\n"); 1148 ret = -ECONNRESET; 1149 goto out; 1150 1151 default: 1152 goto out; 1153 } 1154 } 1155 1156 out: 1157 return ret <= 0 ? ret : -ENODEV; 1158 } 1159 1160 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc) 1161 { 1162 srp_handle_qp_err(cq, wc, "INV RKEY"); 1163 } 1164 1165 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch, 1166 u32 rkey) 1167 { 1168 struct ib_send_wr wr = { 1169 .opcode = IB_WR_LOCAL_INV, 1170 .next = NULL, 1171 .num_sge = 0, 1172 .send_flags = 0, 1173 .ex.invalidate_rkey = rkey, 1174 }; 1175 1176 wr.wr_cqe = &req->reg_cqe; 1177 req->reg_cqe.done = srp_inv_rkey_err_done; 1178 return ib_post_send(ch->qp, &wr, NULL); 1179 } 1180 1181 static void srp_unmap_data(struct scsi_cmnd *scmnd, 1182 struct srp_rdma_ch *ch, 1183 struct srp_request *req) 1184 { 1185 struct srp_target_port *target = ch->target; 1186 struct srp_device *dev = target->srp_host->srp_dev; 1187 struct ib_device *ibdev = dev->dev; 1188 int i, res; 1189 1190 if (!scsi_sglist(scmnd) || 1191 (scmnd->sc_data_direction != DMA_TO_DEVICE && 1192 scmnd->sc_data_direction != DMA_FROM_DEVICE)) 1193 return; 1194 1195 if (dev->use_fast_reg) { 1196 struct srp_fr_desc **pfr; 1197 1198 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { 1199 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey); 1200 if (res < 0) { 1201 shost_printk(KERN_ERR, target->scsi_host, PFX 1202 "Queueing INV WR for rkey %#x failed (%d)\n", 1203 (*pfr)->mr->rkey, res); 1204 queue_work(system_long_wq, 1205 &target->tl_err_work); 1206 } 1207 } 1208 if (req->nmdesc) 1209 srp_fr_pool_put(ch->fr_pool, req->fr_list, 1210 req->nmdesc); 1211 } 1212 1213 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd), 1214 scmnd->sc_data_direction); 1215 } 1216 1217 /** 1218 * srp_claim_req - Take ownership of the scmnd associated with a request. 1219 * @ch: SRP RDMA channel. 1220 * @req: SRP request. 1221 * @sdev: If not NULL, only take ownership for this SCSI device. 1222 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take 1223 * ownership of @req->scmnd if it equals @scmnd. 1224 * 1225 * Return value: 1226 * Either NULL or a pointer to the SCSI command the caller became owner of. 1227 */ 1228 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch, 1229 struct srp_request *req, 1230 struct scsi_device *sdev, 1231 struct scsi_cmnd *scmnd) 1232 { 1233 unsigned long flags; 1234 1235 spin_lock_irqsave(&ch->lock, flags); 1236 if (req->scmnd && 1237 (!sdev || req->scmnd->device == sdev) && 1238 (!scmnd || req->scmnd == scmnd)) { 1239 scmnd = req->scmnd; 1240 req->scmnd = NULL; 1241 } else { 1242 scmnd = NULL; 1243 } 1244 spin_unlock_irqrestore(&ch->lock, flags); 1245 1246 return scmnd; 1247 } 1248 1249 /** 1250 * srp_free_req() - Unmap data and adjust ch->req_lim. 1251 * @ch: SRP RDMA channel. 1252 * @req: Request to be freed. 1253 * @scmnd: SCSI command associated with @req. 1254 * @req_lim_delta: Amount to be added to @target->req_lim. 1255 */ 1256 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req, 1257 struct scsi_cmnd *scmnd, s32 req_lim_delta) 1258 { 1259 unsigned long flags; 1260 1261 srp_unmap_data(scmnd, ch, req); 1262 1263 spin_lock_irqsave(&ch->lock, flags); 1264 ch->req_lim += req_lim_delta; 1265 spin_unlock_irqrestore(&ch->lock, flags); 1266 } 1267 1268 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req, 1269 struct scsi_device *sdev, int result) 1270 { 1271 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL); 1272 1273 if (scmnd) { 1274 srp_free_req(ch, req, scmnd, 0); 1275 scmnd->result = result; 1276 scsi_done(scmnd); 1277 } 1278 } 1279 1280 struct srp_terminate_context { 1281 struct srp_target_port *srp_target; 1282 int scsi_result; 1283 }; 1284 1285 static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr, 1286 bool reserved) 1287 { 1288 struct srp_terminate_context *context = context_ptr; 1289 struct srp_target_port *target = context->srp_target; 1290 u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); 1291 struct srp_rdma_ch *ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 1292 struct srp_request *req = scsi_cmd_priv(scmnd); 1293 1294 srp_finish_req(ch, req, NULL, context->scsi_result); 1295 1296 return true; 1297 } 1298 1299 static void srp_terminate_io(struct srp_rport *rport) 1300 { 1301 struct srp_target_port *target = rport->lld_data; 1302 struct srp_terminate_context context = { .srp_target = target, 1303 .scsi_result = DID_TRANSPORT_FAILFAST << 16 }; 1304 1305 scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, &context); 1306 } 1307 1308 /* Calculate maximum initiator to target information unit length. */ 1309 static uint32_t srp_max_it_iu_len(int cmd_sg_cnt, bool use_imm_data, 1310 uint32_t max_it_iu_size) 1311 { 1312 uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN + 1313 sizeof(struct srp_indirect_buf) + 1314 cmd_sg_cnt * sizeof(struct srp_direct_buf); 1315 1316 if (use_imm_data) 1317 max_iu_len = max(max_iu_len, SRP_IMM_DATA_OFFSET + 1318 srp_max_imm_data); 1319 1320 if (max_it_iu_size) 1321 max_iu_len = min(max_iu_len, max_it_iu_size); 1322 1323 pr_debug("max_iu_len = %d\n", max_iu_len); 1324 1325 return max_iu_len; 1326 } 1327 1328 /* 1329 * It is up to the caller to ensure that srp_rport_reconnect() calls are 1330 * serialized and that no concurrent srp_queuecommand(), srp_abort(), 1331 * srp_reset_device() or srp_reset_host() calls will occur while this function 1332 * is in progress. One way to realize that is not to call this function 1333 * directly but to call srp_reconnect_rport() instead since that last function 1334 * serializes calls of this function via rport->mutex and also blocks 1335 * srp_queuecommand() calls before invoking this function. 1336 */ 1337 static int srp_rport_reconnect(struct srp_rport *rport) 1338 { 1339 struct srp_target_port *target = rport->lld_data; 1340 struct srp_rdma_ch *ch; 1341 uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 1342 srp_use_imm_data, 1343 target->max_it_iu_size); 1344 int i, j, ret = 0; 1345 bool multich = false; 1346 1347 srp_disconnect_target(target); 1348 1349 if (target->state == SRP_TARGET_SCANNING) 1350 return -ENODEV; 1351 1352 /* 1353 * Now get a new local CM ID so that we avoid confusing the target in 1354 * case things are really fouled up. Doing so also ensures that all CM 1355 * callbacks will have finished before a new QP is allocated. 1356 */ 1357 for (i = 0; i < target->ch_count; i++) { 1358 ch = &target->ch[i]; 1359 ret += srp_new_cm_id(ch); 1360 } 1361 { 1362 struct srp_terminate_context context = { 1363 .srp_target = target, .scsi_result = DID_RESET << 16}; 1364 1365 scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, 1366 &context); 1367 } 1368 for (i = 0; i < target->ch_count; i++) { 1369 ch = &target->ch[i]; 1370 /* 1371 * Whether or not creating a new CM ID succeeded, create a new 1372 * QP. This guarantees that all completion callback function 1373 * invocations have finished before request resetting starts. 1374 */ 1375 ret += srp_create_ch_ib(ch); 1376 1377 INIT_LIST_HEAD(&ch->free_tx); 1378 for (j = 0; j < target->queue_size; ++j) 1379 list_add(&ch->tx_ring[j]->list, &ch->free_tx); 1380 } 1381 1382 target->qp_in_error = false; 1383 1384 for (i = 0; i < target->ch_count; i++) { 1385 ch = &target->ch[i]; 1386 if (ret) 1387 break; 1388 ret = srp_connect_ch(ch, max_iu_len, multich); 1389 multich = true; 1390 } 1391 1392 if (ret == 0) 1393 shost_printk(KERN_INFO, target->scsi_host, 1394 PFX "reconnect succeeded\n"); 1395 1396 return ret; 1397 } 1398 1399 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, 1400 unsigned int dma_len, u32 rkey) 1401 { 1402 struct srp_direct_buf *desc = state->desc; 1403 1404 WARN_ON_ONCE(!dma_len); 1405 1406 desc->va = cpu_to_be64(dma_addr); 1407 desc->key = cpu_to_be32(rkey); 1408 desc->len = cpu_to_be32(dma_len); 1409 1410 state->total_len += dma_len; 1411 state->desc++; 1412 state->ndesc++; 1413 } 1414 1415 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc) 1416 { 1417 srp_handle_qp_err(cq, wc, "FAST REG"); 1418 } 1419 1420 /* 1421 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset 1422 * where to start in the first element. If sg_offset_p != NULL then 1423 * *sg_offset_p is updated to the offset in state->sg[retval] of the first 1424 * byte that has not yet been mapped. 1425 */ 1426 static int srp_map_finish_fr(struct srp_map_state *state, 1427 struct srp_request *req, 1428 struct srp_rdma_ch *ch, int sg_nents, 1429 unsigned int *sg_offset_p) 1430 { 1431 struct srp_target_port *target = ch->target; 1432 struct srp_device *dev = target->srp_host->srp_dev; 1433 struct ib_reg_wr wr; 1434 struct srp_fr_desc *desc; 1435 u32 rkey; 1436 int n, err; 1437 1438 if (state->fr.next >= state->fr.end) { 1439 shost_printk(KERN_ERR, ch->target->scsi_host, 1440 PFX "Out of MRs (mr_per_cmd = %d)\n", 1441 ch->target->mr_per_cmd); 1442 return -ENOMEM; 1443 } 1444 1445 WARN_ON_ONCE(!dev->use_fast_reg); 1446 1447 if (sg_nents == 1 && target->global_rkey) { 1448 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1449 1450 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset, 1451 sg_dma_len(state->sg) - sg_offset, 1452 target->global_rkey); 1453 if (sg_offset_p) 1454 *sg_offset_p = 0; 1455 return 1; 1456 } 1457 1458 desc = srp_fr_pool_get(ch->fr_pool); 1459 if (!desc) 1460 return -ENOMEM; 1461 1462 rkey = ib_inc_rkey(desc->mr->rkey); 1463 ib_update_fast_reg_key(desc->mr, rkey); 1464 1465 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p, 1466 dev->mr_page_size); 1467 if (unlikely(n < 0)) { 1468 srp_fr_pool_put(ch->fr_pool, &desc, 1); 1469 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n", 1470 dev_name(&req->scmnd->device->sdev_gendev), sg_nents, 1471 sg_offset_p ? *sg_offset_p : -1, n); 1472 return n; 1473 } 1474 1475 WARN_ON_ONCE(desc->mr->length == 0); 1476 1477 req->reg_cqe.done = srp_reg_mr_err_done; 1478 1479 wr.wr.next = NULL; 1480 wr.wr.opcode = IB_WR_REG_MR; 1481 wr.wr.wr_cqe = &req->reg_cqe; 1482 wr.wr.num_sge = 0; 1483 wr.wr.send_flags = 0; 1484 wr.mr = desc->mr; 1485 wr.key = desc->mr->rkey; 1486 wr.access = (IB_ACCESS_LOCAL_WRITE | 1487 IB_ACCESS_REMOTE_READ | 1488 IB_ACCESS_REMOTE_WRITE); 1489 1490 *state->fr.next++ = desc; 1491 state->nmdesc++; 1492 1493 srp_map_desc(state, desc->mr->iova, 1494 desc->mr->length, desc->mr->rkey); 1495 1496 err = ib_post_send(ch->qp, &wr.wr, NULL); 1497 if (unlikely(err)) { 1498 WARN_ON_ONCE(err == -ENOMEM); 1499 return err; 1500 } 1501 1502 return n; 1503 } 1504 1505 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1506 struct srp_request *req, struct scatterlist *scat, 1507 int count) 1508 { 1509 unsigned int sg_offset = 0; 1510 1511 state->fr.next = req->fr_list; 1512 state->fr.end = req->fr_list + ch->target->mr_per_cmd; 1513 state->sg = scat; 1514 1515 if (count == 0) 1516 return 0; 1517 1518 while (count) { 1519 int i, n; 1520 1521 n = srp_map_finish_fr(state, req, ch, count, &sg_offset); 1522 if (unlikely(n < 0)) 1523 return n; 1524 1525 count -= n; 1526 for (i = 0; i < n; i++) 1527 state->sg = sg_next(state->sg); 1528 } 1529 1530 return 0; 1531 } 1532 1533 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch, 1534 struct srp_request *req, struct scatterlist *scat, 1535 int count) 1536 { 1537 struct srp_target_port *target = ch->target; 1538 struct scatterlist *sg; 1539 int i; 1540 1541 for_each_sg(scat, sg, count, i) { 1542 srp_map_desc(state, sg_dma_address(sg), sg_dma_len(sg), 1543 target->global_rkey); 1544 } 1545 1546 return 0; 1547 } 1548 1549 /* 1550 * Register the indirect data buffer descriptor with the HCA. 1551 * 1552 * Note: since the indirect data buffer descriptor has been allocated with 1553 * kmalloc() it is guaranteed that this buffer is a physically contiguous 1554 * memory buffer. 1555 */ 1556 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req, 1557 void **next_mr, void **end_mr, u32 idb_len, 1558 __be32 *idb_rkey) 1559 { 1560 struct srp_target_port *target = ch->target; 1561 struct srp_device *dev = target->srp_host->srp_dev; 1562 struct srp_map_state state; 1563 struct srp_direct_buf idb_desc; 1564 struct scatterlist idb_sg[1]; 1565 int ret; 1566 1567 memset(&state, 0, sizeof(state)); 1568 memset(&idb_desc, 0, sizeof(idb_desc)); 1569 state.gen.next = next_mr; 1570 state.gen.end = end_mr; 1571 state.desc = &idb_desc; 1572 state.base_dma_addr = req->indirect_dma_addr; 1573 state.dma_len = idb_len; 1574 1575 if (dev->use_fast_reg) { 1576 state.sg = idb_sg; 1577 sg_init_one(idb_sg, req->indirect_desc, idb_len); 1578 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */ 1579 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1580 idb_sg->dma_length = idb_sg->length; /* hack^2 */ 1581 #endif 1582 ret = srp_map_finish_fr(&state, req, ch, 1, NULL); 1583 if (ret < 0) 1584 return ret; 1585 WARN_ON_ONCE(ret < 1); 1586 } else { 1587 return -EINVAL; 1588 } 1589 1590 *idb_rkey = idb_desc.key; 1591 1592 return 0; 1593 } 1594 1595 static void srp_check_mapping(struct srp_map_state *state, 1596 struct srp_rdma_ch *ch, struct srp_request *req, 1597 struct scatterlist *scat, int count) 1598 { 1599 struct srp_device *dev = ch->target->srp_host->srp_dev; 1600 struct srp_fr_desc **pfr; 1601 u64 desc_len = 0, mr_len = 0; 1602 int i; 1603 1604 for (i = 0; i < state->ndesc; i++) 1605 desc_len += be32_to_cpu(req->indirect_desc[i].len); 1606 if (dev->use_fast_reg) 1607 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++) 1608 mr_len += (*pfr)->mr->length; 1609 if (desc_len != scsi_bufflen(req->scmnd) || 1610 mr_len > scsi_bufflen(req->scmnd)) 1611 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n", 1612 scsi_bufflen(req->scmnd), desc_len, mr_len, 1613 state->ndesc, state->nmdesc); 1614 } 1615 1616 /** 1617 * srp_map_data() - map SCSI data buffer onto an SRP request 1618 * @scmnd: SCSI command to map 1619 * @ch: SRP RDMA channel 1620 * @req: SRP request 1621 * 1622 * Returns the length in bytes of the SRP_CMD IU or a negative value if 1623 * mapping failed. The size of any immediate data is not included in the 1624 * return value. 1625 */ 1626 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch, 1627 struct srp_request *req) 1628 { 1629 struct srp_target_port *target = ch->target; 1630 struct scatterlist *scat, *sg; 1631 struct srp_cmd *cmd = req->cmd->buf; 1632 int i, len, nents, count, ret; 1633 struct srp_device *dev; 1634 struct ib_device *ibdev; 1635 struct srp_map_state state; 1636 struct srp_indirect_buf *indirect_hdr; 1637 u64 data_len; 1638 u32 idb_len, table_len; 1639 __be32 idb_rkey; 1640 u8 fmt; 1641 1642 req->cmd->num_sge = 1; 1643 1644 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE) 1645 return sizeof(struct srp_cmd) + cmd->add_cdb_len; 1646 1647 if (scmnd->sc_data_direction != DMA_FROM_DEVICE && 1648 scmnd->sc_data_direction != DMA_TO_DEVICE) { 1649 shost_printk(KERN_WARNING, target->scsi_host, 1650 PFX "Unhandled data direction %d\n", 1651 scmnd->sc_data_direction); 1652 return -EINVAL; 1653 } 1654 1655 nents = scsi_sg_count(scmnd); 1656 scat = scsi_sglist(scmnd); 1657 data_len = scsi_bufflen(scmnd); 1658 1659 dev = target->srp_host->srp_dev; 1660 ibdev = dev->dev; 1661 1662 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction); 1663 if (unlikely(count == 0)) 1664 return -EIO; 1665 1666 if (ch->use_imm_data && 1667 count <= ch->max_imm_sge && 1668 SRP_IMM_DATA_OFFSET + data_len <= ch->max_it_iu_len && 1669 scmnd->sc_data_direction == DMA_TO_DEVICE) { 1670 struct srp_imm_buf *buf; 1671 struct ib_sge *sge = &req->cmd->sge[1]; 1672 1673 fmt = SRP_DATA_DESC_IMM; 1674 len = SRP_IMM_DATA_OFFSET; 1675 req->nmdesc = 0; 1676 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1677 buf->len = cpu_to_be32(data_len); 1678 WARN_ON_ONCE((void *)(buf + 1) > (void *)cmd + len); 1679 for_each_sg(scat, sg, count, i) { 1680 sge[i].addr = sg_dma_address(sg); 1681 sge[i].length = sg_dma_len(sg); 1682 sge[i].lkey = target->lkey; 1683 } 1684 req->cmd->num_sge += count; 1685 goto map_complete; 1686 } 1687 1688 fmt = SRP_DATA_DESC_DIRECT; 1689 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1690 sizeof(struct srp_direct_buf); 1691 1692 if (count == 1 && target->global_rkey) { 1693 /* 1694 * The midlayer only generated a single gather/scatter 1695 * entry, or DMA mapping coalesced everything to a 1696 * single entry. So a direct descriptor along with 1697 * the DMA MR suffices. 1698 */ 1699 struct srp_direct_buf *buf; 1700 1701 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1702 buf->va = cpu_to_be64(sg_dma_address(scat)); 1703 buf->key = cpu_to_be32(target->global_rkey); 1704 buf->len = cpu_to_be32(sg_dma_len(scat)); 1705 1706 req->nmdesc = 0; 1707 goto map_complete; 1708 } 1709 1710 /* 1711 * We have more than one scatter/gather entry, so build our indirect 1712 * descriptor table, trying to merge as many entries as we can. 1713 */ 1714 indirect_hdr = (void *)cmd->add_data + cmd->add_cdb_len; 1715 1716 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr, 1717 target->indirect_size, DMA_TO_DEVICE); 1718 1719 memset(&state, 0, sizeof(state)); 1720 state.desc = req->indirect_desc; 1721 if (dev->use_fast_reg) 1722 ret = srp_map_sg_fr(&state, ch, req, scat, count); 1723 else 1724 ret = srp_map_sg_dma(&state, ch, req, scat, count); 1725 req->nmdesc = state.nmdesc; 1726 if (ret < 0) 1727 goto unmap; 1728 1729 { 1730 DEFINE_DYNAMIC_DEBUG_METADATA(ddm, 1731 "Memory mapping consistency check"); 1732 if (DYNAMIC_DEBUG_BRANCH(ddm)) 1733 srp_check_mapping(&state, ch, req, scat, count); 1734 } 1735 1736 /* We've mapped the request, now pull as much of the indirect 1737 * descriptor table as we can into the command buffer. If this 1738 * target is not using an external indirect table, we are 1739 * guaranteed to fit into the command, as the SCSI layer won't 1740 * give us more S/G entries than we allow. 1741 */ 1742 if (state.ndesc == 1) { 1743 /* 1744 * Memory registration collapsed the sg-list into one entry, 1745 * so use a direct descriptor. 1746 */ 1747 struct srp_direct_buf *buf; 1748 1749 buf = (void *)cmd->add_data + cmd->add_cdb_len; 1750 *buf = req->indirect_desc[0]; 1751 goto map_complete; 1752 } 1753 1754 if (unlikely(target->cmd_sg_cnt < state.ndesc && 1755 !target->allow_ext_sg)) { 1756 shost_printk(KERN_ERR, target->scsi_host, 1757 "Could not fit S/G list into SRP_CMD\n"); 1758 ret = -EIO; 1759 goto unmap; 1760 } 1761 1762 count = min(state.ndesc, target->cmd_sg_cnt); 1763 table_len = state.ndesc * sizeof (struct srp_direct_buf); 1764 idb_len = sizeof(struct srp_indirect_buf) + table_len; 1765 1766 fmt = SRP_DATA_DESC_INDIRECT; 1767 len = sizeof(struct srp_cmd) + cmd->add_cdb_len + 1768 sizeof(struct srp_indirect_buf); 1769 len += count * sizeof (struct srp_direct_buf); 1770 1771 memcpy(indirect_hdr->desc_list, req->indirect_desc, 1772 count * sizeof (struct srp_direct_buf)); 1773 1774 if (!target->global_rkey) { 1775 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end, 1776 idb_len, &idb_rkey); 1777 if (ret < 0) 1778 goto unmap; 1779 req->nmdesc++; 1780 } else { 1781 idb_rkey = cpu_to_be32(target->global_rkey); 1782 } 1783 1784 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr); 1785 indirect_hdr->table_desc.key = idb_rkey; 1786 indirect_hdr->table_desc.len = cpu_to_be32(table_len); 1787 indirect_hdr->len = cpu_to_be32(state.total_len); 1788 1789 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1790 cmd->data_out_desc_cnt = count; 1791 else 1792 cmd->data_in_desc_cnt = count; 1793 1794 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len, 1795 DMA_TO_DEVICE); 1796 1797 map_complete: 1798 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1799 cmd->buf_fmt = fmt << 4; 1800 else 1801 cmd->buf_fmt = fmt; 1802 1803 return len; 1804 1805 unmap: 1806 srp_unmap_data(scmnd, ch, req); 1807 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size) 1808 ret = -E2BIG; 1809 return ret; 1810 } 1811 1812 /* 1813 * Return an IU and possible credit to the free pool 1814 */ 1815 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu, 1816 enum srp_iu_type iu_type) 1817 { 1818 unsigned long flags; 1819 1820 spin_lock_irqsave(&ch->lock, flags); 1821 list_add(&iu->list, &ch->free_tx); 1822 if (iu_type != SRP_IU_RSP) 1823 ++ch->req_lim; 1824 spin_unlock_irqrestore(&ch->lock, flags); 1825 } 1826 1827 /* 1828 * Must be called with ch->lock held to protect req_lim and free_tx. 1829 * If IU is not sent, it must be returned using srp_put_tx_iu(). 1830 * 1831 * Note: 1832 * An upper limit for the number of allocated information units for each 1833 * request type is: 1834 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues 1835 * more than Scsi_Host.can_queue requests. 1836 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE. 1837 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than 1838 * one unanswered SRP request to an initiator. 1839 */ 1840 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch, 1841 enum srp_iu_type iu_type) 1842 { 1843 struct srp_target_port *target = ch->target; 1844 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; 1845 struct srp_iu *iu; 1846 1847 lockdep_assert_held(&ch->lock); 1848 1849 ib_process_cq_direct(ch->send_cq, -1); 1850 1851 if (list_empty(&ch->free_tx)) 1852 return NULL; 1853 1854 /* Initiator responses to target requests do not consume credits */ 1855 if (iu_type != SRP_IU_RSP) { 1856 if (ch->req_lim <= rsv) { 1857 ++target->zero_req_lim; 1858 return NULL; 1859 } 1860 1861 --ch->req_lim; 1862 } 1863 1864 iu = list_first_entry(&ch->free_tx, struct srp_iu, list); 1865 list_del(&iu->list); 1866 return iu; 1867 } 1868 1869 /* 1870 * Note: if this function is called from inside ib_drain_sq() then it will 1871 * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE 1872 * with status IB_WC_SUCCESS then that's a bug. 1873 */ 1874 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc) 1875 { 1876 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1877 struct srp_rdma_ch *ch = cq->cq_context; 1878 1879 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1880 srp_handle_qp_err(cq, wc, "SEND"); 1881 return; 1882 } 1883 1884 lockdep_assert_held(&ch->lock); 1885 1886 list_add(&iu->list, &ch->free_tx); 1887 } 1888 1889 /** 1890 * srp_post_send() - send an SRP information unit 1891 * @ch: RDMA channel over which to send the information unit. 1892 * @iu: Information unit to send. 1893 * @len: Length of the information unit excluding immediate data. 1894 */ 1895 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len) 1896 { 1897 struct srp_target_port *target = ch->target; 1898 struct ib_send_wr wr; 1899 1900 if (WARN_ON_ONCE(iu->num_sge > SRP_MAX_SGE)) 1901 return -EINVAL; 1902 1903 iu->sge[0].addr = iu->dma; 1904 iu->sge[0].length = len; 1905 iu->sge[0].lkey = target->lkey; 1906 1907 iu->cqe.done = srp_send_done; 1908 1909 wr.next = NULL; 1910 wr.wr_cqe = &iu->cqe; 1911 wr.sg_list = &iu->sge[0]; 1912 wr.num_sge = iu->num_sge; 1913 wr.opcode = IB_WR_SEND; 1914 wr.send_flags = IB_SEND_SIGNALED; 1915 1916 return ib_post_send(ch->qp, &wr, NULL); 1917 } 1918 1919 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu) 1920 { 1921 struct srp_target_port *target = ch->target; 1922 struct ib_recv_wr wr; 1923 struct ib_sge list; 1924 1925 list.addr = iu->dma; 1926 list.length = iu->size; 1927 list.lkey = target->lkey; 1928 1929 iu->cqe.done = srp_recv_done; 1930 1931 wr.next = NULL; 1932 wr.wr_cqe = &iu->cqe; 1933 wr.sg_list = &list; 1934 wr.num_sge = 1; 1935 1936 return ib_post_recv(ch->qp, &wr, NULL); 1937 } 1938 1939 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) 1940 { 1941 struct srp_target_port *target = ch->target; 1942 struct srp_request *req; 1943 struct scsi_cmnd *scmnd; 1944 unsigned long flags; 1945 1946 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { 1947 spin_lock_irqsave(&ch->lock, flags); 1948 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1949 if (rsp->tag == ch->tsk_mgmt_tag) { 1950 ch->tsk_mgmt_status = -1; 1951 if (be32_to_cpu(rsp->resp_data_len) >= 4) 1952 ch->tsk_mgmt_status = rsp->data[3]; 1953 complete(&ch->tsk_mgmt_done); 1954 } else { 1955 shost_printk(KERN_ERR, target->scsi_host, 1956 "Received tsk mgmt response too late for tag %#llx\n", 1957 rsp->tag); 1958 } 1959 spin_unlock_irqrestore(&ch->lock, flags); 1960 } else { 1961 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag); 1962 if (scmnd) { 1963 req = scsi_cmd_priv(scmnd); 1964 scmnd = srp_claim_req(ch, req, NULL, scmnd); 1965 } else { 1966 shost_printk(KERN_ERR, target->scsi_host, 1967 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", 1968 rsp->tag, ch - target->ch, ch->qp->qp_num); 1969 1970 spin_lock_irqsave(&ch->lock, flags); 1971 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1972 spin_unlock_irqrestore(&ch->lock, flags); 1973 1974 return; 1975 } 1976 scmnd->result = rsp->status; 1977 1978 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { 1979 memcpy(scmnd->sense_buffer, rsp->data + 1980 be32_to_cpu(rsp->resp_data_len), 1981 min_t(int, be32_to_cpu(rsp->sense_data_len), 1982 SCSI_SENSE_BUFFERSIZE)); 1983 } 1984 1985 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER)) 1986 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt)); 1987 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER)) 1988 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt)); 1989 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER)) 1990 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt)); 1991 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER)) 1992 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt)); 1993 1994 srp_free_req(ch, req, scmnd, 1995 be32_to_cpu(rsp->req_lim_delta)); 1996 1997 scsi_done(scmnd); 1998 } 1999 } 2000 2001 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, 2002 void *rsp, int len) 2003 { 2004 struct srp_target_port *target = ch->target; 2005 struct ib_device *dev = target->srp_host->srp_dev->dev; 2006 unsigned long flags; 2007 struct srp_iu *iu; 2008 int err; 2009 2010 spin_lock_irqsave(&ch->lock, flags); 2011 ch->req_lim += req_delta; 2012 iu = __srp_get_tx_iu(ch, SRP_IU_RSP); 2013 spin_unlock_irqrestore(&ch->lock, flags); 2014 2015 if (!iu) { 2016 shost_printk(KERN_ERR, target->scsi_host, PFX 2017 "no IU available to send response\n"); 2018 return 1; 2019 } 2020 2021 iu->num_sge = 1; 2022 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); 2023 memcpy(iu->buf, rsp, len); 2024 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); 2025 2026 err = srp_post_send(ch, iu, len); 2027 if (err) { 2028 shost_printk(KERN_ERR, target->scsi_host, PFX 2029 "unable to post response: %d\n", err); 2030 srp_put_tx_iu(ch, iu, SRP_IU_RSP); 2031 } 2032 2033 return err; 2034 } 2035 2036 static void srp_process_cred_req(struct srp_rdma_ch *ch, 2037 struct srp_cred_req *req) 2038 { 2039 struct srp_cred_rsp rsp = { 2040 .opcode = SRP_CRED_RSP, 2041 .tag = req->tag, 2042 }; 2043 s32 delta = be32_to_cpu(req->req_lim_delta); 2044 2045 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2046 shost_printk(KERN_ERR, ch->target->scsi_host, PFX 2047 "problems processing SRP_CRED_REQ\n"); 2048 } 2049 2050 static void srp_process_aer_req(struct srp_rdma_ch *ch, 2051 struct srp_aer_req *req) 2052 { 2053 struct srp_target_port *target = ch->target; 2054 struct srp_aer_rsp rsp = { 2055 .opcode = SRP_AER_RSP, 2056 .tag = req->tag, 2057 }; 2058 s32 delta = be32_to_cpu(req->req_lim_delta); 2059 2060 shost_printk(KERN_ERR, target->scsi_host, PFX 2061 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); 2062 2063 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 2064 shost_printk(KERN_ERR, target->scsi_host, PFX 2065 "problems processing SRP_AER_REQ\n"); 2066 } 2067 2068 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) 2069 { 2070 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 2071 struct srp_rdma_ch *ch = cq->cq_context; 2072 struct srp_target_port *target = ch->target; 2073 struct ib_device *dev = target->srp_host->srp_dev->dev; 2074 int res; 2075 u8 opcode; 2076 2077 if (unlikely(wc->status != IB_WC_SUCCESS)) { 2078 srp_handle_qp_err(cq, wc, "RECV"); 2079 return; 2080 } 2081 2082 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len, 2083 DMA_FROM_DEVICE); 2084 2085 opcode = *(u8 *) iu->buf; 2086 2087 if (0) { 2088 shost_printk(KERN_ERR, target->scsi_host, 2089 PFX "recv completion, opcode 0x%02x\n", opcode); 2090 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1, 2091 iu->buf, wc->byte_len, true); 2092 } 2093 2094 switch (opcode) { 2095 case SRP_RSP: 2096 srp_process_rsp(ch, iu->buf); 2097 break; 2098 2099 case SRP_CRED_REQ: 2100 srp_process_cred_req(ch, iu->buf); 2101 break; 2102 2103 case SRP_AER_REQ: 2104 srp_process_aer_req(ch, iu->buf); 2105 break; 2106 2107 case SRP_T_LOGOUT: 2108 /* XXX Handle target logout */ 2109 shost_printk(KERN_WARNING, target->scsi_host, 2110 PFX "Got target logout request\n"); 2111 break; 2112 2113 default: 2114 shost_printk(KERN_WARNING, target->scsi_host, 2115 PFX "Unhandled SRP opcode 0x%02x\n", opcode); 2116 break; 2117 } 2118 2119 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len, 2120 DMA_FROM_DEVICE); 2121 2122 res = srp_post_recv(ch, iu); 2123 if (res != 0) 2124 shost_printk(KERN_ERR, target->scsi_host, 2125 PFX "Recv failed with error code %d\n", res); 2126 } 2127 2128 /** 2129 * srp_tl_err_work() - handle a transport layer error 2130 * @work: Work structure embedded in an SRP target port. 2131 * 2132 * Note: This function may get invoked before the rport has been created, 2133 * hence the target->rport test. 2134 */ 2135 static void srp_tl_err_work(struct work_struct *work) 2136 { 2137 struct srp_target_port *target; 2138 2139 target = container_of(work, struct srp_target_port, tl_err_work); 2140 if (target->rport) 2141 srp_start_tl_fail_timers(target->rport); 2142 } 2143 2144 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 2145 const char *opname) 2146 { 2147 struct srp_rdma_ch *ch = cq->cq_context; 2148 struct srp_target_port *target = ch->target; 2149 2150 if (ch->connected && !target->qp_in_error) { 2151 shost_printk(KERN_ERR, target->scsi_host, 2152 PFX "failed %s status %s (%d) for CQE %p\n", 2153 opname, ib_wc_status_msg(wc->status), wc->status, 2154 wc->wr_cqe); 2155 queue_work(system_long_wq, &target->tl_err_work); 2156 } 2157 target->qp_in_error = true; 2158 } 2159 2160 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) 2161 { 2162 struct request *rq = scsi_cmd_to_rq(scmnd); 2163 struct srp_target_port *target = host_to_target(shost); 2164 struct srp_rdma_ch *ch; 2165 struct srp_request *req = scsi_cmd_priv(scmnd); 2166 struct srp_iu *iu; 2167 struct srp_cmd *cmd; 2168 struct ib_device *dev; 2169 unsigned long flags; 2170 u32 tag; 2171 int len, ret; 2172 2173 scmnd->result = srp_chkready(target->rport); 2174 if (unlikely(scmnd->result)) 2175 goto err; 2176 2177 WARN_ON_ONCE(rq->tag < 0); 2178 tag = blk_mq_unique_tag(rq); 2179 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 2180 2181 spin_lock_irqsave(&ch->lock, flags); 2182 iu = __srp_get_tx_iu(ch, SRP_IU_CMD); 2183 spin_unlock_irqrestore(&ch->lock, flags); 2184 2185 if (!iu) 2186 goto err; 2187 2188 dev = target->srp_host->srp_dev->dev; 2189 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len, 2190 DMA_TO_DEVICE); 2191 2192 cmd = iu->buf; 2193 memset(cmd, 0, sizeof *cmd); 2194 2195 cmd->opcode = SRP_CMD; 2196 int_to_scsilun(scmnd->device->lun, &cmd->lun); 2197 cmd->tag = tag; 2198 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len); 2199 if (unlikely(scmnd->cmd_len > sizeof(cmd->cdb))) { 2200 cmd->add_cdb_len = round_up(scmnd->cmd_len - sizeof(cmd->cdb), 2201 4); 2202 if (WARN_ON_ONCE(cmd->add_cdb_len > SRP_MAX_ADD_CDB_LEN)) 2203 goto err_iu; 2204 } 2205 2206 req->scmnd = scmnd; 2207 req->cmd = iu; 2208 2209 len = srp_map_data(scmnd, ch, req); 2210 if (len < 0) { 2211 shost_printk(KERN_ERR, target->scsi_host, 2212 PFX "Failed to map data (%d)\n", len); 2213 /* 2214 * If we ran out of memory descriptors (-ENOMEM) because an 2215 * application is queuing many requests with more than 2216 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer 2217 * to reduce queue depth temporarily. 2218 */ 2219 scmnd->result = len == -ENOMEM ? 2220 DID_OK << 16 | SAM_STAT_TASK_SET_FULL : DID_ERROR << 16; 2221 goto err_iu; 2222 } 2223 2224 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len, 2225 DMA_TO_DEVICE); 2226 2227 if (srp_post_send(ch, iu, len)) { 2228 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); 2229 scmnd->result = DID_ERROR << 16; 2230 goto err_unmap; 2231 } 2232 2233 return 0; 2234 2235 err_unmap: 2236 srp_unmap_data(scmnd, ch, req); 2237 2238 err_iu: 2239 srp_put_tx_iu(ch, iu, SRP_IU_CMD); 2240 2241 /* 2242 * Avoid that the loops that iterate over the request ring can 2243 * encounter a dangling SCSI command pointer. 2244 */ 2245 req->scmnd = NULL; 2246 2247 err: 2248 if (scmnd->result) { 2249 scsi_done(scmnd); 2250 ret = 0; 2251 } else { 2252 ret = SCSI_MLQUEUE_HOST_BUSY; 2253 } 2254 2255 return ret; 2256 } 2257 2258 /* 2259 * Note: the resources allocated in this function are freed in 2260 * srp_free_ch_ib(). 2261 */ 2262 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch) 2263 { 2264 struct srp_target_port *target = ch->target; 2265 int i; 2266 2267 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring), 2268 GFP_KERNEL); 2269 if (!ch->rx_ring) 2270 goto err_no_ring; 2271 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring), 2272 GFP_KERNEL); 2273 if (!ch->tx_ring) 2274 goto err_no_ring; 2275 2276 for (i = 0; i < target->queue_size; ++i) { 2277 ch->rx_ring[i] = srp_alloc_iu(target->srp_host, 2278 ch->max_ti_iu_len, 2279 GFP_KERNEL, DMA_FROM_DEVICE); 2280 if (!ch->rx_ring[i]) 2281 goto err; 2282 } 2283 2284 for (i = 0; i < target->queue_size; ++i) { 2285 ch->tx_ring[i] = srp_alloc_iu(target->srp_host, 2286 ch->max_it_iu_len, 2287 GFP_KERNEL, DMA_TO_DEVICE); 2288 if (!ch->tx_ring[i]) 2289 goto err; 2290 2291 list_add(&ch->tx_ring[i]->list, &ch->free_tx); 2292 } 2293 2294 return 0; 2295 2296 err: 2297 for (i = 0; i < target->queue_size; ++i) { 2298 srp_free_iu(target->srp_host, ch->rx_ring[i]); 2299 srp_free_iu(target->srp_host, ch->tx_ring[i]); 2300 } 2301 2302 2303 err_no_ring: 2304 kfree(ch->tx_ring); 2305 ch->tx_ring = NULL; 2306 kfree(ch->rx_ring); 2307 ch->rx_ring = NULL; 2308 2309 return -ENOMEM; 2310 } 2311 2312 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) 2313 { 2314 uint64_t T_tr_ns, max_compl_time_ms; 2315 uint32_t rq_tmo_jiffies; 2316 2317 /* 2318 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair, 2319 * table 91), both the QP timeout and the retry count have to be set 2320 * for RC QP's during the RTR to RTS transition. 2321 */ 2322 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) != 2323 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)); 2324 2325 /* 2326 * Set target->rq_tmo_jiffies to one second more than the largest time 2327 * it can take before an error completion is generated. See also 2328 * C9-140..142 in the IBTA spec for more information about how to 2329 * convert the QP Local ACK Timeout value to nanoseconds. 2330 */ 2331 T_tr_ns = 4096 * (1ULL << qp_attr->timeout); 2332 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns; 2333 do_div(max_compl_time_ms, NSEC_PER_MSEC); 2334 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000); 2335 2336 return rq_tmo_jiffies; 2337 } 2338 2339 static void srp_cm_rep_handler(struct ib_cm_id *cm_id, 2340 const struct srp_login_rsp *lrsp, 2341 struct srp_rdma_ch *ch) 2342 { 2343 struct srp_target_port *target = ch->target; 2344 struct ib_qp_attr *qp_attr = NULL; 2345 int attr_mask = 0; 2346 int ret = 0; 2347 int i; 2348 2349 if (lrsp->opcode == SRP_LOGIN_RSP) { 2350 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); 2351 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); 2352 ch->use_imm_data = srp_use_imm_data && 2353 (lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP); 2354 ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 2355 ch->use_imm_data, 2356 target->max_it_iu_size); 2357 WARN_ON_ONCE(ch->max_it_iu_len > 2358 be32_to_cpu(lrsp->max_it_iu_len)); 2359 2360 if (ch->use_imm_data) 2361 shost_printk(KERN_DEBUG, target->scsi_host, 2362 PFX "using immediate data\n"); 2363 2364 /* 2365 * Reserve credits for task management so we don't 2366 * bounce requests back to the SCSI mid-layer. 2367 */ 2368 target->scsi_host->can_queue 2369 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE, 2370 target->scsi_host->can_queue); 2371 target->scsi_host->cmd_per_lun 2372 = min_t(int, target->scsi_host->can_queue, 2373 target->scsi_host->cmd_per_lun); 2374 } else { 2375 shost_printk(KERN_WARNING, target->scsi_host, 2376 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode); 2377 ret = -ECONNRESET; 2378 goto error; 2379 } 2380 2381 if (!ch->rx_ring) { 2382 ret = srp_alloc_iu_bufs(ch); 2383 if (ret) 2384 goto error; 2385 } 2386 2387 for (i = 0; i < target->queue_size; i++) { 2388 struct srp_iu *iu = ch->rx_ring[i]; 2389 2390 ret = srp_post_recv(ch, iu); 2391 if (ret) 2392 goto error; 2393 } 2394 2395 if (!target->using_rdma_cm) { 2396 ret = -ENOMEM; 2397 qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL); 2398 if (!qp_attr) 2399 goto error; 2400 2401 qp_attr->qp_state = IB_QPS_RTR; 2402 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2403 if (ret) 2404 goto error_free; 2405 2406 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2407 if (ret) 2408 goto error_free; 2409 2410 qp_attr->qp_state = IB_QPS_RTS; 2411 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2412 if (ret) 2413 goto error_free; 2414 2415 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); 2416 2417 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2418 if (ret) 2419 goto error_free; 2420 2421 ret = ib_send_cm_rtu(cm_id, NULL, 0); 2422 } 2423 2424 error_free: 2425 kfree(qp_attr); 2426 2427 error: 2428 ch->status = ret; 2429 } 2430 2431 static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id, 2432 const struct ib_cm_event *event, 2433 struct srp_rdma_ch *ch) 2434 { 2435 struct srp_target_port *target = ch->target; 2436 struct Scsi_Host *shost = target->scsi_host; 2437 struct ib_class_port_info *cpi; 2438 int opcode; 2439 u16 dlid; 2440 2441 switch (event->param.rej_rcvd.reason) { 2442 case IB_CM_REJ_PORT_CM_REDIRECT: 2443 cpi = event->param.rej_rcvd.ari; 2444 dlid = be16_to_cpu(cpi->redirect_lid); 2445 sa_path_set_dlid(&ch->ib_cm.path, dlid); 2446 ch->ib_cm.path.pkey = cpi->redirect_pkey; 2447 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; 2448 memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16); 2449 2450 ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; 2451 break; 2452 2453 case IB_CM_REJ_PORT_REDIRECT: 2454 if (srp_target_is_topspin(target)) { 2455 union ib_gid *dgid = &ch->ib_cm.path.dgid; 2456 2457 /* 2458 * Topspin/Cisco SRP gateways incorrectly send 2459 * reject reason code 25 when they mean 24 2460 * (port redirect). 2461 */ 2462 memcpy(dgid->raw, event->param.rej_rcvd.ari, 16); 2463 2464 shost_printk(KERN_DEBUG, shost, 2465 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", 2466 be64_to_cpu(dgid->global.subnet_prefix), 2467 be64_to_cpu(dgid->global.interface_id)); 2468 2469 ch->status = SRP_PORT_REDIRECT; 2470 } else { 2471 shost_printk(KERN_WARNING, shost, 2472 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); 2473 ch->status = -ECONNRESET; 2474 } 2475 break; 2476 2477 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2478 shost_printk(KERN_WARNING, shost, 2479 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2480 ch->status = -ECONNRESET; 2481 break; 2482 2483 case IB_CM_REJ_CONSUMER_DEFINED: 2484 opcode = *(u8 *) event->private_data; 2485 if (opcode == SRP_LOGIN_REJ) { 2486 struct srp_login_rej *rej = event->private_data; 2487 u32 reason = be32_to_cpu(rej->reason); 2488 2489 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2490 shost_printk(KERN_WARNING, shost, 2491 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2492 else 2493 shost_printk(KERN_WARNING, shost, PFX 2494 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n", 2495 target->sgid.raw, 2496 target->ib_cm.orig_dgid.raw, 2497 reason); 2498 } else 2499 shost_printk(KERN_WARNING, shost, 2500 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED," 2501 " opcode 0x%02x\n", opcode); 2502 ch->status = -ECONNRESET; 2503 break; 2504 2505 case IB_CM_REJ_STALE_CONN: 2506 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); 2507 ch->status = SRP_STALE_CONN; 2508 break; 2509 2510 default: 2511 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2512 event->param.rej_rcvd.reason); 2513 ch->status = -ECONNRESET; 2514 } 2515 } 2516 2517 static int srp_ib_cm_handler(struct ib_cm_id *cm_id, 2518 const struct ib_cm_event *event) 2519 { 2520 struct srp_rdma_ch *ch = cm_id->context; 2521 struct srp_target_port *target = ch->target; 2522 int comp = 0; 2523 2524 switch (event->event) { 2525 case IB_CM_REQ_ERROR: 2526 shost_printk(KERN_DEBUG, target->scsi_host, 2527 PFX "Sending CM REQ failed\n"); 2528 comp = 1; 2529 ch->status = -ECONNRESET; 2530 break; 2531 2532 case IB_CM_REP_RECEIVED: 2533 comp = 1; 2534 srp_cm_rep_handler(cm_id, event->private_data, ch); 2535 break; 2536 2537 case IB_CM_REJ_RECEIVED: 2538 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2539 comp = 1; 2540 2541 srp_ib_cm_rej_handler(cm_id, event, ch); 2542 break; 2543 2544 case IB_CM_DREQ_RECEIVED: 2545 shost_printk(KERN_WARNING, target->scsi_host, 2546 PFX "DREQ received - connection closed\n"); 2547 ch->connected = false; 2548 if (ib_send_cm_drep(cm_id, NULL, 0)) 2549 shost_printk(KERN_ERR, target->scsi_host, 2550 PFX "Sending CM DREP failed\n"); 2551 queue_work(system_long_wq, &target->tl_err_work); 2552 break; 2553 2554 case IB_CM_TIMEWAIT_EXIT: 2555 shost_printk(KERN_ERR, target->scsi_host, 2556 PFX "connection closed\n"); 2557 comp = 1; 2558 2559 ch->status = 0; 2560 break; 2561 2562 case IB_CM_MRA_RECEIVED: 2563 case IB_CM_DREQ_ERROR: 2564 case IB_CM_DREP_RECEIVED: 2565 break; 2566 2567 default: 2568 shost_printk(KERN_WARNING, target->scsi_host, 2569 PFX "Unhandled CM event %d\n", event->event); 2570 break; 2571 } 2572 2573 if (comp) 2574 complete(&ch->done); 2575 2576 return 0; 2577 } 2578 2579 static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch, 2580 struct rdma_cm_event *event) 2581 { 2582 struct srp_target_port *target = ch->target; 2583 struct Scsi_Host *shost = target->scsi_host; 2584 int opcode; 2585 2586 switch (event->status) { 2587 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2588 shost_printk(KERN_WARNING, shost, 2589 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2590 ch->status = -ECONNRESET; 2591 break; 2592 2593 case IB_CM_REJ_CONSUMER_DEFINED: 2594 opcode = *(u8 *) event->param.conn.private_data; 2595 if (opcode == SRP_LOGIN_REJ) { 2596 struct srp_login_rej *rej = 2597 (struct srp_login_rej *) 2598 event->param.conn.private_data; 2599 u32 reason = be32_to_cpu(rej->reason); 2600 2601 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2602 shost_printk(KERN_WARNING, shost, 2603 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2604 else 2605 shost_printk(KERN_WARNING, shost, 2606 PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason); 2607 } else { 2608 shost_printk(KERN_WARNING, shost, 2609 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n", 2610 opcode); 2611 } 2612 ch->status = -ECONNRESET; 2613 break; 2614 2615 case IB_CM_REJ_STALE_CONN: 2616 shost_printk(KERN_WARNING, shost, 2617 " REJ reason: stale connection\n"); 2618 ch->status = SRP_STALE_CONN; 2619 break; 2620 2621 default: 2622 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2623 event->status); 2624 ch->status = -ECONNRESET; 2625 break; 2626 } 2627 } 2628 2629 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id, 2630 struct rdma_cm_event *event) 2631 { 2632 struct srp_rdma_ch *ch = cm_id->context; 2633 struct srp_target_port *target = ch->target; 2634 int comp = 0; 2635 2636 switch (event->event) { 2637 case RDMA_CM_EVENT_ADDR_RESOLVED: 2638 ch->status = 0; 2639 comp = 1; 2640 break; 2641 2642 case RDMA_CM_EVENT_ADDR_ERROR: 2643 ch->status = -ENXIO; 2644 comp = 1; 2645 break; 2646 2647 case RDMA_CM_EVENT_ROUTE_RESOLVED: 2648 ch->status = 0; 2649 comp = 1; 2650 break; 2651 2652 case RDMA_CM_EVENT_ROUTE_ERROR: 2653 case RDMA_CM_EVENT_UNREACHABLE: 2654 ch->status = -EHOSTUNREACH; 2655 comp = 1; 2656 break; 2657 2658 case RDMA_CM_EVENT_CONNECT_ERROR: 2659 shost_printk(KERN_DEBUG, target->scsi_host, 2660 PFX "Sending CM REQ failed\n"); 2661 comp = 1; 2662 ch->status = -ECONNRESET; 2663 break; 2664 2665 case RDMA_CM_EVENT_ESTABLISHED: 2666 comp = 1; 2667 srp_cm_rep_handler(NULL, event->param.conn.private_data, ch); 2668 break; 2669 2670 case RDMA_CM_EVENT_REJECTED: 2671 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2672 comp = 1; 2673 2674 srp_rdma_cm_rej_handler(ch, event); 2675 break; 2676 2677 case RDMA_CM_EVENT_DISCONNECTED: 2678 if (ch->connected) { 2679 shost_printk(KERN_WARNING, target->scsi_host, 2680 PFX "received DREQ\n"); 2681 rdma_disconnect(ch->rdma_cm.cm_id); 2682 comp = 1; 2683 ch->status = 0; 2684 queue_work(system_long_wq, &target->tl_err_work); 2685 } 2686 break; 2687 2688 case RDMA_CM_EVENT_TIMEWAIT_EXIT: 2689 shost_printk(KERN_ERR, target->scsi_host, 2690 PFX "connection closed\n"); 2691 2692 comp = 1; 2693 ch->status = 0; 2694 break; 2695 2696 default: 2697 shost_printk(KERN_WARNING, target->scsi_host, 2698 PFX "Unhandled CM event %d\n", event->event); 2699 break; 2700 } 2701 2702 if (comp) 2703 complete(&ch->done); 2704 2705 return 0; 2706 } 2707 2708 /** 2709 * srp_change_queue_depth - setting device queue depth 2710 * @sdev: scsi device struct 2711 * @qdepth: requested queue depth 2712 * 2713 * Returns queue depth. 2714 */ 2715 static int 2716 srp_change_queue_depth(struct scsi_device *sdev, int qdepth) 2717 { 2718 if (!sdev->tagged_supported) 2719 qdepth = 1; 2720 return scsi_change_queue_depth(sdev, qdepth); 2721 } 2722 2723 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, 2724 u8 func, u8 *status) 2725 { 2726 struct srp_target_port *target = ch->target; 2727 struct srp_rport *rport = target->rport; 2728 struct ib_device *dev = target->srp_host->srp_dev->dev; 2729 struct srp_iu *iu; 2730 struct srp_tsk_mgmt *tsk_mgmt; 2731 int res; 2732 2733 if (!ch->connected || target->qp_in_error) 2734 return -1; 2735 2736 /* 2737 * Lock the rport mutex to avoid that srp_create_ch_ib() is 2738 * invoked while a task management function is being sent. 2739 */ 2740 mutex_lock(&rport->mutex); 2741 spin_lock_irq(&ch->lock); 2742 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT); 2743 spin_unlock_irq(&ch->lock); 2744 2745 if (!iu) { 2746 mutex_unlock(&rport->mutex); 2747 2748 return -1; 2749 } 2750 2751 iu->num_sge = 1; 2752 2753 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, 2754 DMA_TO_DEVICE); 2755 tsk_mgmt = iu->buf; 2756 memset(tsk_mgmt, 0, sizeof *tsk_mgmt); 2757 2758 tsk_mgmt->opcode = SRP_TSK_MGMT; 2759 int_to_scsilun(lun, &tsk_mgmt->lun); 2760 tsk_mgmt->tsk_mgmt_func = func; 2761 tsk_mgmt->task_tag = req_tag; 2762 2763 spin_lock_irq(&ch->lock); 2764 ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT; 2765 tsk_mgmt->tag = ch->tsk_mgmt_tag; 2766 spin_unlock_irq(&ch->lock); 2767 2768 init_completion(&ch->tsk_mgmt_done); 2769 2770 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, 2771 DMA_TO_DEVICE); 2772 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) { 2773 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT); 2774 mutex_unlock(&rport->mutex); 2775 2776 return -1; 2777 } 2778 res = wait_for_completion_timeout(&ch->tsk_mgmt_done, 2779 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS)); 2780 if (res > 0 && status) 2781 *status = ch->tsk_mgmt_status; 2782 mutex_unlock(&rport->mutex); 2783 2784 WARN_ON_ONCE(res < 0); 2785 2786 return res > 0 ? 0 : -1; 2787 } 2788 2789 static int srp_abort(struct scsi_cmnd *scmnd) 2790 { 2791 struct srp_target_port *target = host_to_target(scmnd->device->host); 2792 struct srp_request *req = (struct srp_request *) scmnd->host_scribble; 2793 u32 tag; 2794 u16 ch_idx; 2795 struct srp_rdma_ch *ch; 2796 int ret; 2797 2798 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); 2799 2800 if (!req) 2801 return SUCCESS; 2802 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd)); 2803 ch_idx = blk_mq_unique_tag_to_hwq(tag); 2804 if (WARN_ON_ONCE(ch_idx >= target->ch_count)) 2805 return SUCCESS; 2806 ch = &target->ch[ch_idx]; 2807 if (!srp_claim_req(ch, req, NULL, scmnd)) 2808 return SUCCESS; 2809 shost_printk(KERN_ERR, target->scsi_host, 2810 "Sending SRP abort for tag %#x\n", tag); 2811 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun, 2812 SRP_TSK_ABORT_TASK, NULL) == 0) 2813 ret = SUCCESS; 2814 else if (target->rport->state == SRP_RPORT_LOST) 2815 ret = FAST_IO_FAIL; 2816 else 2817 ret = FAILED; 2818 if (ret == SUCCESS) { 2819 srp_free_req(ch, req, scmnd, 0); 2820 scmnd->result = DID_ABORT << 16; 2821 scsi_done(scmnd); 2822 } 2823 2824 return ret; 2825 } 2826 2827 static int srp_reset_device(struct scsi_cmnd *scmnd) 2828 { 2829 struct srp_target_port *target = host_to_target(scmnd->device->host); 2830 struct srp_rdma_ch *ch; 2831 u8 status; 2832 2833 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); 2834 2835 ch = &target->ch[0]; 2836 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun, 2837 SRP_TSK_LUN_RESET, &status)) 2838 return FAILED; 2839 if (status) 2840 return FAILED; 2841 2842 return SUCCESS; 2843 } 2844 2845 static int srp_reset_host(struct scsi_cmnd *scmnd) 2846 { 2847 struct srp_target_port *target = host_to_target(scmnd->device->host); 2848 2849 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n"); 2850 2851 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED; 2852 } 2853 2854 static int srp_target_alloc(struct scsi_target *starget) 2855 { 2856 struct Scsi_Host *shost = dev_to_shost(starget->dev.parent); 2857 struct srp_target_port *target = host_to_target(shost); 2858 2859 if (target->target_can_queue) 2860 starget->can_queue = target->target_can_queue; 2861 return 0; 2862 } 2863 2864 static int srp_slave_configure(struct scsi_device *sdev) 2865 { 2866 struct Scsi_Host *shost = sdev->host; 2867 struct srp_target_port *target = host_to_target(shost); 2868 struct request_queue *q = sdev->request_queue; 2869 unsigned long timeout; 2870 2871 if (sdev->type == TYPE_DISK) { 2872 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies); 2873 blk_queue_rq_timeout(q, timeout); 2874 } 2875 2876 return 0; 2877 } 2878 2879 static ssize_t id_ext_show(struct device *dev, struct device_attribute *attr, 2880 char *buf) 2881 { 2882 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2883 2884 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->id_ext)); 2885 } 2886 2887 static DEVICE_ATTR_RO(id_ext); 2888 2889 static ssize_t ioc_guid_show(struct device *dev, struct device_attribute *attr, 2890 char *buf) 2891 { 2892 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2893 2894 return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid)); 2895 } 2896 2897 static DEVICE_ATTR_RO(ioc_guid); 2898 2899 static ssize_t service_id_show(struct device *dev, 2900 struct device_attribute *attr, char *buf) 2901 { 2902 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2903 2904 if (target->using_rdma_cm) 2905 return -ENOENT; 2906 return sysfs_emit(buf, "0x%016llx\n", 2907 be64_to_cpu(target->ib_cm.service_id)); 2908 } 2909 2910 static DEVICE_ATTR_RO(service_id); 2911 2912 static ssize_t pkey_show(struct device *dev, struct device_attribute *attr, 2913 char *buf) 2914 { 2915 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2916 2917 if (target->using_rdma_cm) 2918 return -ENOENT; 2919 2920 return sysfs_emit(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey)); 2921 } 2922 2923 static DEVICE_ATTR_RO(pkey); 2924 2925 static ssize_t sgid_show(struct device *dev, struct device_attribute *attr, 2926 char *buf) 2927 { 2928 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2929 2930 return sysfs_emit(buf, "%pI6\n", target->sgid.raw); 2931 } 2932 2933 static DEVICE_ATTR_RO(sgid); 2934 2935 static ssize_t dgid_show(struct device *dev, struct device_attribute *attr, 2936 char *buf) 2937 { 2938 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2939 struct srp_rdma_ch *ch = &target->ch[0]; 2940 2941 if (target->using_rdma_cm) 2942 return -ENOENT; 2943 2944 return sysfs_emit(buf, "%pI6\n", ch->ib_cm.path.dgid.raw); 2945 } 2946 2947 static DEVICE_ATTR_RO(dgid); 2948 2949 static ssize_t orig_dgid_show(struct device *dev, struct device_attribute *attr, 2950 char *buf) 2951 { 2952 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2953 2954 if (target->using_rdma_cm) 2955 return -ENOENT; 2956 2957 return sysfs_emit(buf, "%pI6\n", target->ib_cm.orig_dgid.raw); 2958 } 2959 2960 static DEVICE_ATTR_RO(orig_dgid); 2961 2962 static ssize_t req_lim_show(struct device *dev, struct device_attribute *attr, 2963 char *buf) 2964 { 2965 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2966 struct srp_rdma_ch *ch; 2967 int i, req_lim = INT_MAX; 2968 2969 for (i = 0; i < target->ch_count; i++) { 2970 ch = &target->ch[i]; 2971 req_lim = min(req_lim, ch->req_lim); 2972 } 2973 2974 return sysfs_emit(buf, "%d\n", req_lim); 2975 } 2976 2977 static DEVICE_ATTR_RO(req_lim); 2978 2979 static ssize_t zero_req_lim_show(struct device *dev, 2980 struct device_attribute *attr, char *buf) 2981 { 2982 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2983 2984 return sysfs_emit(buf, "%d\n", target->zero_req_lim); 2985 } 2986 2987 static DEVICE_ATTR_RO(zero_req_lim); 2988 2989 static ssize_t local_ib_port_show(struct device *dev, 2990 struct device_attribute *attr, char *buf) 2991 { 2992 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2993 2994 return sysfs_emit(buf, "%d\n", target->srp_host->port); 2995 } 2996 2997 static DEVICE_ATTR_RO(local_ib_port); 2998 2999 static ssize_t local_ib_device_show(struct device *dev, 3000 struct device_attribute *attr, char *buf) 3001 { 3002 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3003 3004 return sysfs_emit(buf, "%s\n", 3005 dev_name(&target->srp_host->srp_dev->dev->dev)); 3006 } 3007 3008 static DEVICE_ATTR_RO(local_ib_device); 3009 3010 static ssize_t ch_count_show(struct device *dev, struct device_attribute *attr, 3011 char *buf) 3012 { 3013 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3014 3015 return sysfs_emit(buf, "%d\n", target->ch_count); 3016 } 3017 3018 static DEVICE_ATTR_RO(ch_count); 3019 3020 static ssize_t comp_vector_show(struct device *dev, 3021 struct device_attribute *attr, char *buf) 3022 { 3023 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3024 3025 return sysfs_emit(buf, "%d\n", target->comp_vector); 3026 } 3027 3028 static DEVICE_ATTR_RO(comp_vector); 3029 3030 static ssize_t tl_retry_count_show(struct device *dev, 3031 struct device_attribute *attr, char *buf) 3032 { 3033 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3034 3035 return sysfs_emit(buf, "%d\n", target->tl_retry_count); 3036 } 3037 3038 static DEVICE_ATTR_RO(tl_retry_count); 3039 3040 static ssize_t cmd_sg_entries_show(struct device *dev, 3041 struct device_attribute *attr, char *buf) 3042 { 3043 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3044 3045 return sysfs_emit(buf, "%u\n", target->cmd_sg_cnt); 3046 } 3047 3048 static DEVICE_ATTR_RO(cmd_sg_entries); 3049 3050 static ssize_t allow_ext_sg_show(struct device *dev, 3051 struct device_attribute *attr, char *buf) 3052 { 3053 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 3054 3055 return sysfs_emit(buf, "%s\n", target->allow_ext_sg ? "true" : "false"); 3056 } 3057 3058 static DEVICE_ATTR_RO(allow_ext_sg); 3059 3060 static struct attribute *srp_host_attrs[] = { 3061 &dev_attr_id_ext.attr, 3062 &dev_attr_ioc_guid.attr, 3063 &dev_attr_service_id.attr, 3064 &dev_attr_pkey.attr, 3065 &dev_attr_sgid.attr, 3066 &dev_attr_dgid.attr, 3067 &dev_attr_orig_dgid.attr, 3068 &dev_attr_req_lim.attr, 3069 &dev_attr_zero_req_lim.attr, 3070 &dev_attr_local_ib_port.attr, 3071 &dev_attr_local_ib_device.attr, 3072 &dev_attr_ch_count.attr, 3073 &dev_attr_comp_vector.attr, 3074 &dev_attr_tl_retry_count.attr, 3075 &dev_attr_cmd_sg_entries.attr, 3076 &dev_attr_allow_ext_sg.attr, 3077 NULL 3078 }; 3079 3080 ATTRIBUTE_GROUPS(srp_host); 3081 3082 static struct scsi_host_template srp_template = { 3083 .module = THIS_MODULE, 3084 .name = "InfiniBand SRP initiator", 3085 .proc_name = DRV_NAME, 3086 .target_alloc = srp_target_alloc, 3087 .slave_configure = srp_slave_configure, 3088 .info = srp_target_info, 3089 .init_cmd_priv = srp_init_cmd_priv, 3090 .exit_cmd_priv = srp_exit_cmd_priv, 3091 .queuecommand = srp_queuecommand, 3092 .change_queue_depth = srp_change_queue_depth, 3093 .eh_timed_out = srp_timed_out, 3094 .eh_abort_handler = srp_abort, 3095 .eh_device_reset_handler = srp_reset_device, 3096 .eh_host_reset_handler = srp_reset_host, 3097 .skip_settle_delay = true, 3098 .sg_tablesize = SRP_DEF_SG_TABLESIZE, 3099 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE, 3100 .this_id = -1, 3101 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE, 3102 .shost_groups = srp_host_groups, 3103 .track_queue_depth = 1, 3104 .cmd_size = sizeof(struct srp_request), 3105 }; 3106 3107 static int srp_sdev_count(struct Scsi_Host *host) 3108 { 3109 struct scsi_device *sdev; 3110 int c = 0; 3111 3112 shost_for_each_device(sdev, host) 3113 c++; 3114 3115 return c; 3116 } 3117 3118 /* 3119 * Return values: 3120 * < 0 upon failure. Caller is responsible for SRP target port cleanup. 3121 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port 3122 * removal has been scheduled. 3123 * 0 and target->state != SRP_TARGET_REMOVED upon success. 3124 */ 3125 static int srp_add_target(struct srp_host *host, struct srp_target_port *target) 3126 { 3127 struct srp_rport_identifiers ids; 3128 struct srp_rport *rport; 3129 3130 target->state = SRP_TARGET_SCANNING; 3131 sprintf(target->target_name, "SRP.T10:%016llX", 3132 be64_to_cpu(target->id_ext)); 3133 3134 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent)) 3135 return -ENODEV; 3136 3137 memcpy(ids.port_id, &target->id_ext, 8); 3138 memcpy(ids.port_id + 8, &target->ioc_guid, 8); 3139 ids.roles = SRP_RPORT_ROLE_TARGET; 3140 rport = srp_rport_add(target->scsi_host, &ids); 3141 if (IS_ERR(rport)) { 3142 scsi_remove_host(target->scsi_host); 3143 return PTR_ERR(rport); 3144 } 3145 3146 rport->lld_data = target; 3147 target->rport = rport; 3148 3149 spin_lock(&host->target_lock); 3150 list_add_tail(&target->list, &host->target_list); 3151 spin_unlock(&host->target_lock); 3152 3153 scsi_scan_target(&target->scsi_host->shost_gendev, 3154 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); 3155 3156 if (srp_connected_ch(target) < target->ch_count || 3157 target->qp_in_error) { 3158 shost_printk(KERN_INFO, target->scsi_host, 3159 PFX "SCSI scan failed - removing SCSI host\n"); 3160 srp_queue_remove_work(target); 3161 goto out; 3162 } 3163 3164 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n", 3165 dev_name(&target->scsi_host->shost_gendev), 3166 srp_sdev_count(target->scsi_host)); 3167 3168 spin_lock_irq(&target->lock); 3169 if (target->state == SRP_TARGET_SCANNING) 3170 target->state = SRP_TARGET_LIVE; 3171 spin_unlock_irq(&target->lock); 3172 3173 out: 3174 return 0; 3175 } 3176 3177 static void srp_release_dev(struct device *dev) 3178 { 3179 struct srp_host *host = 3180 container_of(dev, struct srp_host, dev); 3181 3182 complete(&host->released); 3183 } 3184 3185 static struct class srp_class = { 3186 .name = "infiniband_srp", 3187 .dev_release = srp_release_dev 3188 }; 3189 3190 /** 3191 * srp_conn_unique() - check whether the connection to a target is unique 3192 * @host: SRP host. 3193 * @target: SRP target port. 3194 */ 3195 static bool srp_conn_unique(struct srp_host *host, 3196 struct srp_target_port *target) 3197 { 3198 struct srp_target_port *t; 3199 bool ret = false; 3200 3201 if (target->state == SRP_TARGET_REMOVED) 3202 goto out; 3203 3204 ret = true; 3205 3206 spin_lock(&host->target_lock); 3207 list_for_each_entry(t, &host->target_list, list) { 3208 if (t != target && 3209 target->id_ext == t->id_ext && 3210 target->ioc_guid == t->ioc_guid && 3211 target->initiator_ext == t->initiator_ext) { 3212 ret = false; 3213 break; 3214 } 3215 } 3216 spin_unlock(&host->target_lock); 3217 3218 out: 3219 return ret; 3220 } 3221 3222 /* 3223 * Target ports are added by writing 3224 * 3225 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>, 3226 * pkey=<P_Key>,service_id=<service ID> 3227 * or 3228 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>, 3229 * [src=<IPv4 address>,]dest=<IPv4 address>:<port number> 3230 * 3231 * to the add_target sysfs attribute. 3232 */ 3233 enum { 3234 SRP_OPT_ERR = 0, 3235 SRP_OPT_ID_EXT = 1 << 0, 3236 SRP_OPT_IOC_GUID = 1 << 1, 3237 SRP_OPT_DGID = 1 << 2, 3238 SRP_OPT_PKEY = 1 << 3, 3239 SRP_OPT_SERVICE_ID = 1 << 4, 3240 SRP_OPT_MAX_SECT = 1 << 5, 3241 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, 3242 SRP_OPT_IO_CLASS = 1 << 7, 3243 SRP_OPT_INITIATOR_EXT = 1 << 8, 3244 SRP_OPT_CMD_SG_ENTRIES = 1 << 9, 3245 SRP_OPT_ALLOW_EXT_SG = 1 << 10, 3246 SRP_OPT_SG_TABLESIZE = 1 << 11, 3247 SRP_OPT_COMP_VECTOR = 1 << 12, 3248 SRP_OPT_TL_RETRY_COUNT = 1 << 13, 3249 SRP_OPT_QUEUE_SIZE = 1 << 14, 3250 SRP_OPT_IP_SRC = 1 << 15, 3251 SRP_OPT_IP_DEST = 1 << 16, 3252 SRP_OPT_TARGET_CAN_QUEUE= 1 << 17, 3253 SRP_OPT_MAX_IT_IU_SIZE = 1 << 18, 3254 SRP_OPT_CH_COUNT = 1 << 19, 3255 }; 3256 3257 static unsigned int srp_opt_mandatory[] = { 3258 SRP_OPT_ID_EXT | 3259 SRP_OPT_IOC_GUID | 3260 SRP_OPT_DGID | 3261 SRP_OPT_PKEY | 3262 SRP_OPT_SERVICE_ID, 3263 SRP_OPT_ID_EXT | 3264 SRP_OPT_IOC_GUID | 3265 SRP_OPT_IP_DEST, 3266 }; 3267 3268 static const match_table_t srp_opt_tokens = { 3269 { SRP_OPT_ID_EXT, "id_ext=%s" }, 3270 { SRP_OPT_IOC_GUID, "ioc_guid=%s" }, 3271 { SRP_OPT_DGID, "dgid=%s" }, 3272 { SRP_OPT_PKEY, "pkey=%x" }, 3273 { SRP_OPT_SERVICE_ID, "service_id=%s" }, 3274 { SRP_OPT_MAX_SECT, "max_sect=%d" }, 3275 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, 3276 { SRP_OPT_TARGET_CAN_QUEUE, "target_can_queue=%d" }, 3277 { SRP_OPT_IO_CLASS, "io_class=%x" }, 3278 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, 3279 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, 3280 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, 3281 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, 3282 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" }, 3283 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" }, 3284 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" }, 3285 { SRP_OPT_IP_SRC, "src=%s" }, 3286 { SRP_OPT_IP_DEST, "dest=%s" }, 3287 { SRP_OPT_MAX_IT_IU_SIZE, "max_it_iu_size=%d" }, 3288 { SRP_OPT_CH_COUNT, "ch_count=%u", }, 3289 { SRP_OPT_ERR, NULL } 3290 }; 3291 3292 /** 3293 * srp_parse_in - parse an IP address and port number combination 3294 * @net: [in] Network namespace. 3295 * @sa: [out] Address family, IP address and port number. 3296 * @addr_port_str: [in] IP address and port number. 3297 * @has_port: [out] Whether or not @addr_port_str includes a port number. 3298 * 3299 * Parse the following address formats: 3300 * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5. 3301 * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5. 3302 */ 3303 static int srp_parse_in(struct net *net, struct sockaddr_storage *sa, 3304 const char *addr_port_str, bool *has_port) 3305 { 3306 char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL); 3307 char *port_str; 3308 int ret; 3309 3310 if (!addr) 3311 return -ENOMEM; 3312 port_str = strrchr(addr, ':'); 3313 if (port_str && strchr(port_str, ']')) 3314 port_str = NULL; 3315 if (port_str) 3316 *port_str++ = '\0'; 3317 if (has_port) 3318 *has_port = port_str != NULL; 3319 ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa); 3320 if (ret && addr[0]) { 3321 addr_end = addr + strlen(addr) - 1; 3322 if (addr[0] == '[' && *addr_end == ']') { 3323 *addr_end = '\0'; 3324 ret = inet_pton_with_scope(net, AF_INET6, addr + 1, 3325 port_str, sa); 3326 } 3327 } 3328 kfree(addr); 3329 pr_debug("%s -> %pISpfsc\n", addr_port_str, sa); 3330 return ret; 3331 } 3332 3333 static int srp_parse_options(struct net *net, const char *buf, 3334 struct srp_target_port *target) 3335 { 3336 char *options, *sep_opt; 3337 char *p; 3338 substring_t args[MAX_OPT_ARGS]; 3339 unsigned long long ull; 3340 bool has_port; 3341 int opt_mask = 0; 3342 int token; 3343 int ret = -EINVAL; 3344 int i; 3345 3346 options = kstrdup(buf, GFP_KERNEL); 3347 if (!options) 3348 return -ENOMEM; 3349 3350 sep_opt = options; 3351 while ((p = strsep(&sep_opt, ",\n")) != NULL) { 3352 if (!*p) 3353 continue; 3354 3355 token = match_token(p, srp_opt_tokens, args); 3356 opt_mask |= token; 3357 3358 switch (token) { 3359 case SRP_OPT_ID_EXT: 3360 p = match_strdup(args); 3361 if (!p) { 3362 ret = -ENOMEM; 3363 goto out; 3364 } 3365 ret = kstrtoull(p, 16, &ull); 3366 if (ret) { 3367 pr_warn("invalid id_ext parameter '%s'\n", p); 3368 kfree(p); 3369 goto out; 3370 } 3371 target->id_ext = cpu_to_be64(ull); 3372 kfree(p); 3373 break; 3374 3375 case SRP_OPT_IOC_GUID: 3376 p = match_strdup(args); 3377 if (!p) { 3378 ret = -ENOMEM; 3379 goto out; 3380 } 3381 ret = kstrtoull(p, 16, &ull); 3382 if (ret) { 3383 pr_warn("invalid ioc_guid parameter '%s'\n", p); 3384 kfree(p); 3385 goto out; 3386 } 3387 target->ioc_guid = cpu_to_be64(ull); 3388 kfree(p); 3389 break; 3390 3391 case SRP_OPT_DGID: 3392 p = match_strdup(args); 3393 if (!p) { 3394 ret = -ENOMEM; 3395 goto out; 3396 } 3397 if (strlen(p) != 32) { 3398 pr_warn("bad dest GID parameter '%s'\n", p); 3399 kfree(p); 3400 goto out; 3401 } 3402 3403 ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16); 3404 kfree(p); 3405 if (ret < 0) 3406 goto out; 3407 break; 3408 3409 case SRP_OPT_PKEY: 3410 if (match_hex(args, &token)) { 3411 pr_warn("bad P_Key parameter '%s'\n", p); 3412 goto out; 3413 } 3414 target->ib_cm.pkey = cpu_to_be16(token); 3415 break; 3416 3417 case SRP_OPT_SERVICE_ID: 3418 p = match_strdup(args); 3419 if (!p) { 3420 ret = -ENOMEM; 3421 goto out; 3422 } 3423 ret = kstrtoull(p, 16, &ull); 3424 if (ret) { 3425 pr_warn("bad service_id parameter '%s'\n", p); 3426 kfree(p); 3427 goto out; 3428 } 3429 target->ib_cm.service_id = cpu_to_be64(ull); 3430 kfree(p); 3431 break; 3432 3433 case SRP_OPT_IP_SRC: 3434 p = match_strdup(args); 3435 if (!p) { 3436 ret = -ENOMEM; 3437 goto out; 3438 } 3439 ret = srp_parse_in(net, &target->rdma_cm.src.ss, p, 3440 NULL); 3441 if (ret < 0) { 3442 pr_warn("bad source parameter '%s'\n", p); 3443 kfree(p); 3444 goto out; 3445 } 3446 target->rdma_cm.src_specified = true; 3447 kfree(p); 3448 break; 3449 3450 case SRP_OPT_IP_DEST: 3451 p = match_strdup(args); 3452 if (!p) { 3453 ret = -ENOMEM; 3454 goto out; 3455 } 3456 ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p, 3457 &has_port); 3458 if (!has_port) 3459 ret = -EINVAL; 3460 if (ret < 0) { 3461 pr_warn("bad dest parameter '%s'\n", p); 3462 kfree(p); 3463 goto out; 3464 } 3465 target->using_rdma_cm = true; 3466 kfree(p); 3467 break; 3468 3469 case SRP_OPT_MAX_SECT: 3470 if (match_int(args, &token)) { 3471 pr_warn("bad max sect parameter '%s'\n", p); 3472 goto out; 3473 } 3474 target->scsi_host->max_sectors = token; 3475 break; 3476 3477 case SRP_OPT_QUEUE_SIZE: 3478 if (match_int(args, &token) || token < 1) { 3479 pr_warn("bad queue_size parameter '%s'\n", p); 3480 goto out; 3481 } 3482 target->scsi_host->can_queue = token; 3483 target->queue_size = token + SRP_RSP_SQ_SIZE + 3484 SRP_TSK_MGMT_SQ_SIZE; 3485 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3486 target->scsi_host->cmd_per_lun = token; 3487 break; 3488 3489 case SRP_OPT_MAX_CMD_PER_LUN: 3490 if (match_int(args, &token) || token < 1) { 3491 pr_warn("bad max cmd_per_lun parameter '%s'\n", 3492 p); 3493 goto out; 3494 } 3495 target->scsi_host->cmd_per_lun = token; 3496 break; 3497 3498 case SRP_OPT_TARGET_CAN_QUEUE: 3499 if (match_int(args, &token) || token < 1) { 3500 pr_warn("bad max target_can_queue parameter '%s'\n", 3501 p); 3502 goto out; 3503 } 3504 target->target_can_queue = token; 3505 break; 3506 3507 case SRP_OPT_IO_CLASS: 3508 if (match_hex(args, &token)) { 3509 pr_warn("bad IO class parameter '%s'\n", p); 3510 goto out; 3511 } 3512 if (token != SRP_REV10_IB_IO_CLASS && 3513 token != SRP_REV16A_IB_IO_CLASS) { 3514 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n", 3515 token, SRP_REV10_IB_IO_CLASS, 3516 SRP_REV16A_IB_IO_CLASS); 3517 goto out; 3518 } 3519 target->io_class = token; 3520 break; 3521 3522 case SRP_OPT_INITIATOR_EXT: 3523 p = match_strdup(args); 3524 if (!p) { 3525 ret = -ENOMEM; 3526 goto out; 3527 } 3528 ret = kstrtoull(p, 16, &ull); 3529 if (ret) { 3530 pr_warn("bad initiator_ext value '%s'\n", p); 3531 kfree(p); 3532 goto out; 3533 } 3534 target->initiator_ext = cpu_to_be64(ull); 3535 kfree(p); 3536 break; 3537 3538 case SRP_OPT_CMD_SG_ENTRIES: 3539 if (match_int(args, &token) || token < 1 || token > 255) { 3540 pr_warn("bad max cmd_sg_entries parameter '%s'\n", 3541 p); 3542 goto out; 3543 } 3544 target->cmd_sg_cnt = token; 3545 break; 3546 3547 case SRP_OPT_ALLOW_EXT_SG: 3548 if (match_int(args, &token)) { 3549 pr_warn("bad allow_ext_sg parameter '%s'\n", p); 3550 goto out; 3551 } 3552 target->allow_ext_sg = !!token; 3553 break; 3554 3555 case SRP_OPT_SG_TABLESIZE: 3556 if (match_int(args, &token) || token < 1 || 3557 token > SG_MAX_SEGMENTS) { 3558 pr_warn("bad max sg_tablesize parameter '%s'\n", 3559 p); 3560 goto out; 3561 } 3562 target->sg_tablesize = token; 3563 break; 3564 3565 case SRP_OPT_COMP_VECTOR: 3566 if (match_int(args, &token) || token < 0) { 3567 pr_warn("bad comp_vector parameter '%s'\n", p); 3568 goto out; 3569 } 3570 target->comp_vector = token; 3571 break; 3572 3573 case SRP_OPT_TL_RETRY_COUNT: 3574 if (match_int(args, &token) || token < 2 || token > 7) { 3575 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n", 3576 p); 3577 goto out; 3578 } 3579 target->tl_retry_count = token; 3580 break; 3581 3582 case SRP_OPT_MAX_IT_IU_SIZE: 3583 if (match_int(args, &token) || token < 0) { 3584 pr_warn("bad maximum initiator to target IU size '%s'\n", p); 3585 goto out; 3586 } 3587 target->max_it_iu_size = token; 3588 break; 3589 3590 case SRP_OPT_CH_COUNT: 3591 if (match_int(args, &token) || token < 1) { 3592 pr_warn("bad channel count %s\n", p); 3593 goto out; 3594 } 3595 target->ch_count = token; 3596 break; 3597 3598 default: 3599 pr_warn("unknown parameter or missing value '%s' in target creation request\n", 3600 p); 3601 goto out; 3602 } 3603 } 3604 3605 for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) { 3606 if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) { 3607 ret = 0; 3608 break; 3609 } 3610 } 3611 if (ret) 3612 pr_warn("target creation request is missing one or more parameters\n"); 3613 3614 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue 3615 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3616 pr_warn("cmd_per_lun = %d > queue_size = %d\n", 3617 target->scsi_host->cmd_per_lun, 3618 target->scsi_host->can_queue); 3619 3620 out: 3621 kfree(options); 3622 return ret; 3623 } 3624 3625 static ssize_t add_target_store(struct device *dev, 3626 struct device_attribute *attr, const char *buf, 3627 size_t count) 3628 { 3629 struct srp_host *host = 3630 container_of(dev, struct srp_host, dev); 3631 struct Scsi_Host *target_host; 3632 struct srp_target_port *target; 3633 struct srp_rdma_ch *ch; 3634 struct srp_device *srp_dev = host->srp_dev; 3635 struct ib_device *ibdev = srp_dev->dev; 3636 int ret, i, ch_idx; 3637 unsigned int max_sectors_per_mr, mr_per_cmd = 0; 3638 bool multich = false; 3639 uint32_t max_iu_len; 3640 3641 target_host = scsi_host_alloc(&srp_template, 3642 sizeof (struct srp_target_port)); 3643 if (!target_host) 3644 return -ENOMEM; 3645 3646 target_host->transportt = ib_srp_transport_template; 3647 target_host->max_channel = 0; 3648 target_host->max_id = 1; 3649 target_host->max_lun = -1LL; 3650 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; 3651 target_host->max_segment_size = ib_dma_max_seg_size(ibdev); 3652 3653 if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)) 3654 target_host->virt_boundary_mask = ~srp_dev->mr_page_mask; 3655 3656 target = host_to_target(target_host); 3657 3658 target->net = kobj_ns_grab_current(KOBJ_NS_TYPE_NET); 3659 target->io_class = SRP_REV16A_IB_IO_CLASS; 3660 target->scsi_host = target_host; 3661 target->srp_host = host; 3662 target->lkey = host->srp_dev->pd->local_dma_lkey; 3663 target->global_rkey = host->srp_dev->global_rkey; 3664 target->cmd_sg_cnt = cmd_sg_entries; 3665 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries; 3666 target->allow_ext_sg = allow_ext_sg; 3667 target->tl_retry_count = 7; 3668 target->queue_size = SRP_DEFAULT_QUEUE_SIZE; 3669 3670 /* 3671 * Avoid that the SCSI host can be removed by srp_remove_target() 3672 * before this function returns. 3673 */ 3674 scsi_host_get(target->scsi_host); 3675 3676 ret = mutex_lock_interruptible(&host->add_target_mutex); 3677 if (ret < 0) 3678 goto put; 3679 3680 ret = srp_parse_options(target->net, buf, target); 3681 if (ret) 3682 goto out; 3683 3684 if (!srp_conn_unique(target->srp_host, target)) { 3685 if (target->using_rdma_cm) { 3686 shost_printk(KERN_INFO, target->scsi_host, 3687 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n", 3688 be64_to_cpu(target->id_ext), 3689 be64_to_cpu(target->ioc_guid), 3690 &target->rdma_cm.dst); 3691 } else { 3692 shost_printk(KERN_INFO, target->scsi_host, 3693 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n", 3694 be64_to_cpu(target->id_ext), 3695 be64_to_cpu(target->ioc_guid), 3696 be64_to_cpu(target->initiator_ext)); 3697 } 3698 ret = -EEXIST; 3699 goto out; 3700 } 3701 3702 if (!srp_dev->has_fr && !target->allow_ext_sg && 3703 target->cmd_sg_cnt < target->sg_tablesize) { 3704 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n"); 3705 target->sg_tablesize = target->cmd_sg_cnt; 3706 } 3707 3708 if (srp_dev->use_fast_reg) { 3709 bool gaps_reg = (ibdev->attrs.device_cap_flags & 3710 IB_DEVICE_SG_GAPS_REG); 3711 3712 max_sectors_per_mr = srp_dev->max_pages_per_mr << 3713 (ilog2(srp_dev->mr_page_size) - 9); 3714 if (!gaps_reg) { 3715 /* 3716 * FR can only map one HCA page per entry. If the start 3717 * address is not aligned on a HCA page boundary two 3718 * entries will be used for the head and the tail 3719 * although these two entries combined contain at most 3720 * one HCA page of data. Hence the "+ 1" in the 3721 * calculation below. 3722 * 3723 * The indirect data buffer descriptor is contiguous 3724 * so the memory for that buffer will only be 3725 * registered if register_always is true. Hence add 3726 * one to mr_per_cmd if register_always has been set. 3727 */ 3728 mr_per_cmd = register_always + 3729 (target->scsi_host->max_sectors + 1 + 3730 max_sectors_per_mr - 1) / max_sectors_per_mr; 3731 } else { 3732 mr_per_cmd = register_always + 3733 (target->sg_tablesize + 3734 srp_dev->max_pages_per_mr - 1) / 3735 srp_dev->max_pages_per_mr; 3736 } 3737 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n", 3738 target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size, 3739 max_sectors_per_mr, mr_per_cmd); 3740 } 3741 3742 target_host->sg_tablesize = target->sg_tablesize; 3743 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd; 3744 target->mr_per_cmd = mr_per_cmd; 3745 target->indirect_size = target->sg_tablesize * 3746 sizeof (struct srp_direct_buf); 3747 max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt, 3748 srp_use_imm_data, 3749 target->max_it_iu_size); 3750 3751 INIT_WORK(&target->tl_err_work, srp_tl_err_work); 3752 INIT_WORK(&target->remove_work, srp_remove_work); 3753 spin_lock_init(&target->lock); 3754 ret = rdma_query_gid(ibdev, host->port, 0, &target->sgid); 3755 if (ret) 3756 goto out; 3757 3758 ret = -ENOMEM; 3759 if (target->ch_count == 0) { 3760 target->ch_count = 3761 min(ch_count ?: 3762 max(4 * num_online_nodes(), 3763 ibdev->num_comp_vectors), 3764 num_online_cpus()); 3765 } 3766 3767 target->ch = kcalloc(target->ch_count, sizeof(*target->ch), 3768 GFP_KERNEL); 3769 if (!target->ch) 3770 goto out; 3771 3772 for (ch_idx = 0; ch_idx < target->ch_count; ++ch_idx) { 3773 ch = &target->ch[ch_idx]; 3774 ch->target = target; 3775 ch->comp_vector = ch_idx % ibdev->num_comp_vectors; 3776 spin_lock_init(&ch->lock); 3777 INIT_LIST_HEAD(&ch->free_tx); 3778 ret = srp_new_cm_id(ch); 3779 if (ret) 3780 goto err_disconnect; 3781 3782 ret = srp_create_ch_ib(ch); 3783 if (ret) 3784 goto err_disconnect; 3785 3786 ret = srp_connect_ch(ch, max_iu_len, multich); 3787 if (ret) { 3788 char dst[64]; 3789 3790 if (target->using_rdma_cm) 3791 snprintf(dst, sizeof(dst), "%pIS", 3792 &target->rdma_cm.dst); 3793 else 3794 snprintf(dst, sizeof(dst), "%pI6", 3795 target->ib_cm.orig_dgid.raw); 3796 shost_printk(KERN_ERR, target->scsi_host, 3797 PFX "Connection %d/%d to %s failed\n", 3798 ch_idx, 3799 target->ch_count, dst); 3800 if (ch_idx == 0) { 3801 goto free_ch; 3802 } else { 3803 srp_free_ch_ib(target, ch); 3804 target->ch_count = ch - target->ch; 3805 goto connected; 3806 } 3807 } 3808 multich = true; 3809 } 3810 3811 connected: 3812 target->scsi_host->nr_hw_queues = target->ch_count; 3813 3814 ret = srp_add_target(host, target); 3815 if (ret) 3816 goto err_disconnect; 3817 3818 if (target->state != SRP_TARGET_REMOVED) { 3819 if (target->using_rdma_cm) { 3820 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3821 "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n", 3822 be64_to_cpu(target->id_ext), 3823 be64_to_cpu(target->ioc_guid), 3824 target->sgid.raw, &target->rdma_cm.dst); 3825 } else { 3826 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3827 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n", 3828 be64_to_cpu(target->id_ext), 3829 be64_to_cpu(target->ioc_guid), 3830 be16_to_cpu(target->ib_cm.pkey), 3831 be64_to_cpu(target->ib_cm.service_id), 3832 target->sgid.raw, 3833 target->ib_cm.orig_dgid.raw); 3834 } 3835 } 3836 3837 ret = count; 3838 3839 out: 3840 mutex_unlock(&host->add_target_mutex); 3841 3842 put: 3843 scsi_host_put(target->scsi_host); 3844 if (ret < 0) { 3845 /* 3846 * If a call to srp_remove_target() has not been scheduled, 3847 * drop the network namespace reference now that was obtained 3848 * earlier in this function. 3849 */ 3850 if (target->state != SRP_TARGET_REMOVED) 3851 kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net); 3852 scsi_host_put(target->scsi_host); 3853 } 3854 3855 return ret; 3856 3857 err_disconnect: 3858 srp_disconnect_target(target); 3859 3860 free_ch: 3861 for (i = 0; i < target->ch_count; i++) { 3862 ch = &target->ch[i]; 3863 srp_free_ch_ib(target, ch); 3864 } 3865 3866 kfree(target->ch); 3867 goto out; 3868 } 3869 3870 static DEVICE_ATTR_WO(add_target); 3871 3872 static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr, 3873 char *buf) 3874 { 3875 struct srp_host *host = container_of(dev, struct srp_host, dev); 3876 3877 return sysfs_emit(buf, "%s\n", dev_name(&host->srp_dev->dev->dev)); 3878 } 3879 3880 static DEVICE_ATTR_RO(ibdev); 3881 3882 static ssize_t port_show(struct device *dev, struct device_attribute *attr, 3883 char *buf) 3884 { 3885 struct srp_host *host = container_of(dev, struct srp_host, dev); 3886 3887 return sysfs_emit(buf, "%d\n", host->port); 3888 } 3889 3890 static DEVICE_ATTR_RO(port); 3891 3892 static struct srp_host *srp_add_port(struct srp_device *device, u8 port) 3893 { 3894 struct srp_host *host; 3895 3896 host = kzalloc(sizeof *host, GFP_KERNEL); 3897 if (!host) 3898 return NULL; 3899 3900 INIT_LIST_HEAD(&host->target_list); 3901 spin_lock_init(&host->target_lock); 3902 init_completion(&host->released); 3903 mutex_init(&host->add_target_mutex); 3904 host->srp_dev = device; 3905 host->port = port; 3906 3907 host->dev.class = &srp_class; 3908 host->dev.parent = device->dev->dev.parent; 3909 dev_set_name(&host->dev, "srp-%s-%d", dev_name(&device->dev->dev), 3910 port); 3911 3912 if (device_register(&host->dev)) 3913 goto free_host; 3914 if (device_create_file(&host->dev, &dev_attr_add_target)) 3915 goto err_class; 3916 if (device_create_file(&host->dev, &dev_attr_ibdev)) 3917 goto err_class; 3918 if (device_create_file(&host->dev, &dev_attr_port)) 3919 goto err_class; 3920 3921 return host; 3922 3923 err_class: 3924 device_unregister(&host->dev); 3925 3926 free_host: 3927 kfree(host); 3928 3929 return NULL; 3930 } 3931 3932 static void srp_rename_dev(struct ib_device *device, void *client_data) 3933 { 3934 struct srp_device *srp_dev = client_data; 3935 struct srp_host *host, *tmp_host; 3936 3937 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 3938 char name[IB_DEVICE_NAME_MAX + 8]; 3939 3940 snprintf(name, sizeof(name), "srp-%s-%d", 3941 dev_name(&device->dev), host->port); 3942 device_rename(&host->dev, name); 3943 } 3944 } 3945 3946 static int srp_add_one(struct ib_device *device) 3947 { 3948 struct srp_device *srp_dev; 3949 struct ib_device_attr *attr = &device->attrs; 3950 struct srp_host *host; 3951 int mr_page_shift; 3952 unsigned int p; 3953 u64 max_pages_per_mr; 3954 unsigned int flags = 0; 3955 3956 srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL); 3957 if (!srp_dev) 3958 return -ENOMEM; 3959 3960 /* 3961 * Use the smallest page size supported by the HCA, down to a 3962 * minimum of 4096 bytes. We're unlikely to build large sglists 3963 * out of smaller entries. 3964 */ 3965 mr_page_shift = max(12, ffs(attr->page_size_cap) - 1); 3966 srp_dev->mr_page_size = 1 << mr_page_shift; 3967 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1); 3968 max_pages_per_mr = attr->max_mr_size; 3969 do_div(max_pages_per_mr, srp_dev->mr_page_size); 3970 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__, 3971 attr->max_mr_size, srp_dev->mr_page_size, 3972 max_pages_per_mr, SRP_MAX_PAGES_PER_MR); 3973 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR, 3974 max_pages_per_mr); 3975 3976 srp_dev->has_fr = (attr->device_cap_flags & 3977 IB_DEVICE_MEM_MGT_EXTENSIONS); 3978 if (!never_register && !srp_dev->has_fr) 3979 dev_warn(&device->dev, "FR is not supported\n"); 3980 else if (!never_register && 3981 attr->max_mr_size >= 2 * srp_dev->mr_page_size) 3982 srp_dev->use_fast_reg = srp_dev->has_fr; 3983 3984 if (never_register || !register_always || !srp_dev->has_fr) 3985 flags |= IB_PD_UNSAFE_GLOBAL_RKEY; 3986 3987 if (srp_dev->use_fast_reg) { 3988 srp_dev->max_pages_per_mr = 3989 min_t(u32, srp_dev->max_pages_per_mr, 3990 attr->max_fast_reg_page_list_len); 3991 } 3992 srp_dev->mr_max_size = srp_dev->mr_page_size * 3993 srp_dev->max_pages_per_mr; 3994 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n", 3995 dev_name(&device->dev), mr_page_shift, attr->max_mr_size, 3996 attr->max_fast_reg_page_list_len, 3997 srp_dev->max_pages_per_mr, srp_dev->mr_max_size); 3998 3999 INIT_LIST_HEAD(&srp_dev->dev_list); 4000 4001 srp_dev->dev = device; 4002 srp_dev->pd = ib_alloc_pd(device, flags); 4003 if (IS_ERR(srp_dev->pd)) { 4004 int ret = PTR_ERR(srp_dev->pd); 4005 4006 kfree(srp_dev); 4007 return ret; 4008 } 4009 4010 if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) { 4011 srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey; 4012 WARN_ON_ONCE(srp_dev->global_rkey == 0); 4013 } 4014 4015 rdma_for_each_port (device, p) { 4016 host = srp_add_port(srp_dev, p); 4017 if (host) 4018 list_add_tail(&host->list, &srp_dev->dev_list); 4019 } 4020 4021 ib_set_client_data(device, &srp_client, srp_dev); 4022 return 0; 4023 } 4024 4025 static void srp_remove_one(struct ib_device *device, void *client_data) 4026 { 4027 struct srp_device *srp_dev; 4028 struct srp_host *host, *tmp_host; 4029 struct srp_target_port *target; 4030 4031 srp_dev = client_data; 4032 4033 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 4034 device_unregister(&host->dev); 4035 /* 4036 * Wait for the sysfs entry to go away, so that no new 4037 * target ports can be created. 4038 */ 4039 wait_for_completion(&host->released); 4040 4041 /* 4042 * Remove all target ports. 4043 */ 4044 spin_lock(&host->target_lock); 4045 list_for_each_entry(target, &host->target_list, list) 4046 srp_queue_remove_work(target); 4047 spin_unlock(&host->target_lock); 4048 4049 /* 4050 * Wait for tl_err and target port removal tasks. 4051 */ 4052 flush_workqueue(system_long_wq); 4053 flush_workqueue(srp_remove_wq); 4054 4055 kfree(host); 4056 } 4057 4058 ib_dealloc_pd(srp_dev->pd); 4059 4060 kfree(srp_dev); 4061 } 4062 4063 static struct srp_function_template ib_srp_transport_functions = { 4064 .has_rport_state = true, 4065 .reset_timer_if_blocked = true, 4066 .reconnect_delay = &srp_reconnect_delay, 4067 .fast_io_fail_tmo = &srp_fast_io_fail_tmo, 4068 .dev_loss_tmo = &srp_dev_loss_tmo, 4069 .reconnect = srp_rport_reconnect, 4070 .rport_delete = srp_rport_delete, 4071 .terminate_rport_io = srp_terminate_io, 4072 }; 4073 4074 static int __init srp_init_module(void) 4075 { 4076 int ret; 4077 4078 BUILD_BUG_ON(sizeof(struct srp_aer_req) != 36); 4079 BUILD_BUG_ON(sizeof(struct srp_cmd) != 48); 4080 BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 4); 4081 BUILD_BUG_ON(sizeof(struct srp_indirect_buf) != 20); 4082 BUILD_BUG_ON(sizeof(struct srp_login_req) != 64); 4083 BUILD_BUG_ON(sizeof(struct srp_login_req_rdma) != 56); 4084 BUILD_BUG_ON(sizeof(struct srp_rsp) != 36); 4085 4086 if (srp_sg_tablesize) { 4087 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); 4088 if (!cmd_sg_entries) 4089 cmd_sg_entries = srp_sg_tablesize; 4090 } 4091 4092 if (!cmd_sg_entries) 4093 cmd_sg_entries = SRP_DEF_SG_TABLESIZE; 4094 4095 if (cmd_sg_entries > 255) { 4096 pr_warn("Clamping cmd_sg_entries to 255\n"); 4097 cmd_sg_entries = 255; 4098 } 4099 4100 if (!indirect_sg_entries) 4101 indirect_sg_entries = cmd_sg_entries; 4102 else if (indirect_sg_entries < cmd_sg_entries) { 4103 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n", 4104 cmd_sg_entries); 4105 indirect_sg_entries = cmd_sg_entries; 4106 } 4107 4108 if (indirect_sg_entries > SG_MAX_SEGMENTS) { 4109 pr_warn("Clamping indirect_sg_entries to %u\n", 4110 SG_MAX_SEGMENTS); 4111 indirect_sg_entries = SG_MAX_SEGMENTS; 4112 } 4113 4114 srp_remove_wq = create_workqueue("srp_remove"); 4115 if (!srp_remove_wq) { 4116 ret = -ENOMEM; 4117 goto out; 4118 } 4119 4120 ret = -ENOMEM; 4121 ib_srp_transport_template = 4122 srp_attach_transport(&ib_srp_transport_functions); 4123 if (!ib_srp_transport_template) 4124 goto destroy_wq; 4125 4126 ret = class_register(&srp_class); 4127 if (ret) { 4128 pr_err("couldn't register class infiniband_srp\n"); 4129 goto release_tr; 4130 } 4131 4132 ib_sa_register_client(&srp_sa_client); 4133 4134 ret = ib_register_client(&srp_client); 4135 if (ret) { 4136 pr_err("couldn't register IB client\n"); 4137 goto unreg_sa; 4138 } 4139 4140 out: 4141 return ret; 4142 4143 unreg_sa: 4144 ib_sa_unregister_client(&srp_sa_client); 4145 class_unregister(&srp_class); 4146 4147 release_tr: 4148 srp_release_transport(ib_srp_transport_template); 4149 4150 destroy_wq: 4151 destroy_workqueue(srp_remove_wq); 4152 goto out; 4153 } 4154 4155 static void __exit srp_cleanup_module(void) 4156 { 4157 ib_unregister_client(&srp_client); 4158 ib_sa_unregister_client(&srp_sa_client); 4159 class_unregister(&srp_class); 4160 srp_release_transport(ib_srp_transport_template); 4161 destroy_workqueue(srp_remove_wq); 4162 } 4163 4164 module_init(srp_init_module); 4165 module_exit(srp_cleanup_module); 4166