1 /* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/module.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/err.h> 39 #include <linux/string.h> 40 #include <linux/parser.h> 41 #include <linux/random.h> 42 #include <linux/jiffies.h> 43 #include <rdma/ib_cache.h> 44 45 #include <linux/atomic.h> 46 47 #include <scsi/scsi.h> 48 #include <scsi/scsi_device.h> 49 #include <scsi/scsi_dbg.h> 50 #include <scsi/scsi_tcq.h> 51 #include <scsi/srp.h> 52 #include <scsi/scsi_transport_srp.h> 53 54 #include "ib_srp.h" 55 56 #define DRV_NAME "ib_srp" 57 #define PFX DRV_NAME ": " 58 #define DRV_VERSION "2.0" 59 #define DRV_RELDATE "July 26, 2015" 60 61 MODULE_AUTHOR("Roland Dreier"); 62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); 63 MODULE_LICENSE("Dual BSD/GPL"); 64 MODULE_VERSION(DRV_VERSION); 65 MODULE_INFO(release_date, DRV_RELDATE); 66 67 static unsigned int srp_sg_tablesize; 68 static unsigned int cmd_sg_entries; 69 static unsigned int indirect_sg_entries; 70 static bool allow_ext_sg; 71 static bool prefer_fr = true; 72 static bool register_always = true; 73 static bool never_register; 74 static int topspin_workarounds = 1; 75 76 module_param(srp_sg_tablesize, uint, 0444); 77 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); 78 79 module_param(cmd_sg_entries, uint, 0444); 80 MODULE_PARM_DESC(cmd_sg_entries, 81 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); 82 83 module_param(indirect_sg_entries, uint, 0444); 84 MODULE_PARM_DESC(indirect_sg_entries, 85 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")"); 86 87 module_param(allow_ext_sg, bool, 0444); 88 MODULE_PARM_DESC(allow_ext_sg, 89 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)"); 90 91 module_param(topspin_workarounds, int, 0444); 92 MODULE_PARM_DESC(topspin_workarounds, 93 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0"); 94 95 module_param(prefer_fr, bool, 0444); 96 MODULE_PARM_DESC(prefer_fr, 97 "Whether to use fast registration if both FMR and fast registration are supported"); 98 99 module_param(register_always, bool, 0444); 100 MODULE_PARM_DESC(register_always, 101 "Use memory registration even for contiguous memory regions"); 102 103 module_param(never_register, bool, 0444); 104 MODULE_PARM_DESC(never_register, "Never register memory"); 105 106 static const struct kernel_param_ops srp_tmo_ops; 107 108 static int srp_reconnect_delay = 10; 109 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 110 S_IRUGO | S_IWUSR); 111 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts"); 112 113 static int srp_fast_io_fail_tmo = 15; 114 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo, 115 S_IRUGO | S_IWUSR); 116 MODULE_PARM_DESC(fast_io_fail_tmo, 117 "Number of seconds between the observation of a transport" 118 " layer error and failing all I/O. \"off\" means that this" 119 " functionality is disabled."); 120 121 static int srp_dev_loss_tmo = 600; 122 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo, 123 S_IRUGO | S_IWUSR); 124 MODULE_PARM_DESC(dev_loss_tmo, 125 "Maximum number of seconds that the SRP transport should" 126 " insulate transport layer errors. After this time has been" 127 " exceeded the SCSI host is removed. Should be" 128 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT) 129 " if fast_io_fail_tmo has not been set. \"off\" means that" 130 " this functionality is disabled."); 131 132 static unsigned ch_count; 133 module_param(ch_count, uint, 0444); 134 MODULE_PARM_DESC(ch_count, 135 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA."); 136 137 static void srp_add_one(struct ib_device *device); 138 static void srp_remove_one(struct ib_device *device, void *client_data); 139 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc); 140 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 141 const char *opname); 142 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event); 143 144 static struct scsi_transport_template *ib_srp_transport_template; 145 static struct workqueue_struct *srp_remove_wq; 146 147 static struct ib_client srp_client = { 148 .name = "srp", 149 .add = srp_add_one, 150 .remove = srp_remove_one 151 }; 152 153 static struct ib_sa_client srp_sa_client; 154 155 static int srp_tmo_get(char *buffer, const struct kernel_param *kp) 156 { 157 int tmo = *(int *)kp->arg; 158 159 if (tmo >= 0) 160 return sprintf(buffer, "%d", tmo); 161 else 162 return sprintf(buffer, "off"); 163 } 164 165 static int srp_tmo_set(const char *val, const struct kernel_param *kp) 166 { 167 int tmo, res; 168 169 res = srp_parse_tmo(&tmo, val); 170 if (res) 171 goto out; 172 173 if (kp->arg == &srp_reconnect_delay) 174 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo, 175 srp_dev_loss_tmo); 176 else if (kp->arg == &srp_fast_io_fail_tmo) 177 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo); 178 else 179 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo, 180 tmo); 181 if (res) 182 goto out; 183 *(int *)kp->arg = tmo; 184 185 out: 186 return res; 187 } 188 189 static const struct kernel_param_ops srp_tmo_ops = { 190 .get = srp_tmo_get, 191 .set = srp_tmo_set, 192 }; 193 194 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) 195 { 196 return (struct srp_target_port *) host->hostdata; 197 } 198 199 static const char *srp_target_info(struct Scsi_Host *host) 200 { 201 return host_to_target(host)->target_name; 202 } 203 204 static int srp_target_is_topspin(struct srp_target_port *target) 205 { 206 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad }; 207 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d }; 208 209 return topspin_workarounds && 210 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) || 211 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui)); 212 } 213 214 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size, 215 gfp_t gfp_mask, 216 enum dma_data_direction direction) 217 { 218 struct srp_iu *iu; 219 220 iu = kmalloc(sizeof *iu, gfp_mask); 221 if (!iu) 222 goto out; 223 224 iu->buf = kzalloc(size, gfp_mask); 225 if (!iu->buf) 226 goto out_free_iu; 227 228 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size, 229 direction); 230 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma)) 231 goto out_free_buf; 232 233 iu->size = size; 234 iu->direction = direction; 235 236 return iu; 237 238 out_free_buf: 239 kfree(iu->buf); 240 out_free_iu: 241 kfree(iu); 242 out: 243 return NULL; 244 } 245 246 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu) 247 { 248 if (!iu) 249 return; 250 251 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size, 252 iu->direction); 253 kfree(iu->buf); 254 kfree(iu); 255 } 256 257 static void srp_qp_event(struct ib_event *event, void *context) 258 { 259 pr_debug("QP event %s (%d)\n", 260 ib_event_msg(event->event), event->event); 261 } 262 263 static int srp_init_qp(struct srp_target_port *target, 264 struct ib_qp *qp) 265 { 266 struct ib_qp_attr *attr; 267 int ret; 268 269 attr = kmalloc(sizeof *attr, GFP_KERNEL); 270 if (!attr) 271 return -ENOMEM; 272 273 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 274 target->srp_host->port, 275 be16_to_cpu(target->pkey), 276 &attr->pkey_index); 277 if (ret) 278 goto out; 279 280 attr->qp_state = IB_QPS_INIT; 281 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ | 282 IB_ACCESS_REMOTE_WRITE); 283 attr->port_num = target->srp_host->port; 284 285 ret = ib_modify_qp(qp, attr, 286 IB_QP_STATE | 287 IB_QP_PKEY_INDEX | 288 IB_QP_ACCESS_FLAGS | 289 IB_QP_PORT); 290 291 out: 292 kfree(attr); 293 return ret; 294 } 295 296 static int srp_new_cm_id(struct srp_rdma_ch *ch) 297 { 298 struct srp_target_port *target = ch->target; 299 struct ib_cm_id *new_cm_id; 300 301 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, 302 srp_cm_handler, ch); 303 if (IS_ERR(new_cm_id)) 304 return PTR_ERR(new_cm_id); 305 306 if (ch->cm_id) 307 ib_destroy_cm_id(ch->cm_id); 308 ch->cm_id = new_cm_id; 309 ch->path.sgid = target->sgid; 310 ch->path.dgid = target->orig_dgid; 311 ch->path.pkey = target->pkey; 312 ch->path.service_id = target->service_id; 313 314 return 0; 315 } 316 317 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target) 318 { 319 struct srp_device *dev = target->srp_host->srp_dev; 320 struct ib_fmr_pool_param fmr_param; 321 322 memset(&fmr_param, 0, sizeof(fmr_param)); 323 fmr_param.pool_size = target->mr_pool_size; 324 fmr_param.dirty_watermark = fmr_param.pool_size / 4; 325 fmr_param.cache = 1; 326 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr; 327 fmr_param.page_shift = ilog2(dev->mr_page_size); 328 fmr_param.access = (IB_ACCESS_LOCAL_WRITE | 329 IB_ACCESS_REMOTE_WRITE | 330 IB_ACCESS_REMOTE_READ); 331 332 return ib_create_fmr_pool(dev->pd, &fmr_param); 333 } 334 335 /** 336 * srp_destroy_fr_pool() - free the resources owned by a pool 337 * @pool: Fast registration pool to be destroyed. 338 */ 339 static void srp_destroy_fr_pool(struct srp_fr_pool *pool) 340 { 341 int i; 342 struct srp_fr_desc *d; 343 344 if (!pool) 345 return; 346 347 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 348 if (d->mr) 349 ib_dereg_mr(d->mr); 350 } 351 kfree(pool); 352 } 353 354 /** 355 * srp_create_fr_pool() - allocate and initialize a pool for fast registration 356 * @device: IB device to allocate fast registration descriptors for. 357 * @pd: Protection domain associated with the FR descriptors. 358 * @pool_size: Number of descriptors to allocate. 359 * @max_page_list_len: Maximum fast registration work request page list length. 360 */ 361 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, 362 struct ib_pd *pd, int pool_size, 363 int max_page_list_len) 364 { 365 struct srp_fr_pool *pool; 366 struct srp_fr_desc *d; 367 struct ib_mr *mr; 368 int i, ret = -EINVAL; 369 370 if (pool_size <= 0) 371 goto err; 372 ret = -ENOMEM; 373 pool = kzalloc(sizeof(struct srp_fr_pool) + 374 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL); 375 if (!pool) 376 goto err; 377 pool->size = pool_size; 378 pool->max_page_list_len = max_page_list_len; 379 spin_lock_init(&pool->lock); 380 INIT_LIST_HEAD(&pool->free_list); 381 382 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 383 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 384 max_page_list_len); 385 if (IS_ERR(mr)) { 386 ret = PTR_ERR(mr); 387 goto destroy_pool; 388 } 389 d->mr = mr; 390 list_add_tail(&d->entry, &pool->free_list); 391 } 392 393 out: 394 return pool; 395 396 destroy_pool: 397 srp_destroy_fr_pool(pool); 398 399 err: 400 pool = ERR_PTR(ret); 401 goto out; 402 } 403 404 /** 405 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration 406 * @pool: Pool to obtain descriptor from. 407 */ 408 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool) 409 { 410 struct srp_fr_desc *d = NULL; 411 unsigned long flags; 412 413 spin_lock_irqsave(&pool->lock, flags); 414 if (!list_empty(&pool->free_list)) { 415 d = list_first_entry(&pool->free_list, typeof(*d), entry); 416 list_del(&d->entry); 417 } 418 spin_unlock_irqrestore(&pool->lock, flags); 419 420 return d; 421 } 422 423 /** 424 * srp_fr_pool_put() - put an FR descriptor back in the free list 425 * @pool: Pool the descriptor was allocated from. 426 * @desc: Pointer to an array of fast registration descriptor pointers. 427 * @n: Number of descriptors to put back. 428 * 429 * Note: The caller must already have queued an invalidation request for 430 * desc->mr->rkey before calling this function. 431 */ 432 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc, 433 int n) 434 { 435 unsigned long flags; 436 int i; 437 438 spin_lock_irqsave(&pool->lock, flags); 439 for (i = 0; i < n; i++) 440 list_add(&desc[i]->entry, &pool->free_list); 441 spin_unlock_irqrestore(&pool->lock, flags); 442 } 443 444 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) 445 { 446 struct srp_device *dev = target->srp_host->srp_dev; 447 448 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size, 449 dev->max_pages_per_mr); 450 } 451 452 /** 453 * srp_destroy_qp() - destroy an RDMA queue pair 454 * @qp: RDMA queue pair. 455 * 456 * Drain the qp before destroying it. This avoids that the receive 457 * completion handler can access the queue pair while it is 458 * being destroyed. 459 */ 460 static void srp_destroy_qp(struct ib_qp *qp) 461 { 462 ib_drain_rq(qp); 463 ib_destroy_qp(qp); 464 } 465 466 static int srp_create_ch_ib(struct srp_rdma_ch *ch) 467 { 468 struct srp_target_port *target = ch->target; 469 struct srp_device *dev = target->srp_host->srp_dev; 470 struct ib_qp_init_attr *init_attr; 471 struct ib_cq *recv_cq, *send_cq; 472 struct ib_qp *qp; 473 struct ib_fmr_pool *fmr_pool = NULL; 474 struct srp_fr_pool *fr_pool = NULL; 475 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2; 476 int ret; 477 478 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL); 479 if (!init_attr) 480 return -ENOMEM; 481 482 /* queue_size + 1 for ib_drain_rq() */ 483 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1, 484 ch->comp_vector, IB_POLL_SOFTIRQ); 485 if (IS_ERR(recv_cq)) { 486 ret = PTR_ERR(recv_cq); 487 goto err; 488 } 489 490 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size, 491 ch->comp_vector, IB_POLL_DIRECT); 492 if (IS_ERR(send_cq)) { 493 ret = PTR_ERR(send_cq); 494 goto err_recv_cq; 495 } 496 497 init_attr->event_handler = srp_qp_event; 498 init_attr->cap.max_send_wr = m * target->queue_size; 499 init_attr->cap.max_recv_wr = target->queue_size + 1; 500 init_attr->cap.max_recv_sge = 1; 501 init_attr->cap.max_send_sge = 1; 502 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 503 init_attr->qp_type = IB_QPT_RC; 504 init_attr->send_cq = send_cq; 505 init_attr->recv_cq = recv_cq; 506 507 qp = ib_create_qp(dev->pd, init_attr); 508 if (IS_ERR(qp)) { 509 ret = PTR_ERR(qp); 510 goto err_send_cq; 511 } 512 513 ret = srp_init_qp(target, qp); 514 if (ret) 515 goto err_qp; 516 517 if (dev->use_fast_reg) { 518 fr_pool = srp_alloc_fr_pool(target); 519 if (IS_ERR(fr_pool)) { 520 ret = PTR_ERR(fr_pool); 521 shost_printk(KERN_WARNING, target->scsi_host, PFX 522 "FR pool allocation failed (%d)\n", ret); 523 goto err_qp; 524 } 525 } else if (dev->use_fmr) { 526 fmr_pool = srp_alloc_fmr_pool(target); 527 if (IS_ERR(fmr_pool)) { 528 ret = PTR_ERR(fmr_pool); 529 shost_printk(KERN_WARNING, target->scsi_host, PFX 530 "FMR pool allocation failed (%d)\n", ret); 531 goto err_qp; 532 } 533 } 534 535 if (ch->qp) 536 srp_destroy_qp(ch->qp); 537 if (ch->recv_cq) 538 ib_free_cq(ch->recv_cq); 539 if (ch->send_cq) 540 ib_free_cq(ch->send_cq); 541 542 ch->qp = qp; 543 ch->recv_cq = recv_cq; 544 ch->send_cq = send_cq; 545 546 if (dev->use_fast_reg) { 547 if (ch->fr_pool) 548 srp_destroy_fr_pool(ch->fr_pool); 549 ch->fr_pool = fr_pool; 550 } else if (dev->use_fmr) { 551 if (ch->fmr_pool) 552 ib_destroy_fmr_pool(ch->fmr_pool); 553 ch->fmr_pool = fmr_pool; 554 } 555 556 kfree(init_attr); 557 return 0; 558 559 err_qp: 560 srp_destroy_qp(qp); 561 562 err_send_cq: 563 ib_free_cq(send_cq); 564 565 err_recv_cq: 566 ib_free_cq(recv_cq); 567 568 err: 569 kfree(init_attr); 570 return ret; 571 } 572 573 /* 574 * Note: this function may be called without srp_alloc_iu_bufs() having been 575 * invoked. Hence the ch->[rt]x_ring checks. 576 */ 577 static void srp_free_ch_ib(struct srp_target_port *target, 578 struct srp_rdma_ch *ch) 579 { 580 struct srp_device *dev = target->srp_host->srp_dev; 581 int i; 582 583 if (!ch->target) 584 return; 585 586 if (ch->cm_id) { 587 ib_destroy_cm_id(ch->cm_id); 588 ch->cm_id = NULL; 589 } 590 591 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */ 592 if (!ch->qp) 593 return; 594 595 if (dev->use_fast_reg) { 596 if (ch->fr_pool) 597 srp_destroy_fr_pool(ch->fr_pool); 598 } else if (dev->use_fmr) { 599 if (ch->fmr_pool) 600 ib_destroy_fmr_pool(ch->fmr_pool); 601 } 602 603 srp_destroy_qp(ch->qp); 604 ib_free_cq(ch->send_cq); 605 ib_free_cq(ch->recv_cq); 606 607 /* 608 * Avoid that the SCSI error handler tries to use this channel after 609 * it has been freed. The SCSI error handler can namely continue 610 * trying to perform recovery actions after scsi_remove_host() 611 * returned. 612 */ 613 ch->target = NULL; 614 615 ch->qp = NULL; 616 ch->send_cq = ch->recv_cq = NULL; 617 618 if (ch->rx_ring) { 619 for (i = 0; i < target->queue_size; ++i) 620 srp_free_iu(target->srp_host, ch->rx_ring[i]); 621 kfree(ch->rx_ring); 622 ch->rx_ring = NULL; 623 } 624 if (ch->tx_ring) { 625 for (i = 0; i < target->queue_size; ++i) 626 srp_free_iu(target->srp_host, ch->tx_ring[i]); 627 kfree(ch->tx_ring); 628 ch->tx_ring = NULL; 629 } 630 } 631 632 static void srp_path_rec_completion(int status, 633 struct ib_sa_path_rec *pathrec, 634 void *ch_ptr) 635 { 636 struct srp_rdma_ch *ch = ch_ptr; 637 struct srp_target_port *target = ch->target; 638 639 ch->status = status; 640 if (status) 641 shost_printk(KERN_ERR, target->scsi_host, 642 PFX "Got failed path rec status %d\n", status); 643 else 644 ch->path = *pathrec; 645 complete(&ch->done); 646 } 647 648 static int srp_lookup_path(struct srp_rdma_ch *ch) 649 { 650 struct srp_target_port *target = ch->target; 651 int ret; 652 653 ch->path.numb_path = 1; 654 655 init_completion(&ch->done); 656 657 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client, 658 target->srp_host->srp_dev->dev, 659 target->srp_host->port, 660 &ch->path, 661 IB_SA_PATH_REC_SERVICE_ID | 662 IB_SA_PATH_REC_DGID | 663 IB_SA_PATH_REC_SGID | 664 IB_SA_PATH_REC_NUMB_PATH | 665 IB_SA_PATH_REC_PKEY, 666 SRP_PATH_REC_TIMEOUT_MS, 667 GFP_KERNEL, 668 srp_path_rec_completion, 669 ch, &ch->path_query); 670 if (ch->path_query_id < 0) 671 return ch->path_query_id; 672 673 ret = wait_for_completion_interruptible(&ch->done); 674 if (ret < 0) 675 return ret; 676 677 if (ch->status < 0) 678 shost_printk(KERN_WARNING, target->scsi_host, 679 PFX "Path record query failed\n"); 680 681 return ch->status; 682 } 683 684 static int srp_send_req(struct srp_rdma_ch *ch, bool multich) 685 { 686 struct srp_target_port *target = ch->target; 687 struct { 688 struct ib_cm_req_param param; 689 struct srp_login_req priv; 690 } *req = NULL; 691 int status; 692 693 req = kzalloc(sizeof *req, GFP_KERNEL); 694 if (!req) 695 return -ENOMEM; 696 697 req->param.primary_path = &ch->path; 698 req->param.alternate_path = NULL; 699 req->param.service_id = target->service_id; 700 req->param.qp_num = ch->qp->qp_num; 701 req->param.qp_type = ch->qp->qp_type; 702 req->param.private_data = &req->priv; 703 req->param.private_data_len = sizeof req->priv; 704 req->param.flow_control = 1; 705 706 get_random_bytes(&req->param.starting_psn, 4); 707 req->param.starting_psn &= 0xffffff; 708 709 /* 710 * Pick some arbitrary defaults here; we could make these 711 * module parameters if anyone cared about setting them. 712 */ 713 req->param.responder_resources = 4; 714 req->param.remote_cm_response_timeout = 20; 715 req->param.local_cm_response_timeout = 20; 716 req->param.retry_count = target->tl_retry_count; 717 req->param.rnr_retry_count = 7; 718 req->param.max_cm_retries = 15; 719 720 req->priv.opcode = SRP_LOGIN_REQ; 721 req->priv.tag = 0; 722 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len); 723 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | 724 SRP_BUF_FORMAT_INDIRECT); 725 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI : 726 SRP_MULTICHAN_SINGLE); 727 /* 728 * In the published SRP specification (draft rev. 16a), the 729 * port identifier format is 8 bytes of ID extension followed 730 * by 8 bytes of GUID. Older drafts put the two halves in the 731 * opposite order, so that the GUID comes first. 732 * 733 * Targets conforming to these obsolete drafts can be 734 * recognized by the I/O Class they report. 735 */ 736 if (target->io_class == SRP_REV10_IB_IO_CLASS) { 737 memcpy(req->priv.initiator_port_id, 738 &target->sgid.global.interface_id, 8); 739 memcpy(req->priv.initiator_port_id + 8, 740 &target->initiator_ext, 8); 741 memcpy(req->priv.target_port_id, &target->ioc_guid, 8); 742 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8); 743 } else { 744 memcpy(req->priv.initiator_port_id, 745 &target->initiator_ext, 8); 746 memcpy(req->priv.initiator_port_id + 8, 747 &target->sgid.global.interface_id, 8); 748 memcpy(req->priv.target_port_id, &target->id_ext, 8); 749 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8); 750 } 751 752 /* 753 * Topspin/Cisco SRP targets will reject our login unless we 754 * zero out the first 8 bytes of our initiator port ID and set 755 * the second 8 bytes to the local node GUID. 756 */ 757 if (srp_target_is_topspin(target)) { 758 shost_printk(KERN_DEBUG, target->scsi_host, 759 PFX "Topspin/Cisco initiator port ID workaround " 760 "activated for target GUID %016llx\n", 761 be64_to_cpu(target->ioc_guid)); 762 memset(req->priv.initiator_port_id, 0, 8); 763 memcpy(req->priv.initiator_port_id + 8, 764 &target->srp_host->srp_dev->dev->node_guid, 8); 765 } 766 767 status = ib_send_cm_req(ch->cm_id, &req->param); 768 769 kfree(req); 770 771 return status; 772 } 773 774 static bool srp_queue_remove_work(struct srp_target_port *target) 775 { 776 bool changed = false; 777 778 spin_lock_irq(&target->lock); 779 if (target->state != SRP_TARGET_REMOVED) { 780 target->state = SRP_TARGET_REMOVED; 781 changed = true; 782 } 783 spin_unlock_irq(&target->lock); 784 785 if (changed) 786 queue_work(srp_remove_wq, &target->remove_work); 787 788 return changed; 789 } 790 791 static void srp_disconnect_target(struct srp_target_port *target) 792 { 793 struct srp_rdma_ch *ch; 794 int i; 795 796 /* XXX should send SRP_I_LOGOUT request */ 797 798 for (i = 0; i < target->ch_count; i++) { 799 ch = &target->ch[i]; 800 ch->connected = false; 801 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) { 802 shost_printk(KERN_DEBUG, target->scsi_host, 803 PFX "Sending CM DREQ failed\n"); 804 } 805 } 806 } 807 808 static void srp_free_req_data(struct srp_target_port *target, 809 struct srp_rdma_ch *ch) 810 { 811 struct srp_device *dev = target->srp_host->srp_dev; 812 struct ib_device *ibdev = dev->dev; 813 struct srp_request *req; 814 int i; 815 816 if (!ch->req_ring) 817 return; 818 819 for (i = 0; i < target->req_ring_size; ++i) { 820 req = &ch->req_ring[i]; 821 if (dev->use_fast_reg) { 822 kfree(req->fr_list); 823 } else { 824 kfree(req->fmr_list); 825 kfree(req->map_page); 826 } 827 if (req->indirect_dma_addr) { 828 ib_dma_unmap_single(ibdev, req->indirect_dma_addr, 829 target->indirect_size, 830 DMA_TO_DEVICE); 831 } 832 kfree(req->indirect_desc); 833 } 834 835 kfree(ch->req_ring); 836 ch->req_ring = NULL; 837 } 838 839 static int srp_alloc_req_data(struct srp_rdma_ch *ch) 840 { 841 struct srp_target_port *target = ch->target; 842 struct srp_device *srp_dev = target->srp_host->srp_dev; 843 struct ib_device *ibdev = srp_dev->dev; 844 struct srp_request *req; 845 void *mr_list; 846 dma_addr_t dma_addr; 847 int i, ret = -ENOMEM; 848 849 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring), 850 GFP_KERNEL); 851 if (!ch->req_ring) 852 goto out; 853 854 for (i = 0; i < target->req_ring_size; ++i) { 855 req = &ch->req_ring[i]; 856 mr_list = kmalloc(target->mr_per_cmd * sizeof(void *), 857 GFP_KERNEL); 858 if (!mr_list) 859 goto out; 860 if (srp_dev->use_fast_reg) { 861 req->fr_list = mr_list; 862 } else { 863 req->fmr_list = mr_list; 864 req->map_page = kmalloc(srp_dev->max_pages_per_mr * 865 sizeof(void *), GFP_KERNEL); 866 if (!req->map_page) 867 goto out; 868 } 869 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL); 870 if (!req->indirect_desc) 871 goto out; 872 873 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc, 874 target->indirect_size, 875 DMA_TO_DEVICE); 876 if (ib_dma_mapping_error(ibdev, dma_addr)) 877 goto out; 878 879 req->indirect_dma_addr = dma_addr; 880 } 881 ret = 0; 882 883 out: 884 return ret; 885 } 886 887 /** 888 * srp_del_scsi_host_attr() - Remove attributes defined in the host template. 889 * @shost: SCSI host whose attributes to remove from sysfs. 890 * 891 * Note: Any attributes defined in the host template and that did not exist 892 * before invocation of this function will be ignored. 893 */ 894 static void srp_del_scsi_host_attr(struct Scsi_Host *shost) 895 { 896 struct device_attribute **attr; 897 898 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr) 899 device_remove_file(&shost->shost_dev, *attr); 900 } 901 902 static void srp_remove_target(struct srp_target_port *target) 903 { 904 struct srp_rdma_ch *ch; 905 int i; 906 907 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 908 909 srp_del_scsi_host_attr(target->scsi_host); 910 srp_rport_get(target->rport); 911 srp_remove_host(target->scsi_host); 912 scsi_remove_host(target->scsi_host); 913 srp_stop_rport_timers(target->rport); 914 srp_disconnect_target(target); 915 for (i = 0; i < target->ch_count; i++) { 916 ch = &target->ch[i]; 917 srp_free_ch_ib(target, ch); 918 } 919 cancel_work_sync(&target->tl_err_work); 920 srp_rport_put(target->rport); 921 for (i = 0; i < target->ch_count; i++) { 922 ch = &target->ch[i]; 923 srp_free_req_data(target, ch); 924 } 925 kfree(target->ch); 926 target->ch = NULL; 927 928 spin_lock(&target->srp_host->target_lock); 929 list_del(&target->list); 930 spin_unlock(&target->srp_host->target_lock); 931 932 scsi_host_put(target->scsi_host); 933 } 934 935 static void srp_remove_work(struct work_struct *work) 936 { 937 struct srp_target_port *target = 938 container_of(work, struct srp_target_port, remove_work); 939 940 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 941 942 srp_remove_target(target); 943 } 944 945 static void srp_rport_delete(struct srp_rport *rport) 946 { 947 struct srp_target_port *target = rport->lld_data; 948 949 srp_queue_remove_work(target); 950 } 951 952 /** 953 * srp_connected_ch() - number of connected channels 954 * @target: SRP target port. 955 */ 956 static int srp_connected_ch(struct srp_target_port *target) 957 { 958 int i, c = 0; 959 960 for (i = 0; i < target->ch_count; i++) 961 c += target->ch[i].connected; 962 963 return c; 964 } 965 966 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich) 967 { 968 struct srp_target_port *target = ch->target; 969 int ret; 970 971 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0); 972 973 ret = srp_lookup_path(ch); 974 if (ret) 975 goto out; 976 977 while (1) { 978 init_completion(&ch->done); 979 ret = srp_send_req(ch, multich); 980 if (ret) 981 goto out; 982 ret = wait_for_completion_interruptible(&ch->done); 983 if (ret < 0) 984 goto out; 985 986 /* 987 * The CM event handling code will set status to 988 * SRP_PORT_REDIRECT if we get a port redirect REJ 989 * back, or SRP_DLID_REDIRECT if we get a lid/qp 990 * redirect REJ back. 991 */ 992 ret = ch->status; 993 switch (ret) { 994 case 0: 995 ch->connected = true; 996 goto out; 997 998 case SRP_PORT_REDIRECT: 999 ret = srp_lookup_path(ch); 1000 if (ret) 1001 goto out; 1002 break; 1003 1004 case SRP_DLID_REDIRECT: 1005 break; 1006 1007 case SRP_STALE_CONN: 1008 shost_printk(KERN_ERR, target->scsi_host, PFX 1009 "giving up on stale connection\n"); 1010 ret = -ECONNRESET; 1011 goto out; 1012 1013 default: 1014 goto out; 1015 } 1016 } 1017 1018 out: 1019 return ret <= 0 ? ret : -ENODEV; 1020 } 1021 1022 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc) 1023 { 1024 srp_handle_qp_err(cq, wc, "INV RKEY"); 1025 } 1026 1027 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch, 1028 u32 rkey) 1029 { 1030 struct ib_send_wr *bad_wr; 1031 struct ib_send_wr wr = { 1032 .opcode = IB_WR_LOCAL_INV, 1033 .next = NULL, 1034 .num_sge = 0, 1035 .send_flags = 0, 1036 .ex.invalidate_rkey = rkey, 1037 }; 1038 1039 wr.wr_cqe = &req->reg_cqe; 1040 req->reg_cqe.done = srp_inv_rkey_err_done; 1041 return ib_post_send(ch->qp, &wr, &bad_wr); 1042 } 1043 1044 static void srp_unmap_data(struct scsi_cmnd *scmnd, 1045 struct srp_rdma_ch *ch, 1046 struct srp_request *req) 1047 { 1048 struct srp_target_port *target = ch->target; 1049 struct srp_device *dev = target->srp_host->srp_dev; 1050 struct ib_device *ibdev = dev->dev; 1051 int i, res; 1052 1053 if (!scsi_sglist(scmnd) || 1054 (scmnd->sc_data_direction != DMA_TO_DEVICE && 1055 scmnd->sc_data_direction != DMA_FROM_DEVICE)) 1056 return; 1057 1058 if (dev->use_fast_reg) { 1059 struct srp_fr_desc **pfr; 1060 1061 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { 1062 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey); 1063 if (res < 0) { 1064 shost_printk(KERN_ERR, target->scsi_host, PFX 1065 "Queueing INV WR for rkey %#x failed (%d)\n", 1066 (*pfr)->mr->rkey, res); 1067 queue_work(system_long_wq, 1068 &target->tl_err_work); 1069 } 1070 } 1071 if (req->nmdesc) 1072 srp_fr_pool_put(ch->fr_pool, req->fr_list, 1073 req->nmdesc); 1074 } else if (dev->use_fmr) { 1075 struct ib_pool_fmr **pfmr; 1076 1077 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++) 1078 ib_fmr_pool_unmap(*pfmr); 1079 } 1080 1081 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd), 1082 scmnd->sc_data_direction); 1083 } 1084 1085 /** 1086 * srp_claim_req - Take ownership of the scmnd associated with a request. 1087 * @ch: SRP RDMA channel. 1088 * @req: SRP request. 1089 * @sdev: If not NULL, only take ownership for this SCSI device. 1090 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take 1091 * ownership of @req->scmnd if it equals @scmnd. 1092 * 1093 * Return value: 1094 * Either NULL or a pointer to the SCSI command the caller became owner of. 1095 */ 1096 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch, 1097 struct srp_request *req, 1098 struct scsi_device *sdev, 1099 struct scsi_cmnd *scmnd) 1100 { 1101 unsigned long flags; 1102 1103 spin_lock_irqsave(&ch->lock, flags); 1104 if (req->scmnd && 1105 (!sdev || req->scmnd->device == sdev) && 1106 (!scmnd || req->scmnd == scmnd)) { 1107 scmnd = req->scmnd; 1108 req->scmnd = NULL; 1109 } else { 1110 scmnd = NULL; 1111 } 1112 spin_unlock_irqrestore(&ch->lock, flags); 1113 1114 return scmnd; 1115 } 1116 1117 /** 1118 * srp_free_req() - Unmap data and adjust ch->req_lim. 1119 * @ch: SRP RDMA channel. 1120 * @req: Request to be freed. 1121 * @scmnd: SCSI command associated with @req. 1122 * @req_lim_delta: Amount to be added to @target->req_lim. 1123 */ 1124 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req, 1125 struct scsi_cmnd *scmnd, s32 req_lim_delta) 1126 { 1127 unsigned long flags; 1128 1129 srp_unmap_data(scmnd, ch, req); 1130 1131 spin_lock_irqsave(&ch->lock, flags); 1132 ch->req_lim += req_lim_delta; 1133 spin_unlock_irqrestore(&ch->lock, flags); 1134 } 1135 1136 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req, 1137 struct scsi_device *sdev, int result) 1138 { 1139 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL); 1140 1141 if (scmnd) { 1142 srp_free_req(ch, req, scmnd, 0); 1143 scmnd->result = result; 1144 scmnd->scsi_done(scmnd); 1145 } 1146 } 1147 1148 static void srp_terminate_io(struct srp_rport *rport) 1149 { 1150 struct srp_target_port *target = rport->lld_data; 1151 struct srp_rdma_ch *ch; 1152 struct Scsi_Host *shost = target->scsi_host; 1153 struct scsi_device *sdev; 1154 int i, j; 1155 1156 /* 1157 * Invoking srp_terminate_io() while srp_queuecommand() is running 1158 * is not safe. Hence the warning statement below. 1159 */ 1160 shost_for_each_device(sdev, shost) 1161 WARN_ON_ONCE(sdev->request_queue->request_fn_active); 1162 1163 for (i = 0; i < target->ch_count; i++) { 1164 ch = &target->ch[i]; 1165 1166 for (j = 0; j < target->req_ring_size; ++j) { 1167 struct srp_request *req = &ch->req_ring[j]; 1168 1169 srp_finish_req(ch, req, NULL, 1170 DID_TRANSPORT_FAILFAST << 16); 1171 } 1172 } 1173 } 1174 1175 /* 1176 * It is up to the caller to ensure that srp_rport_reconnect() calls are 1177 * serialized and that no concurrent srp_queuecommand(), srp_abort(), 1178 * srp_reset_device() or srp_reset_host() calls will occur while this function 1179 * is in progress. One way to realize that is not to call this function 1180 * directly but to call srp_reconnect_rport() instead since that last function 1181 * serializes calls of this function via rport->mutex and also blocks 1182 * srp_queuecommand() calls before invoking this function. 1183 */ 1184 static int srp_rport_reconnect(struct srp_rport *rport) 1185 { 1186 struct srp_target_port *target = rport->lld_data; 1187 struct srp_rdma_ch *ch; 1188 int i, j, ret = 0; 1189 bool multich = false; 1190 1191 srp_disconnect_target(target); 1192 1193 if (target->state == SRP_TARGET_SCANNING) 1194 return -ENODEV; 1195 1196 /* 1197 * Now get a new local CM ID so that we avoid confusing the target in 1198 * case things are really fouled up. Doing so also ensures that all CM 1199 * callbacks will have finished before a new QP is allocated. 1200 */ 1201 for (i = 0; i < target->ch_count; i++) { 1202 ch = &target->ch[i]; 1203 ret += srp_new_cm_id(ch); 1204 } 1205 for (i = 0; i < target->ch_count; i++) { 1206 ch = &target->ch[i]; 1207 for (j = 0; j < target->req_ring_size; ++j) { 1208 struct srp_request *req = &ch->req_ring[j]; 1209 1210 srp_finish_req(ch, req, NULL, DID_RESET << 16); 1211 } 1212 } 1213 for (i = 0; i < target->ch_count; i++) { 1214 ch = &target->ch[i]; 1215 /* 1216 * Whether or not creating a new CM ID succeeded, create a new 1217 * QP. This guarantees that all completion callback function 1218 * invocations have finished before request resetting starts. 1219 */ 1220 ret += srp_create_ch_ib(ch); 1221 1222 INIT_LIST_HEAD(&ch->free_tx); 1223 for (j = 0; j < target->queue_size; ++j) 1224 list_add(&ch->tx_ring[j]->list, &ch->free_tx); 1225 } 1226 1227 target->qp_in_error = false; 1228 1229 for (i = 0; i < target->ch_count; i++) { 1230 ch = &target->ch[i]; 1231 if (ret) 1232 break; 1233 ret = srp_connect_ch(ch, multich); 1234 multich = true; 1235 } 1236 1237 if (ret == 0) 1238 shost_printk(KERN_INFO, target->scsi_host, 1239 PFX "reconnect succeeded\n"); 1240 1241 return ret; 1242 } 1243 1244 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, 1245 unsigned int dma_len, u32 rkey) 1246 { 1247 struct srp_direct_buf *desc = state->desc; 1248 1249 WARN_ON_ONCE(!dma_len); 1250 1251 desc->va = cpu_to_be64(dma_addr); 1252 desc->key = cpu_to_be32(rkey); 1253 desc->len = cpu_to_be32(dma_len); 1254 1255 state->total_len += dma_len; 1256 state->desc++; 1257 state->ndesc++; 1258 } 1259 1260 static int srp_map_finish_fmr(struct srp_map_state *state, 1261 struct srp_rdma_ch *ch) 1262 { 1263 struct srp_target_port *target = ch->target; 1264 struct srp_device *dev = target->srp_host->srp_dev; 1265 struct ib_pool_fmr *fmr; 1266 u64 io_addr = 0; 1267 1268 if (state->fmr.next >= state->fmr.end) 1269 return -ENOMEM; 1270 1271 WARN_ON_ONCE(!dev->use_fmr); 1272 1273 if (state->npages == 0) 1274 return 0; 1275 1276 if (state->npages == 1 && target->global_mr) { 1277 srp_map_desc(state, state->base_dma_addr, state->dma_len, 1278 target->global_mr->rkey); 1279 goto reset_state; 1280 } 1281 1282 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages, 1283 state->npages, io_addr); 1284 if (IS_ERR(fmr)) 1285 return PTR_ERR(fmr); 1286 1287 *state->fmr.next++ = fmr; 1288 state->nmdesc++; 1289 1290 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask, 1291 state->dma_len, fmr->fmr->rkey); 1292 1293 reset_state: 1294 state->npages = 0; 1295 state->dma_len = 0; 1296 1297 return 0; 1298 } 1299 1300 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc) 1301 { 1302 srp_handle_qp_err(cq, wc, "FAST REG"); 1303 } 1304 1305 /* 1306 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset 1307 * where to start in the first element. If sg_offset_p != NULL then 1308 * *sg_offset_p is updated to the offset in state->sg[retval] of the first 1309 * byte that has not yet been mapped. 1310 */ 1311 static int srp_map_finish_fr(struct srp_map_state *state, 1312 struct srp_request *req, 1313 struct srp_rdma_ch *ch, int sg_nents, 1314 unsigned int *sg_offset_p) 1315 { 1316 struct srp_target_port *target = ch->target; 1317 struct srp_device *dev = target->srp_host->srp_dev; 1318 struct ib_send_wr *bad_wr; 1319 struct ib_reg_wr wr; 1320 struct srp_fr_desc *desc; 1321 u32 rkey; 1322 int n, err; 1323 1324 if (state->fr.next >= state->fr.end) 1325 return -ENOMEM; 1326 1327 WARN_ON_ONCE(!dev->use_fast_reg); 1328 1329 if (sg_nents == 1 && target->global_mr) { 1330 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1331 1332 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset, 1333 sg_dma_len(state->sg) - sg_offset, 1334 target->global_mr->rkey); 1335 if (sg_offset_p) 1336 *sg_offset_p = 0; 1337 return 1; 1338 } 1339 1340 desc = srp_fr_pool_get(ch->fr_pool); 1341 if (!desc) 1342 return -ENOMEM; 1343 1344 rkey = ib_inc_rkey(desc->mr->rkey); 1345 ib_update_fast_reg_key(desc->mr, rkey); 1346 1347 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p, 1348 dev->mr_page_size); 1349 if (unlikely(n < 0)) { 1350 srp_fr_pool_put(ch->fr_pool, &desc, 1); 1351 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n", 1352 dev_name(&req->scmnd->device->sdev_gendev), sg_nents, 1353 sg_offset_p ? *sg_offset_p : -1, n); 1354 return n; 1355 } 1356 1357 WARN_ON_ONCE(desc->mr->length == 0); 1358 1359 req->reg_cqe.done = srp_reg_mr_err_done; 1360 1361 wr.wr.next = NULL; 1362 wr.wr.opcode = IB_WR_REG_MR; 1363 wr.wr.wr_cqe = &req->reg_cqe; 1364 wr.wr.num_sge = 0; 1365 wr.wr.send_flags = 0; 1366 wr.mr = desc->mr; 1367 wr.key = desc->mr->rkey; 1368 wr.access = (IB_ACCESS_LOCAL_WRITE | 1369 IB_ACCESS_REMOTE_READ | 1370 IB_ACCESS_REMOTE_WRITE); 1371 1372 *state->fr.next++ = desc; 1373 state->nmdesc++; 1374 1375 srp_map_desc(state, desc->mr->iova, 1376 desc->mr->length, desc->mr->rkey); 1377 1378 err = ib_post_send(ch->qp, &wr.wr, &bad_wr); 1379 if (unlikely(err)) { 1380 WARN_ON_ONCE(err == -ENOMEM); 1381 return err; 1382 } 1383 1384 return n; 1385 } 1386 1387 static int srp_map_sg_entry(struct srp_map_state *state, 1388 struct srp_rdma_ch *ch, 1389 struct scatterlist *sg, int sg_index) 1390 { 1391 struct srp_target_port *target = ch->target; 1392 struct srp_device *dev = target->srp_host->srp_dev; 1393 struct ib_device *ibdev = dev->dev; 1394 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg); 1395 unsigned int dma_len = ib_sg_dma_len(ibdev, sg); 1396 unsigned int len = 0; 1397 int ret; 1398 1399 WARN_ON_ONCE(!dma_len); 1400 1401 while (dma_len) { 1402 unsigned offset = dma_addr & ~dev->mr_page_mask; 1403 if (state->npages == dev->max_pages_per_mr || offset != 0) { 1404 ret = srp_map_finish_fmr(state, ch); 1405 if (ret) 1406 return ret; 1407 } 1408 1409 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset); 1410 1411 if (!state->npages) 1412 state->base_dma_addr = dma_addr; 1413 state->pages[state->npages++] = dma_addr & dev->mr_page_mask; 1414 state->dma_len += len; 1415 dma_addr += len; 1416 dma_len -= len; 1417 } 1418 1419 /* 1420 * If the last entry of the MR wasn't a full page, then we need to 1421 * close it out and start a new one -- we can only merge at page 1422 * boundaries. 1423 */ 1424 ret = 0; 1425 if (len != dev->mr_page_size) 1426 ret = srp_map_finish_fmr(state, ch); 1427 return ret; 1428 } 1429 1430 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1431 struct srp_request *req, struct scatterlist *scat, 1432 int count) 1433 { 1434 struct scatterlist *sg; 1435 int i, ret; 1436 1437 state->pages = req->map_page; 1438 state->fmr.next = req->fmr_list; 1439 state->fmr.end = req->fmr_list + ch->target->mr_per_cmd; 1440 1441 for_each_sg(scat, sg, count, i) { 1442 ret = srp_map_sg_entry(state, ch, sg, i); 1443 if (ret) 1444 return ret; 1445 } 1446 1447 ret = srp_map_finish_fmr(state, ch); 1448 if (ret) 1449 return ret; 1450 1451 return 0; 1452 } 1453 1454 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1455 struct srp_request *req, struct scatterlist *scat, 1456 int count) 1457 { 1458 unsigned int sg_offset = 0; 1459 1460 state->fr.next = req->fr_list; 1461 state->fr.end = req->fr_list + ch->target->mr_per_cmd; 1462 state->sg = scat; 1463 1464 if (count == 0) 1465 return 0; 1466 1467 while (count) { 1468 int i, n; 1469 1470 n = srp_map_finish_fr(state, req, ch, count, &sg_offset); 1471 if (unlikely(n < 0)) 1472 return n; 1473 1474 count -= n; 1475 for (i = 0; i < n; i++) 1476 state->sg = sg_next(state->sg); 1477 } 1478 1479 return 0; 1480 } 1481 1482 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch, 1483 struct srp_request *req, struct scatterlist *scat, 1484 int count) 1485 { 1486 struct srp_target_port *target = ch->target; 1487 struct srp_device *dev = target->srp_host->srp_dev; 1488 struct scatterlist *sg; 1489 int i; 1490 1491 for_each_sg(scat, sg, count, i) { 1492 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg), 1493 ib_sg_dma_len(dev->dev, sg), 1494 target->global_mr->rkey); 1495 } 1496 1497 return 0; 1498 } 1499 1500 /* 1501 * Register the indirect data buffer descriptor with the HCA. 1502 * 1503 * Note: since the indirect data buffer descriptor has been allocated with 1504 * kmalloc() it is guaranteed that this buffer is a physically contiguous 1505 * memory buffer. 1506 */ 1507 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req, 1508 void **next_mr, void **end_mr, u32 idb_len, 1509 __be32 *idb_rkey) 1510 { 1511 struct srp_target_port *target = ch->target; 1512 struct srp_device *dev = target->srp_host->srp_dev; 1513 struct srp_map_state state; 1514 struct srp_direct_buf idb_desc; 1515 u64 idb_pages[1]; 1516 struct scatterlist idb_sg[1]; 1517 int ret; 1518 1519 memset(&state, 0, sizeof(state)); 1520 memset(&idb_desc, 0, sizeof(idb_desc)); 1521 state.gen.next = next_mr; 1522 state.gen.end = end_mr; 1523 state.desc = &idb_desc; 1524 state.base_dma_addr = req->indirect_dma_addr; 1525 state.dma_len = idb_len; 1526 1527 if (dev->use_fast_reg) { 1528 state.sg = idb_sg; 1529 sg_init_one(idb_sg, req->indirect_desc, idb_len); 1530 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */ 1531 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1532 idb_sg->dma_length = idb_sg->length; /* hack^2 */ 1533 #endif 1534 ret = srp_map_finish_fr(&state, req, ch, 1, NULL); 1535 if (ret < 0) 1536 return ret; 1537 WARN_ON_ONCE(ret < 1); 1538 } else if (dev->use_fmr) { 1539 state.pages = idb_pages; 1540 state.pages[0] = (req->indirect_dma_addr & 1541 dev->mr_page_mask); 1542 state.npages = 1; 1543 ret = srp_map_finish_fmr(&state, ch); 1544 if (ret < 0) 1545 return ret; 1546 } else { 1547 return -EINVAL; 1548 } 1549 1550 *idb_rkey = idb_desc.key; 1551 1552 return 0; 1553 } 1554 1555 #if defined(DYNAMIC_DATA_DEBUG) 1556 static void srp_check_mapping(struct srp_map_state *state, 1557 struct srp_rdma_ch *ch, struct srp_request *req, 1558 struct scatterlist *scat, int count) 1559 { 1560 struct srp_device *dev = ch->target->srp_host->srp_dev; 1561 struct srp_fr_desc **pfr; 1562 u64 desc_len = 0, mr_len = 0; 1563 int i; 1564 1565 for (i = 0; i < state->ndesc; i++) 1566 desc_len += be32_to_cpu(req->indirect_desc[i].len); 1567 if (dev->use_fast_reg) 1568 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++) 1569 mr_len += (*pfr)->mr->length; 1570 else if (dev->use_fmr) 1571 for (i = 0; i < state->nmdesc; i++) 1572 mr_len += be32_to_cpu(req->indirect_desc[i].len); 1573 if (desc_len != scsi_bufflen(req->scmnd) || 1574 mr_len > scsi_bufflen(req->scmnd)) 1575 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n", 1576 scsi_bufflen(req->scmnd), desc_len, mr_len, 1577 state->ndesc, state->nmdesc); 1578 } 1579 #endif 1580 1581 /** 1582 * srp_map_data() - map SCSI data buffer onto an SRP request 1583 * @scmnd: SCSI command to map 1584 * @ch: SRP RDMA channel 1585 * @req: SRP request 1586 * 1587 * Returns the length in bytes of the SRP_CMD IU or a negative value if 1588 * mapping failed. 1589 */ 1590 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch, 1591 struct srp_request *req) 1592 { 1593 struct srp_target_port *target = ch->target; 1594 struct scatterlist *scat; 1595 struct srp_cmd *cmd = req->cmd->buf; 1596 int len, nents, count, ret; 1597 struct srp_device *dev; 1598 struct ib_device *ibdev; 1599 struct srp_map_state state; 1600 struct srp_indirect_buf *indirect_hdr; 1601 u32 idb_len, table_len; 1602 __be32 idb_rkey; 1603 u8 fmt; 1604 1605 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE) 1606 return sizeof (struct srp_cmd); 1607 1608 if (scmnd->sc_data_direction != DMA_FROM_DEVICE && 1609 scmnd->sc_data_direction != DMA_TO_DEVICE) { 1610 shost_printk(KERN_WARNING, target->scsi_host, 1611 PFX "Unhandled data direction %d\n", 1612 scmnd->sc_data_direction); 1613 return -EINVAL; 1614 } 1615 1616 nents = scsi_sg_count(scmnd); 1617 scat = scsi_sglist(scmnd); 1618 1619 dev = target->srp_host->srp_dev; 1620 ibdev = dev->dev; 1621 1622 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction); 1623 if (unlikely(count == 0)) 1624 return -EIO; 1625 1626 fmt = SRP_DATA_DESC_DIRECT; 1627 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf); 1628 1629 if (count == 1 && target->global_mr) { 1630 /* 1631 * The midlayer only generated a single gather/scatter 1632 * entry, or DMA mapping coalesced everything to a 1633 * single entry. So a direct descriptor along with 1634 * the DMA MR suffices. 1635 */ 1636 struct srp_direct_buf *buf = (void *) cmd->add_data; 1637 1638 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat)); 1639 buf->key = cpu_to_be32(target->global_mr->rkey); 1640 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat)); 1641 1642 req->nmdesc = 0; 1643 goto map_complete; 1644 } 1645 1646 /* 1647 * We have more than one scatter/gather entry, so build our indirect 1648 * descriptor table, trying to merge as many entries as we can. 1649 */ 1650 indirect_hdr = (void *) cmd->add_data; 1651 1652 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr, 1653 target->indirect_size, DMA_TO_DEVICE); 1654 1655 memset(&state, 0, sizeof(state)); 1656 state.desc = req->indirect_desc; 1657 if (dev->use_fast_reg) 1658 ret = srp_map_sg_fr(&state, ch, req, scat, count); 1659 else if (dev->use_fmr) 1660 ret = srp_map_sg_fmr(&state, ch, req, scat, count); 1661 else 1662 ret = srp_map_sg_dma(&state, ch, req, scat, count); 1663 req->nmdesc = state.nmdesc; 1664 if (ret < 0) 1665 goto unmap; 1666 1667 #if defined(DYNAMIC_DEBUG) 1668 { 1669 DEFINE_DYNAMIC_DEBUG_METADATA(ddm, 1670 "Memory mapping consistency check"); 1671 if (unlikely(ddm.flags & _DPRINTK_FLAGS_PRINT)) 1672 srp_check_mapping(&state, ch, req, scat, count); 1673 } 1674 #endif 1675 1676 /* We've mapped the request, now pull as much of the indirect 1677 * descriptor table as we can into the command buffer. If this 1678 * target is not using an external indirect table, we are 1679 * guaranteed to fit into the command, as the SCSI layer won't 1680 * give us more S/G entries than we allow. 1681 */ 1682 if (state.ndesc == 1) { 1683 /* 1684 * Memory registration collapsed the sg-list into one entry, 1685 * so use a direct descriptor. 1686 */ 1687 struct srp_direct_buf *buf = (void *) cmd->add_data; 1688 1689 *buf = req->indirect_desc[0]; 1690 goto map_complete; 1691 } 1692 1693 if (unlikely(target->cmd_sg_cnt < state.ndesc && 1694 !target->allow_ext_sg)) { 1695 shost_printk(KERN_ERR, target->scsi_host, 1696 "Could not fit S/G list into SRP_CMD\n"); 1697 ret = -EIO; 1698 goto unmap; 1699 } 1700 1701 count = min(state.ndesc, target->cmd_sg_cnt); 1702 table_len = state.ndesc * sizeof (struct srp_direct_buf); 1703 idb_len = sizeof(struct srp_indirect_buf) + table_len; 1704 1705 fmt = SRP_DATA_DESC_INDIRECT; 1706 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf); 1707 len += count * sizeof (struct srp_direct_buf); 1708 1709 memcpy(indirect_hdr->desc_list, req->indirect_desc, 1710 count * sizeof (struct srp_direct_buf)); 1711 1712 if (!target->global_mr) { 1713 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end, 1714 idb_len, &idb_rkey); 1715 if (ret < 0) 1716 goto unmap; 1717 req->nmdesc++; 1718 } else { 1719 idb_rkey = cpu_to_be32(target->global_mr->rkey); 1720 } 1721 1722 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr); 1723 indirect_hdr->table_desc.key = idb_rkey; 1724 indirect_hdr->table_desc.len = cpu_to_be32(table_len); 1725 indirect_hdr->len = cpu_to_be32(state.total_len); 1726 1727 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1728 cmd->data_out_desc_cnt = count; 1729 else 1730 cmd->data_in_desc_cnt = count; 1731 1732 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len, 1733 DMA_TO_DEVICE); 1734 1735 map_complete: 1736 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1737 cmd->buf_fmt = fmt << 4; 1738 else 1739 cmd->buf_fmt = fmt; 1740 1741 return len; 1742 1743 unmap: 1744 srp_unmap_data(scmnd, ch, req); 1745 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size) 1746 ret = -E2BIG; 1747 return ret; 1748 } 1749 1750 /* 1751 * Return an IU and possible credit to the free pool 1752 */ 1753 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu, 1754 enum srp_iu_type iu_type) 1755 { 1756 unsigned long flags; 1757 1758 spin_lock_irqsave(&ch->lock, flags); 1759 list_add(&iu->list, &ch->free_tx); 1760 if (iu_type != SRP_IU_RSP) 1761 ++ch->req_lim; 1762 spin_unlock_irqrestore(&ch->lock, flags); 1763 } 1764 1765 /* 1766 * Must be called with ch->lock held to protect req_lim and free_tx. 1767 * If IU is not sent, it must be returned using srp_put_tx_iu(). 1768 * 1769 * Note: 1770 * An upper limit for the number of allocated information units for each 1771 * request type is: 1772 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues 1773 * more than Scsi_Host.can_queue requests. 1774 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE. 1775 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than 1776 * one unanswered SRP request to an initiator. 1777 */ 1778 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch, 1779 enum srp_iu_type iu_type) 1780 { 1781 struct srp_target_port *target = ch->target; 1782 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; 1783 struct srp_iu *iu; 1784 1785 ib_process_cq_direct(ch->send_cq, -1); 1786 1787 if (list_empty(&ch->free_tx)) 1788 return NULL; 1789 1790 /* Initiator responses to target requests do not consume credits */ 1791 if (iu_type != SRP_IU_RSP) { 1792 if (ch->req_lim <= rsv) { 1793 ++target->zero_req_lim; 1794 return NULL; 1795 } 1796 1797 --ch->req_lim; 1798 } 1799 1800 iu = list_first_entry(&ch->free_tx, struct srp_iu, list); 1801 list_del(&iu->list); 1802 return iu; 1803 } 1804 1805 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc) 1806 { 1807 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1808 struct srp_rdma_ch *ch = cq->cq_context; 1809 1810 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1811 srp_handle_qp_err(cq, wc, "SEND"); 1812 return; 1813 } 1814 1815 list_add(&iu->list, &ch->free_tx); 1816 } 1817 1818 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len) 1819 { 1820 struct srp_target_port *target = ch->target; 1821 struct ib_sge list; 1822 struct ib_send_wr wr, *bad_wr; 1823 1824 list.addr = iu->dma; 1825 list.length = len; 1826 list.lkey = target->lkey; 1827 1828 iu->cqe.done = srp_send_done; 1829 1830 wr.next = NULL; 1831 wr.wr_cqe = &iu->cqe; 1832 wr.sg_list = &list; 1833 wr.num_sge = 1; 1834 wr.opcode = IB_WR_SEND; 1835 wr.send_flags = IB_SEND_SIGNALED; 1836 1837 return ib_post_send(ch->qp, &wr, &bad_wr); 1838 } 1839 1840 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu) 1841 { 1842 struct srp_target_port *target = ch->target; 1843 struct ib_recv_wr wr, *bad_wr; 1844 struct ib_sge list; 1845 1846 list.addr = iu->dma; 1847 list.length = iu->size; 1848 list.lkey = target->lkey; 1849 1850 iu->cqe.done = srp_recv_done; 1851 1852 wr.next = NULL; 1853 wr.wr_cqe = &iu->cqe; 1854 wr.sg_list = &list; 1855 wr.num_sge = 1; 1856 1857 return ib_post_recv(ch->qp, &wr, &bad_wr); 1858 } 1859 1860 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) 1861 { 1862 struct srp_target_port *target = ch->target; 1863 struct srp_request *req; 1864 struct scsi_cmnd *scmnd; 1865 unsigned long flags; 1866 1867 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { 1868 spin_lock_irqsave(&ch->lock, flags); 1869 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1870 spin_unlock_irqrestore(&ch->lock, flags); 1871 1872 ch->tsk_mgmt_status = -1; 1873 if (be32_to_cpu(rsp->resp_data_len) >= 4) 1874 ch->tsk_mgmt_status = rsp->data[3]; 1875 complete(&ch->tsk_mgmt_done); 1876 } else { 1877 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag); 1878 if (scmnd) { 1879 req = (void *)scmnd->host_scribble; 1880 scmnd = srp_claim_req(ch, req, NULL, scmnd); 1881 } 1882 if (!scmnd) { 1883 shost_printk(KERN_ERR, target->scsi_host, 1884 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", 1885 rsp->tag, ch - target->ch, ch->qp->qp_num); 1886 1887 spin_lock_irqsave(&ch->lock, flags); 1888 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1889 spin_unlock_irqrestore(&ch->lock, flags); 1890 1891 return; 1892 } 1893 scmnd->result = rsp->status; 1894 1895 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { 1896 memcpy(scmnd->sense_buffer, rsp->data + 1897 be32_to_cpu(rsp->resp_data_len), 1898 min_t(int, be32_to_cpu(rsp->sense_data_len), 1899 SCSI_SENSE_BUFFERSIZE)); 1900 } 1901 1902 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER)) 1903 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt)); 1904 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER)) 1905 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt)); 1906 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER)) 1907 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt)); 1908 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER)) 1909 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt)); 1910 1911 srp_free_req(ch, req, scmnd, 1912 be32_to_cpu(rsp->req_lim_delta)); 1913 1914 scmnd->host_scribble = NULL; 1915 scmnd->scsi_done(scmnd); 1916 } 1917 } 1918 1919 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, 1920 void *rsp, int len) 1921 { 1922 struct srp_target_port *target = ch->target; 1923 struct ib_device *dev = target->srp_host->srp_dev->dev; 1924 unsigned long flags; 1925 struct srp_iu *iu; 1926 int err; 1927 1928 spin_lock_irqsave(&ch->lock, flags); 1929 ch->req_lim += req_delta; 1930 iu = __srp_get_tx_iu(ch, SRP_IU_RSP); 1931 spin_unlock_irqrestore(&ch->lock, flags); 1932 1933 if (!iu) { 1934 shost_printk(KERN_ERR, target->scsi_host, PFX 1935 "no IU available to send response\n"); 1936 return 1; 1937 } 1938 1939 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); 1940 memcpy(iu->buf, rsp, len); 1941 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); 1942 1943 err = srp_post_send(ch, iu, len); 1944 if (err) { 1945 shost_printk(KERN_ERR, target->scsi_host, PFX 1946 "unable to post response: %d\n", err); 1947 srp_put_tx_iu(ch, iu, SRP_IU_RSP); 1948 } 1949 1950 return err; 1951 } 1952 1953 static void srp_process_cred_req(struct srp_rdma_ch *ch, 1954 struct srp_cred_req *req) 1955 { 1956 struct srp_cred_rsp rsp = { 1957 .opcode = SRP_CRED_RSP, 1958 .tag = req->tag, 1959 }; 1960 s32 delta = be32_to_cpu(req->req_lim_delta); 1961 1962 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 1963 shost_printk(KERN_ERR, ch->target->scsi_host, PFX 1964 "problems processing SRP_CRED_REQ\n"); 1965 } 1966 1967 static void srp_process_aer_req(struct srp_rdma_ch *ch, 1968 struct srp_aer_req *req) 1969 { 1970 struct srp_target_port *target = ch->target; 1971 struct srp_aer_rsp rsp = { 1972 .opcode = SRP_AER_RSP, 1973 .tag = req->tag, 1974 }; 1975 s32 delta = be32_to_cpu(req->req_lim_delta); 1976 1977 shost_printk(KERN_ERR, target->scsi_host, PFX 1978 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); 1979 1980 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 1981 shost_printk(KERN_ERR, target->scsi_host, PFX 1982 "problems processing SRP_AER_REQ\n"); 1983 } 1984 1985 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1986 { 1987 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1988 struct srp_rdma_ch *ch = cq->cq_context; 1989 struct srp_target_port *target = ch->target; 1990 struct ib_device *dev = target->srp_host->srp_dev->dev; 1991 int res; 1992 u8 opcode; 1993 1994 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1995 srp_handle_qp_err(cq, wc, "RECV"); 1996 return; 1997 } 1998 1999 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len, 2000 DMA_FROM_DEVICE); 2001 2002 opcode = *(u8 *) iu->buf; 2003 2004 if (0) { 2005 shost_printk(KERN_ERR, target->scsi_host, 2006 PFX "recv completion, opcode 0x%02x\n", opcode); 2007 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1, 2008 iu->buf, wc->byte_len, true); 2009 } 2010 2011 switch (opcode) { 2012 case SRP_RSP: 2013 srp_process_rsp(ch, iu->buf); 2014 break; 2015 2016 case SRP_CRED_REQ: 2017 srp_process_cred_req(ch, iu->buf); 2018 break; 2019 2020 case SRP_AER_REQ: 2021 srp_process_aer_req(ch, iu->buf); 2022 break; 2023 2024 case SRP_T_LOGOUT: 2025 /* XXX Handle target logout */ 2026 shost_printk(KERN_WARNING, target->scsi_host, 2027 PFX "Got target logout request\n"); 2028 break; 2029 2030 default: 2031 shost_printk(KERN_WARNING, target->scsi_host, 2032 PFX "Unhandled SRP opcode 0x%02x\n", opcode); 2033 break; 2034 } 2035 2036 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len, 2037 DMA_FROM_DEVICE); 2038 2039 res = srp_post_recv(ch, iu); 2040 if (res != 0) 2041 shost_printk(KERN_ERR, target->scsi_host, 2042 PFX "Recv failed with error code %d\n", res); 2043 } 2044 2045 /** 2046 * srp_tl_err_work() - handle a transport layer error 2047 * @work: Work structure embedded in an SRP target port. 2048 * 2049 * Note: This function may get invoked before the rport has been created, 2050 * hence the target->rport test. 2051 */ 2052 static void srp_tl_err_work(struct work_struct *work) 2053 { 2054 struct srp_target_port *target; 2055 2056 target = container_of(work, struct srp_target_port, tl_err_work); 2057 if (target->rport) 2058 srp_start_tl_fail_timers(target->rport); 2059 } 2060 2061 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 2062 const char *opname) 2063 { 2064 struct srp_rdma_ch *ch = cq->cq_context; 2065 struct srp_target_port *target = ch->target; 2066 2067 if (ch->connected && !target->qp_in_error) { 2068 shost_printk(KERN_ERR, target->scsi_host, 2069 PFX "failed %s status %s (%d) for CQE %p\n", 2070 opname, ib_wc_status_msg(wc->status), wc->status, 2071 wc->wr_cqe); 2072 queue_work(system_long_wq, &target->tl_err_work); 2073 } 2074 target->qp_in_error = true; 2075 } 2076 2077 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) 2078 { 2079 struct srp_target_port *target = host_to_target(shost); 2080 struct srp_rport *rport = target->rport; 2081 struct srp_rdma_ch *ch; 2082 struct srp_request *req; 2083 struct srp_iu *iu; 2084 struct srp_cmd *cmd; 2085 struct ib_device *dev; 2086 unsigned long flags; 2087 u32 tag; 2088 u16 idx; 2089 int len, ret; 2090 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler; 2091 2092 /* 2093 * The SCSI EH thread is the only context from which srp_queuecommand() 2094 * can get invoked for blocked devices (SDEV_BLOCK / 2095 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by 2096 * locking the rport mutex if invoked from inside the SCSI EH. 2097 */ 2098 if (in_scsi_eh) 2099 mutex_lock(&rport->mutex); 2100 2101 scmnd->result = srp_chkready(target->rport); 2102 if (unlikely(scmnd->result)) 2103 goto err; 2104 2105 WARN_ON_ONCE(scmnd->request->tag < 0); 2106 tag = blk_mq_unique_tag(scmnd->request); 2107 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 2108 idx = blk_mq_unique_tag_to_tag(tag); 2109 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n", 2110 dev_name(&shost->shost_gendev), tag, idx, 2111 target->req_ring_size); 2112 2113 spin_lock_irqsave(&ch->lock, flags); 2114 iu = __srp_get_tx_iu(ch, SRP_IU_CMD); 2115 spin_unlock_irqrestore(&ch->lock, flags); 2116 2117 if (!iu) 2118 goto err; 2119 2120 req = &ch->req_ring[idx]; 2121 dev = target->srp_host->srp_dev->dev; 2122 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len, 2123 DMA_TO_DEVICE); 2124 2125 scmnd->host_scribble = (void *) req; 2126 2127 cmd = iu->buf; 2128 memset(cmd, 0, sizeof *cmd); 2129 2130 cmd->opcode = SRP_CMD; 2131 int_to_scsilun(scmnd->device->lun, &cmd->lun); 2132 cmd->tag = tag; 2133 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len); 2134 2135 req->scmnd = scmnd; 2136 req->cmd = iu; 2137 2138 len = srp_map_data(scmnd, ch, req); 2139 if (len < 0) { 2140 shost_printk(KERN_ERR, target->scsi_host, 2141 PFX "Failed to map data (%d)\n", len); 2142 /* 2143 * If we ran out of memory descriptors (-ENOMEM) because an 2144 * application is queuing many requests with more than 2145 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer 2146 * to reduce queue depth temporarily. 2147 */ 2148 scmnd->result = len == -ENOMEM ? 2149 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16; 2150 goto err_iu; 2151 } 2152 2153 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len, 2154 DMA_TO_DEVICE); 2155 2156 if (srp_post_send(ch, iu, len)) { 2157 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); 2158 goto err_unmap; 2159 } 2160 2161 ret = 0; 2162 2163 unlock_rport: 2164 if (in_scsi_eh) 2165 mutex_unlock(&rport->mutex); 2166 2167 return ret; 2168 2169 err_unmap: 2170 srp_unmap_data(scmnd, ch, req); 2171 2172 err_iu: 2173 srp_put_tx_iu(ch, iu, SRP_IU_CMD); 2174 2175 /* 2176 * Avoid that the loops that iterate over the request ring can 2177 * encounter a dangling SCSI command pointer. 2178 */ 2179 req->scmnd = NULL; 2180 2181 err: 2182 if (scmnd->result) { 2183 scmnd->scsi_done(scmnd); 2184 ret = 0; 2185 } else { 2186 ret = SCSI_MLQUEUE_HOST_BUSY; 2187 } 2188 2189 goto unlock_rport; 2190 } 2191 2192 /* 2193 * Note: the resources allocated in this function are freed in 2194 * srp_free_ch_ib(). 2195 */ 2196 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch) 2197 { 2198 struct srp_target_port *target = ch->target; 2199 int i; 2200 2201 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring), 2202 GFP_KERNEL); 2203 if (!ch->rx_ring) 2204 goto err_no_ring; 2205 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring), 2206 GFP_KERNEL); 2207 if (!ch->tx_ring) 2208 goto err_no_ring; 2209 2210 for (i = 0; i < target->queue_size; ++i) { 2211 ch->rx_ring[i] = srp_alloc_iu(target->srp_host, 2212 ch->max_ti_iu_len, 2213 GFP_KERNEL, DMA_FROM_DEVICE); 2214 if (!ch->rx_ring[i]) 2215 goto err; 2216 } 2217 2218 for (i = 0; i < target->queue_size; ++i) { 2219 ch->tx_ring[i] = srp_alloc_iu(target->srp_host, 2220 target->max_iu_len, 2221 GFP_KERNEL, DMA_TO_DEVICE); 2222 if (!ch->tx_ring[i]) 2223 goto err; 2224 2225 list_add(&ch->tx_ring[i]->list, &ch->free_tx); 2226 } 2227 2228 return 0; 2229 2230 err: 2231 for (i = 0; i < target->queue_size; ++i) { 2232 srp_free_iu(target->srp_host, ch->rx_ring[i]); 2233 srp_free_iu(target->srp_host, ch->tx_ring[i]); 2234 } 2235 2236 2237 err_no_ring: 2238 kfree(ch->tx_ring); 2239 ch->tx_ring = NULL; 2240 kfree(ch->rx_ring); 2241 ch->rx_ring = NULL; 2242 2243 return -ENOMEM; 2244 } 2245 2246 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) 2247 { 2248 uint64_t T_tr_ns, max_compl_time_ms; 2249 uint32_t rq_tmo_jiffies; 2250 2251 /* 2252 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair, 2253 * table 91), both the QP timeout and the retry count have to be set 2254 * for RC QP's during the RTR to RTS transition. 2255 */ 2256 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) != 2257 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)); 2258 2259 /* 2260 * Set target->rq_tmo_jiffies to one second more than the largest time 2261 * it can take before an error completion is generated. See also 2262 * C9-140..142 in the IBTA spec for more information about how to 2263 * convert the QP Local ACK Timeout value to nanoseconds. 2264 */ 2265 T_tr_ns = 4096 * (1ULL << qp_attr->timeout); 2266 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns; 2267 do_div(max_compl_time_ms, NSEC_PER_MSEC); 2268 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000); 2269 2270 return rq_tmo_jiffies; 2271 } 2272 2273 static void srp_cm_rep_handler(struct ib_cm_id *cm_id, 2274 const struct srp_login_rsp *lrsp, 2275 struct srp_rdma_ch *ch) 2276 { 2277 struct srp_target_port *target = ch->target; 2278 struct ib_qp_attr *qp_attr = NULL; 2279 int attr_mask = 0; 2280 int ret; 2281 int i; 2282 2283 if (lrsp->opcode == SRP_LOGIN_RSP) { 2284 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); 2285 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); 2286 2287 /* 2288 * Reserve credits for task management so we don't 2289 * bounce requests back to the SCSI mid-layer. 2290 */ 2291 target->scsi_host->can_queue 2292 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE, 2293 target->scsi_host->can_queue); 2294 target->scsi_host->cmd_per_lun 2295 = min_t(int, target->scsi_host->can_queue, 2296 target->scsi_host->cmd_per_lun); 2297 } else { 2298 shost_printk(KERN_WARNING, target->scsi_host, 2299 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode); 2300 ret = -ECONNRESET; 2301 goto error; 2302 } 2303 2304 if (!ch->rx_ring) { 2305 ret = srp_alloc_iu_bufs(ch); 2306 if (ret) 2307 goto error; 2308 } 2309 2310 ret = -ENOMEM; 2311 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL); 2312 if (!qp_attr) 2313 goto error; 2314 2315 qp_attr->qp_state = IB_QPS_RTR; 2316 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2317 if (ret) 2318 goto error_free; 2319 2320 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2321 if (ret) 2322 goto error_free; 2323 2324 for (i = 0; i < target->queue_size; i++) { 2325 struct srp_iu *iu = ch->rx_ring[i]; 2326 2327 ret = srp_post_recv(ch, iu); 2328 if (ret) 2329 goto error_free; 2330 } 2331 2332 qp_attr->qp_state = IB_QPS_RTS; 2333 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2334 if (ret) 2335 goto error_free; 2336 2337 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); 2338 2339 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2340 if (ret) 2341 goto error_free; 2342 2343 ret = ib_send_cm_rtu(cm_id, NULL, 0); 2344 2345 error_free: 2346 kfree(qp_attr); 2347 2348 error: 2349 ch->status = ret; 2350 } 2351 2352 static void srp_cm_rej_handler(struct ib_cm_id *cm_id, 2353 struct ib_cm_event *event, 2354 struct srp_rdma_ch *ch) 2355 { 2356 struct srp_target_port *target = ch->target; 2357 struct Scsi_Host *shost = target->scsi_host; 2358 struct ib_class_port_info *cpi; 2359 int opcode; 2360 2361 switch (event->param.rej_rcvd.reason) { 2362 case IB_CM_REJ_PORT_CM_REDIRECT: 2363 cpi = event->param.rej_rcvd.ari; 2364 ch->path.dlid = cpi->redirect_lid; 2365 ch->path.pkey = cpi->redirect_pkey; 2366 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; 2367 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16); 2368 2369 ch->status = ch->path.dlid ? 2370 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; 2371 break; 2372 2373 case IB_CM_REJ_PORT_REDIRECT: 2374 if (srp_target_is_topspin(target)) { 2375 /* 2376 * Topspin/Cisco SRP gateways incorrectly send 2377 * reject reason code 25 when they mean 24 2378 * (port redirect). 2379 */ 2380 memcpy(ch->path.dgid.raw, 2381 event->param.rej_rcvd.ari, 16); 2382 2383 shost_printk(KERN_DEBUG, shost, 2384 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", 2385 be64_to_cpu(ch->path.dgid.global.subnet_prefix), 2386 be64_to_cpu(ch->path.dgid.global.interface_id)); 2387 2388 ch->status = SRP_PORT_REDIRECT; 2389 } else { 2390 shost_printk(KERN_WARNING, shost, 2391 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); 2392 ch->status = -ECONNRESET; 2393 } 2394 break; 2395 2396 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2397 shost_printk(KERN_WARNING, shost, 2398 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2399 ch->status = -ECONNRESET; 2400 break; 2401 2402 case IB_CM_REJ_CONSUMER_DEFINED: 2403 opcode = *(u8 *) event->private_data; 2404 if (opcode == SRP_LOGIN_REJ) { 2405 struct srp_login_rej *rej = event->private_data; 2406 u32 reason = be32_to_cpu(rej->reason); 2407 2408 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2409 shost_printk(KERN_WARNING, shost, 2410 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2411 else 2412 shost_printk(KERN_WARNING, shost, PFX 2413 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n", 2414 target->sgid.raw, 2415 target->orig_dgid.raw, reason); 2416 } else 2417 shost_printk(KERN_WARNING, shost, 2418 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED," 2419 " opcode 0x%02x\n", opcode); 2420 ch->status = -ECONNRESET; 2421 break; 2422 2423 case IB_CM_REJ_STALE_CONN: 2424 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); 2425 ch->status = SRP_STALE_CONN; 2426 break; 2427 2428 default: 2429 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2430 event->param.rej_rcvd.reason); 2431 ch->status = -ECONNRESET; 2432 } 2433 } 2434 2435 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) 2436 { 2437 struct srp_rdma_ch *ch = cm_id->context; 2438 struct srp_target_port *target = ch->target; 2439 int comp = 0; 2440 2441 switch (event->event) { 2442 case IB_CM_REQ_ERROR: 2443 shost_printk(KERN_DEBUG, target->scsi_host, 2444 PFX "Sending CM REQ failed\n"); 2445 comp = 1; 2446 ch->status = -ECONNRESET; 2447 break; 2448 2449 case IB_CM_REP_RECEIVED: 2450 comp = 1; 2451 srp_cm_rep_handler(cm_id, event->private_data, ch); 2452 break; 2453 2454 case IB_CM_REJ_RECEIVED: 2455 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2456 comp = 1; 2457 2458 srp_cm_rej_handler(cm_id, event, ch); 2459 break; 2460 2461 case IB_CM_DREQ_RECEIVED: 2462 shost_printk(KERN_WARNING, target->scsi_host, 2463 PFX "DREQ received - connection closed\n"); 2464 ch->connected = false; 2465 if (ib_send_cm_drep(cm_id, NULL, 0)) 2466 shost_printk(KERN_ERR, target->scsi_host, 2467 PFX "Sending CM DREP failed\n"); 2468 queue_work(system_long_wq, &target->tl_err_work); 2469 break; 2470 2471 case IB_CM_TIMEWAIT_EXIT: 2472 shost_printk(KERN_ERR, target->scsi_host, 2473 PFX "connection closed\n"); 2474 comp = 1; 2475 2476 ch->status = 0; 2477 break; 2478 2479 case IB_CM_MRA_RECEIVED: 2480 case IB_CM_DREQ_ERROR: 2481 case IB_CM_DREP_RECEIVED: 2482 break; 2483 2484 default: 2485 shost_printk(KERN_WARNING, target->scsi_host, 2486 PFX "Unhandled CM event %d\n", event->event); 2487 break; 2488 } 2489 2490 if (comp) 2491 complete(&ch->done); 2492 2493 return 0; 2494 } 2495 2496 /** 2497 * srp_change_queue_depth - setting device queue depth 2498 * @sdev: scsi device struct 2499 * @qdepth: requested queue depth 2500 * 2501 * Returns queue depth. 2502 */ 2503 static int 2504 srp_change_queue_depth(struct scsi_device *sdev, int qdepth) 2505 { 2506 if (!sdev->tagged_supported) 2507 qdepth = 1; 2508 return scsi_change_queue_depth(sdev, qdepth); 2509 } 2510 2511 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, 2512 u8 func) 2513 { 2514 struct srp_target_port *target = ch->target; 2515 struct srp_rport *rport = target->rport; 2516 struct ib_device *dev = target->srp_host->srp_dev->dev; 2517 struct srp_iu *iu; 2518 struct srp_tsk_mgmt *tsk_mgmt; 2519 2520 if (!ch->connected || target->qp_in_error) 2521 return -1; 2522 2523 init_completion(&ch->tsk_mgmt_done); 2524 2525 /* 2526 * Lock the rport mutex to avoid that srp_create_ch_ib() is 2527 * invoked while a task management function is being sent. 2528 */ 2529 mutex_lock(&rport->mutex); 2530 spin_lock_irq(&ch->lock); 2531 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT); 2532 spin_unlock_irq(&ch->lock); 2533 2534 if (!iu) { 2535 mutex_unlock(&rport->mutex); 2536 2537 return -1; 2538 } 2539 2540 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, 2541 DMA_TO_DEVICE); 2542 tsk_mgmt = iu->buf; 2543 memset(tsk_mgmt, 0, sizeof *tsk_mgmt); 2544 2545 tsk_mgmt->opcode = SRP_TSK_MGMT; 2546 int_to_scsilun(lun, &tsk_mgmt->lun); 2547 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT; 2548 tsk_mgmt->tsk_mgmt_func = func; 2549 tsk_mgmt->task_tag = req_tag; 2550 2551 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, 2552 DMA_TO_DEVICE); 2553 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) { 2554 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT); 2555 mutex_unlock(&rport->mutex); 2556 2557 return -1; 2558 } 2559 mutex_unlock(&rport->mutex); 2560 2561 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done, 2562 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS))) 2563 return -1; 2564 2565 return 0; 2566 } 2567 2568 static int srp_abort(struct scsi_cmnd *scmnd) 2569 { 2570 struct srp_target_port *target = host_to_target(scmnd->device->host); 2571 struct srp_request *req = (struct srp_request *) scmnd->host_scribble; 2572 u32 tag; 2573 u16 ch_idx; 2574 struct srp_rdma_ch *ch; 2575 int ret; 2576 2577 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); 2578 2579 if (!req) 2580 return SUCCESS; 2581 tag = blk_mq_unique_tag(scmnd->request); 2582 ch_idx = blk_mq_unique_tag_to_hwq(tag); 2583 if (WARN_ON_ONCE(ch_idx >= target->ch_count)) 2584 return SUCCESS; 2585 ch = &target->ch[ch_idx]; 2586 if (!srp_claim_req(ch, req, NULL, scmnd)) 2587 return SUCCESS; 2588 shost_printk(KERN_ERR, target->scsi_host, 2589 "Sending SRP abort for tag %#x\n", tag); 2590 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun, 2591 SRP_TSK_ABORT_TASK) == 0) 2592 ret = SUCCESS; 2593 else if (target->rport->state == SRP_RPORT_LOST) 2594 ret = FAST_IO_FAIL; 2595 else 2596 ret = FAILED; 2597 srp_free_req(ch, req, scmnd, 0); 2598 scmnd->result = DID_ABORT << 16; 2599 scmnd->scsi_done(scmnd); 2600 2601 return ret; 2602 } 2603 2604 static int srp_reset_device(struct scsi_cmnd *scmnd) 2605 { 2606 struct srp_target_port *target = host_to_target(scmnd->device->host); 2607 struct srp_rdma_ch *ch; 2608 int i; 2609 2610 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); 2611 2612 ch = &target->ch[0]; 2613 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun, 2614 SRP_TSK_LUN_RESET)) 2615 return FAILED; 2616 if (ch->tsk_mgmt_status) 2617 return FAILED; 2618 2619 for (i = 0; i < target->ch_count; i++) { 2620 ch = &target->ch[i]; 2621 for (i = 0; i < target->req_ring_size; ++i) { 2622 struct srp_request *req = &ch->req_ring[i]; 2623 2624 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16); 2625 } 2626 } 2627 2628 return SUCCESS; 2629 } 2630 2631 static int srp_reset_host(struct scsi_cmnd *scmnd) 2632 { 2633 struct srp_target_port *target = host_to_target(scmnd->device->host); 2634 2635 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n"); 2636 2637 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED; 2638 } 2639 2640 static int srp_slave_alloc(struct scsi_device *sdev) 2641 { 2642 struct Scsi_Host *shost = sdev->host; 2643 struct srp_target_port *target = host_to_target(shost); 2644 struct srp_device *srp_dev = target->srp_host->srp_dev; 2645 struct ib_device *ibdev = srp_dev->dev; 2646 2647 if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)) 2648 blk_queue_virt_boundary(sdev->request_queue, 2649 ~srp_dev->mr_page_mask); 2650 2651 return 0; 2652 } 2653 2654 static int srp_slave_configure(struct scsi_device *sdev) 2655 { 2656 struct Scsi_Host *shost = sdev->host; 2657 struct srp_target_port *target = host_to_target(shost); 2658 struct request_queue *q = sdev->request_queue; 2659 unsigned long timeout; 2660 2661 if (sdev->type == TYPE_DISK) { 2662 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies); 2663 blk_queue_rq_timeout(q, timeout); 2664 } 2665 2666 return 0; 2667 } 2668 2669 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr, 2670 char *buf) 2671 { 2672 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2673 2674 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext)); 2675 } 2676 2677 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr, 2678 char *buf) 2679 { 2680 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2681 2682 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid)); 2683 } 2684 2685 static ssize_t show_service_id(struct device *dev, 2686 struct device_attribute *attr, char *buf) 2687 { 2688 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2689 2690 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id)); 2691 } 2692 2693 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr, 2694 char *buf) 2695 { 2696 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2697 2698 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey)); 2699 } 2700 2701 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr, 2702 char *buf) 2703 { 2704 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2705 2706 return sprintf(buf, "%pI6\n", target->sgid.raw); 2707 } 2708 2709 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr, 2710 char *buf) 2711 { 2712 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2713 struct srp_rdma_ch *ch = &target->ch[0]; 2714 2715 return sprintf(buf, "%pI6\n", ch->path.dgid.raw); 2716 } 2717 2718 static ssize_t show_orig_dgid(struct device *dev, 2719 struct device_attribute *attr, char *buf) 2720 { 2721 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2722 2723 return sprintf(buf, "%pI6\n", target->orig_dgid.raw); 2724 } 2725 2726 static ssize_t show_req_lim(struct device *dev, 2727 struct device_attribute *attr, char *buf) 2728 { 2729 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2730 struct srp_rdma_ch *ch; 2731 int i, req_lim = INT_MAX; 2732 2733 for (i = 0; i < target->ch_count; i++) { 2734 ch = &target->ch[i]; 2735 req_lim = min(req_lim, ch->req_lim); 2736 } 2737 return sprintf(buf, "%d\n", req_lim); 2738 } 2739 2740 static ssize_t show_zero_req_lim(struct device *dev, 2741 struct device_attribute *attr, char *buf) 2742 { 2743 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2744 2745 return sprintf(buf, "%d\n", target->zero_req_lim); 2746 } 2747 2748 static ssize_t show_local_ib_port(struct device *dev, 2749 struct device_attribute *attr, char *buf) 2750 { 2751 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2752 2753 return sprintf(buf, "%d\n", target->srp_host->port); 2754 } 2755 2756 static ssize_t show_local_ib_device(struct device *dev, 2757 struct device_attribute *attr, char *buf) 2758 { 2759 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2760 2761 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name); 2762 } 2763 2764 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr, 2765 char *buf) 2766 { 2767 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2768 2769 return sprintf(buf, "%d\n", target->ch_count); 2770 } 2771 2772 static ssize_t show_comp_vector(struct device *dev, 2773 struct device_attribute *attr, char *buf) 2774 { 2775 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2776 2777 return sprintf(buf, "%d\n", target->comp_vector); 2778 } 2779 2780 static ssize_t show_tl_retry_count(struct device *dev, 2781 struct device_attribute *attr, char *buf) 2782 { 2783 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2784 2785 return sprintf(buf, "%d\n", target->tl_retry_count); 2786 } 2787 2788 static ssize_t show_cmd_sg_entries(struct device *dev, 2789 struct device_attribute *attr, char *buf) 2790 { 2791 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2792 2793 return sprintf(buf, "%u\n", target->cmd_sg_cnt); 2794 } 2795 2796 static ssize_t show_allow_ext_sg(struct device *dev, 2797 struct device_attribute *attr, char *buf) 2798 { 2799 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2800 2801 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false"); 2802 } 2803 2804 static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL); 2805 static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL); 2806 static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL); 2807 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL); 2808 static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL); 2809 static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL); 2810 static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL); 2811 static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL); 2812 static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL); 2813 static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL); 2814 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL); 2815 static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL); 2816 static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL); 2817 static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL); 2818 static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL); 2819 static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL); 2820 2821 static struct device_attribute *srp_host_attrs[] = { 2822 &dev_attr_id_ext, 2823 &dev_attr_ioc_guid, 2824 &dev_attr_service_id, 2825 &dev_attr_pkey, 2826 &dev_attr_sgid, 2827 &dev_attr_dgid, 2828 &dev_attr_orig_dgid, 2829 &dev_attr_req_lim, 2830 &dev_attr_zero_req_lim, 2831 &dev_attr_local_ib_port, 2832 &dev_attr_local_ib_device, 2833 &dev_attr_ch_count, 2834 &dev_attr_comp_vector, 2835 &dev_attr_tl_retry_count, 2836 &dev_attr_cmd_sg_entries, 2837 &dev_attr_allow_ext_sg, 2838 NULL 2839 }; 2840 2841 static struct scsi_host_template srp_template = { 2842 .module = THIS_MODULE, 2843 .name = "InfiniBand SRP initiator", 2844 .proc_name = DRV_NAME, 2845 .slave_alloc = srp_slave_alloc, 2846 .slave_configure = srp_slave_configure, 2847 .info = srp_target_info, 2848 .queuecommand = srp_queuecommand, 2849 .change_queue_depth = srp_change_queue_depth, 2850 .eh_abort_handler = srp_abort, 2851 .eh_device_reset_handler = srp_reset_device, 2852 .eh_host_reset_handler = srp_reset_host, 2853 .skip_settle_delay = true, 2854 .sg_tablesize = SRP_DEF_SG_TABLESIZE, 2855 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE, 2856 .this_id = -1, 2857 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE, 2858 .use_clustering = ENABLE_CLUSTERING, 2859 .shost_attrs = srp_host_attrs, 2860 .track_queue_depth = 1, 2861 }; 2862 2863 static int srp_sdev_count(struct Scsi_Host *host) 2864 { 2865 struct scsi_device *sdev; 2866 int c = 0; 2867 2868 shost_for_each_device(sdev, host) 2869 c++; 2870 2871 return c; 2872 } 2873 2874 /* 2875 * Return values: 2876 * < 0 upon failure. Caller is responsible for SRP target port cleanup. 2877 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port 2878 * removal has been scheduled. 2879 * 0 and target->state != SRP_TARGET_REMOVED upon success. 2880 */ 2881 static int srp_add_target(struct srp_host *host, struct srp_target_port *target) 2882 { 2883 struct srp_rport_identifiers ids; 2884 struct srp_rport *rport; 2885 2886 target->state = SRP_TARGET_SCANNING; 2887 sprintf(target->target_name, "SRP.T10:%016llX", 2888 be64_to_cpu(target->id_ext)); 2889 2890 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device)) 2891 return -ENODEV; 2892 2893 memcpy(ids.port_id, &target->id_ext, 8); 2894 memcpy(ids.port_id + 8, &target->ioc_guid, 8); 2895 ids.roles = SRP_RPORT_ROLE_TARGET; 2896 rport = srp_rport_add(target->scsi_host, &ids); 2897 if (IS_ERR(rport)) { 2898 scsi_remove_host(target->scsi_host); 2899 return PTR_ERR(rport); 2900 } 2901 2902 rport->lld_data = target; 2903 target->rport = rport; 2904 2905 spin_lock(&host->target_lock); 2906 list_add_tail(&target->list, &host->target_list); 2907 spin_unlock(&host->target_lock); 2908 2909 scsi_scan_target(&target->scsi_host->shost_gendev, 2910 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); 2911 2912 if (srp_connected_ch(target) < target->ch_count || 2913 target->qp_in_error) { 2914 shost_printk(KERN_INFO, target->scsi_host, 2915 PFX "SCSI scan failed - removing SCSI host\n"); 2916 srp_queue_remove_work(target); 2917 goto out; 2918 } 2919 2920 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n", 2921 dev_name(&target->scsi_host->shost_gendev), 2922 srp_sdev_count(target->scsi_host)); 2923 2924 spin_lock_irq(&target->lock); 2925 if (target->state == SRP_TARGET_SCANNING) 2926 target->state = SRP_TARGET_LIVE; 2927 spin_unlock_irq(&target->lock); 2928 2929 out: 2930 return 0; 2931 } 2932 2933 static void srp_release_dev(struct device *dev) 2934 { 2935 struct srp_host *host = 2936 container_of(dev, struct srp_host, dev); 2937 2938 complete(&host->released); 2939 } 2940 2941 static struct class srp_class = { 2942 .name = "infiniband_srp", 2943 .dev_release = srp_release_dev 2944 }; 2945 2946 /** 2947 * srp_conn_unique() - check whether the connection to a target is unique 2948 * @host: SRP host. 2949 * @target: SRP target port. 2950 */ 2951 static bool srp_conn_unique(struct srp_host *host, 2952 struct srp_target_port *target) 2953 { 2954 struct srp_target_port *t; 2955 bool ret = false; 2956 2957 if (target->state == SRP_TARGET_REMOVED) 2958 goto out; 2959 2960 ret = true; 2961 2962 spin_lock(&host->target_lock); 2963 list_for_each_entry(t, &host->target_list, list) { 2964 if (t != target && 2965 target->id_ext == t->id_ext && 2966 target->ioc_guid == t->ioc_guid && 2967 target->initiator_ext == t->initiator_ext) { 2968 ret = false; 2969 break; 2970 } 2971 } 2972 spin_unlock(&host->target_lock); 2973 2974 out: 2975 return ret; 2976 } 2977 2978 /* 2979 * Target ports are added by writing 2980 * 2981 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>, 2982 * pkey=<P_Key>,service_id=<service ID> 2983 * 2984 * to the add_target sysfs attribute. 2985 */ 2986 enum { 2987 SRP_OPT_ERR = 0, 2988 SRP_OPT_ID_EXT = 1 << 0, 2989 SRP_OPT_IOC_GUID = 1 << 1, 2990 SRP_OPT_DGID = 1 << 2, 2991 SRP_OPT_PKEY = 1 << 3, 2992 SRP_OPT_SERVICE_ID = 1 << 4, 2993 SRP_OPT_MAX_SECT = 1 << 5, 2994 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, 2995 SRP_OPT_IO_CLASS = 1 << 7, 2996 SRP_OPT_INITIATOR_EXT = 1 << 8, 2997 SRP_OPT_CMD_SG_ENTRIES = 1 << 9, 2998 SRP_OPT_ALLOW_EXT_SG = 1 << 10, 2999 SRP_OPT_SG_TABLESIZE = 1 << 11, 3000 SRP_OPT_COMP_VECTOR = 1 << 12, 3001 SRP_OPT_TL_RETRY_COUNT = 1 << 13, 3002 SRP_OPT_QUEUE_SIZE = 1 << 14, 3003 SRP_OPT_ALL = (SRP_OPT_ID_EXT | 3004 SRP_OPT_IOC_GUID | 3005 SRP_OPT_DGID | 3006 SRP_OPT_PKEY | 3007 SRP_OPT_SERVICE_ID), 3008 }; 3009 3010 static const match_table_t srp_opt_tokens = { 3011 { SRP_OPT_ID_EXT, "id_ext=%s" }, 3012 { SRP_OPT_IOC_GUID, "ioc_guid=%s" }, 3013 { SRP_OPT_DGID, "dgid=%s" }, 3014 { SRP_OPT_PKEY, "pkey=%x" }, 3015 { SRP_OPT_SERVICE_ID, "service_id=%s" }, 3016 { SRP_OPT_MAX_SECT, "max_sect=%d" }, 3017 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, 3018 { SRP_OPT_IO_CLASS, "io_class=%x" }, 3019 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, 3020 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, 3021 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, 3022 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, 3023 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" }, 3024 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" }, 3025 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" }, 3026 { SRP_OPT_ERR, NULL } 3027 }; 3028 3029 static int srp_parse_options(const char *buf, struct srp_target_port *target) 3030 { 3031 char *options, *sep_opt; 3032 char *p; 3033 char dgid[3]; 3034 substring_t args[MAX_OPT_ARGS]; 3035 int opt_mask = 0; 3036 int token; 3037 int ret = -EINVAL; 3038 int i; 3039 3040 options = kstrdup(buf, GFP_KERNEL); 3041 if (!options) 3042 return -ENOMEM; 3043 3044 sep_opt = options; 3045 while ((p = strsep(&sep_opt, ",\n")) != NULL) { 3046 if (!*p) 3047 continue; 3048 3049 token = match_token(p, srp_opt_tokens, args); 3050 opt_mask |= token; 3051 3052 switch (token) { 3053 case SRP_OPT_ID_EXT: 3054 p = match_strdup(args); 3055 if (!p) { 3056 ret = -ENOMEM; 3057 goto out; 3058 } 3059 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3060 kfree(p); 3061 break; 3062 3063 case SRP_OPT_IOC_GUID: 3064 p = match_strdup(args); 3065 if (!p) { 3066 ret = -ENOMEM; 3067 goto out; 3068 } 3069 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3070 kfree(p); 3071 break; 3072 3073 case SRP_OPT_DGID: 3074 p = match_strdup(args); 3075 if (!p) { 3076 ret = -ENOMEM; 3077 goto out; 3078 } 3079 if (strlen(p) != 32) { 3080 pr_warn("bad dest GID parameter '%s'\n", p); 3081 kfree(p); 3082 goto out; 3083 } 3084 3085 for (i = 0; i < 16; ++i) { 3086 strlcpy(dgid, p + i * 2, sizeof(dgid)); 3087 if (sscanf(dgid, "%hhx", 3088 &target->orig_dgid.raw[i]) < 1) { 3089 ret = -EINVAL; 3090 kfree(p); 3091 goto out; 3092 } 3093 } 3094 kfree(p); 3095 break; 3096 3097 case SRP_OPT_PKEY: 3098 if (match_hex(args, &token)) { 3099 pr_warn("bad P_Key parameter '%s'\n", p); 3100 goto out; 3101 } 3102 target->pkey = cpu_to_be16(token); 3103 break; 3104 3105 case SRP_OPT_SERVICE_ID: 3106 p = match_strdup(args); 3107 if (!p) { 3108 ret = -ENOMEM; 3109 goto out; 3110 } 3111 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3112 kfree(p); 3113 break; 3114 3115 case SRP_OPT_MAX_SECT: 3116 if (match_int(args, &token)) { 3117 pr_warn("bad max sect parameter '%s'\n", p); 3118 goto out; 3119 } 3120 target->scsi_host->max_sectors = token; 3121 break; 3122 3123 case SRP_OPT_QUEUE_SIZE: 3124 if (match_int(args, &token) || token < 1) { 3125 pr_warn("bad queue_size parameter '%s'\n", p); 3126 goto out; 3127 } 3128 target->scsi_host->can_queue = token; 3129 target->queue_size = token + SRP_RSP_SQ_SIZE + 3130 SRP_TSK_MGMT_SQ_SIZE; 3131 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3132 target->scsi_host->cmd_per_lun = token; 3133 break; 3134 3135 case SRP_OPT_MAX_CMD_PER_LUN: 3136 if (match_int(args, &token) || token < 1) { 3137 pr_warn("bad max cmd_per_lun parameter '%s'\n", 3138 p); 3139 goto out; 3140 } 3141 target->scsi_host->cmd_per_lun = token; 3142 break; 3143 3144 case SRP_OPT_IO_CLASS: 3145 if (match_hex(args, &token)) { 3146 pr_warn("bad IO class parameter '%s'\n", p); 3147 goto out; 3148 } 3149 if (token != SRP_REV10_IB_IO_CLASS && 3150 token != SRP_REV16A_IB_IO_CLASS) { 3151 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n", 3152 token, SRP_REV10_IB_IO_CLASS, 3153 SRP_REV16A_IB_IO_CLASS); 3154 goto out; 3155 } 3156 target->io_class = token; 3157 break; 3158 3159 case SRP_OPT_INITIATOR_EXT: 3160 p = match_strdup(args); 3161 if (!p) { 3162 ret = -ENOMEM; 3163 goto out; 3164 } 3165 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3166 kfree(p); 3167 break; 3168 3169 case SRP_OPT_CMD_SG_ENTRIES: 3170 if (match_int(args, &token) || token < 1 || token > 255) { 3171 pr_warn("bad max cmd_sg_entries parameter '%s'\n", 3172 p); 3173 goto out; 3174 } 3175 target->cmd_sg_cnt = token; 3176 break; 3177 3178 case SRP_OPT_ALLOW_EXT_SG: 3179 if (match_int(args, &token)) { 3180 pr_warn("bad allow_ext_sg parameter '%s'\n", p); 3181 goto out; 3182 } 3183 target->allow_ext_sg = !!token; 3184 break; 3185 3186 case SRP_OPT_SG_TABLESIZE: 3187 if (match_int(args, &token) || token < 1 || 3188 token > SG_MAX_SEGMENTS) { 3189 pr_warn("bad max sg_tablesize parameter '%s'\n", 3190 p); 3191 goto out; 3192 } 3193 target->sg_tablesize = token; 3194 break; 3195 3196 case SRP_OPT_COMP_VECTOR: 3197 if (match_int(args, &token) || token < 0) { 3198 pr_warn("bad comp_vector parameter '%s'\n", p); 3199 goto out; 3200 } 3201 target->comp_vector = token; 3202 break; 3203 3204 case SRP_OPT_TL_RETRY_COUNT: 3205 if (match_int(args, &token) || token < 2 || token > 7) { 3206 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n", 3207 p); 3208 goto out; 3209 } 3210 target->tl_retry_count = token; 3211 break; 3212 3213 default: 3214 pr_warn("unknown parameter or missing value '%s' in target creation request\n", 3215 p); 3216 goto out; 3217 } 3218 } 3219 3220 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL) 3221 ret = 0; 3222 else 3223 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i) 3224 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) && 3225 !(srp_opt_tokens[i].token & opt_mask)) 3226 pr_warn("target creation request is missing parameter '%s'\n", 3227 srp_opt_tokens[i].pattern); 3228 3229 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue 3230 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3231 pr_warn("cmd_per_lun = %d > queue_size = %d\n", 3232 target->scsi_host->cmd_per_lun, 3233 target->scsi_host->can_queue); 3234 3235 out: 3236 kfree(options); 3237 return ret; 3238 } 3239 3240 static ssize_t srp_create_target(struct device *dev, 3241 struct device_attribute *attr, 3242 const char *buf, size_t count) 3243 { 3244 struct srp_host *host = 3245 container_of(dev, struct srp_host, dev); 3246 struct Scsi_Host *target_host; 3247 struct srp_target_port *target; 3248 struct srp_rdma_ch *ch; 3249 struct srp_device *srp_dev = host->srp_dev; 3250 struct ib_device *ibdev = srp_dev->dev; 3251 int ret, node_idx, node, cpu, i; 3252 unsigned int max_sectors_per_mr, mr_per_cmd = 0; 3253 bool multich = false; 3254 3255 target_host = scsi_host_alloc(&srp_template, 3256 sizeof (struct srp_target_port)); 3257 if (!target_host) 3258 return -ENOMEM; 3259 3260 target_host->transportt = ib_srp_transport_template; 3261 target_host->max_channel = 0; 3262 target_host->max_id = 1; 3263 target_host->max_lun = -1LL; 3264 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; 3265 3266 target = host_to_target(target_host); 3267 3268 target->io_class = SRP_REV16A_IB_IO_CLASS; 3269 target->scsi_host = target_host; 3270 target->srp_host = host; 3271 target->lkey = host->srp_dev->pd->local_dma_lkey; 3272 target->global_mr = host->srp_dev->global_mr; 3273 target->cmd_sg_cnt = cmd_sg_entries; 3274 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries; 3275 target->allow_ext_sg = allow_ext_sg; 3276 target->tl_retry_count = 7; 3277 target->queue_size = SRP_DEFAULT_QUEUE_SIZE; 3278 3279 /* 3280 * Avoid that the SCSI host can be removed by srp_remove_target() 3281 * before this function returns. 3282 */ 3283 scsi_host_get(target->scsi_host); 3284 3285 mutex_lock(&host->add_target_mutex); 3286 3287 ret = srp_parse_options(buf, target); 3288 if (ret) 3289 goto out; 3290 3291 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE; 3292 3293 if (!srp_conn_unique(target->srp_host, target)) { 3294 shost_printk(KERN_INFO, target->scsi_host, 3295 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n", 3296 be64_to_cpu(target->id_ext), 3297 be64_to_cpu(target->ioc_guid), 3298 be64_to_cpu(target->initiator_ext)); 3299 ret = -EEXIST; 3300 goto out; 3301 } 3302 3303 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg && 3304 target->cmd_sg_cnt < target->sg_tablesize) { 3305 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n"); 3306 target->sg_tablesize = target->cmd_sg_cnt; 3307 } 3308 3309 if (srp_dev->use_fast_reg || srp_dev->use_fmr) { 3310 /* 3311 * FR and FMR can only map one HCA page per entry. If the 3312 * start address is not aligned on a HCA page boundary two 3313 * entries will be used for the head and the tail although 3314 * these two entries combined contain at most one HCA page of 3315 * data. Hence the "+ 1" in the calculation below. 3316 * 3317 * The indirect data buffer descriptor is contiguous so the 3318 * memory for that buffer will only be registered if 3319 * register_always is true. Hence add one to mr_per_cmd if 3320 * register_always has been set. 3321 */ 3322 max_sectors_per_mr = srp_dev->max_pages_per_mr << 3323 (ilog2(srp_dev->mr_page_size) - 9); 3324 mr_per_cmd = register_always + 3325 (target->scsi_host->max_sectors + 1 + 3326 max_sectors_per_mr - 1) / max_sectors_per_mr; 3327 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n", 3328 target->scsi_host->max_sectors, 3329 srp_dev->max_pages_per_mr, srp_dev->mr_page_size, 3330 max_sectors_per_mr, mr_per_cmd); 3331 } 3332 3333 target_host->sg_tablesize = target->sg_tablesize; 3334 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd; 3335 target->mr_per_cmd = mr_per_cmd; 3336 target->indirect_size = target->sg_tablesize * 3337 sizeof (struct srp_direct_buf); 3338 target->max_iu_len = sizeof (struct srp_cmd) + 3339 sizeof (struct srp_indirect_buf) + 3340 target->cmd_sg_cnt * sizeof (struct srp_direct_buf); 3341 3342 INIT_WORK(&target->tl_err_work, srp_tl_err_work); 3343 INIT_WORK(&target->remove_work, srp_remove_work); 3344 spin_lock_init(&target->lock); 3345 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL); 3346 if (ret) 3347 goto out; 3348 3349 ret = -ENOMEM; 3350 target->ch_count = max_t(unsigned, num_online_nodes(), 3351 min(ch_count ? : 3352 min(4 * num_online_nodes(), 3353 ibdev->num_comp_vectors), 3354 num_online_cpus())); 3355 target->ch = kcalloc(target->ch_count, sizeof(*target->ch), 3356 GFP_KERNEL); 3357 if (!target->ch) 3358 goto out; 3359 3360 node_idx = 0; 3361 for_each_online_node(node) { 3362 const int ch_start = (node_idx * target->ch_count / 3363 num_online_nodes()); 3364 const int ch_end = ((node_idx + 1) * target->ch_count / 3365 num_online_nodes()); 3366 const int cv_start = (node_idx * ibdev->num_comp_vectors / 3367 num_online_nodes() + target->comp_vector) 3368 % ibdev->num_comp_vectors; 3369 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors / 3370 num_online_nodes() + target->comp_vector) 3371 % ibdev->num_comp_vectors; 3372 int cpu_idx = 0; 3373 3374 for_each_online_cpu(cpu) { 3375 if (cpu_to_node(cpu) != node) 3376 continue; 3377 if (ch_start + cpu_idx >= ch_end) 3378 continue; 3379 ch = &target->ch[ch_start + cpu_idx]; 3380 ch->target = target; 3381 ch->comp_vector = cv_start == cv_end ? cv_start : 3382 cv_start + cpu_idx % (cv_end - cv_start); 3383 spin_lock_init(&ch->lock); 3384 INIT_LIST_HEAD(&ch->free_tx); 3385 ret = srp_new_cm_id(ch); 3386 if (ret) 3387 goto err_disconnect; 3388 3389 ret = srp_create_ch_ib(ch); 3390 if (ret) 3391 goto err_disconnect; 3392 3393 ret = srp_alloc_req_data(ch); 3394 if (ret) 3395 goto err_disconnect; 3396 3397 ret = srp_connect_ch(ch, multich); 3398 if (ret) { 3399 shost_printk(KERN_ERR, target->scsi_host, 3400 PFX "Connection %d/%d failed\n", 3401 ch_start + cpu_idx, 3402 target->ch_count); 3403 if (node_idx == 0 && cpu_idx == 0) { 3404 goto err_disconnect; 3405 } else { 3406 srp_free_ch_ib(target, ch); 3407 srp_free_req_data(target, ch); 3408 target->ch_count = ch - target->ch; 3409 goto connected; 3410 } 3411 } 3412 3413 multich = true; 3414 cpu_idx++; 3415 } 3416 node_idx++; 3417 } 3418 3419 connected: 3420 target->scsi_host->nr_hw_queues = target->ch_count; 3421 3422 ret = srp_add_target(host, target); 3423 if (ret) 3424 goto err_disconnect; 3425 3426 if (target->state != SRP_TARGET_REMOVED) { 3427 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3428 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n", 3429 be64_to_cpu(target->id_ext), 3430 be64_to_cpu(target->ioc_guid), 3431 be16_to_cpu(target->pkey), 3432 be64_to_cpu(target->service_id), 3433 target->sgid.raw, target->orig_dgid.raw); 3434 } 3435 3436 ret = count; 3437 3438 out: 3439 mutex_unlock(&host->add_target_mutex); 3440 3441 scsi_host_put(target->scsi_host); 3442 if (ret < 0) 3443 scsi_host_put(target->scsi_host); 3444 3445 return ret; 3446 3447 err_disconnect: 3448 srp_disconnect_target(target); 3449 3450 for (i = 0; i < target->ch_count; i++) { 3451 ch = &target->ch[i]; 3452 srp_free_ch_ib(target, ch); 3453 srp_free_req_data(target, ch); 3454 } 3455 3456 kfree(target->ch); 3457 goto out; 3458 } 3459 3460 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target); 3461 3462 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, 3463 char *buf) 3464 { 3465 struct srp_host *host = container_of(dev, struct srp_host, dev); 3466 3467 return sprintf(buf, "%s\n", host->srp_dev->dev->name); 3468 } 3469 3470 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); 3471 3472 static ssize_t show_port(struct device *dev, struct device_attribute *attr, 3473 char *buf) 3474 { 3475 struct srp_host *host = container_of(dev, struct srp_host, dev); 3476 3477 return sprintf(buf, "%d\n", host->port); 3478 } 3479 3480 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL); 3481 3482 static struct srp_host *srp_add_port(struct srp_device *device, u8 port) 3483 { 3484 struct srp_host *host; 3485 3486 host = kzalloc(sizeof *host, GFP_KERNEL); 3487 if (!host) 3488 return NULL; 3489 3490 INIT_LIST_HEAD(&host->target_list); 3491 spin_lock_init(&host->target_lock); 3492 init_completion(&host->released); 3493 mutex_init(&host->add_target_mutex); 3494 host->srp_dev = device; 3495 host->port = port; 3496 3497 host->dev.class = &srp_class; 3498 host->dev.parent = device->dev->dma_device; 3499 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port); 3500 3501 if (device_register(&host->dev)) 3502 goto free_host; 3503 if (device_create_file(&host->dev, &dev_attr_add_target)) 3504 goto err_class; 3505 if (device_create_file(&host->dev, &dev_attr_ibdev)) 3506 goto err_class; 3507 if (device_create_file(&host->dev, &dev_attr_port)) 3508 goto err_class; 3509 3510 return host; 3511 3512 err_class: 3513 device_unregister(&host->dev); 3514 3515 free_host: 3516 kfree(host); 3517 3518 return NULL; 3519 } 3520 3521 static void srp_add_one(struct ib_device *device) 3522 { 3523 struct srp_device *srp_dev; 3524 struct srp_host *host; 3525 int mr_page_shift, p; 3526 u64 max_pages_per_mr; 3527 3528 srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL); 3529 if (!srp_dev) 3530 return; 3531 3532 /* 3533 * Use the smallest page size supported by the HCA, down to a 3534 * minimum of 4096 bytes. We're unlikely to build large sglists 3535 * out of smaller entries. 3536 */ 3537 mr_page_shift = max(12, ffs(device->attrs.page_size_cap) - 1); 3538 srp_dev->mr_page_size = 1 << mr_page_shift; 3539 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1); 3540 max_pages_per_mr = device->attrs.max_mr_size; 3541 do_div(max_pages_per_mr, srp_dev->mr_page_size); 3542 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__, 3543 device->attrs.max_mr_size, srp_dev->mr_page_size, 3544 max_pages_per_mr, SRP_MAX_PAGES_PER_MR); 3545 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR, 3546 max_pages_per_mr); 3547 3548 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr && 3549 device->map_phys_fmr && device->unmap_fmr); 3550 srp_dev->has_fr = (device->attrs.device_cap_flags & 3551 IB_DEVICE_MEM_MGT_EXTENSIONS); 3552 if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) { 3553 dev_warn(&device->dev, "neither FMR nor FR is supported\n"); 3554 } else if (!never_register && 3555 device->attrs.max_mr_size >= 2 * srp_dev->mr_page_size) { 3556 srp_dev->use_fast_reg = (srp_dev->has_fr && 3557 (!srp_dev->has_fmr || prefer_fr)); 3558 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr; 3559 } 3560 3561 if (srp_dev->use_fast_reg) { 3562 srp_dev->max_pages_per_mr = 3563 min_t(u32, srp_dev->max_pages_per_mr, 3564 device->attrs.max_fast_reg_page_list_len); 3565 } 3566 srp_dev->mr_max_size = srp_dev->mr_page_size * 3567 srp_dev->max_pages_per_mr; 3568 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n", 3569 device->name, mr_page_shift, device->attrs.max_mr_size, 3570 device->attrs.max_fast_reg_page_list_len, 3571 srp_dev->max_pages_per_mr, srp_dev->mr_max_size); 3572 3573 INIT_LIST_HEAD(&srp_dev->dev_list); 3574 3575 srp_dev->dev = device; 3576 srp_dev->pd = ib_alloc_pd(device); 3577 if (IS_ERR(srp_dev->pd)) 3578 goto free_dev; 3579 3580 if (never_register || !register_always || 3581 (!srp_dev->has_fmr && !srp_dev->has_fr)) { 3582 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd, 3583 IB_ACCESS_LOCAL_WRITE | 3584 IB_ACCESS_REMOTE_READ | 3585 IB_ACCESS_REMOTE_WRITE); 3586 if (IS_ERR(srp_dev->global_mr)) 3587 goto err_pd; 3588 } 3589 3590 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) { 3591 host = srp_add_port(srp_dev, p); 3592 if (host) 3593 list_add_tail(&host->list, &srp_dev->dev_list); 3594 } 3595 3596 ib_set_client_data(device, &srp_client, srp_dev); 3597 return; 3598 3599 err_pd: 3600 ib_dealloc_pd(srp_dev->pd); 3601 3602 free_dev: 3603 kfree(srp_dev); 3604 } 3605 3606 static void srp_remove_one(struct ib_device *device, void *client_data) 3607 { 3608 struct srp_device *srp_dev; 3609 struct srp_host *host, *tmp_host; 3610 struct srp_target_port *target; 3611 3612 srp_dev = client_data; 3613 if (!srp_dev) 3614 return; 3615 3616 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 3617 device_unregister(&host->dev); 3618 /* 3619 * Wait for the sysfs entry to go away, so that no new 3620 * target ports can be created. 3621 */ 3622 wait_for_completion(&host->released); 3623 3624 /* 3625 * Remove all target ports. 3626 */ 3627 spin_lock(&host->target_lock); 3628 list_for_each_entry(target, &host->target_list, list) 3629 srp_queue_remove_work(target); 3630 spin_unlock(&host->target_lock); 3631 3632 /* 3633 * Wait for tl_err and target port removal tasks. 3634 */ 3635 flush_workqueue(system_long_wq); 3636 flush_workqueue(srp_remove_wq); 3637 3638 kfree(host); 3639 } 3640 3641 if (srp_dev->global_mr) 3642 ib_dereg_mr(srp_dev->global_mr); 3643 ib_dealloc_pd(srp_dev->pd); 3644 3645 kfree(srp_dev); 3646 } 3647 3648 static struct srp_function_template ib_srp_transport_functions = { 3649 .has_rport_state = true, 3650 .reset_timer_if_blocked = true, 3651 .reconnect_delay = &srp_reconnect_delay, 3652 .fast_io_fail_tmo = &srp_fast_io_fail_tmo, 3653 .dev_loss_tmo = &srp_dev_loss_tmo, 3654 .reconnect = srp_rport_reconnect, 3655 .rport_delete = srp_rport_delete, 3656 .terminate_rport_io = srp_terminate_io, 3657 }; 3658 3659 static int __init srp_init_module(void) 3660 { 3661 int ret; 3662 3663 if (srp_sg_tablesize) { 3664 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); 3665 if (!cmd_sg_entries) 3666 cmd_sg_entries = srp_sg_tablesize; 3667 } 3668 3669 if (!cmd_sg_entries) 3670 cmd_sg_entries = SRP_DEF_SG_TABLESIZE; 3671 3672 if (cmd_sg_entries > 255) { 3673 pr_warn("Clamping cmd_sg_entries to 255\n"); 3674 cmd_sg_entries = 255; 3675 } 3676 3677 if (!indirect_sg_entries) 3678 indirect_sg_entries = cmd_sg_entries; 3679 else if (indirect_sg_entries < cmd_sg_entries) { 3680 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n", 3681 cmd_sg_entries); 3682 indirect_sg_entries = cmd_sg_entries; 3683 } 3684 3685 srp_remove_wq = create_workqueue("srp_remove"); 3686 if (!srp_remove_wq) { 3687 ret = -ENOMEM; 3688 goto out; 3689 } 3690 3691 ret = -ENOMEM; 3692 ib_srp_transport_template = 3693 srp_attach_transport(&ib_srp_transport_functions); 3694 if (!ib_srp_transport_template) 3695 goto destroy_wq; 3696 3697 ret = class_register(&srp_class); 3698 if (ret) { 3699 pr_err("couldn't register class infiniband_srp\n"); 3700 goto release_tr; 3701 } 3702 3703 ib_sa_register_client(&srp_sa_client); 3704 3705 ret = ib_register_client(&srp_client); 3706 if (ret) { 3707 pr_err("couldn't register IB client\n"); 3708 goto unreg_sa; 3709 } 3710 3711 out: 3712 return ret; 3713 3714 unreg_sa: 3715 ib_sa_unregister_client(&srp_sa_client); 3716 class_unregister(&srp_class); 3717 3718 release_tr: 3719 srp_release_transport(ib_srp_transport_template); 3720 3721 destroy_wq: 3722 destroy_workqueue(srp_remove_wq); 3723 goto out; 3724 } 3725 3726 static void __exit srp_cleanup_module(void) 3727 { 3728 ib_unregister_client(&srp_client); 3729 ib_sa_unregister_client(&srp_sa_client); 3730 class_unregister(&srp_class); 3731 srp_release_transport(ib_srp_transport_template); 3732 destroy_workqueue(srp_remove_wq); 3733 } 3734 3735 module_init(srp_init_module); 3736 module_exit(srp_cleanup_module); 3737