1 /* 2 * Copyright (c) 2005 Cisco Systems. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 34 35 #include <linux/module.h> 36 #include <linux/init.h> 37 #include <linux/slab.h> 38 #include <linux/err.h> 39 #include <linux/string.h> 40 #include <linux/parser.h> 41 #include <linux/random.h> 42 #include <linux/jiffies.h> 43 #include <rdma/ib_cache.h> 44 45 #include <linux/atomic.h> 46 47 #include <scsi/scsi.h> 48 #include <scsi/scsi_device.h> 49 #include <scsi/scsi_dbg.h> 50 #include <scsi/scsi_tcq.h> 51 #include <scsi/srp.h> 52 #include <scsi/scsi_transport_srp.h> 53 54 #include "ib_srp.h" 55 56 #define DRV_NAME "ib_srp" 57 #define PFX DRV_NAME ": " 58 #define DRV_VERSION "2.0" 59 #define DRV_RELDATE "July 26, 2015" 60 61 MODULE_AUTHOR("Roland Dreier"); 62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator"); 63 MODULE_LICENSE("Dual BSD/GPL"); 64 MODULE_VERSION(DRV_VERSION); 65 MODULE_INFO(release_date, DRV_RELDATE); 66 67 static unsigned int srp_sg_tablesize; 68 static unsigned int cmd_sg_entries; 69 static unsigned int indirect_sg_entries; 70 static bool allow_ext_sg; 71 static bool prefer_fr = true; 72 static bool register_always = true; 73 static bool never_register; 74 static int topspin_workarounds = 1; 75 76 module_param(srp_sg_tablesize, uint, 0444); 77 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries"); 78 79 module_param(cmd_sg_entries, uint, 0444); 80 MODULE_PARM_DESC(cmd_sg_entries, 81 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)"); 82 83 module_param(indirect_sg_entries, uint, 0444); 84 MODULE_PARM_DESC(indirect_sg_entries, 85 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")"); 86 87 module_param(allow_ext_sg, bool, 0444); 88 MODULE_PARM_DESC(allow_ext_sg, 89 "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)"); 90 91 module_param(topspin_workarounds, int, 0444); 92 MODULE_PARM_DESC(topspin_workarounds, 93 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0"); 94 95 module_param(prefer_fr, bool, 0444); 96 MODULE_PARM_DESC(prefer_fr, 97 "Whether to use fast registration if both FMR and fast registration are supported"); 98 99 module_param(register_always, bool, 0444); 100 MODULE_PARM_DESC(register_always, 101 "Use memory registration even for contiguous memory regions"); 102 103 module_param(never_register, bool, 0444); 104 MODULE_PARM_DESC(never_register, "Never register memory"); 105 106 static const struct kernel_param_ops srp_tmo_ops; 107 108 static int srp_reconnect_delay = 10; 109 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay, 110 S_IRUGO | S_IWUSR); 111 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts"); 112 113 static int srp_fast_io_fail_tmo = 15; 114 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo, 115 S_IRUGO | S_IWUSR); 116 MODULE_PARM_DESC(fast_io_fail_tmo, 117 "Number of seconds between the observation of a transport" 118 " layer error and failing all I/O. \"off\" means that this" 119 " functionality is disabled."); 120 121 static int srp_dev_loss_tmo = 600; 122 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo, 123 S_IRUGO | S_IWUSR); 124 MODULE_PARM_DESC(dev_loss_tmo, 125 "Maximum number of seconds that the SRP transport should" 126 " insulate transport layer errors. After this time has been" 127 " exceeded the SCSI host is removed. Should be" 128 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT) 129 " if fast_io_fail_tmo has not been set. \"off\" means that" 130 " this functionality is disabled."); 131 132 static unsigned ch_count; 133 module_param(ch_count, uint, 0444); 134 MODULE_PARM_DESC(ch_count, 135 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA."); 136 137 static void srp_add_one(struct ib_device *device); 138 static void srp_remove_one(struct ib_device *device, void *client_data); 139 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc); 140 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 141 const char *opname); 142 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event); 143 144 static struct scsi_transport_template *ib_srp_transport_template; 145 static struct workqueue_struct *srp_remove_wq; 146 147 static struct ib_client srp_client = { 148 .name = "srp", 149 .add = srp_add_one, 150 .remove = srp_remove_one 151 }; 152 153 static struct ib_sa_client srp_sa_client; 154 155 static int srp_tmo_get(char *buffer, const struct kernel_param *kp) 156 { 157 int tmo = *(int *)kp->arg; 158 159 if (tmo >= 0) 160 return sprintf(buffer, "%d", tmo); 161 else 162 return sprintf(buffer, "off"); 163 } 164 165 static int srp_tmo_set(const char *val, const struct kernel_param *kp) 166 { 167 int tmo, res; 168 169 res = srp_parse_tmo(&tmo, val); 170 if (res) 171 goto out; 172 173 if (kp->arg == &srp_reconnect_delay) 174 res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo, 175 srp_dev_loss_tmo); 176 else if (kp->arg == &srp_fast_io_fail_tmo) 177 res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo); 178 else 179 res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo, 180 tmo); 181 if (res) 182 goto out; 183 *(int *)kp->arg = tmo; 184 185 out: 186 return res; 187 } 188 189 static const struct kernel_param_ops srp_tmo_ops = { 190 .get = srp_tmo_get, 191 .set = srp_tmo_set, 192 }; 193 194 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host) 195 { 196 return (struct srp_target_port *) host->hostdata; 197 } 198 199 static const char *srp_target_info(struct Scsi_Host *host) 200 { 201 return host_to_target(host)->target_name; 202 } 203 204 static int srp_target_is_topspin(struct srp_target_port *target) 205 { 206 static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad }; 207 static const u8 cisco_oui[3] = { 0x00, 0x1b, 0x0d }; 208 209 return topspin_workarounds && 210 (!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) || 211 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui)); 212 } 213 214 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size, 215 gfp_t gfp_mask, 216 enum dma_data_direction direction) 217 { 218 struct srp_iu *iu; 219 220 iu = kmalloc(sizeof *iu, gfp_mask); 221 if (!iu) 222 goto out; 223 224 iu->buf = kzalloc(size, gfp_mask); 225 if (!iu->buf) 226 goto out_free_iu; 227 228 iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size, 229 direction); 230 if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma)) 231 goto out_free_buf; 232 233 iu->size = size; 234 iu->direction = direction; 235 236 return iu; 237 238 out_free_buf: 239 kfree(iu->buf); 240 out_free_iu: 241 kfree(iu); 242 out: 243 return NULL; 244 } 245 246 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu) 247 { 248 if (!iu) 249 return; 250 251 ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size, 252 iu->direction); 253 kfree(iu->buf); 254 kfree(iu); 255 } 256 257 static void srp_qp_event(struct ib_event *event, void *context) 258 { 259 pr_debug("QP event %s (%d)\n", 260 ib_event_msg(event->event), event->event); 261 } 262 263 static int srp_init_qp(struct srp_target_port *target, 264 struct ib_qp *qp) 265 { 266 struct ib_qp_attr *attr; 267 int ret; 268 269 attr = kmalloc(sizeof *attr, GFP_KERNEL); 270 if (!attr) 271 return -ENOMEM; 272 273 ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev, 274 target->srp_host->port, 275 be16_to_cpu(target->pkey), 276 &attr->pkey_index); 277 if (ret) 278 goto out; 279 280 attr->qp_state = IB_QPS_INIT; 281 attr->qp_access_flags = (IB_ACCESS_REMOTE_READ | 282 IB_ACCESS_REMOTE_WRITE); 283 attr->port_num = target->srp_host->port; 284 285 ret = ib_modify_qp(qp, attr, 286 IB_QP_STATE | 287 IB_QP_PKEY_INDEX | 288 IB_QP_ACCESS_FLAGS | 289 IB_QP_PORT); 290 291 out: 292 kfree(attr); 293 return ret; 294 } 295 296 static int srp_new_cm_id(struct srp_rdma_ch *ch) 297 { 298 struct srp_target_port *target = ch->target; 299 struct ib_cm_id *new_cm_id; 300 301 new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev, 302 srp_cm_handler, ch); 303 if (IS_ERR(new_cm_id)) 304 return PTR_ERR(new_cm_id); 305 306 if (ch->cm_id) 307 ib_destroy_cm_id(ch->cm_id); 308 ch->cm_id = new_cm_id; 309 ch->path.sgid = target->sgid; 310 ch->path.dgid = target->orig_dgid; 311 ch->path.pkey = target->pkey; 312 ch->path.service_id = target->service_id; 313 314 return 0; 315 } 316 317 static struct ib_fmr_pool *srp_alloc_fmr_pool(struct srp_target_port *target) 318 { 319 struct srp_device *dev = target->srp_host->srp_dev; 320 struct ib_fmr_pool_param fmr_param; 321 322 memset(&fmr_param, 0, sizeof(fmr_param)); 323 fmr_param.pool_size = target->mr_pool_size; 324 fmr_param.dirty_watermark = fmr_param.pool_size / 4; 325 fmr_param.cache = 1; 326 fmr_param.max_pages_per_fmr = dev->max_pages_per_mr; 327 fmr_param.page_shift = ilog2(dev->mr_page_size); 328 fmr_param.access = (IB_ACCESS_LOCAL_WRITE | 329 IB_ACCESS_REMOTE_WRITE | 330 IB_ACCESS_REMOTE_READ); 331 332 return ib_create_fmr_pool(dev->pd, &fmr_param); 333 } 334 335 /** 336 * srp_destroy_fr_pool() - free the resources owned by a pool 337 * @pool: Fast registration pool to be destroyed. 338 */ 339 static void srp_destroy_fr_pool(struct srp_fr_pool *pool) 340 { 341 int i; 342 struct srp_fr_desc *d; 343 344 if (!pool) 345 return; 346 347 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 348 if (d->mr) 349 ib_dereg_mr(d->mr); 350 } 351 kfree(pool); 352 } 353 354 /** 355 * srp_create_fr_pool() - allocate and initialize a pool for fast registration 356 * @device: IB device to allocate fast registration descriptors for. 357 * @pd: Protection domain associated with the FR descriptors. 358 * @pool_size: Number of descriptors to allocate. 359 * @max_page_list_len: Maximum fast registration work request page list length. 360 */ 361 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device, 362 struct ib_pd *pd, int pool_size, 363 int max_page_list_len) 364 { 365 struct srp_fr_pool *pool; 366 struct srp_fr_desc *d; 367 struct ib_mr *mr; 368 int i, ret = -EINVAL; 369 370 if (pool_size <= 0) 371 goto err; 372 ret = -ENOMEM; 373 pool = kzalloc(sizeof(struct srp_fr_pool) + 374 pool_size * sizeof(struct srp_fr_desc), GFP_KERNEL); 375 if (!pool) 376 goto err; 377 pool->size = pool_size; 378 pool->max_page_list_len = max_page_list_len; 379 spin_lock_init(&pool->lock); 380 INIT_LIST_HEAD(&pool->free_list); 381 382 for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) { 383 mr = ib_alloc_mr(pd, IB_MR_TYPE_MEM_REG, 384 max_page_list_len); 385 if (IS_ERR(mr)) { 386 ret = PTR_ERR(mr); 387 goto destroy_pool; 388 } 389 d->mr = mr; 390 list_add_tail(&d->entry, &pool->free_list); 391 } 392 393 out: 394 return pool; 395 396 destroy_pool: 397 srp_destroy_fr_pool(pool); 398 399 err: 400 pool = ERR_PTR(ret); 401 goto out; 402 } 403 404 /** 405 * srp_fr_pool_get() - obtain a descriptor suitable for fast registration 406 * @pool: Pool to obtain descriptor from. 407 */ 408 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool) 409 { 410 struct srp_fr_desc *d = NULL; 411 unsigned long flags; 412 413 spin_lock_irqsave(&pool->lock, flags); 414 if (!list_empty(&pool->free_list)) { 415 d = list_first_entry(&pool->free_list, typeof(*d), entry); 416 list_del(&d->entry); 417 } 418 spin_unlock_irqrestore(&pool->lock, flags); 419 420 return d; 421 } 422 423 /** 424 * srp_fr_pool_put() - put an FR descriptor back in the free list 425 * @pool: Pool the descriptor was allocated from. 426 * @desc: Pointer to an array of fast registration descriptor pointers. 427 * @n: Number of descriptors to put back. 428 * 429 * Note: The caller must already have queued an invalidation request for 430 * desc->mr->rkey before calling this function. 431 */ 432 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc, 433 int n) 434 { 435 unsigned long flags; 436 int i; 437 438 spin_lock_irqsave(&pool->lock, flags); 439 for (i = 0; i < n; i++) 440 list_add(&desc[i]->entry, &pool->free_list); 441 spin_unlock_irqrestore(&pool->lock, flags); 442 } 443 444 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target) 445 { 446 struct srp_device *dev = target->srp_host->srp_dev; 447 448 return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size, 449 dev->max_pages_per_mr); 450 } 451 452 /** 453 * srp_destroy_qp() - destroy an RDMA queue pair 454 * @qp: RDMA queue pair. 455 * 456 * Drain the qp before destroying it. This avoids that the receive 457 * completion handler can access the queue pair while it is 458 * being destroyed. 459 */ 460 static void srp_destroy_qp(struct ib_qp *qp) 461 { 462 ib_drain_rq(qp); 463 ib_destroy_qp(qp); 464 } 465 466 static int srp_create_ch_ib(struct srp_rdma_ch *ch) 467 { 468 struct srp_target_port *target = ch->target; 469 struct srp_device *dev = target->srp_host->srp_dev; 470 struct ib_qp_init_attr *init_attr; 471 struct ib_cq *recv_cq, *send_cq; 472 struct ib_qp *qp; 473 struct ib_fmr_pool *fmr_pool = NULL; 474 struct srp_fr_pool *fr_pool = NULL; 475 const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2; 476 int ret; 477 478 init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL); 479 if (!init_attr) 480 return -ENOMEM; 481 482 /* queue_size + 1 for ib_drain_rq() */ 483 recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1, 484 ch->comp_vector, IB_POLL_SOFTIRQ); 485 if (IS_ERR(recv_cq)) { 486 ret = PTR_ERR(recv_cq); 487 goto err; 488 } 489 490 send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size, 491 ch->comp_vector, IB_POLL_DIRECT); 492 if (IS_ERR(send_cq)) { 493 ret = PTR_ERR(send_cq); 494 goto err_recv_cq; 495 } 496 497 init_attr->event_handler = srp_qp_event; 498 init_attr->cap.max_send_wr = m * target->queue_size; 499 init_attr->cap.max_recv_wr = target->queue_size + 1; 500 init_attr->cap.max_recv_sge = 1; 501 init_attr->cap.max_send_sge = 1; 502 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR; 503 init_attr->qp_type = IB_QPT_RC; 504 init_attr->send_cq = send_cq; 505 init_attr->recv_cq = recv_cq; 506 507 qp = ib_create_qp(dev->pd, init_attr); 508 if (IS_ERR(qp)) { 509 ret = PTR_ERR(qp); 510 goto err_send_cq; 511 } 512 513 ret = srp_init_qp(target, qp); 514 if (ret) 515 goto err_qp; 516 517 if (dev->use_fast_reg) { 518 fr_pool = srp_alloc_fr_pool(target); 519 if (IS_ERR(fr_pool)) { 520 ret = PTR_ERR(fr_pool); 521 shost_printk(KERN_WARNING, target->scsi_host, PFX 522 "FR pool allocation failed (%d)\n", ret); 523 goto err_qp; 524 } 525 } else if (dev->use_fmr) { 526 fmr_pool = srp_alloc_fmr_pool(target); 527 if (IS_ERR(fmr_pool)) { 528 ret = PTR_ERR(fmr_pool); 529 shost_printk(KERN_WARNING, target->scsi_host, PFX 530 "FMR pool allocation failed (%d)\n", ret); 531 goto err_qp; 532 } 533 } 534 535 if (ch->qp) 536 srp_destroy_qp(ch->qp); 537 if (ch->recv_cq) 538 ib_free_cq(ch->recv_cq); 539 if (ch->send_cq) 540 ib_free_cq(ch->send_cq); 541 542 ch->qp = qp; 543 ch->recv_cq = recv_cq; 544 ch->send_cq = send_cq; 545 546 if (dev->use_fast_reg) { 547 if (ch->fr_pool) 548 srp_destroy_fr_pool(ch->fr_pool); 549 ch->fr_pool = fr_pool; 550 } else if (dev->use_fmr) { 551 if (ch->fmr_pool) 552 ib_destroy_fmr_pool(ch->fmr_pool); 553 ch->fmr_pool = fmr_pool; 554 } 555 556 kfree(init_attr); 557 return 0; 558 559 err_qp: 560 srp_destroy_qp(qp); 561 562 err_send_cq: 563 ib_free_cq(send_cq); 564 565 err_recv_cq: 566 ib_free_cq(recv_cq); 567 568 err: 569 kfree(init_attr); 570 return ret; 571 } 572 573 /* 574 * Note: this function may be called without srp_alloc_iu_bufs() having been 575 * invoked. Hence the ch->[rt]x_ring checks. 576 */ 577 static void srp_free_ch_ib(struct srp_target_port *target, 578 struct srp_rdma_ch *ch) 579 { 580 struct srp_device *dev = target->srp_host->srp_dev; 581 int i; 582 583 if (!ch->target) 584 return; 585 586 if (ch->cm_id) { 587 ib_destroy_cm_id(ch->cm_id); 588 ch->cm_id = NULL; 589 } 590 591 /* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */ 592 if (!ch->qp) 593 return; 594 595 if (dev->use_fast_reg) { 596 if (ch->fr_pool) 597 srp_destroy_fr_pool(ch->fr_pool); 598 } else if (dev->use_fmr) { 599 if (ch->fmr_pool) 600 ib_destroy_fmr_pool(ch->fmr_pool); 601 } 602 603 srp_destroy_qp(ch->qp); 604 ib_free_cq(ch->send_cq); 605 ib_free_cq(ch->recv_cq); 606 607 /* 608 * Avoid that the SCSI error handler tries to use this channel after 609 * it has been freed. The SCSI error handler can namely continue 610 * trying to perform recovery actions after scsi_remove_host() 611 * returned. 612 */ 613 ch->target = NULL; 614 615 ch->qp = NULL; 616 ch->send_cq = ch->recv_cq = NULL; 617 618 if (ch->rx_ring) { 619 for (i = 0; i < target->queue_size; ++i) 620 srp_free_iu(target->srp_host, ch->rx_ring[i]); 621 kfree(ch->rx_ring); 622 ch->rx_ring = NULL; 623 } 624 if (ch->tx_ring) { 625 for (i = 0; i < target->queue_size; ++i) 626 srp_free_iu(target->srp_host, ch->tx_ring[i]); 627 kfree(ch->tx_ring); 628 ch->tx_ring = NULL; 629 } 630 } 631 632 static void srp_path_rec_completion(int status, 633 struct ib_sa_path_rec *pathrec, 634 void *ch_ptr) 635 { 636 struct srp_rdma_ch *ch = ch_ptr; 637 struct srp_target_port *target = ch->target; 638 639 ch->status = status; 640 if (status) 641 shost_printk(KERN_ERR, target->scsi_host, 642 PFX "Got failed path rec status %d\n", status); 643 else 644 ch->path = *pathrec; 645 complete(&ch->done); 646 } 647 648 static int srp_lookup_path(struct srp_rdma_ch *ch) 649 { 650 struct srp_target_port *target = ch->target; 651 int ret; 652 653 ch->path.numb_path = 1; 654 655 init_completion(&ch->done); 656 657 ch->path_query_id = ib_sa_path_rec_get(&srp_sa_client, 658 target->srp_host->srp_dev->dev, 659 target->srp_host->port, 660 &ch->path, 661 IB_SA_PATH_REC_SERVICE_ID | 662 IB_SA_PATH_REC_DGID | 663 IB_SA_PATH_REC_SGID | 664 IB_SA_PATH_REC_NUMB_PATH | 665 IB_SA_PATH_REC_PKEY, 666 SRP_PATH_REC_TIMEOUT_MS, 667 GFP_KERNEL, 668 srp_path_rec_completion, 669 ch, &ch->path_query); 670 if (ch->path_query_id < 0) 671 return ch->path_query_id; 672 673 ret = wait_for_completion_interruptible(&ch->done); 674 if (ret < 0) 675 return ret; 676 677 if (ch->status < 0) 678 shost_printk(KERN_WARNING, target->scsi_host, 679 PFX "Path record query failed\n"); 680 681 return ch->status; 682 } 683 684 static int srp_send_req(struct srp_rdma_ch *ch, bool multich) 685 { 686 struct srp_target_port *target = ch->target; 687 struct { 688 struct ib_cm_req_param param; 689 struct srp_login_req priv; 690 } *req = NULL; 691 int status; 692 693 req = kzalloc(sizeof *req, GFP_KERNEL); 694 if (!req) 695 return -ENOMEM; 696 697 req->param.primary_path = &ch->path; 698 req->param.alternate_path = NULL; 699 req->param.service_id = target->service_id; 700 req->param.qp_num = ch->qp->qp_num; 701 req->param.qp_type = ch->qp->qp_type; 702 req->param.private_data = &req->priv; 703 req->param.private_data_len = sizeof req->priv; 704 req->param.flow_control = 1; 705 706 get_random_bytes(&req->param.starting_psn, 4); 707 req->param.starting_psn &= 0xffffff; 708 709 /* 710 * Pick some arbitrary defaults here; we could make these 711 * module parameters if anyone cared about setting them. 712 */ 713 req->param.responder_resources = 4; 714 req->param.remote_cm_response_timeout = 20; 715 req->param.local_cm_response_timeout = 20; 716 req->param.retry_count = target->tl_retry_count; 717 req->param.rnr_retry_count = 7; 718 req->param.max_cm_retries = 15; 719 720 req->priv.opcode = SRP_LOGIN_REQ; 721 req->priv.tag = 0; 722 req->priv.req_it_iu_len = cpu_to_be32(target->max_iu_len); 723 req->priv.req_buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT | 724 SRP_BUF_FORMAT_INDIRECT); 725 req->priv.req_flags = (multich ? SRP_MULTICHAN_MULTI : 726 SRP_MULTICHAN_SINGLE); 727 /* 728 * In the published SRP specification (draft rev. 16a), the 729 * port identifier format is 8 bytes of ID extension followed 730 * by 8 bytes of GUID. Older drafts put the two halves in the 731 * opposite order, so that the GUID comes first. 732 * 733 * Targets conforming to these obsolete drafts can be 734 * recognized by the I/O Class they report. 735 */ 736 if (target->io_class == SRP_REV10_IB_IO_CLASS) { 737 memcpy(req->priv.initiator_port_id, 738 &target->sgid.global.interface_id, 8); 739 memcpy(req->priv.initiator_port_id + 8, 740 &target->initiator_ext, 8); 741 memcpy(req->priv.target_port_id, &target->ioc_guid, 8); 742 memcpy(req->priv.target_port_id + 8, &target->id_ext, 8); 743 } else { 744 memcpy(req->priv.initiator_port_id, 745 &target->initiator_ext, 8); 746 memcpy(req->priv.initiator_port_id + 8, 747 &target->sgid.global.interface_id, 8); 748 memcpy(req->priv.target_port_id, &target->id_ext, 8); 749 memcpy(req->priv.target_port_id + 8, &target->ioc_guid, 8); 750 } 751 752 /* 753 * Topspin/Cisco SRP targets will reject our login unless we 754 * zero out the first 8 bytes of our initiator port ID and set 755 * the second 8 bytes to the local node GUID. 756 */ 757 if (srp_target_is_topspin(target)) { 758 shost_printk(KERN_DEBUG, target->scsi_host, 759 PFX "Topspin/Cisco initiator port ID workaround " 760 "activated for target GUID %016llx\n", 761 be64_to_cpu(target->ioc_guid)); 762 memset(req->priv.initiator_port_id, 0, 8); 763 memcpy(req->priv.initiator_port_id + 8, 764 &target->srp_host->srp_dev->dev->node_guid, 8); 765 } 766 767 status = ib_send_cm_req(ch->cm_id, &req->param); 768 769 kfree(req); 770 771 return status; 772 } 773 774 static bool srp_queue_remove_work(struct srp_target_port *target) 775 { 776 bool changed = false; 777 778 spin_lock_irq(&target->lock); 779 if (target->state != SRP_TARGET_REMOVED) { 780 target->state = SRP_TARGET_REMOVED; 781 changed = true; 782 } 783 spin_unlock_irq(&target->lock); 784 785 if (changed) 786 queue_work(srp_remove_wq, &target->remove_work); 787 788 return changed; 789 } 790 791 static void srp_disconnect_target(struct srp_target_port *target) 792 { 793 struct srp_rdma_ch *ch; 794 int i; 795 796 /* XXX should send SRP_I_LOGOUT request */ 797 798 for (i = 0; i < target->ch_count; i++) { 799 ch = &target->ch[i]; 800 ch->connected = false; 801 if (ch->cm_id && ib_send_cm_dreq(ch->cm_id, NULL, 0)) { 802 shost_printk(KERN_DEBUG, target->scsi_host, 803 PFX "Sending CM DREQ failed\n"); 804 } 805 } 806 } 807 808 static void srp_free_req_data(struct srp_target_port *target, 809 struct srp_rdma_ch *ch) 810 { 811 struct srp_device *dev = target->srp_host->srp_dev; 812 struct ib_device *ibdev = dev->dev; 813 struct srp_request *req; 814 int i; 815 816 if (!ch->req_ring) 817 return; 818 819 for (i = 0; i < target->req_ring_size; ++i) { 820 req = &ch->req_ring[i]; 821 if (dev->use_fast_reg) { 822 kfree(req->fr_list); 823 } else { 824 kfree(req->fmr_list); 825 kfree(req->map_page); 826 } 827 if (req->indirect_dma_addr) { 828 ib_dma_unmap_single(ibdev, req->indirect_dma_addr, 829 target->indirect_size, 830 DMA_TO_DEVICE); 831 } 832 kfree(req->indirect_desc); 833 } 834 835 kfree(ch->req_ring); 836 ch->req_ring = NULL; 837 } 838 839 static int srp_alloc_req_data(struct srp_rdma_ch *ch) 840 { 841 struct srp_target_port *target = ch->target; 842 struct srp_device *srp_dev = target->srp_host->srp_dev; 843 struct ib_device *ibdev = srp_dev->dev; 844 struct srp_request *req; 845 void *mr_list; 846 dma_addr_t dma_addr; 847 int i, ret = -ENOMEM; 848 849 ch->req_ring = kcalloc(target->req_ring_size, sizeof(*ch->req_ring), 850 GFP_KERNEL); 851 if (!ch->req_ring) 852 goto out; 853 854 for (i = 0; i < target->req_ring_size; ++i) { 855 req = &ch->req_ring[i]; 856 mr_list = kmalloc(target->mr_per_cmd * sizeof(void *), 857 GFP_KERNEL); 858 if (!mr_list) 859 goto out; 860 if (srp_dev->use_fast_reg) { 861 req->fr_list = mr_list; 862 } else { 863 req->fmr_list = mr_list; 864 req->map_page = kmalloc(srp_dev->max_pages_per_mr * 865 sizeof(void *), GFP_KERNEL); 866 if (!req->map_page) 867 goto out; 868 } 869 req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL); 870 if (!req->indirect_desc) 871 goto out; 872 873 dma_addr = ib_dma_map_single(ibdev, req->indirect_desc, 874 target->indirect_size, 875 DMA_TO_DEVICE); 876 if (ib_dma_mapping_error(ibdev, dma_addr)) 877 goto out; 878 879 req->indirect_dma_addr = dma_addr; 880 } 881 ret = 0; 882 883 out: 884 return ret; 885 } 886 887 /** 888 * srp_del_scsi_host_attr() - Remove attributes defined in the host template. 889 * @shost: SCSI host whose attributes to remove from sysfs. 890 * 891 * Note: Any attributes defined in the host template and that did not exist 892 * before invocation of this function will be ignored. 893 */ 894 static void srp_del_scsi_host_attr(struct Scsi_Host *shost) 895 { 896 struct device_attribute **attr; 897 898 for (attr = shost->hostt->shost_attrs; attr && *attr; ++attr) 899 device_remove_file(&shost->shost_dev, *attr); 900 } 901 902 static void srp_remove_target(struct srp_target_port *target) 903 { 904 struct srp_rdma_ch *ch; 905 int i; 906 907 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 908 909 srp_del_scsi_host_attr(target->scsi_host); 910 srp_rport_get(target->rport); 911 srp_remove_host(target->scsi_host); 912 scsi_remove_host(target->scsi_host); 913 srp_stop_rport_timers(target->rport); 914 srp_disconnect_target(target); 915 for (i = 0; i < target->ch_count; i++) { 916 ch = &target->ch[i]; 917 srp_free_ch_ib(target, ch); 918 } 919 cancel_work_sync(&target->tl_err_work); 920 srp_rport_put(target->rport); 921 for (i = 0; i < target->ch_count; i++) { 922 ch = &target->ch[i]; 923 srp_free_req_data(target, ch); 924 } 925 kfree(target->ch); 926 target->ch = NULL; 927 928 spin_lock(&target->srp_host->target_lock); 929 list_del(&target->list); 930 spin_unlock(&target->srp_host->target_lock); 931 932 scsi_host_put(target->scsi_host); 933 } 934 935 static void srp_remove_work(struct work_struct *work) 936 { 937 struct srp_target_port *target = 938 container_of(work, struct srp_target_port, remove_work); 939 940 WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED); 941 942 srp_remove_target(target); 943 } 944 945 static void srp_rport_delete(struct srp_rport *rport) 946 { 947 struct srp_target_port *target = rport->lld_data; 948 949 srp_queue_remove_work(target); 950 } 951 952 /** 953 * srp_connected_ch() - number of connected channels 954 * @target: SRP target port. 955 */ 956 static int srp_connected_ch(struct srp_target_port *target) 957 { 958 int i, c = 0; 959 960 for (i = 0; i < target->ch_count; i++) 961 c += target->ch[i].connected; 962 963 return c; 964 } 965 966 static int srp_connect_ch(struct srp_rdma_ch *ch, bool multich) 967 { 968 struct srp_target_port *target = ch->target; 969 int ret; 970 971 WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0); 972 973 ret = srp_lookup_path(ch); 974 if (ret) 975 goto out; 976 977 while (1) { 978 init_completion(&ch->done); 979 ret = srp_send_req(ch, multich); 980 if (ret) 981 goto out; 982 ret = wait_for_completion_interruptible(&ch->done); 983 if (ret < 0) 984 goto out; 985 986 /* 987 * The CM event handling code will set status to 988 * SRP_PORT_REDIRECT if we get a port redirect REJ 989 * back, or SRP_DLID_REDIRECT if we get a lid/qp 990 * redirect REJ back. 991 */ 992 ret = ch->status; 993 switch (ret) { 994 case 0: 995 ch->connected = true; 996 goto out; 997 998 case SRP_PORT_REDIRECT: 999 ret = srp_lookup_path(ch); 1000 if (ret) 1001 goto out; 1002 break; 1003 1004 case SRP_DLID_REDIRECT: 1005 break; 1006 1007 case SRP_STALE_CONN: 1008 shost_printk(KERN_ERR, target->scsi_host, PFX 1009 "giving up on stale connection\n"); 1010 ret = -ECONNRESET; 1011 goto out; 1012 1013 default: 1014 goto out; 1015 } 1016 } 1017 1018 out: 1019 return ret <= 0 ? ret : -ENODEV; 1020 } 1021 1022 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc) 1023 { 1024 srp_handle_qp_err(cq, wc, "INV RKEY"); 1025 } 1026 1027 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch, 1028 u32 rkey) 1029 { 1030 struct ib_send_wr *bad_wr; 1031 struct ib_send_wr wr = { 1032 .opcode = IB_WR_LOCAL_INV, 1033 .next = NULL, 1034 .num_sge = 0, 1035 .send_flags = 0, 1036 .ex.invalidate_rkey = rkey, 1037 }; 1038 1039 wr.wr_cqe = &req->reg_cqe; 1040 req->reg_cqe.done = srp_inv_rkey_err_done; 1041 return ib_post_send(ch->qp, &wr, &bad_wr); 1042 } 1043 1044 static void srp_unmap_data(struct scsi_cmnd *scmnd, 1045 struct srp_rdma_ch *ch, 1046 struct srp_request *req) 1047 { 1048 struct srp_target_port *target = ch->target; 1049 struct srp_device *dev = target->srp_host->srp_dev; 1050 struct ib_device *ibdev = dev->dev; 1051 int i, res; 1052 1053 if (!scsi_sglist(scmnd) || 1054 (scmnd->sc_data_direction != DMA_TO_DEVICE && 1055 scmnd->sc_data_direction != DMA_FROM_DEVICE)) 1056 return; 1057 1058 if (dev->use_fast_reg) { 1059 struct srp_fr_desc **pfr; 1060 1061 for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) { 1062 res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey); 1063 if (res < 0) { 1064 shost_printk(KERN_ERR, target->scsi_host, PFX 1065 "Queueing INV WR for rkey %#x failed (%d)\n", 1066 (*pfr)->mr->rkey, res); 1067 queue_work(system_long_wq, 1068 &target->tl_err_work); 1069 } 1070 } 1071 if (req->nmdesc) 1072 srp_fr_pool_put(ch->fr_pool, req->fr_list, 1073 req->nmdesc); 1074 } else if (dev->use_fmr) { 1075 struct ib_pool_fmr **pfmr; 1076 1077 for (i = req->nmdesc, pfmr = req->fmr_list; i > 0; i--, pfmr++) 1078 ib_fmr_pool_unmap(*pfmr); 1079 } 1080 1081 ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd), 1082 scmnd->sc_data_direction); 1083 } 1084 1085 /** 1086 * srp_claim_req - Take ownership of the scmnd associated with a request. 1087 * @ch: SRP RDMA channel. 1088 * @req: SRP request. 1089 * @sdev: If not NULL, only take ownership for this SCSI device. 1090 * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take 1091 * ownership of @req->scmnd if it equals @scmnd. 1092 * 1093 * Return value: 1094 * Either NULL or a pointer to the SCSI command the caller became owner of. 1095 */ 1096 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch, 1097 struct srp_request *req, 1098 struct scsi_device *sdev, 1099 struct scsi_cmnd *scmnd) 1100 { 1101 unsigned long flags; 1102 1103 spin_lock_irqsave(&ch->lock, flags); 1104 if (req->scmnd && 1105 (!sdev || req->scmnd->device == sdev) && 1106 (!scmnd || req->scmnd == scmnd)) { 1107 scmnd = req->scmnd; 1108 req->scmnd = NULL; 1109 } else { 1110 scmnd = NULL; 1111 } 1112 spin_unlock_irqrestore(&ch->lock, flags); 1113 1114 return scmnd; 1115 } 1116 1117 /** 1118 * srp_free_req() - Unmap data and adjust ch->req_lim. 1119 * @ch: SRP RDMA channel. 1120 * @req: Request to be freed. 1121 * @scmnd: SCSI command associated with @req. 1122 * @req_lim_delta: Amount to be added to @target->req_lim. 1123 */ 1124 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req, 1125 struct scsi_cmnd *scmnd, s32 req_lim_delta) 1126 { 1127 unsigned long flags; 1128 1129 srp_unmap_data(scmnd, ch, req); 1130 1131 spin_lock_irqsave(&ch->lock, flags); 1132 ch->req_lim += req_lim_delta; 1133 spin_unlock_irqrestore(&ch->lock, flags); 1134 } 1135 1136 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req, 1137 struct scsi_device *sdev, int result) 1138 { 1139 struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL); 1140 1141 if (scmnd) { 1142 srp_free_req(ch, req, scmnd, 0); 1143 scmnd->result = result; 1144 scmnd->scsi_done(scmnd); 1145 } 1146 } 1147 1148 static void srp_terminate_io(struct srp_rport *rport) 1149 { 1150 struct srp_target_port *target = rport->lld_data; 1151 struct srp_rdma_ch *ch; 1152 struct Scsi_Host *shost = target->scsi_host; 1153 struct scsi_device *sdev; 1154 int i, j; 1155 1156 /* 1157 * Invoking srp_terminate_io() while srp_queuecommand() is running 1158 * is not safe. Hence the warning statement below. 1159 */ 1160 shost_for_each_device(sdev, shost) 1161 WARN_ON_ONCE(sdev->request_queue->request_fn_active); 1162 1163 for (i = 0; i < target->ch_count; i++) { 1164 ch = &target->ch[i]; 1165 1166 for (j = 0; j < target->req_ring_size; ++j) { 1167 struct srp_request *req = &ch->req_ring[j]; 1168 1169 srp_finish_req(ch, req, NULL, 1170 DID_TRANSPORT_FAILFAST << 16); 1171 } 1172 } 1173 } 1174 1175 /* 1176 * It is up to the caller to ensure that srp_rport_reconnect() calls are 1177 * serialized and that no concurrent srp_queuecommand(), srp_abort(), 1178 * srp_reset_device() or srp_reset_host() calls will occur while this function 1179 * is in progress. One way to realize that is not to call this function 1180 * directly but to call srp_reconnect_rport() instead since that last function 1181 * serializes calls of this function via rport->mutex and also blocks 1182 * srp_queuecommand() calls before invoking this function. 1183 */ 1184 static int srp_rport_reconnect(struct srp_rport *rport) 1185 { 1186 struct srp_target_port *target = rport->lld_data; 1187 struct srp_rdma_ch *ch; 1188 int i, j, ret = 0; 1189 bool multich = false; 1190 1191 srp_disconnect_target(target); 1192 1193 if (target->state == SRP_TARGET_SCANNING) 1194 return -ENODEV; 1195 1196 /* 1197 * Now get a new local CM ID so that we avoid confusing the target in 1198 * case things are really fouled up. Doing so also ensures that all CM 1199 * callbacks will have finished before a new QP is allocated. 1200 */ 1201 for (i = 0; i < target->ch_count; i++) { 1202 ch = &target->ch[i]; 1203 ret += srp_new_cm_id(ch); 1204 } 1205 for (i = 0; i < target->ch_count; i++) { 1206 ch = &target->ch[i]; 1207 for (j = 0; j < target->req_ring_size; ++j) { 1208 struct srp_request *req = &ch->req_ring[j]; 1209 1210 srp_finish_req(ch, req, NULL, DID_RESET << 16); 1211 } 1212 } 1213 for (i = 0; i < target->ch_count; i++) { 1214 ch = &target->ch[i]; 1215 /* 1216 * Whether or not creating a new CM ID succeeded, create a new 1217 * QP. This guarantees that all completion callback function 1218 * invocations have finished before request resetting starts. 1219 */ 1220 ret += srp_create_ch_ib(ch); 1221 1222 INIT_LIST_HEAD(&ch->free_tx); 1223 for (j = 0; j < target->queue_size; ++j) 1224 list_add(&ch->tx_ring[j]->list, &ch->free_tx); 1225 } 1226 1227 target->qp_in_error = false; 1228 1229 for (i = 0; i < target->ch_count; i++) { 1230 ch = &target->ch[i]; 1231 if (ret) 1232 break; 1233 ret = srp_connect_ch(ch, multich); 1234 multich = true; 1235 } 1236 1237 if (ret == 0) 1238 shost_printk(KERN_INFO, target->scsi_host, 1239 PFX "reconnect succeeded\n"); 1240 1241 return ret; 1242 } 1243 1244 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr, 1245 unsigned int dma_len, u32 rkey) 1246 { 1247 struct srp_direct_buf *desc = state->desc; 1248 1249 WARN_ON_ONCE(!dma_len); 1250 1251 desc->va = cpu_to_be64(dma_addr); 1252 desc->key = cpu_to_be32(rkey); 1253 desc->len = cpu_to_be32(dma_len); 1254 1255 state->total_len += dma_len; 1256 state->desc++; 1257 state->ndesc++; 1258 } 1259 1260 static int srp_map_finish_fmr(struct srp_map_state *state, 1261 struct srp_rdma_ch *ch) 1262 { 1263 struct srp_target_port *target = ch->target; 1264 struct srp_device *dev = target->srp_host->srp_dev; 1265 struct ib_pool_fmr *fmr; 1266 u64 io_addr = 0; 1267 1268 if (state->fmr.next >= state->fmr.end) 1269 return -ENOMEM; 1270 1271 WARN_ON_ONCE(!dev->use_fmr); 1272 1273 if (state->npages == 0) 1274 return 0; 1275 1276 if (state->npages == 1 && target->global_mr) { 1277 srp_map_desc(state, state->base_dma_addr, state->dma_len, 1278 target->global_mr->rkey); 1279 goto reset_state; 1280 } 1281 1282 fmr = ib_fmr_pool_map_phys(ch->fmr_pool, state->pages, 1283 state->npages, io_addr); 1284 if (IS_ERR(fmr)) 1285 return PTR_ERR(fmr); 1286 1287 *state->fmr.next++ = fmr; 1288 state->nmdesc++; 1289 1290 srp_map_desc(state, state->base_dma_addr & ~dev->mr_page_mask, 1291 state->dma_len, fmr->fmr->rkey); 1292 1293 reset_state: 1294 state->npages = 0; 1295 state->dma_len = 0; 1296 1297 return 0; 1298 } 1299 1300 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc) 1301 { 1302 srp_handle_qp_err(cq, wc, "FAST REG"); 1303 } 1304 1305 /* 1306 * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset 1307 * where to start in the first element. If sg_offset_p != NULL then 1308 * *sg_offset_p is updated to the offset in state->sg[retval] of the first 1309 * byte that has not yet been mapped. 1310 */ 1311 static int srp_map_finish_fr(struct srp_map_state *state, 1312 struct srp_request *req, 1313 struct srp_rdma_ch *ch, int sg_nents, 1314 unsigned int *sg_offset_p) 1315 { 1316 struct srp_target_port *target = ch->target; 1317 struct srp_device *dev = target->srp_host->srp_dev; 1318 struct ib_send_wr *bad_wr; 1319 struct ib_reg_wr wr; 1320 struct srp_fr_desc *desc; 1321 u32 rkey; 1322 int n, err; 1323 1324 if (state->fr.next >= state->fr.end) 1325 return -ENOMEM; 1326 1327 WARN_ON_ONCE(!dev->use_fast_reg); 1328 1329 if (sg_nents == 1 && target->global_mr) { 1330 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1331 1332 srp_map_desc(state, sg_dma_address(state->sg) + sg_offset, 1333 sg_dma_len(state->sg) - sg_offset, 1334 target->global_mr->rkey); 1335 if (sg_offset_p) 1336 *sg_offset_p = 0; 1337 return 1; 1338 } 1339 1340 desc = srp_fr_pool_get(ch->fr_pool); 1341 if (!desc) 1342 return -ENOMEM; 1343 1344 rkey = ib_inc_rkey(desc->mr->rkey); 1345 ib_update_fast_reg_key(desc->mr, rkey); 1346 1347 n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p, 1348 dev->mr_page_size); 1349 if (unlikely(n < 0)) { 1350 srp_fr_pool_put(ch->fr_pool, &desc, 1); 1351 pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n", 1352 dev_name(&req->scmnd->device->sdev_gendev), sg_nents, 1353 sg_offset_p ? *sg_offset_p : -1, n); 1354 return n; 1355 } 1356 1357 WARN_ON_ONCE(desc->mr->length == 0); 1358 1359 req->reg_cqe.done = srp_reg_mr_err_done; 1360 1361 wr.wr.next = NULL; 1362 wr.wr.opcode = IB_WR_REG_MR; 1363 wr.wr.wr_cqe = &req->reg_cqe; 1364 wr.wr.num_sge = 0; 1365 wr.wr.send_flags = 0; 1366 wr.mr = desc->mr; 1367 wr.key = desc->mr->rkey; 1368 wr.access = (IB_ACCESS_LOCAL_WRITE | 1369 IB_ACCESS_REMOTE_READ | 1370 IB_ACCESS_REMOTE_WRITE); 1371 1372 *state->fr.next++ = desc; 1373 state->nmdesc++; 1374 1375 srp_map_desc(state, desc->mr->iova, 1376 desc->mr->length, desc->mr->rkey); 1377 1378 err = ib_post_send(ch->qp, &wr.wr, &bad_wr); 1379 if (unlikely(err)) { 1380 WARN_ON_ONCE(err == -ENOMEM); 1381 return err; 1382 } 1383 1384 return n; 1385 } 1386 1387 static int srp_map_sg_entry(struct srp_map_state *state, 1388 struct srp_rdma_ch *ch, 1389 struct scatterlist *sg, int sg_index) 1390 { 1391 struct srp_target_port *target = ch->target; 1392 struct srp_device *dev = target->srp_host->srp_dev; 1393 struct ib_device *ibdev = dev->dev; 1394 dma_addr_t dma_addr = ib_sg_dma_address(ibdev, sg); 1395 unsigned int dma_len = ib_sg_dma_len(ibdev, sg); 1396 unsigned int len = 0; 1397 int ret; 1398 1399 WARN_ON_ONCE(!dma_len); 1400 1401 while (dma_len) { 1402 unsigned offset = dma_addr & ~dev->mr_page_mask; 1403 if (state->npages == dev->max_pages_per_mr || offset != 0) { 1404 ret = srp_map_finish_fmr(state, ch); 1405 if (ret) 1406 return ret; 1407 } 1408 1409 len = min_t(unsigned int, dma_len, dev->mr_page_size - offset); 1410 1411 if (!state->npages) 1412 state->base_dma_addr = dma_addr; 1413 state->pages[state->npages++] = dma_addr & dev->mr_page_mask; 1414 state->dma_len += len; 1415 dma_addr += len; 1416 dma_len -= len; 1417 } 1418 1419 /* 1420 * If the last entry of the MR wasn't a full page, then we need to 1421 * close it out and start a new one -- we can only merge at page 1422 * boundaries. 1423 */ 1424 ret = 0; 1425 if (len != dev->mr_page_size) 1426 ret = srp_map_finish_fmr(state, ch); 1427 return ret; 1428 } 1429 1430 static int srp_map_sg_fmr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1431 struct srp_request *req, struct scatterlist *scat, 1432 int count) 1433 { 1434 struct scatterlist *sg; 1435 int i, ret; 1436 1437 state->pages = req->map_page; 1438 state->fmr.next = req->fmr_list; 1439 state->fmr.end = req->fmr_list + ch->target->mr_per_cmd; 1440 1441 for_each_sg(scat, sg, count, i) { 1442 ret = srp_map_sg_entry(state, ch, sg, i); 1443 if (ret) 1444 return ret; 1445 } 1446 1447 ret = srp_map_finish_fmr(state, ch); 1448 if (ret) 1449 return ret; 1450 1451 return 0; 1452 } 1453 1454 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch, 1455 struct srp_request *req, struct scatterlist *scat, 1456 int count) 1457 { 1458 unsigned int sg_offset = 0; 1459 1460 state->desc = req->indirect_desc; 1461 state->fr.next = req->fr_list; 1462 state->fr.end = req->fr_list + ch->target->mr_per_cmd; 1463 state->sg = scat; 1464 1465 if (count == 0) 1466 return 0; 1467 1468 while (count) { 1469 int i, n; 1470 1471 n = srp_map_finish_fr(state, req, ch, count, &sg_offset); 1472 if (unlikely(n < 0)) 1473 return n; 1474 1475 count -= n; 1476 for (i = 0; i < n; i++) 1477 state->sg = sg_next(state->sg); 1478 } 1479 1480 return 0; 1481 } 1482 1483 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch, 1484 struct srp_request *req, struct scatterlist *scat, 1485 int count) 1486 { 1487 struct srp_target_port *target = ch->target; 1488 struct srp_device *dev = target->srp_host->srp_dev; 1489 struct scatterlist *sg; 1490 int i; 1491 1492 state->desc = req->indirect_desc; 1493 for_each_sg(scat, sg, count, i) { 1494 srp_map_desc(state, ib_sg_dma_address(dev->dev, sg), 1495 ib_sg_dma_len(dev->dev, sg), 1496 target->global_mr->rkey); 1497 } 1498 1499 return 0; 1500 } 1501 1502 /* 1503 * Register the indirect data buffer descriptor with the HCA. 1504 * 1505 * Note: since the indirect data buffer descriptor has been allocated with 1506 * kmalloc() it is guaranteed that this buffer is a physically contiguous 1507 * memory buffer. 1508 */ 1509 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req, 1510 void **next_mr, void **end_mr, u32 idb_len, 1511 __be32 *idb_rkey) 1512 { 1513 struct srp_target_port *target = ch->target; 1514 struct srp_device *dev = target->srp_host->srp_dev; 1515 struct srp_map_state state; 1516 struct srp_direct_buf idb_desc; 1517 u64 idb_pages[1]; 1518 struct scatterlist idb_sg[1]; 1519 int ret; 1520 1521 memset(&state, 0, sizeof(state)); 1522 memset(&idb_desc, 0, sizeof(idb_desc)); 1523 state.gen.next = next_mr; 1524 state.gen.end = end_mr; 1525 state.desc = &idb_desc; 1526 state.base_dma_addr = req->indirect_dma_addr; 1527 state.dma_len = idb_len; 1528 1529 if (dev->use_fast_reg) { 1530 state.sg = idb_sg; 1531 sg_init_one(idb_sg, req->indirect_desc, idb_len); 1532 idb_sg->dma_address = req->indirect_dma_addr; /* hack! */ 1533 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1534 idb_sg->dma_length = idb_sg->length; /* hack^2 */ 1535 #endif 1536 ret = srp_map_finish_fr(&state, req, ch, 1, NULL); 1537 if (ret < 0) 1538 return ret; 1539 WARN_ON_ONCE(ret < 1); 1540 } else if (dev->use_fmr) { 1541 state.pages = idb_pages; 1542 state.pages[0] = (req->indirect_dma_addr & 1543 dev->mr_page_mask); 1544 state.npages = 1; 1545 ret = srp_map_finish_fmr(&state, ch); 1546 if (ret < 0) 1547 return ret; 1548 } else { 1549 return -EINVAL; 1550 } 1551 1552 *idb_rkey = idb_desc.key; 1553 1554 return 0; 1555 } 1556 1557 #if defined(DYNAMIC_DATA_DEBUG) 1558 static void srp_check_mapping(struct srp_map_state *state, 1559 struct srp_rdma_ch *ch, struct srp_request *req, 1560 struct scatterlist *scat, int count) 1561 { 1562 struct srp_device *dev = ch->target->srp_host->srp_dev; 1563 struct srp_fr_desc **pfr; 1564 u64 desc_len = 0, mr_len = 0; 1565 int i; 1566 1567 for (i = 0; i < state->ndesc; i++) 1568 desc_len += be32_to_cpu(req->indirect_desc[i].len); 1569 if (dev->use_fast_reg) 1570 for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++) 1571 mr_len += (*pfr)->mr->length; 1572 else if (dev->use_fmr) 1573 for (i = 0; i < state->nmdesc; i++) 1574 mr_len += be32_to_cpu(req->indirect_desc[i].len); 1575 if (desc_len != scsi_bufflen(req->scmnd) || 1576 mr_len > scsi_bufflen(req->scmnd)) 1577 pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n", 1578 scsi_bufflen(req->scmnd), desc_len, mr_len, 1579 state->ndesc, state->nmdesc); 1580 } 1581 #endif 1582 1583 /** 1584 * srp_map_data() - map SCSI data buffer onto an SRP request 1585 * @scmnd: SCSI command to map 1586 * @ch: SRP RDMA channel 1587 * @req: SRP request 1588 * 1589 * Returns the length in bytes of the SRP_CMD IU or a negative value if 1590 * mapping failed. 1591 */ 1592 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch, 1593 struct srp_request *req) 1594 { 1595 struct srp_target_port *target = ch->target; 1596 struct scatterlist *scat; 1597 struct srp_cmd *cmd = req->cmd->buf; 1598 int len, nents, count, ret; 1599 struct srp_device *dev; 1600 struct ib_device *ibdev; 1601 struct srp_map_state state; 1602 struct srp_indirect_buf *indirect_hdr; 1603 u32 idb_len, table_len; 1604 __be32 idb_rkey; 1605 u8 fmt; 1606 1607 if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE) 1608 return sizeof (struct srp_cmd); 1609 1610 if (scmnd->sc_data_direction != DMA_FROM_DEVICE && 1611 scmnd->sc_data_direction != DMA_TO_DEVICE) { 1612 shost_printk(KERN_WARNING, target->scsi_host, 1613 PFX "Unhandled data direction %d\n", 1614 scmnd->sc_data_direction); 1615 return -EINVAL; 1616 } 1617 1618 nents = scsi_sg_count(scmnd); 1619 scat = scsi_sglist(scmnd); 1620 1621 dev = target->srp_host->srp_dev; 1622 ibdev = dev->dev; 1623 1624 count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction); 1625 if (unlikely(count == 0)) 1626 return -EIO; 1627 1628 fmt = SRP_DATA_DESC_DIRECT; 1629 len = sizeof (struct srp_cmd) + sizeof (struct srp_direct_buf); 1630 1631 if (count == 1 && target->global_mr) { 1632 /* 1633 * The midlayer only generated a single gather/scatter 1634 * entry, or DMA mapping coalesced everything to a 1635 * single entry. So a direct descriptor along with 1636 * the DMA MR suffices. 1637 */ 1638 struct srp_direct_buf *buf = (void *) cmd->add_data; 1639 1640 buf->va = cpu_to_be64(ib_sg_dma_address(ibdev, scat)); 1641 buf->key = cpu_to_be32(target->global_mr->rkey); 1642 buf->len = cpu_to_be32(ib_sg_dma_len(ibdev, scat)); 1643 1644 req->nmdesc = 0; 1645 goto map_complete; 1646 } 1647 1648 /* 1649 * We have more than one scatter/gather entry, so build our indirect 1650 * descriptor table, trying to merge as many entries as we can. 1651 */ 1652 indirect_hdr = (void *) cmd->add_data; 1653 1654 ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr, 1655 target->indirect_size, DMA_TO_DEVICE); 1656 1657 memset(&state, 0, sizeof(state)); 1658 if (dev->use_fast_reg) 1659 ret = srp_map_sg_fr(&state, ch, req, scat, count); 1660 else if (dev->use_fmr) 1661 ret = srp_map_sg_fmr(&state, ch, req, scat, count); 1662 else 1663 ret = srp_map_sg_dma(&state, ch, req, scat, count); 1664 req->nmdesc = state.nmdesc; 1665 if (ret < 0) 1666 goto unmap; 1667 1668 #if defined(DYNAMIC_DEBUG) 1669 { 1670 DEFINE_DYNAMIC_DEBUG_METADATA(ddm, 1671 "Memory mapping consistency check"); 1672 if (unlikely(ddm.flags & _DPRINTK_FLAGS_PRINT)) 1673 srp_check_mapping(&state, ch, req, scat, count); 1674 } 1675 #endif 1676 1677 /* We've mapped the request, now pull as much of the indirect 1678 * descriptor table as we can into the command buffer. If this 1679 * target is not using an external indirect table, we are 1680 * guaranteed to fit into the command, as the SCSI layer won't 1681 * give us more S/G entries than we allow. 1682 */ 1683 if (state.ndesc == 1) { 1684 /* 1685 * Memory registration collapsed the sg-list into one entry, 1686 * so use a direct descriptor. 1687 */ 1688 struct srp_direct_buf *buf = (void *) cmd->add_data; 1689 1690 *buf = req->indirect_desc[0]; 1691 goto map_complete; 1692 } 1693 1694 if (unlikely(target->cmd_sg_cnt < state.ndesc && 1695 !target->allow_ext_sg)) { 1696 shost_printk(KERN_ERR, target->scsi_host, 1697 "Could not fit S/G list into SRP_CMD\n"); 1698 ret = -EIO; 1699 goto unmap; 1700 } 1701 1702 count = min(state.ndesc, target->cmd_sg_cnt); 1703 table_len = state.ndesc * sizeof (struct srp_direct_buf); 1704 idb_len = sizeof(struct srp_indirect_buf) + table_len; 1705 1706 fmt = SRP_DATA_DESC_INDIRECT; 1707 len = sizeof(struct srp_cmd) + sizeof (struct srp_indirect_buf); 1708 len += count * sizeof (struct srp_direct_buf); 1709 1710 memcpy(indirect_hdr->desc_list, req->indirect_desc, 1711 count * sizeof (struct srp_direct_buf)); 1712 1713 if (!target->global_mr) { 1714 ret = srp_map_idb(ch, req, state.gen.next, state.gen.end, 1715 idb_len, &idb_rkey); 1716 if (ret < 0) 1717 goto unmap; 1718 req->nmdesc++; 1719 } else { 1720 idb_rkey = cpu_to_be32(target->global_mr->rkey); 1721 } 1722 1723 indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr); 1724 indirect_hdr->table_desc.key = idb_rkey; 1725 indirect_hdr->table_desc.len = cpu_to_be32(table_len); 1726 indirect_hdr->len = cpu_to_be32(state.total_len); 1727 1728 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1729 cmd->data_out_desc_cnt = count; 1730 else 1731 cmd->data_in_desc_cnt = count; 1732 1733 ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len, 1734 DMA_TO_DEVICE); 1735 1736 map_complete: 1737 if (scmnd->sc_data_direction == DMA_TO_DEVICE) 1738 cmd->buf_fmt = fmt << 4; 1739 else 1740 cmd->buf_fmt = fmt; 1741 1742 return len; 1743 1744 unmap: 1745 srp_unmap_data(scmnd, ch, req); 1746 if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size) 1747 ret = -E2BIG; 1748 return ret; 1749 } 1750 1751 /* 1752 * Return an IU and possible credit to the free pool 1753 */ 1754 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu, 1755 enum srp_iu_type iu_type) 1756 { 1757 unsigned long flags; 1758 1759 spin_lock_irqsave(&ch->lock, flags); 1760 list_add(&iu->list, &ch->free_tx); 1761 if (iu_type != SRP_IU_RSP) 1762 ++ch->req_lim; 1763 spin_unlock_irqrestore(&ch->lock, flags); 1764 } 1765 1766 /* 1767 * Must be called with ch->lock held to protect req_lim and free_tx. 1768 * If IU is not sent, it must be returned using srp_put_tx_iu(). 1769 * 1770 * Note: 1771 * An upper limit for the number of allocated information units for each 1772 * request type is: 1773 * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues 1774 * more than Scsi_Host.can_queue requests. 1775 * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE. 1776 * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than 1777 * one unanswered SRP request to an initiator. 1778 */ 1779 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch, 1780 enum srp_iu_type iu_type) 1781 { 1782 struct srp_target_port *target = ch->target; 1783 s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE; 1784 struct srp_iu *iu; 1785 1786 ib_process_cq_direct(ch->send_cq, -1); 1787 1788 if (list_empty(&ch->free_tx)) 1789 return NULL; 1790 1791 /* Initiator responses to target requests do not consume credits */ 1792 if (iu_type != SRP_IU_RSP) { 1793 if (ch->req_lim <= rsv) { 1794 ++target->zero_req_lim; 1795 return NULL; 1796 } 1797 1798 --ch->req_lim; 1799 } 1800 1801 iu = list_first_entry(&ch->free_tx, struct srp_iu, list); 1802 list_del(&iu->list); 1803 return iu; 1804 } 1805 1806 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc) 1807 { 1808 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1809 struct srp_rdma_ch *ch = cq->cq_context; 1810 1811 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1812 srp_handle_qp_err(cq, wc, "SEND"); 1813 return; 1814 } 1815 1816 list_add(&iu->list, &ch->free_tx); 1817 } 1818 1819 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len) 1820 { 1821 struct srp_target_port *target = ch->target; 1822 struct ib_sge list; 1823 struct ib_send_wr wr, *bad_wr; 1824 1825 list.addr = iu->dma; 1826 list.length = len; 1827 list.lkey = target->lkey; 1828 1829 iu->cqe.done = srp_send_done; 1830 1831 wr.next = NULL; 1832 wr.wr_cqe = &iu->cqe; 1833 wr.sg_list = &list; 1834 wr.num_sge = 1; 1835 wr.opcode = IB_WR_SEND; 1836 wr.send_flags = IB_SEND_SIGNALED; 1837 1838 return ib_post_send(ch->qp, &wr, &bad_wr); 1839 } 1840 1841 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu) 1842 { 1843 struct srp_target_port *target = ch->target; 1844 struct ib_recv_wr wr, *bad_wr; 1845 struct ib_sge list; 1846 1847 list.addr = iu->dma; 1848 list.length = iu->size; 1849 list.lkey = target->lkey; 1850 1851 iu->cqe.done = srp_recv_done; 1852 1853 wr.next = NULL; 1854 wr.wr_cqe = &iu->cqe; 1855 wr.sg_list = &list; 1856 wr.num_sge = 1; 1857 1858 return ib_post_recv(ch->qp, &wr, &bad_wr); 1859 } 1860 1861 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp) 1862 { 1863 struct srp_target_port *target = ch->target; 1864 struct srp_request *req; 1865 struct scsi_cmnd *scmnd; 1866 unsigned long flags; 1867 1868 if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) { 1869 spin_lock_irqsave(&ch->lock, flags); 1870 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1871 spin_unlock_irqrestore(&ch->lock, flags); 1872 1873 ch->tsk_mgmt_status = -1; 1874 if (be32_to_cpu(rsp->resp_data_len) >= 4) 1875 ch->tsk_mgmt_status = rsp->data[3]; 1876 complete(&ch->tsk_mgmt_done); 1877 } else { 1878 scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag); 1879 if (scmnd) { 1880 req = (void *)scmnd->host_scribble; 1881 scmnd = srp_claim_req(ch, req, NULL, scmnd); 1882 } 1883 if (!scmnd) { 1884 shost_printk(KERN_ERR, target->scsi_host, 1885 "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n", 1886 rsp->tag, ch - target->ch, ch->qp->qp_num); 1887 1888 spin_lock_irqsave(&ch->lock, flags); 1889 ch->req_lim += be32_to_cpu(rsp->req_lim_delta); 1890 spin_unlock_irqrestore(&ch->lock, flags); 1891 1892 return; 1893 } 1894 scmnd->result = rsp->status; 1895 1896 if (rsp->flags & SRP_RSP_FLAG_SNSVALID) { 1897 memcpy(scmnd->sense_buffer, rsp->data + 1898 be32_to_cpu(rsp->resp_data_len), 1899 min_t(int, be32_to_cpu(rsp->sense_data_len), 1900 SCSI_SENSE_BUFFERSIZE)); 1901 } 1902 1903 if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER)) 1904 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt)); 1905 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DIOVER)) 1906 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_in_res_cnt)); 1907 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER)) 1908 scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt)); 1909 else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOOVER)) 1910 scsi_set_resid(scmnd, -be32_to_cpu(rsp->data_out_res_cnt)); 1911 1912 srp_free_req(ch, req, scmnd, 1913 be32_to_cpu(rsp->req_lim_delta)); 1914 1915 scmnd->host_scribble = NULL; 1916 scmnd->scsi_done(scmnd); 1917 } 1918 } 1919 1920 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta, 1921 void *rsp, int len) 1922 { 1923 struct srp_target_port *target = ch->target; 1924 struct ib_device *dev = target->srp_host->srp_dev->dev; 1925 unsigned long flags; 1926 struct srp_iu *iu; 1927 int err; 1928 1929 spin_lock_irqsave(&ch->lock, flags); 1930 ch->req_lim += req_delta; 1931 iu = __srp_get_tx_iu(ch, SRP_IU_RSP); 1932 spin_unlock_irqrestore(&ch->lock, flags); 1933 1934 if (!iu) { 1935 shost_printk(KERN_ERR, target->scsi_host, PFX 1936 "no IU available to send response\n"); 1937 return 1; 1938 } 1939 1940 ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE); 1941 memcpy(iu->buf, rsp, len); 1942 ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE); 1943 1944 err = srp_post_send(ch, iu, len); 1945 if (err) { 1946 shost_printk(KERN_ERR, target->scsi_host, PFX 1947 "unable to post response: %d\n", err); 1948 srp_put_tx_iu(ch, iu, SRP_IU_RSP); 1949 } 1950 1951 return err; 1952 } 1953 1954 static void srp_process_cred_req(struct srp_rdma_ch *ch, 1955 struct srp_cred_req *req) 1956 { 1957 struct srp_cred_rsp rsp = { 1958 .opcode = SRP_CRED_RSP, 1959 .tag = req->tag, 1960 }; 1961 s32 delta = be32_to_cpu(req->req_lim_delta); 1962 1963 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 1964 shost_printk(KERN_ERR, ch->target->scsi_host, PFX 1965 "problems processing SRP_CRED_REQ\n"); 1966 } 1967 1968 static void srp_process_aer_req(struct srp_rdma_ch *ch, 1969 struct srp_aer_req *req) 1970 { 1971 struct srp_target_port *target = ch->target; 1972 struct srp_aer_rsp rsp = { 1973 .opcode = SRP_AER_RSP, 1974 .tag = req->tag, 1975 }; 1976 s32 delta = be32_to_cpu(req->req_lim_delta); 1977 1978 shost_printk(KERN_ERR, target->scsi_host, PFX 1979 "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun)); 1980 1981 if (srp_response_common(ch, delta, &rsp, sizeof(rsp))) 1982 shost_printk(KERN_ERR, target->scsi_host, PFX 1983 "problems processing SRP_AER_REQ\n"); 1984 } 1985 1986 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc) 1987 { 1988 struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe); 1989 struct srp_rdma_ch *ch = cq->cq_context; 1990 struct srp_target_port *target = ch->target; 1991 struct ib_device *dev = target->srp_host->srp_dev->dev; 1992 int res; 1993 u8 opcode; 1994 1995 if (unlikely(wc->status != IB_WC_SUCCESS)) { 1996 srp_handle_qp_err(cq, wc, "RECV"); 1997 return; 1998 } 1999 2000 ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len, 2001 DMA_FROM_DEVICE); 2002 2003 opcode = *(u8 *) iu->buf; 2004 2005 if (0) { 2006 shost_printk(KERN_ERR, target->scsi_host, 2007 PFX "recv completion, opcode 0x%02x\n", opcode); 2008 print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1, 2009 iu->buf, wc->byte_len, true); 2010 } 2011 2012 switch (opcode) { 2013 case SRP_RSP: 2014 srp_process_rsp(ch, iu->buf); 2015 break; 2016 2017 case SRP_CRED_REQ: 2018 srp_process_cred_req(ch, iu->buf); 2019 break; 2020 2021 case SRP_AER_REQ: 2022 srp_process_aer_req(ch, iu->buf); 2023 break; 2024 2025 case SRP_T_LOGOUT: 2026 /* XXX Handle target logout */ 2027 shost_printk(KERN_WARNING, target->scsi_host, 2028 PFX "Got target logout request\n"); 2029 break; 2030 2031 default: 2032 shost_printk(KERN_WARNING, target->scsi_host, 2033 PFX "Unhandled SRP opcode 0x%02x\n", opcode); 2034 break; 2035 } 2036 2037 ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len, 2038 DMA_FROM_DEVICE); 2039 2040 res = srp_post_recv(ch, iu); 2041 if (res != 0) 2042 shost_printk(KERN_ERR, target->scsi_host, 2043 PFX "Recv failed with error code %d\n", res); 2044 } 2045 2046 /** 2047 * srp_tl_err_work() - handle a transport layer error 2048 * @work: Work structure embedded in an SRP target port. 2049 * 2050 * Note: This function may get invoked before the rport has been created, 2051 * hence the target->rport test. 2052 */ 2053 static void srp_tl_err_work(struct work_struct *work) 2054 { 2055 struct srp_target_port *target; 2056 2057 target = container_of(work, struct srp_target_port, tl_err_work); 2058 if (target->rport) 2059 srp_start_tl_fail_timers(target->rport); 2060 } 2061 2062 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc, 2063 const char *opname) 2064 { 2065 struct srp_rdma_ch *ch = cq->cq_context; 2066 struct srp_target_port *target = ch->target; 2067 2068 if (ch->connected && !target->qp_in_error) { 2069 shost_printk(KERN_ERR, target->scsi_host, 2070 PFX "failed %s status %s (%d) for CQE %p\n", 2071 opname, ib_wc_status_msg(wc->status), wc->status, 2072 wc->wr_cqe); 2073 queue_work(system_long_wq, &target->tl_err_work); 2074 } 2075 target->qp_in_error = true; 2076 } 2077 2078 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd) 2079 { 2080 struct srp_target_port *target = host_to_target(shost); 2081 struct srp_rport *rport = target->rport; 2082 struct srp_rdma_ch *ch; 2083 struct srp_request *req; 2084 struct srp_iu *iu; 2085 struct srp_cmd *cmd; 2086 struct ib_device *dev; 2087 unsigned long flags; 2088 u32 tag; 2089 u16 idx; 2090 int len, ret; 2091 const bool in_scsi_eh = !in_interrupt() && current == shost->ehandler; 2092 2093 /* 2094 * The SCSI EH thread is the only context from which srp_queuecommand() 2095 * can get invoked for blocked devices (SDEV_BLOCK / 2096 * SDEV_CREATED_BLOCK). Avoid racing with srp_reconnect_rport() by 2097 * locking the rport mutex if invoked from inside the SCSI EH. 2098 */ 2099 if (in_scsi_eh) 2100 mutex_lock(&rport->mutex); 2101 2102 scmnd->result = srp_chkready(target->rport); 2103 if (unlikely(scmnd->result)) 2104 goto err; 2105 2106 WARN_ON_ONCE(scmnd->request->tag < 0); 2107 tag = blk_mq_unique_tag(scmnd->request); 2108 ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)]; 2109 idx = blk_mq_unique_tag_to_tag(tag); 2110 WARN_ONCE(idx >= target->req_ring_size, "%s: tag %#x: idx %d >= %d\n", 2111 dev_name(&shost->shost_gendev), tag, idx, 2112 target->req_ring_size); 2113 2114 spin_lock_irqsave(&ch->lock, flags); 2115 iu = __srp_get_tx_iu(ch, SRP_IU_CMD); 2116 spin_unlock_irqrestore(&ch->lock, flags); 2117 2118 if (!iu) 2119 goto err; 2120 2121 req = &ch->req_ring[idx]; 2122 dev = target->srp_host->srp_dev->dev; 2123 ib_dma_sync_single_for_cpu(dev, iu->dma, target->max_iu_len, 2124 DMA_TO_DEVICE); 2125 2126 scmnd->host_scribble = (void *) req; 2127 2128 cmd = iu->buf; 2129 memset(cmd, 0, sizeof *cmd); 2130 2131 cmd->opcode = SRP_CMD; 2132 int_to_scsilun(scmnd->device->lun, &cmd->lun); 2133 cmd->tag = tag; 2134 memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len); 2135 2136 req->scmnd = scmnd; 2137 req->cmd = iu; 2138 2139 len = srp_map_data(scmnd, ch, req); 2140 if (len < 0) { 2141 shost_printk(KERN_ERR, target->scsi_host, 2142 PFX "Failed to map data (%d)\n", len); 2143 /* 2144 * If we ran out of memory descriptors (-ENOMEM) because an 2145 * application is queuing many requests with more than 2146 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer 2147 * to reduce queue depth temporarily. 2148 */ 2149 scmnd->result = len == -ENOMEM ? 2150 DID_OK << 16 | QUEUE_FULL << 1 : DID_ERROR << 16; 2151 goto err_iu; 2152 } 2153 2154 ib_dma_sync_single_for_device(dev, iu->dma, target->max_iu_len, 2155 DMA_TO_DEVICE); 2156 2157 if (srp_post_send(ch, iu, len)) { 2158 shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n"); 2159 goto err_unmap; 2160 } 2161 2162 ret = 0; 2163 2164 unlock_rport: 2165 if (in_scsi_eh) 2166 mutex_unlock(&rport->mutex); 2167 2168 return ret; 2169 2170 err_unmap: 2171 srp_unmap_data(scmnd, ch, req); 2172 2173 err_iu: 2174 srp_put_tx_iu(ch, iu, SRP_IU_CMD); 2175 2176 /* 2177 * Avoid that the loops that iterate over the request ring can 2178 * encounter a dangling SCSI command pointer. 2179 */ 2180 req->scmnd = NULL; 2181 2182 err: 2183 if (scmnd->result) { 2184 scmnd->scsi_done(scmnd); 2185 ret = 0; 2186 } else { 2187 ret = SCSI_MLQUEUE_HOST_BUSY; 2188 } 2189 2190 goto unlock_rport; 2191 } 2192 2193 /* 2194 * Note: the resources allocated in this function are freed in 2195 * srp_free_ch_ib(). 2196 */ 2197 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch) 2198 { 2199 struct srp_target_port *target = ch->target; 2200 int i; 2201 2202 ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring), 2203 GFP_KERNEL); 2204 if (!ch->rx_ring) 2205 goto err_no_ring; 2206 ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring), 2207 GFP_KERNEL); 2208 if (!ch->tx_ring) 2209 goto err_no_ring; 2210 2211 for (i = 0; i < target->queue_size; ++i) { 2212 ch->rx_ring[i] = srp_alloc_iu(target->srp_host, 2213 ch->max_ti_iu_len, 2214 GFP_KERNEL, DMA_FROM_DEVICE); 2215 if (!ch->rx_ring[i]) 2216 goto err; 2217 } 2218 2219 for (i = 0; i < target->queue_size; ++i) { 2220 ch->tx_ring[i] = srp_alloc_iu(target->srp_host, 2221 target->max_iu_len, 2222 GFP_KERNEL, DMA_TO_DEVICE); 2223 if (!ch->tx_ring[i]) 2224 goto err; 2225 2226 list_add(&ch->tx_ring[i]->list, &ch->free_tx); 2227 } 2228 2229 return 0; 2230 2231 err: 2232 for (i = 0; i < target->queue_size; ++i) { 2233 srp_free_iu(target->srp_host, ch->rx_ring[i]); 2234 srp_free_iu(target->srp_host, ch->tx_ring[i]); 2235 } 2236 2237 2238 err_no_ring: 2239 kfree(ch->tx_ring); 2240 ch->tx_ring = NULL; 2241 kfree(ch->rx_ring); 2242 ch->rx_ring = NULL; 2243 2244 return -ENOMEM; 2245 } 2246 2247 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask) 2248 { 2249 uint64_t T_tr_ns, max_compl_time_ms; 2250 uint32_t rq_tmo_jiffies; 2251 2252 /* 2253 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair, 2254 * table 91), both the QP timeout and the retry count have to be set 2255 * for RC QP's during the RTR to RTS transition. 2256 */ 2257 WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) != 2258 (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)); 2259 2260 /* 2261 * Set target->rq_tmo_jiffies to one second more than the largest time 2262 * it can take before an error completion is generated. See also 2263 * C9-140..142 in the IBTA spec for more information about how to 2264 * convert the QP Local ACK Timeout value to nanoseconds. 2265 */ 2266 T_tr_ns = 4096 * (1ULL << qp_attr->timeout); 2267 max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns; 2268 do_div(max_compl_time_ms, NSEC_PER_MSEC); 2269 rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000); 2270 2271 return rq_tmo_jiffies; 2272 } 2273 2274 static void srp_cm_rep_handler(struct ib_cm_id *cm_id, 2275 const struct srp_login_rsp *lrsp, 2276 struct srp_rdma_ch *ch) 2277 { 2278 struct srp_target_port *target = ch->target; 2279 struct ib_qp_attr *qp_attr = NULL; 2280 int attr_mask = 0; 2281 int ret; 2282 int i; 2283 2284 if (lrsp->opcode == SRP_LOGIN_RSP) { 2285 ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len); 2286 ch->req_lim = be32_to_cpu(lrsp->req_lim_delta); 2287 2288 /* 2289 * Reserve credits for task management so we don't 2290 * bounce requests back to the SCSI mid-layer. 2291 */ 2292 target->scsi_host->can_queue 2293 = min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE, 2294 target->scsi_host->can_queue); 2295 target->scsi_host->cmd_per_lun 2296 = min_t(int, target->scsi_host->can_queue, 2297 target->scsi_host->cmd_per_lun); 2298 } else { 2299 shost_printk(KERN_WARNING, target->scsi_host, 2300 PFX "Unhandled RSP opcode %#x\n", lrsp->opcode); 2301 ret = -ECONNRESET; 2302 goto error; 2303 } 2304 2305 if (!ch->rx_ring) { 2306 ret = srp_alloc_iu_bufs(ch); 2307 if (ret) 2308 goto error; 2309 } 2310 2311 ret = -ENOMEM; 2312 qp_attr = kmalloc(sizeof *qp_attr, GFP_KERNEL); 2313 if (!qp_attr) 2314 goto error; 2315 2316 qp_attr->qp_state = IB_QPS_RTR; 2317 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2318 if (ret) 2319 goto error_free; 2320 2321 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2322 if (ret) 2323 goto error_free; 2324 2325 for (i = 0; i < target->queue_size; i++) { 2326 struct srp_iu *iu = ch->rx_ring[i]; 2327 2328 ret = srp_post_recv(ch, iu); 2329 if (ret) 2330 goto error_free; 2331 } 2332 2333 qp_attr->qp_state = IB_QPS_RTS; 2334 ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask); 2335 if (ret) 2336 goto error_free; 2337 2338 target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask); 2339 2340 ret = ib_modify_qp(ch->qp, qp_attr, attr_mask); 2341 if (ret) 2342 goto error_free; 2343 2344 ret = ib_send_cm_rtu(cm_id, NULL, 0); 2345 2346 error_free: 2347 kfree(qp_attr); 2348 2349 error: 2350 ch->status = ret; 2351 } 2352 2353 static void srp_cm_rej_handler(struct ib_cm_id *cm_id, 2354 struct ib_cm_event *event, 2355 struct srp_rdma_ch *ch) 2356 { 2357 struct srp_target_port *target = ch->target; 2358 struct Scsi_Host *shost = target->scsi_host; 2359 struct ib_class_port_info *cpi; 2360 int opcode; 2361 2362 switch (event->param.rej_rcvd.reason) { 2363 case IB_CM_REJ_PORT_CM_REDIRECT: 2364 cpi = event->param.rej_rcvd.ari; 2365 ch->path.dlid = cpi->redirect_lid; 2366 ch->path.pkey = cpi->redirect_pkey; 2367 cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff; 2368 memcpy(ch->path.dgid.raw, cpi->redirect_gid, 16); 2369 2370 ch->status = ch->path.dlid ? 2371 SRP_DLID_REDIRECT : SRP_PORT_REDIRECT; 2372 break; 2373 2374 case IB_CM_REJ_PORT_REDIRECT: 2375 if (srp_target_is_topspin(target)) { 2376 /* 2377 * Topspin/Cisco SRP gateways incorrectly send 2378 * reject reason code 25 when they mean 24 2379 * (port redirect). 2380 */ 2381 memcpy(ch->path.dgid.raw, 2382 event->param.rej_rcvd.ari, 16); 2383 2384 shost_printk(KERN_DEBUG, shost, 2385 PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n", 2386 be64_to_cpu(ch->path.dgid.global.subnet_prefix), 2387 be64_to_cpu(ch->path.dgid.global.interface_id)); 2388 2389 ch->status = SRP_PORT_REDIRECT; 2390 } else { 2391 shost_printk(KERN_WARNING, shost, 2392 " REJ reason: IB_CM_REJ_PORT_REDIRECT\n"); 2393 ch->status = -ECONNRESET; 2394 } 2395 break; 2396 2397 case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID: 2398 shost_printk(KERN_WARNING, shost, 2399 " REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n"); 2400 ch->status = -ECONNRESET; 2401 break; 2402 2403 case IB_CM_REJ_CONSUMER_DEFINED: 2404 opcode = *(u8 *) event->private_data; 2405 if (opcode == SRP_LOGIN_REJ) { 2406 struct srp_login_rej *rej = event->private_data; 2407 u32 reason = be32_to_cpu(rej->reason); 2408 2409 if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE) 2410 shost_printk(KERN_WARNING, shost, 2411 PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n"); 2412 else 2413 shost_printk(KERN_WARNING, shost, PFX 2414 "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n", 2415 target->sgid.raw, 2416 target->orig_dgid.raw, reason); 2417 } else 2418 shost_printk(KERN_WARNING, shost, 2419 " REJ reason: IB_CM_REJ_CONSUMER_DEFINED," 2420 " opcode 0x%02x\n", opcode); 2421 ch->status = -ECONNRESET; 2422 break; 2423 2424 case IB_CM_REJ_STALE_CONN: 2425 shost_printk(KERN_WARNING, shost, " REJ reason: stale connection\n"); 2426 ch->status = SRP_STALE_CONN; 2427 break; 2428 2429 default: 2430 shost_printk(KERN_WARNING, shost, " REJ reason 0x%x\n", 2431 event->param.rej_rcvd.reason); 2432 ch->status = -ECONNRESET; 2433 } 2434 } 2435 2436 static int srp_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event) 2437 { 2438 struct srp_rdma_ch *ch = cm_id->context; 2439 struct srp_target_port *target = ch->target; 2440 int comp = 0; 2441 2442 switch (event->event) { 2443 case IB_CM_REQ_ERROR: 2444 shost_printk(KERN_DEBUG, target->scsi_host, 2445 PFX "Sending CM REQ failed\n"); 2446 comp = 1; 2447 ch->status = -ECONNRESET; 2448 break; 2449 2450 case IB_CM_REP_RECEIVED: 2451 comp = 1; 2452 srp_cm_rep_handler(cm_id, event->private_data, ch); 2453 break; 2454 2455 case IB_CM_REJ_RECEIVED: 2456 shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n"); 2457 comp = 1; 2458 2459 srp_cm_rej_handler(cm_id, event, ch); 2460 break; 2461 2462 case IB_CM_DREQ_RECEIVED: 2463 shost_printk(KERN_WARNING, target->scsi_host, 2464 PFX "DREQ received - connection closed\n"); 2465 ch->connected = false; 2466 if (ib_send_cm_drep(cm_id, NULL, 0)) 2467 shost_printk(KERN_ERR, target->scsi_host, 2468 PFX "Sending CM DREP failed\n"); 2469 queue_work(system_long_wq, &target->tl_err_work); 2470 break; 2471 2472 case IB_CM_TIMEWAIT_EXIT: 2473 shost_printk(KERN_ERR, target->scsi_host, 2474 PFX "connection closed\n"); 2475 comp = 1; 2476 2477 ch->status = 0; 2478 break; 2479 2480 case IB_CM_MRA_RECEIVED: 2481 case IB_CM_DREQ_ERROR: 2482 case IB_CM_DREP_RECEIVED: 2483 break; 2484 2485 default: 2486 shost_printk(KERN_WARNING, target->scsi_host, 2487 PFX "Unhandled CM event %d\n", event->event); 2488 break; 2489 } 2490 2491 if (comp) 2492 complete(&ch->done); 2493 2494 return 0; 2495 } 2496 2497 /** 2498 * srp_change_queue_depth - setting device queue depth 2499 * @sdev: scsi device struct 2500 * @qdepth: requested queue depth 2501 * 2502 * Returns queue depth. 2503 */ 2504 static int 2505 srp_change_queue_depth(struct scsi_device *sdev, int qdepth) 2506 { 2507 if (!sdev->tagged_supported) 2508 qdepth = 1; 2509 return scsi_change_queue_depth(sdev, qdepth); 2510 } 2511 2512 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun, 2513 u8 func) 2514 { 2515 struct srp_target_port *target = ch->target; 2516 struct srp_rport *rport = target->rport; 2517 struct ib_device *dev = target->srp_host->srp_dev->dev; 2518 struct srp_iu *iu; 2519 struct srp_tsk_mgmt *tsk_mgmt; 2520 2521 if (!ch->connected || target->qp_in_error) 2522 return -1; 2523 2524 init_completion(&ch->tsk_mgmt_done); 2525 2526 /* 2527 * Lock the rport mutex to avoid that srp_create_ch_ib() is 2528 * invoked while a task management function is being sent. 2529 */ 2530 mutex_lock(&rport->mutex); 2531 spin_lock_irq(&ch->lock); 2532 iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT); 2533 spin_unlock_irq(&ch->lock); 2534 2535 if (!iu) { 2536 mutex_unlock(&rport->mutex); 2537 2538 return -1; 2539 } 2540 2541 ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt, 2542 DMA_TO_DEVICE); 2543 tsk_mgmt = iu->buf; 2544 memset(tsk_mgmt, 0, sizeof *tsk_mgmt); 2545 2546 tsk_mgmt->opcode = SRP_TSK_MGMT; 2547 int_to_scsilun(lun, &tsk_mgmt->lun); 2548 tsk_mgmt->tag = req_tag | SRP_TAG_TSK_MGMT; 2549 tsk_mgmt->tsk_mgmt_func = func; 2550 tsk_mgmt->task_tag = req_tag; 2551 2552 ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt, 2553 DMA_TO_DEVICE); 2554 if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) { 2555 srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT); 2556 mutex_unlock(&rport->mutex); 2557 2558 return -1; 2559 } 2560 mutex_unlock(&rport->mutex); 2561 2562 if (!wait_for_completion_timeout(&ch->tsk_mgmt_done, 2563 msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS))) 2564 return -1; 2565 2566 return 0; 2567 } 2568 2569 static int srp_abort(struct scsi_cmnd *scmnd) 2570 { 2571 struct srp_target_port *target = host_to_target(scmnd->device->host); 2572 struct srp_request *req = (struct srp_request *) scmnd->host_scribble; 2573 u32 tag; 2574 u16 ch_idx; 2575 struct srp_rdma_ch *ch; 2576 int ret; 2577 2578 shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n"); 2579 2580 if (!req) 2581 return SUCCESS; 2582 tag = blk_mq_unique_tag(scmnd->request); 2583 ch_idx = blk_mq_unique_tag_to_hwq(tag); 2584 if (WARN_ON_ONCE(ch_idx >= target->ch_count)) 2585 return SUCCESS; 2586 ch = &target->ch[ch_idx]; 2587 if (!srp_claim_req(ch, req, NULL, scmnd)) 2588 return SUCCESS; 2589 shost_printk(KERN_ERR, target->scsi_host, 2590 "Sending SRP abort for tag %#x\n", tag); 2591 if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun, 2592 SRP_TSK_ABORT_TASK) == 0) 2593 ret = SUCCESS; 2594 else if (target->rport->state == SRP_RPORT_LOST) 2595 ret = FAST_IO_FAIL; 2596 else 2597 ret = FAILED; 2598 srp_free_req(ch, req, scmnd, 0); 2599 scmnd->result = DID_ABORT << 16; 2600 scmnd->scsi_done(scmnd); 2601 2602 return ret; 2603 } 2604 2605 static int srp_reset_device(struct scsi_cmnd *scmnd) 2606 { 2607 struct srp_target_port *target = host_to_target(scmnd->device->host); 2608 struct srp_rdma_ch *ch; 2609 int i; 2610 2611 shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n"); 2612 2613 ch = &target->ch[0]; 2614 if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun, 2615 SRP_TSK_LUN_RESET)) 2616 return FAILED; 2617 if (ch->tsk_mgmt_status) 2618 return FAILED; 2619 2620 for (i = 0; i < target->ch_count; i++) { 2621 ch = &target->ch[i]; 2622 for (i = 0; i < target->req_ring_size; ++i) { 2623 struct srp_request *req = &ch->req_ring[i]; 2624 2625 srp_finish_req(ch, req, scmnd->device, DID_RESET << 16); 2626 } 2627 } 2628 2629 return SUCCESS; 2630 } 2631 2632 static int srp_reset_host(struct scsi_cmnd *scmnd) 2633 { 2634 struct srp_target_port *target = host_to_target(scmnd->device->host); 2635 2636 shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n"); 2637 2638 return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED; 2639 } 2640 2641 static int srp_slave_alloc(struct scsi_device *sdev) 2642 { 2643 struct Scsi_Host *shost = sdev->host; 2644 struct srp_target_port *target = host_to_target(shost); 2645 struct srp_device *srp_dev = target->srp_host->srp_dev; 2646 struct ib_device *ibdev = srp_dev->dev; 2647 2648 if (!(ibdev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)) 2649 blk_queue_virt_boundary(sdev->request_queue, 2650 ~srp_dev->mr_page_mask); 2651 2652 return 0; 2653 } 2654 2655 static int srp_slave_configure(struct scsi_device *sdev) 2656 { 2657 struct Scsi_Host *shost = sdev->host; 2658 struct srp_target_port *target = host_to_target(shost); 2659 struct request_queue *q = sdev->request_queue; 2660 unsigned long timeout; 2661 2662 if (sdev->type == TYPE_DISK) { 2663 timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies); 2664 blk_queue_rq_timeout(q, timeout); 2665 } 2666 2667 return 0; 2668 } 2669 2670 static ssize_t show_id_ext(struct device *dev, struct device_attribute *attr, 2671 char *buf) 2672 { 2673 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2674 2675 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->id_ext)); 2676 } 2677 2678 static ssize_t show_ioc_guid(struct device *dev, struct device_attribute *attr, 2679 char *buf) 2680 { 2681 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2682 2683 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid)); 2684 } 2685 2686 static ssize_t show_service_id(struct device *dev, 2687 struct device_attribute *attr, char *buf) 2688 { 2689 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2690 2691 return sprintf(buf, "0x%016llx\n", be64_to_cpu(target->service_id)); 2692 } 2693 2694 static ssize_t show_pkey(struct device *dev, struct device_attribute *attr, 2695 char *buf) 2696 { 2697 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2698 2699 return sprintf(buf, "0x%04x\n", be16_to_cpu(target->pkey)); 2700 } 2701 2702 static ssize_t show_sgid(struct device *dev, struct device_attribute *attr, 2703 char *buf) 2704 { 2705 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2706 2707 return sprintf(buf, "%pI6\n", target->sgid.raw); 2708 } 2709 2710 static ssize_t show_dgid(struct device *dev, struct device_attribute *attr, 2711 char *buf) 2712 { 2713 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2714 struct srp_rdma_ch *ch = &target->ch[0]; 2715 2716 return sprintf(buf, "%pI6\n", ch->path.dgid.raw); 2717 } 2718 2719 static ssize_t show_orig_dgid(struct device *dev, 2720 struct device_attribute *attr, char *buf) 2721 { 2722 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2723 2724 return sprintf(buf, "%pI6\n", target->orig_dgid.raw); 2725 } 2726 2727 static ssize_t show_req_lim(struct device *dev, 2728 struct device_attribute *attr, char *buf) 2729 { 2730 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2731 struct srp_rdma_ch *ch; 2732 int i, req_lim = INT_MAX; 2733 2734 for (i = 0; i < target->ch_count; i++) { 2735 ch = &target->ch[i]; 2736 req_lim = min(req_lim, ch->req_lim); 2737 } 2738 return sprintf(buf, "%d\n", req_lim); 2739 } 2740 2741 static ssize_t show_zero_req_lim(struct device *dev, 2742 struct device_attribute *attr, char *buf) 2743 { 2744 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2745 2746 return sprintf(buf, "%d\n", target->zero_req_lim); 2747 } 2748 2749 static ssize_t show_local_ib_port(struct device *dev, 2750 struct device_attribute *attr, char *buf) 2751 { 2752 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2753 2754 return sprintf(buf, "%d\n", target->srp_host->port); 2755 } 2756 2757 static ssize_t show_local_ib_device(struct device *dev, 2758 struct device_attribute *attr, char *buf) 2759 { 2760 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2761 2762 return sprintf(buf, "%s\n", target->srp_host->srp_dev->dev->name); 2763 } 2764 2765 static ssize_t show_ch_count(struct device *dev, struct device_attribute *attr, 2766 char *buf) 2767 { 2768 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2769 2770 return sprintf(buf, "%d\n", target->ch_count); 2771 } 2772 2773 static ssize_t show_comp_vector(struct device *dev, 2774 struct device_attribute *attr, char *buf) 2775 { 2776 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2777 2778 return sprintf(buf, "%d\n", target->comp_vector); 2779 } 2780 2781 static ssize_t show_tl_retry_count(struct device *dev, 2782 struct device_attribute *attr, char *buf) 2783 { 2784 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2785 2786 return sprintf(buf, "%d\n", target->tl_retry_count); 2787 } 2788 2789 static ssize_t show_cmd_sg_entries(struct device *dev, 2790 struct device_attribute *attr, char *buf) 2791 { 2792 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2793 2794 return sprintf(buf, "%u\n", target->cmd_sg_cnt); 2795 } 2796 2797 static ssize_t show_allow_ext_sg(struct device *dev, 2798 struct device_attribute *attr, char *buf) 2799 { 2800 struct srp_target_port *target = host_to_target(class_to_shost(dev)); 2801 2802 return sprintf(buf, "%s\n", target->allow_ext_sg ? "true" : "false"); 2803 } 2804 2805 static DEVICE_ATTR(id_ext, S_IRUGO, show_id_ext, NULL); 2806 static DEVICE_ATTR(ioc_guid, S_IRUGO, show_ioc_guid, NULL); 2807 static DEVICE_ATTR(service_id, S_IRUGO, show_service_id, NULL); 2808 static DEVICE_ATTR(pkey, S_IRUGO, show_pkey, NULL); 2809 static DEVICE_ATTR(sgid, S_IRUGO, show_sgid, NULL); 2810 static DEVICE_ATTR(dgid, S_IRUGO, show_dgid, NULL); 2811 static DEVICE_ATTR(orig_dgid, S_IRUGO, show_orig_dgid, NULL); 2812 static DEVICE_ATTR(req_lim, S_IRUGO, show_req_lim, NULL); 2813 static DEVICE_ATTR(zero_req_lim, S_IRUGO, show_zero_req_lim, NULL); 2814 static DEVICE_ATTR(local_ib_port, S_IRUGO, show_local_ib_port, NULL); 2815 static DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL); 2816 static DEVICE_ATTR(ch_count, S_IRUGO, show_ch_count, NULL); 2817 static DEVICE_ATTR(comp_vector, S_IRUGO, show_comp_vector, NULL); 2818 static DEVICE_ATTR(tl_retry_count, S_IRUGO, show_tl_retry_count, NULL); 2819 static DEVICE_ATTR(cmd_sg_entries, S_IRUGO, show_cmd_sg_entries, NULL); 2820 static DEVICE_ATTR(allow_ext_sg, S_IRUGO, show_allow_ext_sg, NULL); 2821 2822 static struct device_attribute *srp_host_attrs[] = { 2823 &dev_attr_id_ext, 2824 &dev_attr_ioc_guid, 2825 &dev_attr_service_id, 2826 &dev_attr_pkey, 2827 &dev_attr_sgid, 2828 &dev_attr_dgid, 2829 &dev_attr_orig_dgid, 2830 &dev_attr_req_lim, 2831 &dev_attr_zero_req_lim, 2832 &dev_attr_local_ib_port, 2833 &dev_attr_local_ib_device, 2834 &dev_attr_ch_count, 2835 &dev_attr_comp_vector, 2836 &dev_attr_tl_retry_count, 2837 &dev_attr_cmd_sg_entries, 2838 &dev_attr_allow_ext_sg, 2839 NULL 2840 }; 2841 2842 static struct scsi_host_template srp_template = { 2843 .module = THIS_MODULE, 2844 .name = "InfiniBand SRP initiator", 2845 .proc_name = DRV_NAME, 2846 .slave_alloc = srp_slave_alloc, 2847 .slave_configure = srp_slave_configure, 2848 .info = srp_target_info, 2849 .queuecommand = srp_queuecommand, 2850 .change_queue_depth = srp_change_queue_depth, 2851 .eh_abort_handler = srp_abort, 2852 .eh_device_reset_handler = srp_reset_device, 2853 .eh_host_reset_handler = srp_reset_host, 2854 .skip_settle_delay = true, 2855 .sg_tablesize = SRP_DEF_SG_TABLESIZE, 2856 .can_queue = SRP_DEFAULT_CMD_SQ_SIZE, 2857 .this_id = -1, 2858 .cmd_per_lun = SRP_DEFAULT_CMD_SQ_SIZE, 2859 .use_clustering = ENABLE_CLUSTERING, 2860 .shost_attrs = srp_host_attrs, 2861 .track_queue_depth = 1, 2862 }; 2863 2864 static int srp_sdev_count(struct Scsi_Host *host) 2865 { 2866 struct scsi_device *sdev; 2867 int c = 0; 2868 2869 shost_for_each_device(sdev, host) 2870 c++; 2871 2872 return c; 2873 } 2874 2875 /* 2876 * Return values: 2877 * < 0 upon failure. Caller is responsible for SRP target port cleanup. 2878 * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port 2879 * removal has been scheduled. 2880 * 0 and target->state != SRP_TARGET_REMOVED upon success. 2881 */ 2882 static int srp_add_target(struct srp_host *host, struct srp_target_port *target) 2883 { 2884 struct srp_rport_identifiers ids; 2885 struct srp_rport *rport; 2886 2887 target->state = SRP_TARGET_SCANNING; 2888 sprintf(target->target_name, "SRP.T10:%016llX", 2889 be64_to_cpu(target->id_ext)); 2890 2891 if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dma_device)) 2892 return -ENODEV; 2893 2894 memcpy(ids.port_id, &target->id_ext, 8); 2895 memcpy(ids.port_id + 8, &target->ioc_guid, 8); 2896 ids.roles = SRP_RPORT_ROLE_TARGET; 2897 rport = srp_rport_add(target->scsi_host, &ids); 2898 if (IS_ERR(rport)) { 2899 scsi_remove_host(target->scsi_host); 2900 return PTR_ERR(rport); 2901 } 2902 2903 rport->lld_data = target; 2904 target->rport = rport; 2905 2906 spin_lock(&host->target_lock); 2907 list_add_tail(&target->list, &host->target_list); 2908 spin_unlock(&host->target_lock); 2909 2910 scsi_scan_target(&target->scsi_host->shost_gendev, 2911 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL); 2912 2913 if (srp_connected_ch(target) < target->ch_count || 2914 target->qp_in_error) { 2915 shost_printk(KERN_INFO, target->scsi_host, 2916 PFX "SCSI scan failed - removing SCSI host\n"); 2917 srp_queue_remove_work(target); 2918 goto out; 2919 } 2920 2921 pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n", 2922 dev_name(&target->scsi_host->shost_gendev), 2923 srp_sdev_count(target->scsi_host)); 2924 2925 spin_lock_irq(&target->lock); 2926 if (target->state == SRP_TARGET_SCANNING) 2927 target->state = SRP_TARGET_LIVE; 2928 spin_unlock_irq(&target->lock); 2929 2930 out: 2931 return 0; 2932 } 2933 2934 static void srp_release_dev(struct device *dev) 2935 { 2936 struct srp_host *host = 2937 container_of(dev, struct srp_host, dev); 2938 2939 complete(&host->released); 2940 } 2941 2942 static struct class srp_class = { 2943 .name = "infiniband_srp", 2944 .dev_release = srp_release_dev 2945 }; 2946 2947 /** 2948 * srp_conn_unique() - check whether the connection to a target is unique 2949 * @host: SRP host. 2950 * @target: SRP target port. 2951 */ 2952 static bool srp_conn_unique(struct srp_host *host, 2953 struct srp_target_port *target) 2954 { 2955 struct srp_target_port *t; 2956 bool ret = false; 2957 2958 if (target->state == SRP_TARGET_REMOVED) 2959 goto out; 2960 2961 ret = true; 2962 2963 spin_lock(&host->target_lock); 2964 list_for_each_entry(t, &host->target_list, list) { 2965 if (t != target && 2966 target->id_ext == t->id_ext && 2967 target->ioc_guid == t->ioc_guid && 2968 target->initiator_ext == t->initiator_ext) { 2969 ret = false; 2970 break; 2971 } 2972 } 2973 spin_unlock(&host->target_lock); 2974 2975 out: 2976 return ret; 2977 } 2978 2979 /* 2980 * Target ports are added by writing 2981 * 2982 * id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>, 2983 * pkey=<P_Key>,service_id=<service ID> 2984 * 2985 * to the add_target sysfs attribute. 2986 */ 2987 enum { 2988 SRP_OPT_ERR = 0, 2989 SRP_OPT_ID_EXT = 1 << 0, 2990 SRP_OPT_IOC_GUID = 1 << 1, 2991 SRP_OPT_DGID = 1 << 2, 2992 SRP_OPT_PKEY = 1 << 3, 2993 SRP_OPT_SERVICE_ID = 1 << 4, 2994 SRP_OPT_MAX_SECT = 1 << 5, 2995 SRP_OPT_MAX_CMD_PER_LUN = 1 << 6, 2996 SRP_OPT_IO_CLASS = 1 << 7, 2997 SRP_OPT_INITIATOR_EXT = 1 << 8, 2998 SRP_OPT_CMD_SG_ENTRIES = 1 << 9, 2999 SRP_OPT_ALLOW_EXT_SG = 1 << 10, 3000 SRP_OPT_SG_TABLESIZE = 1 << 11, 3001 SRP_OPT_COMP_VECTOR = 1 << 12, 3002 SRP_OPT_TL_RETRY_COUNT = 1 << 13, 3003 SRP_OPT_QUEUE_SIZE = 1 << 14, 3004 SRP_OPT_ALL = (SRP_OPT_ID_EXT | 3005 SRP_OPT_IOC_GUID | 3006 SRP_OPT_DGID | 3007 SRP_OPT_PKEY | 3008 SRP_OPT_SERVICE_ID), 3009 }; 3010 3011 static const match_table_t srp_opt_tokens = { 3012 { SRP_OPT_ID_EXT, "id_ext=%s" }, 3013 { SRP_OPT_IOC_GUID, "ioc_guid=%s" }, 3014 { SRP_OPT_DGID, "dgid=%s" }, 3015 { SRP_OPT_PKEY, "pkey=%x" }, 3016 { SRP_OPT_SERVICE_ID, "service_id=%s" }, 3017 { SRP_OPT_MAX_SECT, "max_sect=%d" }, 3018 { SRP_OPT_MAX_CMD_PER_LUN, "max_cmd_per_lun=%d" }, 3019 { SRP_OPT_IO_CLASS, "io_class=%x" }, 3020 { SRP_OPT_INITIATOR_EXT, "initiator_ext=%s" }, 3021 { SRP_OPT_CMD_SG_ENTRIES, "cmd_sg_entries=%u" }, 3022 { SRP_OPT_ALLOW_EXT_SG, "allow_ext_sg=%u" }, 3023 { SRP_OPT_SG_TABLESIZE, "sg_tablesize=%u" }, 3024 { SRP_OPT_COMP_VECTOR, "comp_vector=%u" }, 3025 { SRP_OPT_TL_RETRY_COUNT, "tl_retry_count=%u" }, 3026 { SRP_OPT_QUEUE_SIZE, "queue_size=%d" }, 3027 { SRP_OPT_ERR, NULL } 3028 }; 3029 3030 static int srp_parse_options(const char *buf, struct srp_target_port *target) 3031 { 3032 char *options, *sep_opt; 3033 char *p; 3034 char dgid[3]; 3035 substring_t args[MAX_OPT_ARGS]; 3036 int opt_mask = 0; 3037 int token; 3038 int ret = -EINVAL; 3039 int i; 3040 3041 options = kstrdup(buf, GFP_KERNEL); 3042 if (!options) 3043 return -ENOMEM; 3044 3045 sep_opt = options; 3046 while ((p = strsep(&sep_opt, ",\n")) != NULL) { 3047 if (!*p) 3048 continue; 3049 3050 token = match_token(p, srp_opt_tokens, args); 3051 opt_mask |= token; 3052 3053 switch (token) { 3054 case SRP_OPT_ID_EXT: 3055 p = match_strdup(args); 3056 if (!p) { 3057 ret = -ENOMEM; 3058 goto out; 3059 } 3060 target->id_ext = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3061 kfree(p); 3062 break; 3063 3064 case SRP_OPT_IOC_GUID: 3065 p = match_strdup(args); 3066 if (!p) { 3067 ret = -ENOMEM; 3068 goto out; 3069 } 3070 target->ioc_guid = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3071 kfree(p); 3072 break; 3073 3074 case SRP_OPT_DGID: 3075 p = match_strdup(args); 3076 if (!p) { 3077 ret = -ENOMEM; 3078 goto out; 3079 } 3080 if (strlen(p) != 32) { 3081 pr_warn("bad dest GID parameter '%s'\n", p); 3082 kfree(p); 3083 goto out; 3084 } 3085 3086 for (i = 0; i < 16; ++i) { 3087 strlcpy(dgid, p + i * 2, sizeof(dgid)); 3088 if (sscanf(dgid, "%hhx", 3089 &target->orig_dgid.raw[i]) < 1) { 3090 ret = -EINVAL; 3091 kfree(p); 3092 goto out; 3093 } 3094 } 3095 kfree(p); 3096 break; 3097 3098 case SRP_OPT_PKEY: 3099 if (match_hex(args, &token)) { 3100 pr_warn("bad P_Key parameter '%s'\n", p); 3101 goto out; 3102 } 3103 target->pkey = cpu_to_be16(token); 3104 break; 3105 3106 case SRP_OPT_SERVICE_ID: 3107 p = match_strdup(args); 3108 if (!p) { 3109 ret = -ENOMEM; 3110 goto out; 3111 } 3112 target->service_id = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3113 kfree(p); 3114 break; 3115 3116 case SRP_OPT_MAX_SECT: 3117 if (match_int(args, &token)) { 3118 pr_warn("bad max sect parameter '%s'\n", p); 3119 goto out; 3120 } 3121 target->scsi_host->max_sectors = token; 3122 break; 3123 3124 case SRP_OPT_QUEUE_SIZE: 3125 if (match_int(args, &token) || token < 1) { 3126 pr_warn("bad queue_size parameter '%s'\n", p); 3127 goto out; 3128 } 3129 target->scsi_host->can_queue = token; 3130 target->queue_size = token + SRP_RSP_SQ_SIZE + 3131 SRP_TSK_MGMT_SQ_SIZE; 3132 if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3133 target->scsi_host->cmd_per_lun = token; 3134 break; 3135 3136 case SRP_OPT_MAX_CMD_PER_LUN: 3137 if (match_int(args, &token) || token < 1) { 3138 pr_warn("bad max cmd_per_lun parameter '%s'\n", 3139 p); 3140 goto out; 3141 } 3142 target->scsi_host->cmd_per_lun = token; 3143 break; 3144 3145 case SRP_OPT_IO_CLASS: 3146 if (match_hex(args, &token)) { 3147 pr_warn("bad IO class parameter '%s'\n", p); 3148 goto out; 3149 } 3150 if (token != SRP_REV10_IB_IO_CLASS && 3151 token != SRP_REV16A_IB_IO_CLASS) { 3152 pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n", 3153 token, SRP_REV10_IB_IO_CLASS, 3154 SRP_REV16A_IB_IO_CLASS); 3155 goto out; 3156 } 3157 target->io_class = token; 3158 break; 3159 3160 case SRP_OPT_INITIATOR_EXT: 3161 p = match_strdup(args); 3162 if (!p) { 3163 ret = -ENOMEM; 3164 goto out; 3165 } 3166 target->initiator_ext = cpu_to_be64(simple_strtoull(p, NULL, 16)); 3167 kfree(p); 3168 break; 3169 3170 case SRP_OPT_CMD_SG_ENTRIES: 3171 if (match_int(args, &token) || token < 1 || token > 255) { 3172 pr_warn("bad max cmd_sg_entries parameter '%s'\n", 3173 p); 3174 goto out; 3175 } 3176 target->cmd_sg_cnt = token; 3177 break; 3178 3179 case SRP_OPT_ALLOW_EXT_SG: 3180 if (match_int(args, &token)) { 3181 pr_warn("bad allow_ext_sg parameter '%s'\n", p); 3182 goto out; 3183 } 3184 target->allow_ext_sg = !!token; 3185 break; 3186 3187 case SRP_OPT_SG_TABLESIZE: 3188 if (match_int(args, &token) || token < 1 || 3189 token > SG_MAX_SEGMENTS) { 3190 pr_warn("bad max sg_tablesize parameter '%s'\n", 3191 p); 3192 goto out; 3193 } 3194 target->sg_tablesize = token; 3195 break; 3196 3197 case SRP_OPT_COMP_VECTOR: 3198 if (match_int(args, &token) || token < 0) { 3199 pr_warn("bad comp_vector parameter '%s'\n", p); 3200 goto out; 3201 } 3202 target->comp_vector = token; 3203 break; 3204 3205 case SRP_OPT_TL_RETRY_COUNT: 3206 if (match_int(args, &token) || token < 2 || token > 7) { 3207 pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n", 3208 p); 3209 goto out; 3210 } 3211 target->tl_retry_count = token; 3212 break; 3213 3214 default: 3215 pr_warn("unknown parameter or missing value '%s' in target creation request\n", 3216 p); 3217 goto out; 3218 } 3219 } 3220 3221 if ((opt_mask & SRP_OPT_ALL) == SRP_OPT_ALL) 3222 ret = 0; 3223 else 3224 for (i = 0; i < ARRAY_SIZE(srp_opt_tokens); ++i) 3225 if ((srp_opt_tokens[i].token & SRP_OPT_ALL) && 3226 !(srp_opt_tokens[i].token & opt_mask)) 3227 pr_warn("target creation request is missing parameter '%s'\n", 3228 srp_opt_tokens[i].pattern); 3229 3230 if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue 3231 && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN)) 3232 pr_warn("cmd_per_lun = %d > queue_size = %d\n", 3233 target->scsi_host->cmd_per_lun, 3234 target->scsi_host->can_queue); 3235 3236 out: 3237 kfree(options); 3238 return ret; 3239 } 3240 3241 static ssize_t srp_create_target(struct device *dev, 3242 struct device_attribute *attr, 3243 const char *buf, size_t count) 3244 { 3245 struct srp_host *host = 3246 container_of(dev, struct srp_host, dev); 3247 struct Scsi_Host *target_host; 3248 struct srp_target_port *target; 3249 struct srp_rdma_ch *ch; 3250 struct srp_device *srp_dev = host->srp_dev; 3251 struct ib_device *ibdev = srp_dev->dev; 3252 int ret, node_idx, node, cpu, i; 3253 unsigned int max_sectors_per_mr, mr_per_cmd = 0; 3254 bool multich = false; 3255 3256 target_host = scsi_host_alloc(&srp_template, 3257 sizeof (struct srp_target_port)); 3258 if (!target_host) 3259 return -ENOMEM; 3260 3261 target_host->transportt = ib_srp_transport_template; 3262 target_host->max_channel = 0; 3263 target_host->max_id = 1; 3264 target_host->max_lun = -1LL; 3265 target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb; 3266 3267 target = host_to_target(target_host); 3268 3269 target->io_class = SRP_REV16A_IB_IO_CLASS; 3270 target->scsi_host = target_host; 3271 target->srp_host = host; 3272 target->lkey = host->srp_dev->pd->local_dma_lkey; 3273 target->global_mr = host->srp_dev->global_mr; 3274 target->cmd_sg_cnt = cmd_sg_entries; 3275 target->sg_tablesize = indirect_sg_entries ? : cmd_sg_entries; 3276 target->allow_ext_sg = allow_ext_sg; 3277 target->tl_retry_count = 7; 3278 target->queue_size = SRP_DEFAULT_QUEUE_SIZE; 3279 3280 /* 3281 * Avoid that the SCSI host can be removed by srp_remove_target() 3282 * before this function returns. 3283 */ 3284 scsi_host_get(target->scsi_host); 3285 3286 mutex_lock(&host->add_target_mutex); 3287 3288 ret = srp_parse_options(buf, target); 3289 if (ret) 3290 goto out; 3291 3292 target->req_ring_size = target->queue_size - SRP_TSK_MGMT_SQ_SIZE; 3293 3294 if (!srp_conn_unique(target->srp_host, target)) { 3295 shost_printk(KERN_INFO, target->scsi_host, 3296 PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n", 3297 be64_to_cpu(target->id_ext), 3298 be64_to_cpu(target->ioc_guid), 3299 be64_to_cpu(target->initiator_ext)); 3300 ret = -EEXIST; 3301 goto out; 3302 } 3303 3304 if (!srp_dev->has_fmr && !srp_dev->has_fr && !target->allow_ext_sg && 3305 target->cmd_sg_cnt < target->sg_tablesize) { 3306 pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n"); 3307 target->sg_tablesize = target->cmd_sg_cnt; 3308 } 3309 3310 if (srp_dev->use_fast_reg || srp_dev->use_fmr) { 3311 /* 3312 * FR and FMR can only map one HCA page per entry. If the 3313 * start address is not aligned on a HCA page boundary two 3314 * entries will be used for the head and the tail although 3315 * these two entries combined contain at most one HCA page of 3316 * data. Hence the "+ 1" in the calculation below. 3317 * 3318 * The indirect data buffer descriptor is contiguous so the 3319 * memory for that buffer will only be registered if 3320 * register_always is true. Hence add one to mr_per_cmd if 3321 * register_always has been set. 3322 */ 3323 max_sectors_per_mr = srp_dev->max_pages_per_mr << 3324 (ilog2(srp_dev->mr_page_size) - 9); 3325 mr_per_cmd = register_always + 3326 (target->scsi_host->max_sectors + 1 + 3327 max_sectors_per_mr - 1) / max_sectors_per_mr; 3328 pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n", 3329 target->scsi_host->max_sectors, 3330 srp_dev->max_pages_per_mr, srp_dev->mr_page_size, 3331 max_sectors_per_mr, mr_per_cmd); 3332 } 3333 3334 target_host->sg_tablesize = target->sg_tablesize; 3335 target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd; 3336 target->mr_per_cmd = mr_per_cmd; 3337 target->indirect_size = target->sg_tablesize * 3338 sizeof (struct srp_direct_buf); 3339 target->max_iu_len = sizeof (struct srp_cmd) + 3340 sizeof (struct srp_indirect_buf) + 3341 target->cmd_sg_cnt * sizeof (struct srp_direct_buf); 3342 3343 INIT_WORK(&target->tl_err_work, srp_tl_err_work); 3344 INIT_WORK(&target->remove_work, srp_remove_work); 3345 spin_lock_init(&target->lock); 3346 ret = ib_query_gid(ibdev, host->port, 0, &target->sgid, NULL); 3347 if (ret) 3348 goto out; 3349 3350 ret = -ENOMEM; 3351 target->ch_count = max_t(unsigned, num_online_nodes(), 3352 min(ch_count ? : 3353 min(4 * num_online_nodes(), 3354 ibdev->num_comp_vectors), 3355 num_online_cpus())); 3356 target->ch = kcalloc(target->ch_count, sizeof(*target->ch), 3357 GFP_KERNEL); 3358 if (!target->ch) 3359 goto out; 3360 3361 node_idx = 0; 3362 for_each_online_node(node) { 3363 const int ch_start = (node_idx * target->ch_count / 3364 num_online_nodes()); 3365 const int ch_end = ((node_idx + 1) * target->ch_count / 3366 num_online_nodes()); 3367 const int cv_start = (node_idx * ibdev->num_comp_vectors / 3368 num_online_nodes() + target->comp_vector) 3369 % ibdev->num_comp_vectors; 3370 const int cv_end = ((node_idx + 1) * ibdev->num_comp_vectors / 3371 num_online_nodes() + target->comp_vector) 3372 % ibdev->num_comp_vectors; 3373 int cpu_idx = 0; 3374 3375 for_each_online_cpu(cpu) { 3376 if (cpu_to_node(cpu) != node) 3377 continue; 3378 if (ch_start + cpu_idx >= ch_end) 3379 continue; 3380 ch = &target->ch[ch_start + cpu_idx]; 3381 ch->target = target; 3382 ch->comp_vector = cv_start == cv_end ? cv_start : 3383 cv_start + cpu_idx % (cv_end - cv_start); 3384 spin_lock_init(&ch->lock); 3385 INIT_LIST_HEAD(&ch->free_tx); 3386 ret = srp_new_cm_id(ch); 3387 if (ret) 3388 goto err_disconnect; 3389 3390 ret = srp_create_ch_ib(ch); 3391 if (ret) 3392 goto err_disconnect; 3393 3394 ret = srp_alloc_req_data(ch); 3395 if (ret) 3396 goto err_disconnect; 3397 3398 ret = srp_connect_ch(ch, multich); 3399 if (ret) { 3400 shost_printk(KERN_ERR, target->scsi_host, 3401 PFX "Connection %d/%d failed\n", 3402 ch_start + cpu_idx, 3403 target->ch_count); 3404 if (node_idx == 0 && cpu_idx == 0) { 3405 goto err_disconnect; 3406 } else { 3407 srp_free_ch_ib(target, ch); 3408 srp_free_req_data(target, ch); 3409 target->ch_count = ch - target->ch; 3410 goto connected; 3411 } 3412 } 3413 3414 multich = true; 3415 cpu_idx++; 3416 } 3417 node_idx++; 3418 } 3419 3420 connected: 3421 target->scsi_host->nr_hw_queues = target->ch_count; 3422 3423 ret = srp_add_target(host, target); 3424 if (ret) 3425 goto err_disconnect; 3426 3427 if (target->state != SRP_TARGET_REMOVED) { 3428 shost_printk(KERN_DEBUG, target->scsi_host, PFX 3429 "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n", 3430 be64_to_cpu(target->id_ext), 3431 be64_to_cpu(target->ioc_guid), 3432 be16_to_cpu(target->pkey), 3433 be64_to_cpu(target->service_id), 3434 target->sgid.raw, target->orig_dgid.raw); 3435 } 3436 3437 ret = count; 3438 3439 out: 3440 mutex_unlock(&host->add_target_mutex); 3441 3442 scsi_host_put(target->scsi_host); 3443 if (ret < 0) 3444 scsi_host_put(target->scsi_host); 3445 3446 return ret; 3447 3448 err_disconnect: 3449 srp_disconnect_target(target); 3450 3451 for (i = 0; i < target->ch_count; i++) { 3452 ch = &target->ch[i]; 3453 srp_free_ch_ib(target, ch); 3454 srp_free_req_data(target, ch); 3455 } 3456 3457 kfree(target->ch); 3458 goto out; 3459 } 3460 3461 static DEVICE_ATTR(add_target, S_IWUSR, NULL, srp_create_target); 3462 3463 static ssize_t show_ibdev(struct device *dev, struct device_attribute *attr, 3464 char *buf) 3465 { 3466 struct srp_host *host = container_of(dev, struct srp_host, dev); 3467 3468 return sprintf(buf, "%s\n", host->srp_dev->dev->name); 3469 } 3470 3471 static DEVICE_ATTR(ibdev, S_IRUGO, show_ibdev, NULL); 3472 3473 static ssize_t show_port(struct device *dev, struct device_attribute *attr, 3474 char *buf) 3475 { 3476 struct srp_host *host = container_of(dev, struct srp_host, dev); 3477 3478 return sprintf(buf, "%d\n", host->port); 3479 } 3480 3481 static DEVICE_ATTR(port, S_IRUGO, show_port, NULL); 3482 3483 static struct srp_host *srp_add_port(struct srp_device *device, u8 port) 3484 { 3485 struct srp_host *host; 3486 3487 host = kzalloc(sizeof *host, GFP_KERNEL); 3488 if (!host) 3489 return NULL; 3490 3491 INIT_LIST_HEAD(&host->target_list); 3492 spin_lock_init(&host->target_lock); 3493 init_completion(&host->released); 3494 mutex_init(&host->add_target_mutex); 3495 host->srp_dev = device; 3496 host->port = port; 3497 3498 host->dev.class = &srp_class; 3499 host->dev.parent = device->dev->dma_device; 3500 dev_set_name(&host->dev, "srp-%s-%d", device->dev->name, port); 3501 3502 if (device_register(&host->dev)) 3503 goto free_host; 3504 if (device_create_file(&host->dev, &dev_attr_add_target)) 3505 goto err_class; 3506 if (device_create_file(&host->dev, &dev_attr_ibdev)) 3507 goto err_class; 3508 if (device_create_file(&host->dev, &dev_attr_port)) 3509 goto err_class; 3510 3511 return host; 3512 3513 err_class: 3514 device_unregister(&host->dev); 3515 3516 free_host: 3517 kfree(host); 3518 3519 return NULL; 3520 } 3521 3522 static void srp_add_one(struct ib_device *device) 3523 { 3524 struct srp_device *srp_dev; 3525 struct srp_host *host; 3526 int mr_page_shift, p; 3527 u64 max_pages_per_mr; 3528 3529 srp_dev = kmalloc(sizeof *srp_dev, GFP_KERNEL); 3530 if (!srp_dev) 3531 return; 3532 3533 /* 3534 * Use the smallest page size supported by the HCA, down to a 3535 * minimum of 4096 bytes. We're unlikely to build large sglists 3536 * out of smaller entries. 3537 */ 3538 mr_page_shift = max(12, ffs(device->attrs.page_size_cap) - 1); 3539 srp_dev->mr_page_size = 1 << mr_page_shift; 3540 srp_dev->mr_page_mask = ~((u64) srp_dev->mr_page_size - 1); 3541 max_pages_per_mr = device->attrs.max_mr_size; 3542 do_div(max_pages_per_mr, srp_dev->mr_page_size); 3543 pr_debug("%s: %llu / %u = %llu <> %u\n", __func__, 3544 device->attrs.max_mr_size, srp_dev->mr_page_size, 3545 max_pages_per_mr, SRP_MAX_PAGES_PER_MR); 3546 srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR, 3547 max_pages_per_mr); 3548 3549 srp_dev->has_fmr = (device->alloc_fmr && device->dealloc_fmr && 3550 device->map_phys_fmr && device->unmap_fmr); 3551 srp_dev->has_fr = (device->attrs.device_cap_flags & 3552 IB_DEVICE_MEM_MGT_EXTENSIONS); 3553 if (!never_register && !srp_dev->has_fmr && !srp_dev->has_fr) { 3554 dev_warn(&device->dev, "neither FMR nor FR is supported\n"); 3555 } else if (!never_register && 3556 device->attrs.max_mr_size >= 2 * srp_dev->mr_page_size) { 3557 srp_dev->use_fast_reg = (srp_dev->has_fr && 3558 (!srp_dev->has_fmr || prefer_fr)); 3559 srp_dev->use_fmr = !srp_dev->use_fast_reg && srp_dev->has_fmr; 3560 } 3561 3562 if (srp_dev->use_fast_reg) { 3563 srp_dev->max_pages_per_mr = 3564 min_t(u32, srp_dev->max_pages_per_mr, 3565 device->attrs.max_fast_reg_page_list_len); 3566 } 3567 srp_dev->mr_max_size = srp_dev->mr_page_size * 3568 srp_dev->max_pages_per_mr; 3569 pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n", 3570 device->name, mr_page_shift, device->attrs.max_mr_size, 3571 device->attrs.max_fast_reg_page_list_len, 3572 srp_dev->max_pages_per_mr, srp_dev->mr_max_size); 3573 3574 INIT_LIST_HEAD(&srp_dev->dev_list); 3575 3576 srp_dev->dev = device; 3577 srp_dev->pd = ib_alloc_pd(device); 3578 if (IS_ERR(srp_dev->pd)) 3579 goto free_dev; 3580 3581 if (never_register || !register_always || 3582 (!srp_dev->has_fmr && !srp_dev->has_fr)) { 3583 srp_dev->global_mr = ib_get_dma_mr(srp_dev->pd, 3584 IB_ACCESS_LOCAL_WRITE | 3585 IB_ACCESS_REMOTE_READ | 3586 IB_ACCESS_REMOTE_WRITE); 3587 if (IS_ERR(srp_dev->global_mr)) 3588 goto err_pd; 3589 } else { 3590 srp_dev->global_mr = NULL; 3591 } 3592 3593 for (p = rdma_start_port(device); p <= rdma_end_port(device); ++p) { 3594 host = srp_add_port(srp_dev, p); 3595 if (host) 3596 list_add_tail(&host->list, &srp_dev->dev_list); 3597 } 3598 3599 ib_set_client_data(device, &srp_client, srp_dev); 3600 return; 3601 3602 err_pd: 3603 ib_dealloc_pd(srp_dev->pd); 3604 3605 free_dev: 3606 kfree(srp_dev); 3607 } 3608 3609 static void srp_remove_one(struct ib_device *device, void *client_data) 3610 { 3611 struct srp_device *srp_dev; 3612 struct srp_host *host, *tmp_host; 3613 struct srp_target_port *target; 3614 3615 srp_dev = client_data; 3616 if (!srp_dev) 3617 return; 3618 3619 list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) { 3620 device_unregister(&host->dev); 3621 /* 3622 * Wait for the sysfs entry to go away, so that no new 3623 * target ports can be created. 3624 */ 3625 wait_for_completion(&host->released); 3626 3627 /* 3628 * Remove all target ports. 3629 */ 3630 spin_lock(&host->target_lock); 3631 list_for_each_entry(target, &host->target_list, list) 3632 srp_queue_remove_work(target); 3633 spin_unlock(&host->target_lock); 3634 3635 /* 3636 * Wait for tl_err and target port removal tasks. 3637 */ 3638 flush_workqueue(system_long_wq); 3639 flush_workqueue(srp_remove_wq); 3640 3641 kfree(host); 3642 } 3643 3644 if (srp_dev->global_mr) 3645 ib_dereg_mr(srp_dev->global_mr); 3646 ib_dealloc_pd(srp_dev->pd); 3647 3648 kfree(srp_dev); 3649 } 3650 3651 static struct srp_function_template ib_srp_transport_functions = { 3652 .has_rport_state = true, 3653 .reset_timer_if_blocked = true, 3654 .reconnect_delay = &srp_reconnect_delay, 3655 .fast_io_fail_tmo = &srp_fast_io_fail_tmo, 3656 .dev_loss_tmo = &srp_dev_loss_tmo, 3657 .reconnect = srp_rport_reconnect, 3658 .rport_delete = srp_rport_delete, 3659 .terminate_rport_io = srp_terminate_io, 3660 }; 3661 3662 static int __init srp_init_module(void) 3663 { 3664 int ret; 3665 3666 if (srp_sg_tablesize) { 3667 pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n"); 3668 if (!cmd_sg_entries) 3669 cmd_sg_entries = srp_sg_tablesize; 3670 } 3671 3672 if (!cmd_sg_entries) 3673 cmd_sg_entries = SRP_DEF_SG_TABLESIZE; 3674 3675 if (cmd_sg_entries > 255) { 3676 pr_warn("Clamping cmd_sg_entries to 255\n"); 3677 cmd_sg_entries = 255; 3678 } 3679 3680 if (!indirect_sg_entries) 3681 indirect_sg_entries = cmd_sg_entries; 3682 else if (indirect_sg_entries < cmd_sg_entries) { 3683 pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n", 3684 cmd_sg_entries); 3685 indirect_sg_entries = cmd_sg_entries; 3686 } 3687 3688 srp_remove_wq = create_workqueue("srp_remove"); 3689 if (!srp_remove_wq) { 3690 ret = -ENOMEM; 3691 goto out; 3692 } 3693 3694 ret = -ENOMEM; 3695 ib_srp_transport_template = 3696 srp_attach_transport(&ib_srp_transport_functions); 3697 if (!ib_srp_transport_template) 3698 goto destroy_wq; 3699 3700 ret = class_register(&srp_class); 3701 if (ret) { 3702 pr_err("couldn't register class infiniband_srp\n"); 3703 goto release_tr; 3704 } 3705 3706 ib_sa_register_client(&srp_sa_client); 3707 3708 ret = ib_register_client(&srp_client); 3709 if (ret) { 3710 pr_err("couldn't register IB client\n"); 3711 goto unreg_sa; 3712 } 3713 3714 out: 3715 return ret; 3716 3717 unreg_sa: 3718 ib_sa_unregister_client(&srp_sa_client); 3719 class_unregister(&srp_class); 3720 3721 release_tr: 3722 srp_release_transport(ib_srp_transport_template); 3723 3724 destroy_wq: 3725 destroy_workqueue(srp_remove_wq); 3726 goto out; 3727 } 3728 3729 static void __exit srp_cleanup_module(void) 3730 { 3731 ib_unregister_client(&srp_client); 3732 ib_sa_unregister_client(&srp_sa_client); 3733 class_unregister(&srp_class); 3734 srp_release_transport(ib_srp_transport_template); 3735 destroy_workqueue(srp_remove_wq); 3736 } 3737 3738 module_init(srp_init_module); 3739 module_exit(srp_cleanup_module); 3740