1 /*
2  * Copyright (c) 2005 Cisco Systems.  All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34 
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/err.h>
39 #include <linux/string.h>
40 #include <linux/parser.h>
41 #include <linux/random.h>
42 #include <linux/jiffies.h>
43 #include <linux/lockdep.h>
44 #include <linux/inet.h>
45 #include <rdma/ib_cache.h>
46 
47 #include <linux/atomic.h>
48 
49 #include <scsi/scsi.h>
50 #include <scsi/scsi_device.h>
51 #include <scsi/scsi_dbg.h>
52 #include <scsi/scsi_tcq.h>
53 #include <scsi/srp.h>
54 #include <scsi/scsi_transport_srp.h>
55 
56 #include "ib_srp.h"
57 
58 #define DRV_NAME	"ib_srp"
59 #define PFX		DRV_NAME ": "
60 
61 MODULE_AUTHOR("Roland Dreier");
62 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol initiator");
63 MODULE_LICENSE("Dual BSD/GPL");
64 
65 static unsigned int srp_sg_tablesize;
66 static unsigned int cmd_sg_entries;
67 static unsigned int indirect_sg_entries;
68 static bool allow_ext_sg;
69 static bool register_always = true;
70 static bool never_register;
71 static int topspin_workarounds = 1;
72 
73 module_param(srp_sg_tablesize, uint, 0444);
74 MODULE_PARM_DESC(srp_sg_tablesize, "Deprecated name for cmd_sg_entries");
75 
76 module_param(cmd_sg_entries, uint, 0444);
77 MODULE_PARM_DESC(cmd_sg_entries,
78 		 "Default number of gather/scatter entries in the SRP command (default is 12, max 255)");
79 
80 module_param(indirect_sg_entries, uint, 0444);
81 MODULE_PARM_DESC(indirect_sg_entries,
82 		 "Default max number of gather/scatter entries (default is 12, max is " __stringify(SG_MAX_SEGMENTS) ")");
83 
84 module_param(allow_ext_sg, bool, 0444);
85 MODULE_PARM_DESC(allow_ext_sg,
86 		  "Default behavior when there are more than cmd_sg_entries S/G entries after mapping; fails the request when false (default false)");
87 
88 module_param(topspin_workarounds, int, 0444);
89 MODULE_PARM_DESC(topspin_workarounds,
90 		 "Enable workarounds for Topspin/Cisco SRP target bugs if != 0");
91 
92 module_param(register_always, bool, 0444);
93 MODULE_PARM_DESC(register_always,
94 		 "Use memory registration even for contiguous memory regions");
95 
96 module_param(never_register, bool, 0444);
97 MODULE_PARM_DESC(never_register, "Never register memory");
98 
99 static const struct kernel_param_ops srp_tmo_ops;
100 
101 static int srp_reconnect_delay = 10;
102 module_param_cb(reconnect_delay, &srp_tmo_ops, &srp_reconnect_delay,
103 		S_IRUGO | S_IWUSR);
104 MODULE_PARM_DESC(reconnect_delay, "Time between successive reconnect attempts");
105 
106 static int srp_fast_io_fail_tmo = 15;
107 module_param_cb(fast_io_fail_tmo, &srp_tmo_ops, &srp_fast_io_fail_tmo,
108 		S_IRUGO | S_IWUSR);
109 MODULE_PARM_DESC(fast_io_fail_tmo,
110 		 "Number of seconds between the observation of a transport"
111 		 " layer error and failing all I/O. \"off\" means that this"
112 		 " functionality is disabled.");
113 
114 static int srp_dev_loss_tmo = 600;
115 module_param_cb(dev_loss_tmo, &srp_tmo_ops, &srp_dev_loss_tmo,
116 		S_IRUGO | S_IWUSR);
117 MODULE_PARM_DESC(dev_loss_tmo,
118 		 "Maximum number of seconds that the SRP transport should"
119 		 " insulate transport layer errors. After this time has been"
120 		 " exceeded the SCSI host is removed. Should be"
121 		 " between 1 and " __stringify(SCSI_DEVICE_BLOCK_MAX_TIMEOUT)
122 		 " if fast_io_fail_tmo has not been set. \"off\" means that"
123 		 " this functionality is disabled.");
124 
125 static bool srp_use_imm_data = true;
126 module_param_named(use_imm_data, srp_use_imm_data, bool, 0644);
127 MODULE_PARM_DESC(use_imm_data,
128 		 "Whether or not to request permission to use immediate data during SRP login.");
129 
130 static unsigned int srp_max_imm_data = 8 * 1024;
131 module_param_named(max_imm_data, srp_max_imm_data, uint, 0644);
132 MODULE_PARM_DESC(max_imm_data, "Maximum immediate data size.");
133 
134 static unsigned ch_count;
135 module_param(ch_count, uint, 0444);
136 MODULE_PARM_DESC(ch_count,
137 		 "Number of RDMA channels to use for communication with an SRP target. Using more than one channel improves performance if the HCA supports multiple completion vectors. The default value is the minimum of four times the number of online CPU sockets and the number of completion vectors supported by the HCA.");
138 
139 static int srp_add_one(struct ib_device *device);
140 static void srp_remove_one(struct ib_device *device, void *client_data);
141 static void srp_rename_dev(struct ib_device *device, void *client_data);
142 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc);
143 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
144 		const char *opname);
145 static int srp_ib_cm_handler(struct ib_cm_id *cm_id,
146 			     const struct ib_cm_event *event);
147 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
148 			       struct rdma_cm_event *event);
149 
150 static struct scsi_transport_template *ib_srp_transport_template;
151 static struct workqueue_struct *srp_remove_wq;
152 
153 static struct ib_client srp_client = {
154 	.name   = "srp",
155 	.add    = srp_add_one,
156 	.remove = srp_remove_one,
157 	.rename = srp_rename_dev
158 };
159 
160 static struct ib_sa_client srp_sa_client;
161 
162 static int srp_tmo_get(char *buffer, const struct kernel_param *kp)
163 {
164 	int tmo = *(int *)kp->arg;
165 
166 	if (tmo >= 0)
167 		return sysfs_emit(buffer, "%d\n", tmo);
168 	else
169 		return sysfs_emit(buffer, "off\n");
170 }
171 
172 static int srp_tmo_set(const char *val, const struct kernel_param *kp)
173 {
174 	int tmo, res;
175 
176 	res = srp_parse_tmo(&tmo, val);
177 	if (res)
178 		goto out;
179 
180 	if (kp->arg == &srp_reconnect_delay)
181 		res = srp_tmo_valid(tmo, srp_fast_io_fail_tmo,
182 				    srp_dev_loss_tmo);
183 	else if (kp->arg == &srp_fast_io_fail_tmo)
184 		res = srp_tmo_valid(srp_reconnect_delay, tmo, srp_dev_loss_tmo);
185 	else
186 		res = srp_tmo_valid(srp_reconnect_delay, srp_fast_io_fail_tmo,
187 				    tmo);
188 	if (res)
189 		goto out;
190 	*(int *)kp->arg = tmo;
191 
192 out:
193 	return res;
194 }
195 
196 static const struct kernel_param_ops srp_tmo_ops = {
197 	.get = srp_tmo_get,
198 	.set = srp_tmo_set,
199 };
200 
201 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
202 {
203 	return (struct srp_target_port *) host->hostdata;
204 }
205 
206 static const char *srp_target_info(struct Scsi_Host *host)
207 {
208 	return host_to_target(host)->target_name;
209 }
210 
211 static int srp_target_is_topspin(struct srp_target_port *target)
212 {
213 	static const u8 topspin_oui[3] = { 0x00, 0x05, 0xad };
214 	static const u8 cisco_oui[3]   = { 0x00, 0x1b, 0x0d };
215 
216 	return topspin_workarounds &&
217 		(!memcmp(&target->ioc_guid, topspin_oui, sizeof topspin_oui) ||
218 		 !memcmp(&target->ioc_guid, cisco_oui, sizeof cisco_oui));
219 }
220 
221 static struct srp_iu *srp_alloc_iu(struct srp_host *host, size_t size,
222 				   gfp_t gfp_mask,
223 				   enum dma_data_direction direction)
224 {
225 	struct srp_iu *iu;
226 
227 	iu = kmalloc(sizeof *iu, gfp_mask);
228 	if (!iu)
229 		goto out;
230 
231 	iu->buf = kzalloc(size, gfp_mask);
232 	if (!iu->buf)
233 		goto out_free_iu;
234 
235 	iu->dma = ib_dma_map_single(host->srp_dev->dev, iu->buf, size,
236 				    direction);
237 	if (ib_dma_mapping_error(host->srp_dev->dev, iu->dma))
238 		goto out_free_buf;
239 
240 	iu->size      = size;
241 	iu->direction = direction;
242 
243 	return iu;
244 
245 out_free_buf:
246 	kfree(iu->buf);
247 out_free_iu:
248 	kfree(iu);
249 out:
250 	return NULL;
251 }
252 
253 static void srp_free_iu(struct srp_host *host, struct srp_iu *iu)
254 {
255 	if (!iu)
256 		return;
257 
258 	ib_dma_unmap_single(host->srp_dev->dev, iu->dma, iu->size,
259 			    iu->direction);
260 	kfree(iu->buf);
261 	kfree(iu);
262 }
263 
264 static void srp_qp_event(struct ib_event *event, void *context)
265 {
266 	pr_debug("QP event %s (%d)\n",
267 		 ib_event_msg(event->event), event->event);
268 }
269 
270 static int srp_init_ib_qp(struct srp_target_port *target,
271 			  struct ib_qp *qp)
272 {
273 	struct ib_qp_attr *attr;
274 	int ret;
275 
276 	attr = kmalloc(sizeof *attr, GFP_KERNEL);
277 	if (!attr)
278 		return -ENOMEM;
279 
280 	ret = ib_find_cached_pkey(target->srp_host->srp_dev->dev,
281 				  target->srp_host->port,
282 				  be16_to_cpu(target->ib_cm.pkey),
283 				  &attr->pkey_index);
284 	if (ret)
285 		goto out;
286 
287 	attr->qp_state        = IB_QPS_INIT;
288 	attr->qp_access_flags = (IB_ACCESS_REMOTE_READ |
289 				    IB_ACCESS_REMOTE_WRITE);
290 	attr->port_num        = target->srp_host->port;
291 
292 	ret = ib_modify_qp(qp, attr,
293 			   IB_QP_STATE		|
294 			   IB_QP_PKEY_INDEX	|
295 			   IB_QP_ACCESS_FLAGS	|
296 			   IB_QP_PORT);
297 
298 out:
299 	kfree(attr);
300 	return ret;
301 }
302 
303 static int srp_new_ib_cm_id(struct srp_rdma_ch *ch)
304 {
305 	struct srp_target_port *target = ch->target;
306 	struct ib_cm_id *new_cm_id;
307 
308 	new_cm_id = ib_create_cm_id(target->srp_host->srp_dev->dev,
309 				    srp_ib_cm_handler, ch);
310 	if (IS_ERR(new_cm_id))
311 		return PTR_ERR(new_cm_id);
312 
313 	if (ch->ib_cm.cm_id)
314 		ib_destroy_cm_id(ch->ib_cm.cm_id);
315 	ch->ib_cm.cm_id = new_cm_id;
316 	if (rdma_cap_opa_ah(target->srp_host->srp_dev->dev,
317 			    target->srp_host->port))
318 		ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_OPA;
319 	else
320 		ch->ib_cm.path.rec_type = SA_PATH_REC_TYPE_IB;
321 	ch->ib_cm.path.sgid = target->sgid;
322 	ch->ib_cm.path.dgid = target->ib_cm.orig_dgid;
323 	ch->ib_cm.path.pkey = target->ib_cm.pkey;
324 	ch->ib_cm.path.service_id = target->ib_cm.service_id;
325 
326 	return 0;
327 }
328 
329 static int srp_new_rdma_cm_id(struct srp_rdma_ch *ch)
330 {
331 	struct srp_target_port *target = ch->target;
332 	struct rdma_cm_id *new_cm_id;
333 	int ret;
334 
335 	new_cm_id = rdma_create_id(target->net, srp_rdma_cm_handler, ch,
336 				   RDMA_PS_TCP, IB_QPT_RC);
337 	if (IS_ERR(new_cm_id)) {
338 		ret = PTR_ERR(new_cm_id);
339 		new_cm_id = NULL;
340 		goto out;
341 	}
342 
343 	init_completion(&ch->done);
344 	ret = rdma_resolve_addr(new_cm_id, target->rdma_cm.src_specified ?
345 				&target->rdma_cm.src.sa : NULL,
346 				&target->rdma_cm.dst.sa,
347 				SRP_PATH_REC_TIMEOUT_MS);
348 	if (ret) {
349 		pr_err("No route available from %pISpsc to %pISpsc (%d)\n",
350 		       &target->rdma_cm.src, &target->rdma_cm.dst, ret);
351 		goto out;
352 	}
353 	ret = wait_for_completion_interruptible(&ch->done);
354 	if (ret < 0)
355 		goto out;
356 
357 	ret = ch->status;
358 	if (ret) {
359 		pr_err("Resolving address %pISpsc failed (%d)\n",
360 		       &target->rdma_cm.dst, ret);
361 		goto out;
362 	}
363 
364 	swap(ch->rdma_cm.cm_id, new_cm_id);
365 
366 out:
367 	if (new_cm_id)
368 		rdma_destroy_id(new_cm_id);
369 
370 	return ret;
371 }
372 
373 static int srp_new_cm_id(struct srp_rdma_ch *ch)
374 {
375 	struct srp_target_port *target = ch->target;
376 
377 	return target->using_rdma_cm ? srp_new_rdma_cm_id(ch) :
378 		srp_new_ib_cm_id(ch);
379 }
380 
381 /**
382  * srp_destroy_fr_pool() - free the resources owned by a pool
383  * @pool: Fast registration pool to be destroyed.
384  */
385 static void srp_destroy_fr_pool(struct srp_fr_pool *pool)
386 {
387 	int i;
388 	struct srp_fr_desc *d;
389 
390 	if (!pool)
391 		return;
392 
393 	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
394 		if (d->mr)
395 			ib_dereg_mr(d->mr);
396 	}
397 	kfree(pool);
398 }
399 
400 /**
401  * srp_create_fr_pool() - allocate and initialize a pool for fast registration
402  * @device:            IB device to allocate fast registration descriptors for.
403  * @pd:                Protection domain associated with the FR descriptors.
404  * @pool_size:         Number of descriptors to allocate.
405  * @max_page_list_len: Maximum fast registration work request page list length.
406  */
407 static struct srp_fr_pool *srp_create_fr_pool(struct ib_device *device,
408 					      struct ib_pd *pd, int pool_size,
409 					      int max_page_list_len)
410 {
411 	struct srp_fr_pool *pool;
412 	struct srp_fr_desc *d;
413 	struct ib_mr *mr;
414 	int i, ret = -EINVAL;
415 	enum ib_mr_type mr_type;
416 
417 	if (pool_size <= 0)
418 		goto err;
419 	ret = -ENOMEM;
420 	pool = kzalloc(struct_size(pool, desc, pool_size), GFP_KERNEL);
421 	if (!pool)
422 		goto err;
423 	pool->size = pool_size;
424 	pool->max_page_list_len = max_page_list_len;
425 	spin_lock_init(&pool->lock);
426 	INIT_LIST_HEAD(&pool->free_list);
427 
428 	if (device->attrs.kernel_cap_flags & IBK_SG_GAPS_REG)
429 		mr_type = IB_MR_TYPE_SG_GAPS;
430 	else
431 		mr_type = IB_MR_TYPE_MEM_REG;
432 
433 	for (i = 0, d = &pool->desc[0]; i < pool->size; i++, d++) {
434 		mr = ib_alloc_mr(pd, mr_type, max_page_list_len);
435 		if (IS_ERR(mr)) {
436 			ret = PTR_ERR(mr);
437 			if (ret == -ENOMEM)
438 				pr_info("%s: ib_alloc_mr() failed. Try to reduce max_cmd_per_lun, max_sect or ch_count\n",
439 					dev_name(&device->dev));
440 			goto destroy_pool;
441 		}
442 		d->mr = mr;
443 		list_add_tail(&d->entry, &pool->free_list);
444 	}
445 
446 out:
447 	return pool;
448 
449 destroy_pool:
450 	srp_destroy_fr_pool(pool);
451 
452 err:
453 	pool = ERR_PTR(ret);
454 	goto out;
455 }
456 
457 /**
458  * srp_fr_pool_get() - obtain a descriptor suitable for fast registration
459  * @pool: Pool to obtain descriptor from.
460  */
461 static struct srp_fr_desc *srp_fr_pool_get(struct srp_fr_pool *pool)
462 {
463 	struct srp_fr_desc *d = NULL;
464 	unsigned long flags;
465 
466 	spin_lock_irqsave(&pool->lock, flags);
467 	if (!list_empty(&pool->free_list)) {
468 		d = list_first_entry(&pool->free_list, typeof(*d), entry);
469 		list_del(&d->entry);
470 	}
471 	spin_unlock_irqrestore(&pool->lock, flags);
472 
473 	return d;
474 }
475 
476 /**
477  * srp_fr_pool_put() - put an FR descriptor back in the free list
478  * @pool: Pool the descriptor was allocated from.
479  * @desc: Pointer to an array of fast registration descriptor pointers.
480  * @n:    Number of descriptors to put back.
481  *
482  * Note: The caller must already have queued an invalidation request for
483  * desc->mr->rkey before calling this function.
484  */
485 static void srp_fr_pool_put(struct srp_fr_pool *pool, struct srp_fr_desc **desc,
486 			    int n)
487 {
488 	unsigned long flags;
489 	int i;
490 
491 	spin_lock_irqsave(&pool->lock, flags);
492 	for (i = 0; i < n; i++)
493 		list_add(&desc[i]->entry, &pool->free_list);
494 	spin_unlock_irqrestore(&pool->lock, flags);
495 }
496 
497 static struct srp_fr_pool *srp_alloc_fr_pool(struct srp_target_port *target)
498 {
499 	struct srp_device *dev = target->srp_host->srp_dev;
500 
501 	return srp_create_fr_pool(dev->dev, dev->pd, target->mr_pool_size,
502 				  dev->max_pages_per_mr);
503 }
504 
505 /**
506  * srp_destroy_qp() - destroy an RDMA queue pair
507  * @ch: SRP RDMA channel.
508  *
509  * Drain the qp before destroying it.  This avoids that the receive
510  * completion handler can access the queue pair while it is
511  * being destroyed.
512  */
513 static void srp_destroy_qp(struct srp_rdma_ch *ch)
514 {
515 	spin_lock_irq(&ch->lock);
516 	ib_process_cq_direct(ch->send_cq, -1);
517 	spin_unlock_irq(&ch->lock);
518 
519 	ib_drain_qp(ch->qp);
520 	ib_destroy_qp(ch->qp);
521 }
522 
523 static int srp_create_ch_ib(struct srp_rdma_ch *ch)
524 {
525 	struct srp_target_port *target = ch->target;
526 	struct srp_device *dev = target->srp_host->srp_dev;
527 	const struct ib_device_attr *attr = &dev->dev->attrs;
528 	struct ib_qp_init_attr *init_attr;
529 	struct ib_cq *recv_cq, *send_cq;
530 	struct ib_qp *qp;
531 	struct srp_fr_pool *fr_pool = NULL;
532 	const int m = 1 + dev->use_fast_reg * target->mr_per_cmd * 2;
533 	int ret;
534 
535 	init_attr = kzalloc(sizeof *init_attr, GFP_KERNEL);
536 	if (!init_attr)
537 		return -ENOMEM;
538 
539 	/* queue_size + 1 for ib_drain_rq() */
540 	recv_cq = ib_alloc_cq(dev->dev, ch, target->queue_size + 1,
541 				ch->comp_vector, IB_POLL_SOFTIRQ);
542 	if (IS_ERR(recv_cq)) {
543 		ret = PTR_ERR(recv_cq);
544 		goto err;
545 	}
546 
547 	send_cq = ib_alloc_cq(dev->dev, ch, m * target->queue_size,
548 				ch->comp_vector, IB_POLL_DIRECT);
549 	if (IS_ERR(send_cq)) {
550 		ret = PTR_ERR(send_cq);
551 		goto err_recv_cq;
552 	}
553 
554 	init_attr->event_handler       = srp_qp_event;
555 	init_attr->cap.max_send_wr     = m * target->queue_size;
556 	init_attr->cap.max_recv_wr     = target->queue_size + 1;
557 	init_attr->cap.max_recv_sge    = 1;
558 	init_attr->cap.max_send_sge    = min(SRP_MAX_SGE, attr->max_send_sge);
559 	init_attr->sq_sig_type         = IB_SIGNAL_REQ_WR;
560 	init_attr->qp_type             = IB_QPT_RC;
561 	init_attr->send_cq             = send_cq;
562 	init_attr->recv_cq             = recv_cq;
563 
564 	ch->max_imm_sge = min(init_attr->cap.max_send_sge - 1U, 255U);
565 
566 	if (target->using_rdma_cm) {
567 		ret = rdma_create_qp(ch->rdma_cm.cm_id, dev->pd, init_attr);
568 		qp = ch->rdma_cm.cm_id->qp;
569 	} else {
570 		qp = ib_create_qp(dev->pd, init_attr);
571 		if (!IS_ERR(qp)) {
572 			ret = srp_init_ib_qp(target, qp);
573 			if (ret)
574 				ib_destroy_qp(qp);
575 		} else {
576 			ret = PTR_ERR(qp);
577 		}
578 	}
579 	if (ret) {
580 		pr_err("QP creation failed for dev %s: %d\n",
581 		       dev_name(&dev->dev->dev), ret);
582 		goto err_send_cq;
583 	}
584 
585 	if (dev->use_fast_reg) {
586 		fr_pool = srp_alloc_fr_pool(target);
587 		if (IS_ERR(fr_pool)) {
588 			ret = PTR_ERR(fr_pool);
589 			shost_printk(KERN_WARNING, target->scsi_host, PFX
590 				     "FR pool allocation failed (%d)\n", ret);
591 			goto err_qp;
592 		}
593 	}
594 
595 	if (ch->qp)
596 		srp_destroy_qp(ch);
597 	if (ch->recv_cq)
598 		ib_free_cq(ch->recv_cq);
599 	if (ch->send_cq)
600 		ib_free_cq(ch->send_cq);
601 
602 	ch->qp = qp;
603 	ch->recv_cq = recv_cq;
604 	ch->send_cq = send_cq;
605 
606 	if (dev->use_fast_reg) {
607 		if (ch->fr_pool)
608 			srp_destroy_fr_pool(ch->fr_pool);
609 		ch->fr_pool = fr_pool;
610 	}
611 
612 	kfree(init_attr);
613 	return 0;
614 
615 err_qp:
616 	if (target->using_rdma_cm)
617 		rdma_destroy_qp(ch->rdma_cm.cm_id);
618 	else
619 		ib_destroy_qp(qp);
620 
621 err_send_cq:
622 	ib_free_cq(send_cq);
623 
624 err_recv_cq:
625 	ib_free_cq(recv_cq);
626 
627 err:
628 	kfree(init_attr);
629 	return ret;
630 }
631 
632 /*
633  * Note: this function may be called without srp_alloc_iu_bufs() having been
634  * invoked. Hence the ch->[rt]x_ring checks.
635  */
636 static void srp_free_ch_ib(struct srp_target_port *target,
637 			   struct srp_rdma_ch *ch)
638 {
639 	struct srp_device *dev = target->srp_host->srp_dev;
640 	int i;
641 
642 	if (!ch->target)
643 		return;
644 
645 	if (target->using_rdma_cm) {
646 		if (ch->rdma_cm.cm_id) {
647 			rdma_destroy_id(ch->rdma_cm.cm_id);
648 			ch->rdma_cm.cm_id = NULL;
649 		}
650 	} else {
651 		if (ch->ib_cm.cm_id) {
652 			ib_destroy_cm_id(ch->ib_cm.cm_id);
653 			ch->ib_cm.cm_id = NULL;
654 		}
655 	}
656 
657 	/* If srp_new_cm_id() succeeded but srp_create_ch_ib() not, return. */
658 	if (!ch->qp)
659 		return;
660 
661 	if (dev->use_fast_reg) {
662 		if (ch->fr_pool)
663 			srp_destroy_fr_pool(ch->fr_pool);
664 	}
665 
666 	srp_destroy_qp(ch);
667 	ib_free_cq(ch->send_cq);
668 	ib_free_cq(ch->recv_cq);
669 
670 	/*
671 	 * Avoid that the SCSI error handler tries to use this channel after
672 	 * it has been freed. The SCSI error handler can namely continue
673 	 * trying to perform recovery actions after scsi_remove_host()
674 	 * returned.
675 	 */
676 	ch->target = NULL;
677 
678 	ch->qp = NULL;
679 	ch->send_cq = ch->recv_cq = NULL;
680 
681 	if (ch->rx_ring) {
682 		for (i = 0; i < target->queue_size; ++i)
683 			srp_free_iu(target->srp_host, ch->rx_ring[i]);
684 		kfree(ch->rx_ring);
685 		ch->rx_ring = NULL;
686 	}
687 	if (ch->tx_ring) {
688 		for (i = 0; i < target->queue_size; ++i)
689 			srp_free_iu(target->srp_host, ch->tx_ring[i]);
690 		kfree(ch->tx_ring);
691 		ch->tx_ring = NULL;
692 	}
693 }
694 
695 static void srp_path_rec_completion(int status,
696 				    struct sa_path_rec *pathrec,
697 				    unsigned int num_paths, void *ch_ptr)
698 {
699 	struct srp_rdma_ch *ch = ch_ptr;
700 	struct srp_target_port *target = ch->target;
701 
702 	ch->status = status;
703 	if (status)
704 		shost_printk(KERN_ERR, target->scsi_host,
705 			     PFX "Got failed path rec status %d\n", status);
706 	else
707 		ch->ib_cm.path = *pathrec;
708 	complete(&ch->done);
709 }
710 
711 static int srp_ib_lookup_path(struct srp_rdma_ch *ch)
712 {
713 	struct srp_target_port *target = ch->target;
714 	int ret;
715 
716 	ch->ib_cm.path.numb_path = 1;
717 
718 	init_completion(&ch->done);
719 
720 	ch->ib_cm.path_query_id = ib_sa_path_rec_get(&srp_sa_client,
721 					       target->srp_host->srp_dev->dev,
722 					       target->srp_host->port,
723 					       &ch->ib_cm.path,
724 					       IB_SA_PATH_REC_SERVICE_ID |
725 					       IB_SA_PATH_REC_DGID	 |
726 					       IB_SA_PATH_REC_SGID	 |
727 					       IB_SA_PATH_REC_NUMB_PATH	 |
728 					       IB_SA_PATH_REC_PKEY,
729 					       SRP_PATH_REC_TIMEOUT_MS,
730 					       GFP_KERNEL,
731 					       srp_path_rec_completion,
732 					       ch, &ch->ib_cm.path_query);
733 	if (ch->ib_cm.path_query_id < 0)
734 		return ch->ib_cm.path_query_id;
735 
736 	ret = wait_for_completion_interruptible(&ch->done);
737 	if (ret < 0)
738 		return ret;
739 
740 	if (ch->status < 0)
741 		shost_printk(KERN_WARNING, target->scsi_host,
742 			     PFX "Path record query failed: sgid %pI6, dgid %pI6, pkey %#04x, service_id %#16llx\n",
743 			     ch->ib_cm.path.sgid.raw, ch->ib_cm.path.dgid.raw,
744 			     be16_to_cpu(target->ib_cm.pkey),
745 			     be64_to_cpu(target->ib_cm.service_id));
746 
747 	return ch->status;
748 }
749 
750 static int srp_rdma_lookup_path(struct srp_rdma_ch *ch)
751 {
752 	struct srp_target_port *target = ch->target;
753 	int ret;
754 
755 	init_completion(&ch->done);
756 
757 	ret = rdma_resolve_route(ch->rdma_cm.cm_id, SRP_PATH_REC_TIMEOUT_MS);
758 	if (ret)
759 		return ret;
760 
761 	wait_for_completion_interruptible(&ch->done);
762 
763 	if (ch->status != 0)
764 		shost_printk(KERN_WARNING, target->scsi_host,
765 			     PFX "Path resolution failed\n");
766 
767 	return ch->status;
768 }
769 
770 static int srp_lookup_path(struct srp_rdma_ch *ch)
771 {
772 	struct srp_target_port *target = ch->target;
773 
774 	return target->using_rdma_cm ? srp_rdma_lookup_path(ch) :
775 		srp_ib_lookup_path(ch);
776 }
777 
778 static u8 srp_get_subnet_timeout(struct srp_host *host)
779 {
780 	struct ib_port_attr attr;
781 	int ret;
782 	u8 subnet_timeout = 18;
783 
784 	ret = ib_query_port(host->srp_dev->dev, host->port, &attr);
785 	if (ret == 0)
786 		subnet_timeout = attr.subnet_timeout;
787 
788 	if (unlikely(subnet_timeout < 15))
789 		pr_warn("%s: subnet timeout %d may cause SRP login to fail.\n",
790 			dev_name(&host->srp_dev->dev->dev), subnet_timeout);
791 
792 	return subnet_timeout;
793 }
794 
795 static int srp_send_req(struct srp_rdma_ch *ch, uint32_t max_iu_len,
796 			bool multich)
797 {
798 	struct srp_target_port *target = ch->target;
799 	struct {
800 		struct rdma_conn_param	  rdma_param;
801 		struct srp_login_req_rdma rdma_req;
802 		struct ib_cm_req_param	  ib_param;
803 		struct srp_login_req	  ib_req;
804 	} *req = NULL;
805 	char *ipi, *tpi;
806 	int status;
807 
808 	req = kzalloc(sizeof *req, GFP_KERNEL);
809 	if (!req)
810 		return -ENOMEM;
811 
812 	req->ib_param.flow_control = 1;
813 	req->ib_param.retry_count = target->tl_retry_count;
814 
815 	/*
816 	 * Pick some arbitrary defaults here; we could make these
817 	 * module parameters if anyone cared about setting them.
818 	 */
819 	req->ib_param.responder_resources = 4;
820 	req->ib_param.rnr_retry_count = 7;
821 	req->ib_param.max_cm_retries = 15;
822 
823 	req->ib_req.opcode = SRP_LOGIN_REQ;
824 	req->ib_req.tag = 0;
825 	req->ib_req.req_it_iu_len = cpu_to_be32(max_iu_len);
826 	req->ib_req.req_buf_fmt	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
827 					      SRP_BUF_FORMAT_INDIRECT);
828 	req->ib_req.req_flags = (multich ? SRP_MULTICHAN_MULTI :
829 				 SRP_MULTICHAN_SINGLE);
830 	if (srp_use_imm_data) {
831 		req->ib_req.req_flags |= SRP_IMMED_REQUESTED;
832 		req->ib_req.imm_data_offset = cpu_to_be16(SRP_IMM_DATA_OFFSET);
833 	}
834 
835 	if (target->using_rdma_cm) {
836 		req->rdma_param.flow_control = req->ib_param.flow_control;
837 		req->rdma_param.responder_resources =
838 			req->ib_param.responder_resources;
839 		req->rdma_param.initiator_depth = req->ib_param.initiator_depth;
840 		req->rdma_param.retry_count = req->ib_param.retry_count;
841 		req->rdma_param.rnr_retry_count = req->ib_param.rnr_retry_count;
842 		req->rdma_param.private_data = &req->rdma_req;
843 		req->rdma_param.private_data_len = sizeof(req->rdma_req);
844 
845 		req->rdma_req.opcode = req->ib_req.opcode;
846 		req->rdma_req.tag = req->ib_req.tag;
847 		req->rdma_req.req_it_iu_len = req->ib_req.req_it_iu_len;
848 		req->rdma_req.req_buf_fmt = req->ib_req.req_buf_fmt;
849 		req->rdma_req.req_flags	= req->ib_req.req_flags;
850 		req->rdma_req.imm_data_offset = req->ib_req.imm_data_offset;
851 
852 		ipi = req->rdma_req.initiator_port_id;
853 		tpi = req->rdma_req.target_port_id;
854 	} else {
855 		u8 subnet_timeout;
856 
857 		subnet_timeout = srp_get_subnet_timeout(target->srp_host);
858 
859 		req->ib_param.primary_path = &ch->ib_cm.path;
860 		req->ib_param.alternate_path = NULL;
861 		req->ib_param.service_id = target->ib_cm.service_id;
862 		get_random_bytes(&req->ib_param.starting_psn, 4);
863 		req->ib_param.starting_psn &= 0xffffff;
864 		req->ib_param.qp_num = ch->qp->qp_num;
865 		req->ib_param.qp_type = ch->qp->qp_type;
866 		req->ib_param.local_cm_response_timeout = subnet_timeout + 2;
867 		req->ib_param.remote_cm_response_timeout = subnet_timeout + 2;
868 		req->ib_param.private_data = &req->ib_req;
869 		req->ib_param.private_data_len = sizeof(req->ib_req);
870 
871 		ipi = req->ib_req.initiator_port_id;
872 		tpi = req->ib_req.target_port_id;
873 	}
874 
875 	/*
876 	 * In the published SRP specification (draft rev. 16a), the
877 	 * port identifier format is 8 bytes of ID extension followed
878 	 * by 8 bytes of GUID.  Older drafts put the two halves in the
879 	 * opposite order, so that the GUID comes first.
880 	 *
881 	 * Targets conforming to these obsolete drafts can be
882 	 * recognized by the I/O Class they report.
883 	 */
884 	if (target->io_class == SRP_REV10_IB_IO_CLASS) {
885 		memcpy(ipi,     &target->sgid.global.interface_id, 8);
886 		memcpy(ipi + 8, &target->initiator_ext, 8);
887 		memcpy(tpi,     &target->ioc_guid, 8);
888 		memcpy(tpi + 8, &target->id_ext, 8);
889 	} else {
890 		memcpy(ipi,     &target->initiator_ext, 8);
891 		memcpy(ipi + 8, &target->sgid.global.interface_id, 8);
892 		memcpy(tpi,     &target->id_ext, 8);
893 		memcpy(tpi + 8, &target->ioc_guid, 8);
894 	}
895 
896 	/*
897 	 * Topspin/Cisco SRP targets will reject our login unless we
898 	 * zero out the first 8 bytes of our initiator port ID and set
899 	 * the second 8 bytes to the local node GUID.
900 	 */
901 	if (srp_target_is_topspin(target)) {
902 		shost_printk(KERN_DEBUG, target->scsi_host,
903 			     PFX "Topspin/Cisco initiator port ID workaround "
904 			     "activated for target GUID %016llx\n",
905 			     be64_to_cpu(target->ioc_guid));
906 		memset(ipi, 0, 8);
907 		memcpy(ipi + 8, &target->srp_host->srp_dev->dev->node_guid, 8);
908 	}
909 
910 	if (target->using_rdma_cm)
911 		status = rdma_connect(ch->rdma_cm.cm_id, &req->rdma_param);
912 	else
913 		status = ib_send_cm_req(ch->ib_cm.cm_id, &req->ib_param);
914 
915 	kfree(req);
916 
917 	return status;
918 }
919 
920 static bool srp_queue_remove_work(struct srp_target_port *target)
921 {
922 	bool changed = false;
923 
924 	spin_lock_irq(&target->lock);
925 	if (target->state != SRP_TARGET_REMOVED) {
926 		target->state = SRP_TARGET_REMOVED;
927 		changed = true;
928 	}
929 	spin_unlock_irq(&target->lock);
930 
931 	if (changed)
932 		queue_work(srp_remove_wq, &target->remove_work);
933 
934 	return changed;
935 }
936 
937 static void srp_disconnect_target(struct srp_target_port *target)
938 {
939 	struct srp_rdma_ch *ch;
940 	int i, ret;
941 
942 	/* XXX should send SRP_I_LOGOUT request */
943 
944 	for (i = 0; i < target->ch_count; i++) {
945 		ch = &target->ch[i];
946 		ch->connected = false;
947 		ret = 0;
948 		if (target->using_rdma_cm) {
949 			if (ch->rdma_cm.cm_id)
950 				rdma_disconnect(ch->rdma_cm.cm_id);
951 		} else {
952 			if (ch->ib_cm.cm_id)
953 				ret = ib_send_cm_dreq(ch->ib_cm.cm_id,
954 						      NULL, 0);
955 		}
956 		if (ret < 0) {
957 			shost_printk(KERN_DEBUG, target->scsi_host,
958 				     PFX "Sending CM DREQ failed\n");
959 		}
960 	}
961 }
962 
963 static int srp_exit_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
964 {
965 	struct srp_target_port *target = host_to_target(shost);
966 	struct srp_device *dev = target->srp_host->srp_dev;
967 	struct ib_device *ibdev = dev->dev;
968 	struct srp_request *req = scsi_cmd_priv(cmd);
969 
970 	kfree(req->fr_list);
971 	if (req->indirect_dma_addr) {
972 		ib_dma_unmap_single(ibdev, req->indirect_dma_addr,
973 				    target->indirect_size,
974 				    DMA_TO_DEVICE);
975 	}
976 	kfree(req->indirect_desc);
977 
978 	return 0;
979 }
980 
981 static int srp_init_cmd_priv(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
982 {
983 	struct srp_target_port *target = host_to_target(shost);
984 	struct srp_device *srp_dev = target->srp_host->srp_dev;
985 	struct ib_device *ibdev = srp_dev->dev;
986 	struct srp_request *req = scsi_cmd_priv(cmd);
987 	dma_addr_t dma_addr;
988 	int ret = -ENOMEM;
989 
990 	if (srp_dev->use_fast_reg) {
991 		req->fr_list = kmalloc_array(target->mr_per_cmd, sizeof(void *),
992 					GFP_KERNEL);
993 		if (!req->fr_list)
994 			goto out;
995 	}
996 	req->indirect_desc = kmalloc(target->indirect_size, GFP_KERNEL);
997 	if (!req->indirect_desc)
998 		goto out;
999 
1000 	dma_addr = ib_dma_map_single(ibdev, req->indirect_desc,
1001 				     target->indirect_size,
1002 				     DMA_TO_DEVICE);
1003 	if (ib_dma_mapping_error(ibdev, dma_addr)) {
1004 		srp_exit_cmd_priv(shost, cmd);
1005 		goto out;
1006 	}
1007 
1008 	req->indirect_dma_addr = dma_addr;
1009 	ret = 0;
1010 
1011 out:
1012 	return ret;
1013 }
1014 
1015 /**
1016  * srp_del_scsi_host_attr() - Remove attributes defined in the host template.
1017  * @shost: SCSI host whose attributes to remove from sysfs.
1018  *
1019  * Note: Any attributes defined in the host template and that did not exist
1020  * before invocation of this function will be ignored.
1021  */
1022 static void srp_del_scsi_host_attr(struct Scsi_Host *shost)
1023 {
1024 	const struct attribute_group **g;
1025 	struct attribute **attr;
1026 
1027 	for (g = shost->hostt->shost_groups; *g; ++g) {
1028 		for (attr = (*g)->attrs; *attr; ++attr) {
1029 			struct device_attribute *dev_attr =
1030 				container_of(*attr, typeof(*dev_attr), attr);
1031 
1032 			device_remove_file(&shost->shost_dev, dev_attr);
1033 		}
1034 	}
1035 }
1036 
1037 static void srp_remove_target(struct srp_target_port *target)
1038 {
1039 	struct srp_rdma_ch *ch;
1040 	int i;
1041 
1042 	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
1043 
1044 	srp_del_scsi_host_attr(target->scsi_host);
1045 	srp_rport_get(target->rport);
1046 	srp_remove_host(target->scsi_host);
1047 	scsi_remove_host(target->scsi_host);
1048 	srp_stop_rport_timers(target->rport);
1049 	srp_disconnect_target(target);
1050 	kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
1051 	for (i = 0; i < target->ch_count; i++) {
1052 		ch = &target->ch[i];
1053 		srp_free_ch_ib(target, ch);
1054 	}
1055 	cancel_work_sync(&target->tl_err_work);
1056 	srp_rport_put(target->rport);
1057 	kfree(target->ch);
1058 	target->ch = NULL;
1059 
1060 	spin_lock(&target->srp_host->target_lock);
1061 	list_del(&target->list);
1062 	spin_unlock(&target->srp_host->target_lock);
1063 
1064 	scsi_host_put(target->scsi_host);
1065 }
1066 
1067 static void srp_remove_work(struct work_struct *work)
1068 {
1069 	struct srp_target_port *target =
1070 		container_of(work, struct srp_target_port, remove_work);
1071 
1072 	WARN_ON_ONCE(target->state != SRP_TARGET_REMOVED);
1073 
1074 	srp_remove_target(target);
1075 }
1076 
1077 static void srp_rport_delete(struct srp_rport *rport)
1078 {
1079 	struct srp_target_port *target = rport->lld_data;
1080 
1081 	srp_queue_remove_work(target);
1082 }
1083 
1084 /**
1085  * srp_connected_ch() - number of connected channels
1086  * @target: SRP target port.
1087  */
1088 static int srp_connected_ch(struct srp_target_port *target)
1089 {
1090 	int i, c = 0;
1091 
1092 	for (i = 0; i < target->ch_count; i++)
1093 		c += target->ch[i].connected;
1094 
1095 	return c;
1096 }
1097 
1098 static int srp_connect_ch(struct srp_rdma_ch *ch, uint32_t max_iu_len,
1099 			  bool multich)
1100 {
1101 	struct srp_target_port *target = ch->target;
1102 	int ret;
1103 
1104 	WARN_ON_ONCE(!multich && srp_connected_ch(target) > 0);
1105 
1106 	ret = srp_lookup_path(ch);
1107 	if (ret)
1108 		goto out;
1109 
1110 	while (1) {
1111 		init_completion(&ch->done);
1112 		ret = srp_send_req(ch, max_iu_len, multich);
1113 		if (ret)
1114 			goto out;
1115 		ret = wait_for_completion_interruptible(&ch->done);
1116 		if (ret < 0)
1117 			goto out;
1118 
1119 		/*
1120 		 * The CM event handling code will set status to
1121 		 * SRP_PORT_REDIRECT if we get a port redirect REJ
1122 		 * back, or SRP_DLID_REDIRECT if we get a lid/qp
1123 		 * redirect REJ back.
1124 		 */
1125 		ret = ch->status;
1126 		switch (ret) {
1127 		case 0:
1128 			ch->connected = true;
1129 			goto out;
1130 
1131 		case SRP_PORT_REDIRECT:
1132 			ret = srp_lookup_path(ch);
1133 			if (ret)
1134 				goto out;
1135 			break;
1136 
1137 		case SRP_DLID_REDIRECT:
1138 			break;
1139 
1140 		case SRP_STALE_CONN:
1141 			shost_printk(KERN_ERR, target->scsi_host, PFX
1142 				     "giving up on stale connection\n");
1143 			ret = -ECONNRESET;
1144 			goto out;
1145 
1146 		default:
1147 			goto out;
1148 		}
1149 	}
1150 
1151 out:
1152 	return ret <= 0 ? ret : -ENODEV;
1153 }
1154 
1155 static void srp_inv_rkey_err_done(struct ib_cq *cq, struct ib_wc *wc)
1156 {
1157 	srp_handle_qp_err(cq, wc, "INV RKEY");
1158 }
1159 
1160 static int srp_inv_rkey(struct srp_request *req, struct srp_rdma_ch *ch,
1161 		u32 rkey)
1162 {
1163 	struct ib_send_wr wr = {
1164 		.opcode		    = IB_WR_LOCAL_INV,
1165 		.next		    = NULL,
1166 		.num_sge	    = 0,
1167 		.send_flags	    = 0,
1168 		.ex.invalidate_rkey = rkey,
1169 	};
1170 
1171 	wr.wr_cqe = &req->reg_cqe;
1172 	req->reg_cqe.done = srp_inv_rkey_err_done;
1173 	return ib_post_send(ch->qp, &wr, NULL);
1174 }
1175 
1176 static void srp_unmap_data(struct scsi_cmnd *scmnd,
1177 			   struct srp_rdma_ch *ch,
1178 			   struct srp_request *req)
1179 {
1180 	struct srp_target_port *target = ch->target;
1181 	struct srp_device *dev = target->srp_host->srp_dev;
1182 	struct ib_device *ibdev = dev->dev;
1183 	int i, res;
1184 
1185 	if (!scsi_sglist(scmnd) ||
1186 	    (scmnd->sc_data_direction != DMA_TO_DEVICE &&
1187 	     scmnd->sc_data_direction != DMA_FROM_DEVICE))
1188 		return;
1189 
1190 	if (dev->use_fast_reg) {
1191 		struct srp_fr_desc **pfr;
1192 
1193 		for (i = req->nmdesc, pfr = req->fr_list; i > 0; i--, pfr++) {
1194 			res = srp_inv_rkey(req, ch, (*pfr)->mr->rkey);
1195 			if (res < 0) {
1196 				shost_printk(KERN_ERR, target->scsi_host, PFX
1197 				  "Queueing INV WR for rkey %#x failed (%d)\n",
1198 				  (*pfr)->mr->rkey, res);
1199 				queue_work(system_long_wq,
1200 					   &target->tl_err_work);
1201 			}
1202 		}
1203 		if (req->nmdesc)
1204 			srp_fr_pool_put(ch->fr_pool, req->fr_list,
1205 					req->nmdesc);
1206 	}
1207 
1208 	ib_dma_unmap_sg(ibdev, scsi_sglist(scmnd), scsi_sg_count(scmnd),
1209 			scmnd->sc_data_direction);
1210 }
1211 
1212 /**
1213  * srp_claim_req - Take ownership of the scmnd associated with a request.
1214  * @ch: SRP RDMA channel.
1215  * @req: SRP request.
1216  * @sdev: If not NULL, only take ownership for this SCSI device.
1217  * @scmnd: If NULL, take ownership of @req->scmnd. If not NULL, only take
1218  *         ownership of @req->scmnd if it equals @scmnd.
1219  *
1220  * Return value:
1221  * Either NULL or a pointer to the SCSI command the caller became owner of.
1222  */
1223 static struct scsi_cmnd *srp_claim_req(struct srp_rdma_ch *ch,
1224 				       struct srp_request *req,
1225 				       struct scsi_device *sdev,
1226 				       struct scsi_cmnd *scmnd)
1227 {
1228 	unsigned long flags;
1229 
1230 	spin_lock_irqsave(&ch->lock, flags);
1231 	if (req->scmnd &&
1232 	    (!sdev || req->scmnd->device == sdev) &&
1233 	    (!scmnd || req->scmnd == scmnd)) {
1234 		scmnd = req->scmnd;
1235 		req->scmnd = NULL;
1236 	} else {
1237 		scmnd = NULL;
1238 	}
1239 	spin_unlock_irqrestore(&ch->lock, flags);
1240 
1241 	return scmnd;
1242 }
1243 
1244 /**
1245  * srp_free_req() - Unmap data and adjust ch->req_lim.
1246  * @ch:     SRP RDMA channel.
1247  * @req:    Request to be freed.
1248  * @scmnd:  SCSI command associated with @req.
1249  * @req_lim_delta: Amount to be added to @target->req_lim.
1250  */
1251 static void srp_free_req(struct srp_rdma_ch *ch, struct srp_request *req,
1252 			 struct scsi_cmnd *scmnd, s32 req_lim_delta)
1253 {
1254 	unsigned long flags;
1255 
1256 	srp_unmap_data(scmnd, ch, req);
1257 
1258 	spin_lock_irqsave(&ch->lock, flags);
1259 	ch->req_lim += req_lim_delta;
1260 	spin_unlock_irqrestore(&ch->lock, flags);
1261 }
1262 
1263 static void srp_finish_req(struct srp_rdma_ch *ch, struct srp_request *req,
1264 			   struct scsi_device *sdev, int result)
1265 {
1266 	struct scsi_cmnd *scmnd = srp_claim_req(ch, req, sdev, NULL);
1267 
1268 	if (scmnd) {
1269 		srp_free_req(ch, req, scmnd, 0);
1270 		scmnd->result = result;
1271 		scsi_done(scmnd);
1272 	}
1273 }
1274 
1275 struct srp_terminate_context {
1276 	struct srp_target_port *srp_target;
1277 	int scsi_result;
1278 };
1279 
1280 static bool srp_terminate_cmd(struct scsi_cmnd *scmnd, void *context_ptr)
1281 {
1282 	struct srp_terminate_context *context = context_ptr;
1283 	struct srp_target_port *target = context->srp_target;
1284 	u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd));
1285 	struct srp_rdma_ch *ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
1286 	struct srp_request *req = scsi_cmd_priv(scmnd);
1287 
1288 	srp_finish_req(ch, req, NULL, context->scsi_result);
1289 
1290 	return true;
1291 }
1292 
1293 static void srp_terminate_io(struct srp_rport *rport)
1294 {
1295 	struct srp_target_port *target = rport->lld_data;
1296 	struct srp_terminate_context context = { .srp_target = target,
1297 		.scsi_result = DID_TRANSPORT_FAILFAST << 16 };
1298 
1299 	scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd, &context);
1300 }
1301 
1302 /* Calculate maximum initiator to target information unit length. */
1303 static uint32_t srp_max_it_iu_len(int cmd_sg_cnt, bool use_imm_data,
1304 				  uint32_t max_it_iu_size)
1305 {
1306 	uint32_t max_iu_len = sizeof(struct srp_cmd) + SRP_MAX_ADD_CDB_LEN +
1307 		sizeof(struct srp_indirect_buf) +
1308 		cmd_sg_cnt * sizeof(struct srp_direct_buf);
1309 
1310 	if (use_imm_data)
1311 		max_iu_len = max(max_iu_len, SRP_IMM_DATA_OFFSET +
1312 				 srp_max_imm_data);
1313 
1314 	if (max_it_iu_size)
1315 		max_iu_len = min(max_iu_len, max_it_iu_size);
1316 
1317 	pr_debug("max_iu_len = %d\n", max_iu_len);
1318 
1319 	return max_iu_len;
1320 }
1321 
1322 /*
1323  * It is up to the caller to ensure that srp_rport_reconnect() calls are
1324  * serialized and that no concurrent srp_queuecommand(), srp_abort(),
1325  * srp_reset_device() or srp_reset_host() calls will occur while this function
1326  * is in progress. One way to realize that is not to call this function
1327  * directly but to call srp_reconnect_rport() instead since that last function
1328  * serializes calls of this function via rport->mutex and also blocks
1329  * srp_queuecommand() calls before invoking this function.
1330  */
1331 static int srp_rport_reconnect(struct srp_rport *rport)
1332 {
1333 	struct srp_target_port *target = rport->lld_data;
1334 	struct srp_rdma_ch *ch;
1335 	uint32_t max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt,
1336 						srp_use_imm_data,
1337 						target->max_it_iu_size);
1338 	int i, j, ret = 0;
1339 	bool multich = false;
1340 
1341 	srp_disconnect_target(target);
1342 
1343 	if (target->state == SRP_TARGET_SCANNING)
1344 		return -ENODEV;
1345 
1346 	/*
1347 	 * Now get a new local CM ID so that we avoid confusing the target in
1348 	 * case things are really fouled up. Doing so also ensures that all CM
1349 	 * callbacks will have finished before a new QP is allocated.
1350 	 */
1351 	for (i = 0; i < target->ch_count; i++) {
1352 		ch = &target->ch[i];
1353 		ret += srp_new_cm_id(ch);
1354 	}
1355 	{
1356 		struct srp_terminate_context context = {
1357 			.srp_target = target, .scsi_result = DID_RESET << 16};
1358 
1359 		scsi_host_busy_iter(target->scsi_host, srp_terminate_cmd,
1360 				    &context);
1361 	}
1362 	for (i = 0; i < target->ch_count; i++) {
1363 		ch = &target->ch[i];
1364 		/*
1365 		 * Whether or not creating a new CM ID succeeded, create a new
1366 		 * QP. This guarantees that all completion callback function
1367 		 * invocations have finished before request resetting starts.
1368 		 */
1369 		ret += srp_create_ch_ib(ch);
1370 
1371 		INIT_LIST_HEAD(&ch->free_tx);
1372 		for (j = 0; j < target->queue_size; ++j)
1373 			list_add(&ch->tx_ring[j]->list, &ch->free_tx);
1374 	}
1375 
1376 	target->qp_in_error = false;
1377 
1378 	for (i = 0; i < target->ch_count; i++) {
1379 		ch = &target->ch[i];
1380 		if (ret)
1381 			break;
1382 		ret = srp_connect_ch(ch, max_iu_len, multich);
1383 		multich = true;
1384 	}
1385 
1386 	if (ret == 0)
1387 		shost_printk(KERN_INFO, target->scsi_host,
1388 			     PFX "reconnect succeeded\n");
1389 
1390 	return ret;
1391 }
1392 
1393 static void srp_map_desc(struct srp_map_state *state, dma_addr_t dma_addr,
1394 			 unsigned int dma_len, u32 rkey)
1395 {
1396 	struct srp_direct_buf *desc = state->desc;
1397 
1398 	WARN_ON_ONCE(!dma_len);
1399 
1400 	desc->va = cpu_to_be64(dma_addr);
1401 	desc->key = cpu_to_be32(rkey);
1402 	desc->len = cpu_to_be32(dma_len);
1403 
1404 	state->total_len += dma_len;
1405 	state->desc++;
1406 	state->ndesc++;
1407 }
1408 
1409 static void srp_reg_mr_err_done(struct ib_cq *cq, struct ib_wc *wc)
1410 {
1411 	srp_handle_qp_err(cq, wc, "FAST REG");
1412 }
1413 
1414 /*
1415  * Map up to sg_nents elements of state->sg where *sg_offset_p is the offset
1416  * where to start in the first element. If sg_offset_p != NULL then
1417  * *sg_offset_p is updated to the offset in state->sg[retval] of the first
1418  * byte that has not yet been mapped.
1419  */
1420 static int srp_map_finish_fr(struct srp_map_state *state,
1421 			     struct srp_request *req,
1422 			     struct srp_rdma_ch *ch, int sg_nents,
1423 			     unsigned int *sg_offset_p)
1424 {
1425 	struct srp_target_port *target = ch->target;
1426 	struct srp_device *dev = target->srp_host->srp_dev;
1427 	struct ib_reg_wr wr;
1428 	struct srp_fr_desc *desc;
1429 	u32 rkey;
1430 	int n, err;
1431 
1432 	if (state->fr.next >= state->fr.end) {
1433 		shost_printk(KERN_ERR, ch->target->scsi_host,
1434 			     PFX "Out of MRs (mr_per_cmd = %d)\n",
1435 			     ch->target->mr_per_cmd);
1436 		return -ENOMEM;
1437 	}
1438 
1439 	WARN_ON_ONCE(!dev->use_fast_reg);
1440 
1441 	if (sg_nents == 1 && target->global_rkey) {
1442 		unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
1443 
1444 		srp_map_desc(state, sg_dma_address(state->sg) + sg_offset,
1445 			     sg_dma_len(state->sg) - sg_offset,
1446 			     target->global_rkey);
1447 		if (sg_offset_p)
1448 			*sg_offset_p = 0;
1449 		return 1;
1450 	}
1451 
1452 	desc = srp_fr_pool_get(ch->fr_pool);
1453 	if (!desc)
1454 		return -ENOMEM;
1455 
1456 	rkey = ib_inc_rkey(desc->mr->rkey);
1457 	ib_update_fast_reg_key(desc->mr, rkey);
1458 
1459 	n = ib_map_mr_sg(desc->mr, state->sg, sg_nents, sg_offset_p,
1460 			 dev->mr_page_size);
1461 	if (unlikely(n < 0)) {
1462 		srp_fr_pool_put(ch->fr_pool, &desc, 1);
1463 		pr_debug("%s: ib_map_mr_sg(%d, %d) returned %d.\n",
1464 			 dev_name(&req->scmnd->device->sdev_gendev), sg_nents,
1465 			 sg_offset_p ? *sg_offset_p : -1, n);
1466 		return n;
1467 	}
1468 
1469 	WARN_ON_ONCE(desc->mr->length == 0);
1470 
1471 	req->reg_cqe.done = srp_reg_mr_err_done;
1472 
1473 	wr.wr.next = NULL;
1474 	wr.wr.opcode = IB_WR_REG_MR;
1475 	wr.wr.wr_cqe = &req->reg_cqe;
1476 	wr.wr.num_sge = 0;
1477 	wr.wr.send_flags = 0;
1478 	wr.mr = desc->mr;
1479 	wr.key = desc->mr->rkey;
1480 	wr.access = (IB_ACCESS_LOCAL_WRITE |
1481 		     IB_ACCESS_REMOTE_READ |
1482 		     IB_ACCESS_REMOTE_WRITE);
1483 
1484 	*state->fr.next++ = desc;
1485 	state->nmdesc++;
1486 
1487 	srp_map_desc(state, desc->mr->iova,
1488 		     desc->mr->length, desc->mr->rkey);
1489 
1490 	err = ib_post_send(ch->qp, &wr.wr, NULL);
1491 	if (unlikely(err)) {
1492 		WARN_ON_ONCE(err == -ENOMEM);
1493 		return err;
1494 	}
1495 
1496 	return n;
1497 }
1498 
1499 static int srp_map_sg_fr(struct srp_map_state *state, struct srp_rdma_ch *ch,
1500 			 struct srp_request *req, struct scatterlist *scat,
1501 			 int count)
1502 {
1503 	unsigned int sg_offset = 0;
1504 
1505 	state->fr.next = req->fr_list;
1506 	state->fr.end = req->fr_list + ch->target->mr_per_cmd;
1507 	state->sg = scat;
1508 
1509 	if (count == 0)
1510 		return 0;
1511 
1512 	while (count) {
1513 		int i, n;
1514 
1515 		n = srp_map_finish_fr(state, req, ch, count, &sg_offset);
1516 		if (unlikely(n < 0))
1517 			return n;
1518 
1519 		count -= n;
1520 		for (i = 0; i < n; i++)
1521 			state->sg = sg_next(state->sg);
1522 	}
1523 
1524 	return 0;
1525 }
1526 
1527 static int srp_map_sg_dma(struct srp_map_state *state, struct srp_rdma_ch *ch,
1528 			  struct srp_request *req, struct scatterlist *scat,
1529 			  int count)
1530 {
1531 	struct srp_target_port *target = ch->target;
1532 	struct scatterlist *sg;
1533 	int i;
1534 
1535 	for_each_sg(scat, sg, count, i) {
1536 		srp_map_desc(state, sg_dma_address(sg), sg_dma_len(sg),
1537 			     target->global_rkey);
1538 	}
1539 
1540 	return 0;
1541 }
1542 
1543 /*
1544  * Register the indirect data buffer descriptor with the HCA.
1545  *
1546  * Note: since the indirect data buffer descriptor has been allocated with
1547  * kmalloc() it is guaranteed that this buffer is a physically contiguous
1548  * memory buffer.
1549  */
1550 static int srp_map_idb(struct srp_rdma_ch *ch, struct srp_request *req,
1551 		       void **next_mr, void **end_mr, u32 idb_len,
1552 		       __be32 *idb_rkey)
1553 {
1554 	struct srp_target_port *target = ch->target;
1555 	struct srp_device *dev = target->srp_host->srp_dev;
1556 	struct srp_map_state state;
1557 	struct srp_direct_buf idb_desc;
1558 	struct scatterlist idb_sg[1];
1559 	int ret;
1560 
1561 	memset(&state, 0, sizeof(state));
1562 	memset(&idb_desc, 0, sizeof(idb_desc));
1563 	state.gen.next = next_mr;
1564 	state.gen.end = end_mr;
1565 	state.desc = &idb_desc;
1566 	state.base_dma_addr = req->indirect_dma_addr;
1567 	state.dma_len = idb_len;
1568 
1569 	if (dev->use_fast_reg) {
1570 		state.sg = idb_sg;
1571 		sg_init_one(idb_sg, req->indirect_desc, idb_len);
1572 		idb_sg->dma_address = req->indirect_dma_addr; /* hack! */
1573 #ifdef CONFIG_NEED_SG_DMA_LENGTH
1574 		idb_sg->dma_length = idb_sg->length;	      /* hack^2 */
1575 #endif
1576 		ret = srp_map_finish_fr(&state, req, ch, 1, NULL);
1577 		if (ret < 0)
1578 			return ret;
1579 		WARN_ON_ONCE(ret < 1);
1580 	} else {
1581 		return -EINVAL;
1582 	}
1583 
1584 	*idb_rkey = idb_desc.key;
1585 
1586 	return 0;
1587 }
1588 
1589 static void srp_check_mapping(struct srp_map_state *state,
1590 			      struct srp_rdma_ch *ch, struct srp_request *req,
1591 			      struct scatterlist *scat, int count)
1592 {
1593 	struct srp_device *dev = ch->target->srp_host->srp_dev;
1594 	struct srp_fr_desc **pfr;
1595 	u64 desc_len = 0, mr_len = 0;
1596 	int i;
1597 
1598 	for (i = 0; i < state->ndesc; i++)
1599 		desc_len += be32_to_cpu(req->indirect_desc[i].len);
1600 	if (dev->use_fast_reg)
1601 		for (i = 0, pfr = req->fr_list; i < state->nmdesc; i++, pfr++)
1602 			mr_len += (*pfr)->mr->length;
1603 	if (desc_len != scsi_bufflen(req->scmnd) ||
1604 	    mr_len > scsi_bufflen(req->scmnd))
1605 		pr_err("Inconsistent: scsi len %d <> desc len %lld <> mr len %lld; ndesc %d; nmdesc = %d\n",
1606 		       scsi_bufflen(req->scmnd), desc_len, mr_len,
1607 		       state->ndesc, state->nmdesc);
1608 }
1609 
1610 /**
1611  * srp_map_data() - map SCSI data buffer onto an SRP request
1612  * @scmnd: SCSI command to map
1613  * @ch: SRP RDMA channel
1614  * @req: SRP request
1615  *
1616  * Returns the length in bytes of the SRP_CMD IU or a negative value if
1617  * mapping failed. The size of any immediate data is not included in the
1618  * return value.
1619  */
1620 static int srp_map_data(struct scsi_cmnd *scmnd, struct srp_rdma_ch *ch,
1621 			struct srp_request *req)
1622 {
1623 	struct srp_target_port *target = ch->target;
1624 	struct scatterlist *scat, *sg;
1625 	struct srp_cmd *cmd = req->cmd->buf;
1626 	int i, len, nents, count, ret;
1627 	struct srp_device *dev;
1628 	struct ib_device *ibdev;
1629 	struct srp_map_state state;
1630 	struct srp_indirect_buf *indirect_hdr;
1631 	u64 data_len;
1632 	u32 idb_len, table_len;
1633 	__be32 idb_rkey;
1634 	u8 fmt;
1635 
1636 	req->cmd->num_sge = 1;
1637 
1638 	if (!scsi_sglist(scmnd) || scmnd->sc_data_direction == DMA_NONE)
1639 		return sizeof(struct srp_cmd) + cmd->add_cdb_len;
1640 
1641 	if (scmnd->sc_data_direction != DMA_FROM_DEVICE &&
1642 	    scmnd->sc_data_direction != DMA_TO_DEVICE) {
1643 		shost_printk(KERN_WARNING, target->scsi_host,
1644 			     PFX "Unhandled data direction %d\n",
1645 			     scmnd->sc_data_direction);
1646 		return -EINVAL;
1647 	}
1648 
1649 	nents = scsi_sg_count(scmnd);
1650 	scat  = scsi_sglist(scmnd);
1651 	data_len = scsi_bufflen(scmnd);
1652 
1653 	dev = target->srp_host->srp_dev;
1654 	ibdev = dev->dev;
1655 
1656 	count = ib_dma_map_sg(ibdev, scat, nents, scmnd->sc_data_direction);
1657 	if (unlikely(count == 0))
1658 		return -EIO;
1659 
1660 	if (ch->use_imm_data &&
1661 	    count <= ch->max_imm_sge &&
1662 	    SRP_IMM_DATA_OFFSET + data_len <= ch->max_it_iu_len &&
1663 	    scmnd->sc_data_direction == DMA_TO_DEVICE) {
1664 		struct srp_imm_buf *buf;
1665 		struct ib_sge *sge = &req->cmd->sge[1];
1666 
1667 		fmt = SRP_DATA_DESC_IMM;
1668 		len = SRP_IMM_DATA_OFFSET;
1669 		req->nmdesc = 0;
1670 		buf = (void *)cmd->add_data + cmd->add_cdb_len;
1671 		buf->len = cpu_to_be32(data_len);
1672 		WARN_ON_ONCE((void *)(buf + 1) > (void *)cmd + len);
1673 		for_each_sg(scat, sg, count, i) {
1674 			sge[i].addr   = sg_dma_address(sg);
1675 			sge[i].length = sg_dma_len(sg);
1676 			sge[i].lkey   = target->lkey;
1677 		}
1678 		req->cmd->num_sge += count;
1679 		goto map_complete;
1680 	}
1681 
1682 	fmt = SRP_DATA_DESC_DIRECT;
1683 	len = sizeof(struct srp_cmd) + cmd->add_cdb_len +
1684 		sizeof(struct srp_direct_buf);
1685 
1686 	if (count == 1 && target->global_rkey) {
1687 		/*
1688 		 * The midlayer only generated a single gather/scatter
1689 		 * entry, or DMA mapping coalesced everything to a
1690 		 * single entry.  So a direct descriptor along with
1691 		 * the DMA MR suffices.
1692 		 */
1693 		struct srp_direct_buf *buf;
1694 
1695 		buf = (void *)cmd->add_data + cmd->add_cdb_len;
1696 		buf->va  = cpu_to_be64(sg_dma_address(scat));
1697 		buf->key = cpu_to_be32(target->global_rkey);
1698 		buf->len = cpu_to_be32(sg_dma_len(scat));
1699 
1700 		req->nmdesc = 0;
1701 		goto map_complete;
1702 	}
1703 
1704 	/*
1705 	 * We have more than one scatter/gather entry, so build our indirect
1706 	 * descriptor table, trying to merge as many entries as we can.
1707 	 */
1708 	indirect_hdr = (void *)cmd->add_data + cmd->add_cdb_len;
1709 
1710 	ib_dma_sync_single_for_cpu(ibdev, req->indirect_dma_addr,
1711 				   target->indirect_size, DMA_TO_DEVICE);
1712 
1713 	memset(&state, 0, sizeof(state));
1714 	state.desc = req->indirect_desc;
1715 	if (dev->use_fast_reg)
1716 		ret = srp_map_sg_fr(&state, ch, req, scat, count);
1717 	else
1718 		ret = srp_map_sg_dma(&state, ch, req, scat, count);
1719 	req->nmdesc = state.nmdesc;
1720 	if (ret < 0)
1721 		goto unmap;
1722 
1723 	{
1724 		DEFINE_DYNAMIC_DEBUG_METADATA(ddm,
1725 			"Memory mapping consistency check");
1726 		if (DYNAMIC_DEBUG_BRANCH(ddm))
1727 			srp_check_mapping(&state, ch, req, scat, count);
1728 	}
1729 
1730 	/* We've mapped the request, now pull as much of the indirect
1731 	 * descriptor table as we can into the command buffer. If this
1732 	 * target is not using an external indirect table, we are
1733 	 * guaranteed to fit into the command, as the SCSI layer won't
1734 	 * give us more S/G entries than we allow.
1735 	 */
1736 	if (state.ndesc == 1) {
1737 		/*
1738 		 * Memory registration collapsed the sg-list into one entry,
1739 		 * so use a direct descriptor.
1740 		 */
1741 		struct srp_direct_buf *buf;
1742 
1743 		buf = (void *)cmd->add_data + cmd->add_cdb_len;
1744 		*buf = req->indirect_desc[0];
1745 		goto map_complete;
1746 	}
1747 
1748 	if (unlikely(target->cmd_sg_cnt < state.ndesc &&
1749 						!target->allow_ext_sg)) {
1750 		shost_printk(KERN_ERR, target->scsi_host,
1751 			     "Could not fit S/G list into SRP_CMD\n");
1752 		ret = -EIO;
1753 		goto unmap;
1754 	}
1755 
1756 	count = min(state.ndesc, target->cmd_sg_cnt);
1757 	table_len = state.ndesc * sizeof (struct srp_direct_buf);
1758 	idb_len = sizeof(struct srp_indirect_buf) + table_len;
1759 
1760 	fmt = SRP_DATA_DESC_INDIRECT;
1761 	len = sizeof(struct srp_cmd) + cmd->add_cdb_len +
1762 		sizeof(struct srp_indirect_buf);
1763 	len += count * sizeof (struct srp_direct_buf);
1764 
1765 	memcpy(indirect_hdr->desc_list, req->indirect_desc,
1766 	       count * sizeof (struct srp_direct_buf));
1767 
1768 	if (!target->global_rkey) {
1769 		ret = srp_map_idb(ch, req, state.gen.next, state.gen.end,
1770 				  idb_len, &idb_rkey);
1771 		if (ret < 0)
1772 			goto unmap;
1773 		req->nmdesc++;
1774 	} else {
1775 		idb_rkey = cpu_to_be32(target->global_rkey);
1776 	}
1777 
1778 	indirect_hdr->table_desc.va = cpu_to_be64(req->indirect_dma_addr);
1779 	indirect_hdr->table_desc.key = idb_rkey;
1780 	indirect_hdr->table_desc.len = cpu_to_be32(table_len);
1781 	indirect_hdr->len = cpu_to_be32(state.total_len);
1782 
1783 	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1784 		cmd->data_out_desc_cnt = count;
1785 	else
1786 		cmd->data_in_desc_cnt = count;
1787 
1788 	ib_dma_sync_single_for_device(ibdev, req->indirect_dma_addr, table_len,
1789 				      DMA_TO_DEVICE);
1790 
1791 map_complete:
1792 	if (scmnd->sc_data_direction == DMA_TO_DEVICE)
1793 		cmd->buf_fmt = fmt << 4;
1794 	else
1795 		cmd->buf_fmt = fmt;
1796 
1797 	return len;
1798 
1799 unmap:
1800 	srp_unmap_data(scmnd, ch, req);
1801 	if (ret == -ENOMEM && req->nmdesc >= target->mr_pool_size)
1802 		ret = -E2BIG;
1803 	return ret;
1804 }
1805 
1806 /*
1807  * Return an IU and possible credit to the free pool
1808  */
1809 static void srp_put_tx_iu(struct srp_rdma_ch *ch, struct srp_iu *iu,
1810 			  enum srp_iu_type iu_type)
1811 {
1812 	unsigned long flags;
1813 
1814 	spin_lock_irqsave(&ch->lock, flags);
1815 	list_add(&iu->list, &ch->free_tx);
1816 	if (iu_type != SRP_IU_RSP)
1817 		++ch->req_lim;
1818 	spin_unlock_irqrestore(&ch->lock, flags);
1819 }
1820 
1821 /*
1822  * Must be called with ch->lock held to protect req_lim and free_tx.
1823  * If IU is not sent, it must be returned using srp_put_tx_iu().
1824  *
1825  * Note:
1826  * An upper limit for the number of allocated information units for each
1827  * request type is:
1828  * - SRP_IU_CMD: SRP_CMD_SQ_SIZE, since the SCSI mid-layer never queues
1829  *   more than Scsi_Host.can_queue requests.
1830  * - SRP_IU_TSK_MGMT: SRP_TSK_MGMT_SQ_SIZE.
1831  * - SRP_IU_RSP: 1, since a conforming SRP target never sends more than
1832  *   one unanswered SRP request to an initiator.
1833  */
1834 static struct srp_iu *__srp_get_tx_iu(struct srp_rdma_ch *ch,
1835 				      enum srp_iu_type iu_type)
1836 {
1837 	struct srp_target_port *target = ch->target;
1838 	s32 rsv = (iu_type == SRP_IU_TSK_MGMT) ? 0 : SRP_TSK_MGMT_SQ_SIZE;
1839 	struct srp_iu *iu;
1840 
1841 	lockdep_assert_held(&ch->lock);
1842 
1843 	ib_process_cq_direct(ch->send_cq, -1);
1844 
1845 	if (list_empty(&ch->free_tx))
1846 		return NULL;
1847 
1848 	/* Initiator responses to target requests do not consume credits */
1849 	if (iu_type != SRP_IU_RSP) {
1850 		if (ch->req_lim <= rsv) {
1851 			++target->zero_req_lim;
1852 			return NULL;
1853 		}
1854 
1855 		--ch->req_lim;
1856 	}
1857 
1858 	iu = list_first_entry(&ch->free_tx, struct srp_iu, list);
1859 	list_del(&iu->list);
1860 	return iu;
1861 }
1862 
1863 /*
1864  * Note: if this function is called from inside ib_drain_sq() then it will
1865  * be called without ch->lock being held. If ib_drain_sq() dequeues a WQE
1866  * with status IB_WC_SUCCESS then that's a bug.
1867  */
1868 static void srp_send_done(struct ib_cq *cq, struct ib_wc *wc)
1869 {
1870 	struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
1871 	struct srp_rdma_ch *ch = cq->cq_context;
1872 
1873 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1874 		srp_handle_qp_err(cq, wc, "SEND");
1875 		return;
1876 	}
1877 
1878 	lockdep_assert_held(&ch->lock);
1879 
1880 	list_add(&iu->list, &ch->free_tx);
1881 }
1882 
1883 /**
1884  * srp_post_send() - send an SRP information unit
1885  * @ch: RDMA channel over which to send the information unit.
1886  * @iu: Information unit to send.
1887  * @len: Length of the information unit excluding immediate data.
1888  */
1889 static int srp_post_send(struct srp_rdma_ch *ch, struct srp_iu *iu, int len)
1890 {
1891 	struct srp_target_port *target = ch->target;
1892 	struct ib_send_wr wr;
1893 
1894 	if (WARN_ON_ONCE(iu->num_sge > SRP_MAX_SGE))
1895 		return -EINVAL;
1896 
1897 	iu->sge[0].addr   = iu->dma;
1898 	iu->sge[0].length = len;
1899 	iu->sge[0].lkey   = target->lkey;
1900 
1901 	iu->cqe.done = srp_send_done;
1902 
1903 	wr.next       = NULL;
1904 	wr.wr_cqe     = &iu->cqe;
1905 	wr.sg_list    = &iu->sge[0];
1906 	wr.num_sge    = iu->num_sge;
1907 	wr.opcode     = IB_WR_SEND;
1908 	wr.send_flags = IB_SEND_SIGNALED;
1909 
1910 	return ib_post_send(ch->qp, &wr, NULL);
1911 }
1912 
1913 static int srp_post_recv(struct srp_rdma_ch *ch, struct srp_iu *iu)
1914 {
1915 	struct srp_target_port *target = ch->target;
1916 	struct ib_recv_wr wr;
1917 	struct ib_sge list;
1918 
1919 	list.addr   = iu->dma;
1920 	list.length = iu->size;
1921 	list.lkey   = target->lkey;
1922 
1923 	iu->cqe.done = srp_recv_done;
1924 
1925 	wr.next     = NULL;
1926 	wr.wr_cqe   = &iu->cqe;
1927 	wr.sg_list  = &list;
1928 	wr.num_sge  = 1;
1929 
1930 	return ib_post_recv(ch->qp, &wr, NULL);
1931 }
1932 
1933 static void srp_process_rsp(struct srp_rdma_ch *ch, struct srp_rsp *rsp)
1934 {
1935 	struct srp_target_port *target = ch->target;
1936 	struct srp_request *req;
1937 	struct scsi_cmnd *scmnd;
1938 	unsigned long flags;
1939 
1940 	if (unlikely(rsp->tag & SRP_TAG_TSK_MGMT)) {
1941 		spin_lock_irqsave(&ch->lock, flags);
1942 		ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1943 		if (rsp->tag == ch->tsk_mgmt_tag) {
1944 			ch->tsk_mgmt_status = -1;
1945 			if (be32_to_cpu(rsp->resp_data_len) >= 4)
1946 				ch->tsk_mgmt_status = rsp->data[3];
1947 			complete(&ch->tsk_mgmt_done);
1948 		} else {
1949 			shost_printk(KERN_ERR, target->scsi_host,
1950 				     "Received tsk mgmt response too late for tag %#llx\n",
1951 				     rsp->tag);
1952 		}
1953 		spin_unlock_irqrestore(&ch->lock, flags);
1954 	} else {
1955 		scmnd = scsi_host_find_tag(target->scsi_host, rsp->tag);
1956 		if (scmnd) {
1957 			req = scsi_cmd_priv(scmnd);
1958 			scmnd = srp_claim_req(ch, req, NULL, scmnd);
1959 		}
1960 		if (!scmnd) {
1961 			shost_printk(KERN_ERR, target->scsi_host,
1962 				     "Null scmnd for RSP w/tag %#016llx received on ch %td / QP %#x\n",
1963 				     rsp->tag, ch - target->ch, ch->qp->qp_num);
1964 
1965 			spin_lock_irqsave(&ch->lock, flags);
1966 			ch->req_lim += be32_to_cpu(rsp->req_lim_delta);
1967 			spin_unlock_irqrestore(&ch->lock, flags);
1968 
1969 			return;
1970 		}
1971 		scmnd->result = rsp->status;
1972 
1973 		if (rsp->flags & SRP_RSP_FLAG_SNSVALID) {
1974 			memcpy(scmnd->sense_buffer, rsp->data +
1975 			       be32_to_cpu(rsp->resp_data_len),
1976 			       min_t(int, be32_to_cpu(rsp->sense_data_len),
1977 				     SCSI_SENSE_BUFFERSIZE));
1978 		}
1979 
1980 		if (unlikely(rsp->flags & SRP_RSP_FLAG_DIUNDER))
1981 			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_in_res_cnt));
1982 		else if (unlikely(rsp->flags & SRP_RSP_FLAG_DOUNDER))
1983 			scsi_set_resid(scmnd, be32_to_cpu(rsp->data_out_res_cnt));
1984 
1985 		srp_free_req(ch, req, scmnd,
1986 			     be32_to_cpu(rsp->req_lim_delta));
1987 
1988 		scsi_done(scmnd);
1989 	}
1990 }
1991 
1992 static int srp_response_common(struct srp_rdma_ch *ch, s32 req_delta,
1993 			       void *rsp, int len)
1994 {
1995 	struct srp_target_port *target = ch->target;
1996 	struct ib_device *dev = target->srp_host->srp_dev->dev;
1997 	unsigned long flags;
1998 	struct srp_iu *iu;
1999 	int err;
2000 
2001 	spin_lock_irqsave(&ch->lock, flags);
2002 	ch->req_lim += req_delta;
2003 	iu = __srp_get_tx_iu(ch, SRP_IU_RSP);
2004 	spin_unlock_irqrestore(&ch->lock, flags);
2005 
2006 	if (!iu) {
2007 		shost_printk(KERN_ERR, target->scsi_host, PFX
2008 			     "no IU available to send response\n");
2009 		return 1;
2010 	}
2011 
2012 	iu->num_sge = 1;
2013 	ib_dma_sync_single_for_cpu(dev, iu->dma, len, DMA_TO_DEVICE);
2014 	memcpy(iu->buf, rsp, len);
2015 	ib_dma_sync_single_for_device(dev, iu->dma, len, DMA_TO_DEVICE);
2016 
2017 	err = srp_post_send(ch, iu, len);
2018 	if (err) {
2019 		shost_printk(KERN_ERR, target->scsi_host, PFX
2020 			     "unable to post response: %d\n", err);
2021 		srp_put_tx_iu(ch, iu, SRP_IU_RSP);
2022 	}
2023 
2024 	return err;
2025 }
2026 
2027 static void srp_process_cred_req(struct srp_rdma_ch *ch,
2028 				 struct srp_cred_req *req)
2029 {
2030 	struct srp_cred_rsp rsp = {
2031 		.opcode = SRP_CRED_RSP,
2032 		.tag = req->tag,
2033 	};
2034 	s32 delta = be32_to_cpu(req->req_lim_delta);
2035 
2036 	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2037 		shost_printk(KERN_ERR, ch->target->scsi_host, PFX
2038 			     "problems processing SRP_CRED_REQ\n");
2039 }
2040 
2041 static void srp_process_aer_req(struct srp_rdma_ch *ch,
2042 				struct srp_aer_req *req)
2043 {
2044 	struct srp_target_port *target = ch->target;
2045 	struct srp_aer_rsp rsp = {
2046 		.opcode = SRP_AER_RSP,
2047 		.tag = req->tag,
2048 	};
2049 	s32 delta = be32_to_cpu(req->req_lim_delta);
2050 
2051 	shost_printk(KERN_ERR, target->scsi_host, PFX
2052 		     "ignoring AER for LUN %llu\n", scsilun_to_int(&req->lun));
2053 
2054 	if (srp_response_common(ch, delta, &rsp, sizeof(rsp)))
2055 		shost_printk(KERN_ERR, target->scsi_host, PFX
2056 			     "problems processing SRP_AER_REQ\n");
2057 }
2058 
2059 static void srp_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2060 {
2061 	struct srp_iu *iu = container_of(wc->wr_cqe, struct srp_iu, cqe);
2062 	struct srp_rdma_ch *ch = cq->cq_context;
2063 	struct srp_target_port *target = ch->target;
2064 	struct ib_device *dev = target->srp_host->srp_dev->dev;
2065 	int res;
2066 	u8 opcode;
2067 
2068 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
2069 		srp_handle_qp_err(cq, wc, "RECV");
2070 		return;
2071 	}
2072 
2073 	ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_ti_iu_len,
2074 				   DMA_FROM_DEVICE);
2075 
2076 	opcode = *(u8 *) iu->buf;
2077 
2078 	if (0) {
2079 		shost_printk(KERN_ERR, target->scsi_host,
2080 			     PFX "recv completion, opcode 0x%02x\n", opcode);
2081 		print_hex_dump(KERN_ERR, "", DUMP_PREFIX_OFFSET, 8, 1,
2082 			       iu->buf, wc->byte_len, true);
2083 	}
2084 
2085 	switch (opcode) {
2086 	case SRP_RSP:
2087 		srp_process_rsp(ch, iu->buf);
2088 		break;
2089 
2090 	case SRP_CRED_REQ:
2091 		srp_process_cred_req(ch, iu->buf);
2092 		break;
2093 
2094 	case SRP_AER_REQ:
2095 		srp_process_aer_req(ch, iu->buf);
2096 		break;
2097 
2098 	case SRP_T_LOGOUT:
2099 		/* XXX Handle target logout */
2100 		shost_printk(KERN_WARNING, target->scsi_host,
2101 			     PFX "Got target logout request\n");
2102 		break;
2103 
2104 	default:
2105 		shost_printk(KERN_WARNING, target->scsi_host,
2106 			     PFX "Unhandled SRP opcode 0x%02x\n", opcode);
2107 		break;
2108 	}
2109 
2110 	ib_dma_sync_single_for_device(dev, iu->dma, ch->max_ti_iu_len,
2111 				      DMA_FROM_DEVICE);
2112 
2113 	res = srp_post_recv(ch, iu);
2114 	if (res != 0)
2115 		shost_printk(KERN_ERR, target->scsi_host,
2116 			     PFX "Recv failed with error code %d\n", res);
2117 }
2118 
2119 /**
2120  * srp_tl_err_work() - handle a transport layer error
2121  * @work: Work structure embedded in an SRP target port.
2122  *
2123  * Note: This function may get invoked before the rport has been created,
2124  * hence the target->rport test.
2125  */
2126 static void srp_tl_err_work(struct work_struct *work)
2127 {
2128 	struct srp_target_port *target;
2129 
2130 	target = container_of(work, struct srp_target_port, tl_err_work);
2131 	if (target->rport)
2132 		srp_start_tl_fail_timers(target->rport);
2133 }
2134 
2135 static void srp_handle_qp_err(struct ib_cq *cq, struct ib_wc *wc,
2136 		const char *opname)
2137 {
2138 	struct srp_rdma_ch *ch = cq->cq_context;
2139 	struct srp_target_port *target = ch->target;
2140 
2141 	if (ch->connected && !target->qp_in_error) {
2142 		shost_printk(KERN_ERR, target->scsi_host,
2143 			     PFX "failed %s status %s (%d) for CQE %p\n",
2144 			     opname, ib_wc_status_msg(wc->status), wc->status,
2145 			     wc->wr_cqe);
2146 		queue_work(system_long_wq, &target->tl_err_work);
2147 	}
2148 	target->qp_in_error = true;
2149 }
2150 
2151 static int srp_queuecommand(struct Scsi_Host *shost, struct scsi_cmnd *scmnd)
2152 {
2153 	struct request *rq = scsi_cmd_to_rq(scmnd);
2154 	struct srp_target_port *target = host_to_target(shost);
2155 	struct srp_rdma_ch *ch;
2156 	struct srp_request *req = scsi_cmd_priv(scmnd);
2157 	struct srp_iu *iu;
2158 	struct srp_cmd *cmd;
2159 	struct ib_device *dev;
2160 	unsigned long flags;
2161 	u32 tag;
2162 	int len, ret;
2163 
2164 	scmnd->result = srp_chkready(target->rport);
2165 	if (unlikely(scmnd->result))
2166 		goto err;
2167 
2168 	WARN_ON_ONCE(rq->tag < 0);
2169 	tag = blk_mq_unique_tag(rq);
2170 	ch = &target->ch[blk_mq_unique_tag_to_hwq(tag)];
2171 
2172 	spin_lock_irqsave(&ch->lock, flags);
2173 	iu = __srp_get_tx_iu(ch, SRP_IU_CMD);
2174 	spin_unlock_irqrestore(&ch->lock, flags);
2175 
2176 	if (!iu)
2177 		goto err;
2178 
2179 	dev = target->srp_host->srp_dev->dev;
2180 	ib_dma_sync_single_for_cpu(dev, iu->dma, ch->max_it_iu_len,
2181 				   DMA_TO_DEVICE);
2182 
2183 	cmd = iu->buf;
2184 	memset(cmd, 0, sizeof *cmd);
2185 
2186 	cmd->opcode = SRP_CMD;
2187 	int_to_scsilun(scmnd->device->lun, &cmd->lun);
2188 	cmd->tag    = tag;
2189 	memcpy(cmd->cdb, scmnd->cmnd, scmnd->cmd_len);
2190 	if (unlikely(scmnd->cmd_len > sizeof(cmd->cdb))) {
2191 		cmd->add_cdb_len = round_up(scmnd->cmd_len - sizeof(cmd->cdb),
2192 					    4);
2193 		if (WARN_ON_ONCE(cmd->add_cdb_len > SRP_MAX_ADD_CDB_LEN))
2194 			goto err_iu;
2195 	}
2196 
2197 	req->scmnd    = scmnd;
2198 	req->cmd      = iu;
2199 
2200 	len = srp_map_data(scmnd, ch, req);
2201 	if (len < 0) {
2202 		shost_printk(KERN_ERR, target->scsi_host,
2203 			     PFX "Failed to map data (%d)\n", len);
2204 		/*
2205 		 * If we ran out of memory descriptors (-ENOMEM) because an
2206 		 * application is queuing many requests with more than
2207 		 * max_pages_per_mr sg-list elements, tell the SCSI mid-layer
2208 		 * to reduce queue depth temporarily.
2209 		 */
2210 		scmnd->result = len == -ENOMEM ?
2211 			DID_OK << 16 | SAM_STAT_TASK_SET_FULL : DID_ERROR << 16;
2212 		goto err_iu;
2213 	}
2214 
2215 	ib_dma_sync_single_for_device(dev, iu->dma, ch->max_it_iu_len,
2216 				      DMA_TO_DEVICE);
2217 
2218 	if (srp_post_send(ch, iu, len)) {
2219 		shost_printk(KERN_ERR, target->scsi_host, PFX "Send failed\n");
2220 		scmnd->result = DID_ERROR << 16;
2221 		goto err_unmap;
2222 	}
2223 
2224 	return 0;
2225 
2226 err_unmap:
2227 	srp_unmap_data(scmnd, ch, req);
2228 
2229 err_iu:
2230 	srp_put_tx_iu(ch, iu, SRP_IU_CMD);
2231 
2232 	/*
2233 	 * Avoid that the loops that iterate over the request ring can
2234 	 * encounter a dangling SCSI command pointer.
2235 	 */
2236 	req->scmnd = NULL;
2237 
2238 err:
2239 	if (scmnd->result) {
2240 		scsi_done(scmnd);
2241 		ret = 0;
2242 	} else {
2243 		ret = SCSI_MLQUEUE_HOST_BUSY;
2244 	}
2245 
2246 	return ret;
2247 }
2248 
2249 /*
2250  * Note: the resources allocated in this function are freed in
2251  * srp_free_ch_ib().
2252  */
2253 static int srp_alloc_iu_bufs(struct srp_rdma_ch *ch)
2254 {
2255 	struct srp_target_port *target = ch->target;
2256 	int i;
2257 
2258 	ch->rx_ring = kcalloc(target->queue_size, sizeof(*ch->rx_ring),
2259 			      GFP_KERNEL);
2260 	if (!ch->rx_ring)
2261 		goto err_no_ring;
2262 	ch->tx_ring = kcalloc(target->queue_size, sizeof(*ch->tx_ring),
2263 			      GFP_KERNEL);
2264 	if (!ch->tx_ring)
2265 		goto err_no_ring;
2266 
2267 	for (i = 0; i < target->queue_size; ++i) {
2268 		ch->rx_ring[i] = srp_alloc_iu(target->srp_host,
2269 					      ch->max_ti_iu_len,
2270 					      GFP_KERNEL, DMA_FROM_DEVICE);
2271 		if (!ch->rx_ring[i])
2272 			goto err;
2273 	}
2274 
2275 	for (i = 0; i < target->queue_size; ++i) {
2276 		ch->tx_ring[i] = srp_alloc_iu(target->srp_host,
2277 					      ch->max_it_iu_len,
2278 					      GFP_KERNEL, DMA_TO_DEVICE);
2279 		if (!ch->tx_ring[i])
2280 			goto err;
2281 
2282 		list_add(&ch->tx_ring[i]->list, &ch->free_tx);
2283 	}
2284 
2285 	return 0;
2286 
2287 err:
2288 	for (i = 0; i < target->queue_size; ++i) {
2289 		srp_free_iu(target->srp_host, ch->rx_ring[i]);
2290 		srp_free_iu(target->srp_host, ch->tx_ring[i]);
2291 	}
2292 
2293 
2294 err_no_ring:
2295 	kfree(ch->tx_ring);
2296 	ch->tx_ring = NULL;
2297 	kfree(ch->rx_ring);
2298 	ch->rx_ring = NULL;
2299 
2300 	return -ENOMEM;
2301 }
2302 
2303 static uint32_t srp_compute_rq_tmo(struct ib_qp_attr *qp_attr, int attr_mask)
2304 {
2305 	uint64_t T_tr_ns, max_compl_time_ms;
2306 	uint32_t rq_tmo_jiffies;
2307 
2308 	/*
2309 	 * According to section 11.2.4.2 in the IBTA spec (Modify Queue Pair,
2310 	 * table 91), both the QP timeout and the retry count have to be set
2311 	 * for RC QP's during the RTR to RTS transition.
2312 	 */
2313 	WARN_ON_ONCE((attr_mask & (IB_QP_TIMEOUT | IB_QP_RETRY_CNT)) !=
2314 		     (IB_QP_TIMEOUT | IB_QP_RETRY_CNT));
2315 
2316 	/*
2317 	 * Set target->rq_tmo_jiffies to one second more than the largest time
2318 	 * it can take before an error completion is generated. See also
2319 	 * C9-140..142 in the IBTA spec for more information about how to
2320 	 * convert the QP Local ACK Timeout value to nanoseconds.
2321 	 */
2322 	T_tr_ns = 4096 * (1ULL << qp_attr->timeout);
2323 	max_compl_time_ms = qp_attr->retry_cnt * 4 * T_tr_ns;
2324 	do_div(max_compl_time_ms, NSEC_PER_MSEC);
2325 	rq_tmo_jiffies = msecs_to_jiffies(max_compl_time_ms + 1000);
2326 
2327 	return rq_tmo_jiffies;
2328 }
2329 
2330 static void srp_cm_rep_handler(struct ib_cm_id *cm_id,
2331 			       const struct srp_login_rsp *lrsp,
2332 			       struct srp_rdma_ch *ch)
2333 {
2334 	struct srp_target_port *target = ch->target;
2335 	struct ib_qp_attr *qp_attr = NULL;
2336 	int attr_mask = 0;
2337 	int ret = 0;
2338 	int i;
2339 
2340 	if (lrsp->opcode == SRP_LOGIN_RSP) {
2341 		ch->max_ti_iu_len = be32_to_cpu(lrsp->max_ti_iu_len);
2342 		ch->req_lim       = be32_to_cpu(lrsp->req_lim_delta);
2343 		ch->use_imm_data  = srp_use_imm_data &&
2344 			(lrsp->rsp_flags & SRP_LOGIN_RSP_IMMED_SUPP);
2345 		ch->max_it_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt,
2346 						      ch->use_imm_data,
2347 						      target->max_it_iu_size);
2348 		WARN_ON_ONCE(ch->max_it_iu_len >
2349 			     be32_to_cpu(lrsp->max_it_iu_len));
2350 
2351 		if (ch->use_imm_data)
2352 			shost_printk(KERN_DEBUG, target->scsi_host,
2353 				     PFX "using immediate data\n");
2354 
2355 		/*
2356 		 * Reserve credits for task management so we don't
2357 		 * bounce requests back to the SCSI mid-layer.
2358 		 */
2359 		target->scsi_host->can_queue
2360 			= min(ch->req_lim - SRP_TSK_MGMT_SQ_SIZE,
2361 			      target->scsi_host->can_queue);
2362 		target->scsi_host->cmd_per_lun
2363 			= min_t(int, target->scsi_host->can_queue,
2364 				target->scsi_host->cmd_per_lun);
2365 	} else {
2366 		shost_printk(KERN_WARNING, target->scsi_host,
2367 			     PFX "Unhandled RSP opcode %#x\n", lrsp->opcode);
2368 		ret = -ECONNRESET;
2369 		goto error;
2370 	}
2371 
2372 	if (!ch->rx_ring) {
2373 		ret = srp_alloc_iu_bufs(ch);
2374 		if (ret)
2375 			goto error;
2376 	}
2377 
2378 	for (i = 0; i < target->queue_size; i++) {
2379 		struct srp_iu *iu = ch->rx_ring[i];
2380 
2381 		ret = srp_post_recv(ch, iu);
2382 		if (ret)
2383 			goto error;
2384 	}
2385 
2386 	if (!target->using_rdma_cm) {
2387 		ret = -ENOMEM;
2388 		qp_attr = kmalloc(sizeof(*qp_attr), GFP_KERNEL);
2389 		if (!qp_attr)
2390 			goto error;
2391 
2392 		qp_attr->qp_state = IB_QPS_RTR;
2393 		ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2394 		if (ret)
2395 			goto error_free;
2396 
2397 		ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2398 		if (ret)
2399 			goto error_free;
2400 
2401 		qp_attr->qp_state = IB_QPS_RTS;
2402 		ret = ib_cm_init_qp_attr(cm_id, qp_attr, &attr_mask);
2403 		if (ret)
2404 			goto error_free;
2405 
2406 		target->rq_tmo_jiffies = srp_compute_rq_tmo(qp_attr, attr_mask);
2407 
2408 		ret = ib_modify_qp(ch->qp, qp_attr, attr_mask);
2409 		if (ret)
2410 			goto error_free;
2411 
2412 		ret = ib_send_cm_rtu(cm_id, NULL, 0);
2413 	}
2414 
2415 error_free:
2416 	kfree(qp_attr);
2417 
2418 error:
2419 	ch->status = ret;
2420 }
2421 
2422 static void srp_ib_cm_rej_handler(struct ib_cm_id *cm_id,
2423 				  const struct ib_cm_event *event,
2424 				  struct srp_rdma_ch *ch)
2425 {
2426 	struct srp_target_port *target = ch->target;
2427 	struct Scsi_Host *shost = target->scsi_host;
2428 	struct ib_class_port_info *cpi;
2429 	int opcode;
2430 	u16 dlid;
2431 
2432 	switch (event->param.rej_rcvd.reason) {
2433 	case IB_CM_REJ_PORT_CM_REDIRECT:
2434 		cpi = event->param.rej_rcvd.ari;
2435 		dlid = be16_to_cpu(cpi->redirect_lid);
2436 		sa_path_set_dlid(&ch->ib_cm.path, dlid);
2437 		ch->ib_cm.path.pkey = cpi->redirect_pkey;
2438 		cm_id->remote_cm_qpn = be32_to_cpu(cpi->redirect_qp) & 0x00ffffff;
2439 		memcpy(ch->ib_cm.path.dgid.raw, cpi->redirect_gid, 16);
2440 
2441 		ch->status = dlid ? SRP_DLID_REDIRECT : SRP_PORT_REDIRECT;
2442 		break;
2443 
2444 	case IB_CM_REJ_PORT_REDIRECT:
2445 		if (srp_target_is_topspin(target)) {
2446 			union ib_gid *dgid = &ch->ib_cm.path.dgid;
2447 
2448 			/*
2449 			 * Topspin/Cisco SRP gateways incorrectly send
2450 			 * reject reason code 25 when they mean 24
2451 			 * (port redirect).
2452 			 */
2453 			memcpy(dgid->raw, event->param.rej_rcvd.ari, 16);
2454 
2455 			shost_printk(KERN_DEBUG, shost,
2456 				     PFX "Topspin/Cisco redirect to target port GID %016llx%016llx\n",
2457 				     be64_to_cpu(dgid->global.subnet_prefix),
2458 				     be64_to_cpu(dgid->global.interface_id));
2459 
2460 			ch->status = SRP_PORT_REDIRECT;
2461 		} else {
2462 			shost_printk(KERN_WARNING, shost,
2463 				     "  REJ reason: IB_CM_REJ_PORT_REDIRECT\n");
2464 			ch->status = -ECONNRESET;
2465 		}
2466 		break;
2467 
2468 	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2469 		shost_printk(KERN_WARNING, shost,
2470 			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2471 		ch->status = -ECONNRESET;
2472 		break;
2473 
2474 	case IB_CM_REJ_CONSUMER_DEFINED:
2475 		opcode = *(u8 *) event->private_data;
2476 		if (opcode == SRP_LOGIN_REJ) {
2477 			struct srp_login_rej *rej = event->private_data;
2478 			u32 reason = be32_to_cpu(rej->reason);
2479 
2480 			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2481 				shost_printk(KERN_WARNING, shost,
2482 					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2483 			else
2484 				shost_printk(KERN_WARNING, shost, PFX
2485 					     "SRP LOGIN from %pI6 to %pI6 REJECTED, reason 0x%08x\n",
2486 					     target->sgid.raw,
2487 					     target->ib_cm.orig_dgid.raw,
2488 					     reason);
2489 		} else
2490 			shost_printk(KERN_WARNING, shost,
2491 				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED,"
2492 				     " opcode 0x%02x\n", opcode);
2493 		ch->status = -ECONNRESET;
2494 		break;
2495 
2496 	case IB_CM_REJ_STALE_CONN:
2497 		shost_printk(KERN_WARNING, shost, "  REJ reason: stale connection\n");
2498 		ch->status = SRP_STALE_CONN;
2499 		break;
2500 
2501 	default:
2502 		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2503 			     event->param.rej_rcvd.reason);
2504 		ch->status = -ECONNRESET;
2505 	}
2506 }
2507 
2508 static int srp_ib_cm_handler(struct ib_cm_id *cm_id,
2509 			     const struct ib_cm_event *event)
2510 {
2511 	struct srp_rdma_ch *ch = cm_id->context;
2512 	struct srp_target_port *target = ch->target;
2513 	int comp = 0;
2514 
2515 	switch (event->event) {
2516 	case IB_CM_REQ_ERROR:
2517 		shost_printk(KERN_DEBUG, target->scsi_host,
2518 			     PFX "Sending CM REQ failed\n");
2519 		comp = 1;
2520 		ch->status = -ECONNRESET;
2521 		break;
2522 
2523 	case IB_CM_REP_RECEIVED:
2524 		comp = 1;
2525 		srp_cm_rep_handler(cm_id, event->private_data, ch);
2526 		break;
2527 
2528 	case IB_CM_REJ_RECEIVED:
2529 		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2530 		comp = 1;
2531 
2532 		srp_ib_cm_rej_handler(cm_id, event, ch);
2533 		break;
2534 
2535 	case IB_CM_DREQ_RECEIVED:
2536 		shost_printk(KERN_WARNING, target->scsi_host,
2537 			     PFX "DREQ received - connection closed\n");
2538 		ch->connected = false;
2539 		if (ib_send_cm_drep(cm_id, NULL, 0))
2540 			shost_printk(KERN_ERR, target->scsi_host,
2541 				     PFX "Sending CM DREP failed\n");
2542 		queue_work(system_long_wq, &target->tl_err_work);
2543 		break;
2544 
2545 	case IB_CM_TIMEWAIT_EXIT:
2546 		shost_printk(KERN_ERR, target->scsi_host,
2547 			     PFX "connection closed\n");
2548 		comp = 1;
2549 
2550 		ch->status = 0;
2551 		break;
2552 
2553 	case IB_CM_MRA_RECEIVED:
2554 	case IB_CM_DREQ_ERROR:
2555 	case IB_CM_DREP_RECEIVED:
2556 		break;
2557 
2558 	default:
2559 		shost_printk(KERN_WARNING, target->scsi_host,
2560 			     PFX "Unhandled CM event %d\n", event->event);
2561 		break;
2562 	}
2563 
2564 	if (comp)
2565 		complete(&ch->done);
2566 
2567 	return 0;
2568 }
2569 
2570 static void srp_rdma_cm_rej_handler(struct srp_rdma_ch *ch,
2571 				    struct rdma_cm_event *event)
2572 {
2573 	struct srp_target_port *target = ch->target;
2574 	struct Scsi_Host *shost = target->scsi_host;
2575 	int opcode;
2576 
2577 	switch (event->status) {
2578 	case IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID:
2579 		shost_printk(KERN_WARNING, shost,
2580 			    "  REJ reason: IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID\n");
2581 		ch->status = -ECONNRESET;
2582 		break;
2583 
2584 	case IB_CM_REJ_CONSUMER_DEFINED:
2585 		opcode = *(u8 *) event->param.conn.private_data;
2586 		if (opcode == SRP_LOGIN_REJ) {
2587 			struct srp_login_rej *rej =
2588 				(struct srp_login_rej *)
2589 				event->param.conn.private_data;
2590 			u32 reason = be32_to_cpu(rej->reason);
2591 
2592 			if (reason == SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE)
2593 				shost_printk(KERN_WARNING, shost,
2594 					     PFX "SRP_LOGIN_REJ: requested max_it_iu_len too large\n");
2595 			else
2596 				shost_printk(KERN_WARNING, shost,
2597 					    PFX "SRP LOGIN REJECTED, reason 0x%08x\n", reason);
2598 		} else {
2599 			shost_printk(KERN_WARNING, shost,
2600 				     "  REJ reason: IB_CM_REJ_CONSUMER_DEFINED, opcode 0x%02x\n",
2601 				     opcode);
2602 		}
2603 		ch->status = -ECONNRESET;
2604 		break;
2605 
2606 	case IB_CM_REJ_STALE_CONN:
2607 		shost_printk(KERN_WARNING, shost,
2608 			     "  REJ reason: stale connection\n");
2609 		ch->status = SRP_STALE_CONN;
2610 		break;
2611 
2612 	default:
2613 		shost_printk(KERN_WARNING, shost, "  REJ reason 0x%x\n",
2614 			     event->status);
2615 		ch->status = -ECONNRESET;
2616 		break;
2617 	}
2618 }
2619 
2620 static int srp_rdma_cm_handler(struct rdma_cm_id *cm_id,
2621 			       struct rdma_cm_event *event)
2622 {
2623 	struct srp_rdma_ch *ch = cm_id->context;
2624 	struct srp_target_port *target = ch->target;
2625 	int comp = 0;
2626 
2627 	switch (event->event) {
2628 	case RDMA_CM_EVENT_ADDR_RESOLVED:
2629 		ch->status = 0;
2630 		comp = 1;
2631 		break;
2632 
2633 	case RDMA_CM_EVENT_ADDR_ERROR:
2634 		ch->status = -ENXIO;
2635 		comp = 1;
2636 		break;
2637 
2638 	case RDMA_CM_EVENT_ROUTE_RESOLVED:
2639 		ch->status = 0;
2640 		comp = 1;
2641 		break;
2642 
2643 	case RDMA_CM_EVENT_ROUTE_ERROR:
2644 	case RDMA_CM_EVENT_UNREACHABLE:
2645 		ch->status = -EHOSTUNREACH;
2646 		comp = 1;
2647 		break;
2648 
2649 	case RDMA_CM_EVENT_CONNECT_ERROR:
2650 		shost_printk(KERN_DEBUG, target->scsi_host,
2651 			     PFX "Sending CM REQ failed\n");
2652 		comp = 1;
2653 		ch->status = -ECONNRESET;
2654 		break;
2655 
2656 	case RDMA_CM_EVENT_ESTABLISHED:
2657 		comp = 1;
2658 		srp_cm_rep_handler(NULL, event->param.conn.private_data, ch);
2659 		break;
2660 
2661 	case RDMA_CM_EVENT_REJECTED:
2662 		shost_printk(KERN_DEBUG, target->scsi_host, PFX "REJ received\n");
2663 		comp = 1;
2664 
2665 		srp_rdma_cm_rej_handler(ch, event);
2666 		break;
2667 
2668 	case RDMA_CM_EVENT_DISCONNECTED:
2669 		if (ch->connected) {
2670 			shost_printk(KERN_WARNING, target->scsi_host,
2671 				     PFX "received DREQ\n");
2672 			rdma_disconnect(ch->rdma_cm.cm_id);
2673 			comp = 1;
2674 			ch->status = 0;
2675 			queue_work(system_long_wq, &target->tl_err_work);
2676 		}
2677 		break;
2678 
2679 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
2680 		shost_printk(KERN_ERR, target->scsi_host,
2681 			     PFX "connection closed\n");
2682 
2683 		comp = 1;
2684 		ch->status = 0;
2685 		break;
2686 
2687 	default:
2688 		shost_printk(KERN_WARNING, target->scsi_host,
2689 			     PFX "Unhandled CM event %d\n", event->event);
2690 		break;
2691 	}
2692 
2693 	if (comp)
2694 		complete(&ch->done);
2695 
2696 	return 0;
2697 }
2698 
2699 /**
2700  * srp_change_queue_depth - setting device queue depth
2701  * @sdev: scsi device struct
2702  * @qdepth: requested queue depth
2703  *
2704  * Returns queue depth.
2705  */
2706 static int
2707 srp_change_queue_depth(struct scsi_device *sdev, int qdepth)
2708 {
2709 	if (!sdev->tagged_supported)
2710 		qdepth = 1;
2711 	return scsi_change_queue_depth(sdev, qdepth);
2712 }
2713 
2714 static int srp_send_tsk_mgmt(struct srp_rdma_ch *ch, u64 req_tag, u64 lun,
2715 			     u8 func, u8 *status)
2716 {
2717 	struct srp_target_port *target = ch->target;
2718 	struct srp_rport *rport = target->rport;
2719 	struct ib_device *dev = target->srp_host->srp_dev->dev;
2720 	struct srp_iu *iu;
2721 	struct srp_tsk_mgmt *tsk_mgmt;
2722 	int res;
2723 
2724 	if (!ch->connected || target->qp_in_error)
2725 		return -1;
2726 
2727 	/*
2728 	 * Lock the rport mutex to avoid that srp_create_ch_ib() is
2729 	 * invoked while a task management function is being sent.
2730 	 */
2731 	mutex_lock(&rport->mutex);
2732 	spin_lock_irq(&ch->lock);
2733 	iu = __srp_get_tx_iu(ch, SRP_IU_TSK_MGMT);
2734 	spin_unlock_irq(&ch->lock);
2735 
2736 	if (!iu) {
2737 		mutex_unlock(&rport->mutex);
2738 
2739 		return -1;
2740 	}
2741 
2742 	iu->num_sge = 1;
2743 
2744 	ib_dma_sync_single_for_cpu(dev, iu->dma, sizeof *tsk_mgmt,
2745 				   DMA_TO_DEVICE);
2746 	tsk_mgmt = iu->buf;
2747 	memset(tsk_mgmt, 0, sizeof *tsk_mgmt);
2748 
2749 	tsk_mgmt->opcode 	= SRP_TSK_MGMT;
2750 	int_to_scsilun(lun, &tsk_mgmt->lun);
2751 	tsk_mgmt->tsk_mgmt_func = func;
2752 	tsk_mgmt->task_tag	= req_tag;
2753 
2754 	spin_lock_irq(&ch->lock);
2755 	ch->tsk_mgmt_tag = (ch->tsk_mgmt_tag + 1) | SRP_TAG_TSK_MGMT;
2756 	tsk_mgmt->tag = ch->tsk_mgmt_tag;
2757 	spin_unlock_irq(&ch->lock);
2758 
2759 	init_completion(&ch->tsk_mgmt_done);
2760 
2761 	ib_dma_sync_single_for_device(dev, iu->dma, sizeof *tsk_mgmt,
2762 				      DMA_TO_DEVICE);
2763 	if (srp_post_send(ch, iu, sizeof(*tsk_mgmt))) {
2764 		srp_put_tx_iu(ch, iu, SRP_IU_TSK_MGMT);
2765 		mutex_unlock(&rport->mutex);
2766 
2767 		return -1;
2768 	}
2769 	res = wait_for_completion_timeout(&ch->tsk_mgmt_done,
2770 					msecs_to_jiffies(SRP_ABORT_TIMEOUT_MS));
2771 	if (res > 0 && status)
2772 		*status = ch->tsk_mgmt_status;
2773 	mutex_unlock(&rport->mutex);
2774 
2775 	WARN_ON_ONCE(res < 0);
2776 
2777 	return res > 0 ? 0 : -1;
2778 }
2779 
2780 static int srp_abort(struct scsi_cmnd *scmnd)
2781 {
2782 	struct srp_target_port *target = host_to_target(scmnd->device->host);
2783 	struct srp_request *req = scsi_cmd_priv(scmnd);
2784 	u32 tag;
2785 	u16 ch_idx;
2786 	struct srp_rdma_ch *ch;
2787 
2788 	shost_printk(KERN_ERR, target->scsi_host, "SRP abort called\n");
2789 
2790 	tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmnd));
2791 	ch_idx = blk_mq_unique_tag_to_hwq(tag);
2792 	if (WARN_ON_ONCE(ch_idx >= target->ch_count))
2793 		return SUCCESS;
2794 	ch = &target->ch[ch_idx];
2795 	if (!srp_claim_req(ch, req, NULL, scmnd))
2796 		return SUCCESS;
2797 	shost_printk(KERN_ERR, target->scsi_host,
2798 		     "Sending SRP abort for tag %#x\n", tag);
2799 	if (srp_send_tsk_mgmt(ch, tag, scmnd->device->lun,
2800 			      SRP_TSK_ABORT_TASK, NULL) == 0) {
2801 		srp_free_req(ch, req, scmnd, 0);
2802 		return SUCCESS;
2803 	}
2804 	if (target->rport->state == SRP_RPORT_LOST)
2805 		return FAST_IO_FAIL;
2806 
2807 	return FAILED;
2808 }
2809 
2810 static int srp_reset_device(struct scsi_cmnd *scmnd)
2811 {
2812 	struct srp_target_port *target = host_to_target(scmnd->device->host);
2813 	struct srp_rdma_ch *ch;
2814 	u8 status;
2815 
2816 	shost_printk(KERN_ERR, target->scsi_host, "SRP reset_device called\n");
2817 
2818 	ch = &target->ch[0];
2819 	if (srp_send_tsk_mgmt(ch, SRP_TAG_NO_REQ, scmnd->device->lun,
2820 			      SRP_TSK_LUN_RESET, &status))
2821 		return FAILED;
2822 	if (status)
2823 		return FAILED;
2824 
2825 	return SUCCESS;
2826 }
2827 
2828 static int srp_reset_host(struct scsi_cmnd *scmnd)
2829 {
2830 	struct srp_target_port *target = host_to_target(scmnd->device->host);
2831 
2832 	shost_printk(KERN_ERR, target->scsi_host, PFX "SRP reset_host called\n");
2833 
2834 	return srp_reconnect_rport(target->rport) == 0 ? SUCCESS : FAILED;
2835 }
2836 
2837 static int srp_target_alloc(struct scsi_target *starget)
2838 {
2839 	struct Scsi_Host *shost = dev_to_shost(starget->dev.parent);
2840 	struct srp_target_port *target = host_to_target(shost);
2841 
2842 	if (target->target_can_queue)
2843 		starget->can_queue = target->target_can_queue;
2844 	return 0;
2845 }
2846 
2847 static int srp_slave_configure(struct scsi_device *sdev)
2848 {
2849 	struct Scsi_Host *shost = sdev->host;
2850 	struct srp_target_port *target = host_to_target(shost);
2851 	struct request_queue *q = sdev->request_queue;
2852 	unsigned long timeout;
2853 
2854 	if (sdev->type == TYPE_DISK) {
2855 		timeout = max_t(unsigned, 30 * HZ, target->rq_tmo_jiffies);
2856 		blk_queue_rq_timeout(q, timeout);
2857 	}
2858 
2859 	return 0;
2860 }
2861 
2862 static ssize_t id_ext_show(struct device *dev, struct device_attribute *attr,
2863 			   char *buf)
2864 {
2865 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2866 
2867 	return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->id_ext));
2868 }
2869 
2870 static DEVICE_ATTR_RO(id_ext);
2871 
2872 static ssize_t ioc_guid_show(struct device *dev, struct device_attribute *attr,
2873 			     char *buf)
2874 {
2875 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2876 
2877 	return sysfs_emit(buf, "0x%016llx\n", be64_to_cpu(target->ioc_guid));
2878 }
2879 
2880 static DEVICE_ATTR_RO(ioc_guid);
2881 
2882 static ssize_t service_id_show(struct device *dev,
2883 			       struct device_attribute *attr, char *buf)
2884 {
2885 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2886 
2887 	if (target->using_rdma_cm)
2888 		return -ENOENT;
2889 	return sysfs_emit(buf, "0x%016llx\n",
2890 			  be64_to_cpu(target->ib_cm.service_id));
2891 }
2892 
2893 static DEVICE_ATTR_RO(service_id);
2894 
2895 static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,
2896 			 char *buf)
2897 {
2898 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2899 
2900 	if (target->using_rdma_cm)
2901 		return -ENOENT;
2902 
2903 	return sysfs_emit(buf, "0x%04x\n", be16_to_cpu(target->ib_cm.pkey));
2904 }
2905 
2906 static DEVICE_ATTR_RO(pkey);
2907 
2908 static ssize_t sgid_show(struct device *dev, struct device_attribute *attr,
2909 			 char *buf)
2910 {
2911 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2912 
2913 	return sysfs_emit(buf, "%pI6\n", target->sgid.raw);
2914 }
2915 
2916 static DEVICE_ATTR_RO(sgid);
2917 
2918 static ssize_t dgid_show(struct device *dev, struct device_attribute *attr,
2919 			 char *buf)
2920 {
2921 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2922 	struct srp_rdma_ch *ch = &target->ch[0];
2923 
2924 	if (target->using_rdma_cm)
2925 		return -ENOENT;
2926 
2927 	return sysfs_emit(buf, "%pI6\n", ch->ib_cm.path.dgid.raw);
2928 }
2929 
2930 static DEVICE_ATTR_RO(dgid);
2931 
2932 static ssize_t orig_dgid_show(struct device *dev, struct device_attribute *attr,
2933 			      char *buf)
2934 {
2935 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2936 
2937 	if (target->using_rdma_cm)
2938 		return -ENOENT;
2939 
2940 	return sysfs_emit(buf, "%pI6\n", target->ib_cm.orig_dgid.raw);
2941 }
2942 
2943 static DEVICE_ATTR_RO(orig_dgid);
2944 
2945 static ssize_t req_lim_show(struct device *dev, struct device_attribute *attr,
2946 			    char *buf)
2947 {
2948 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2949 	struct srp_rdma_ch *ch;
2950 	int i, req_lim = INT_MAX;
2951 
2952 	for (i = 0; i < target->ch_count; i++) {
2953 		ch = &target->ch[i];
2954 		req_lim = min(req_lim, ch->req_lim);
2955 	}
2956 
2957 	return sysfs_emit(buf, "%d\n", req_lim);
2958 }
2959 
2960 static DEVICE_ATTR_RO(req_lim);
2961 
2962 static ssize_t zero_req_lim_show(struct device *dev,
2963 				 struct device_attribute *attr, char *buf)
2964 {
2965 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2966 
2967 	return sysfs_emit(buf, "%d\n", target->zero_req_lim);
2968 }
2969 
2970 static DEVICE_ATTR_RO(zero_req_lim);
2971 
2972 static ssize_t local_ib_port_show(struct device *dev,
2973 				  struct device_attribute *attr, char *buf)
2974 {
2975 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2976 
2977 	return sysfs_emit(buf, "%u\n", target->srp_host->port);
2978 }
2979 
2980 static DEVICE_ATTR_RO(local_ib_port);
2981 
2982 static ssize_t local_ib_device_show(struct device *dev,
2983 				    struct device_attribute *attr, char *buf)
2984 {
2985 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2986 
2987 	return sysfs_emit(buf, "%s\n",
2988 			  dev_name(&target->srp_host->srp_dev->dev->dev));
2989 }
2990 
2991 static DEVICE_ATTR_RO(local_ib_device);
2992 
2993 static ssize_t ch_count_show(struct device *dev, struct device_attribute *attr,
2994 			     char *buf)
2995 {
2996 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
2997 
2998 	return sysfs_emit(buf, "%d\n", target->ch_count);
2999 }
3000 
3001 static DEVICE_ATTR_RO(ch_count);
3002 
3003 static ssize_t comp_vector_show(struct device *dev,
3004 				struct device_attribute *attr, char *buf)
3005 {
3006 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3007 
3008 	return sysfs_emit(buf, "%d\n", target->comp_vector);
3009 }
3010 
3011 static DEVICE_ATTR_RO(comp_vector);
3012 
3013 static ssize_t tl_retry_count_show(struct device *dev,
3014 				   struct device_attribute *attr, char *buf)
3015 {
3016 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3017 
3018 	return sysfs_emit(buf, "%d\n", target->tl_retry_count);
3019 }
3020 
3021 static DEVICE_ATTR_RO(tl_retry_count);
3022 
3023 static ssize_t cmd_sg_entries_show(struct device *dev,
3024 				   struct device_attribute *attr, char *buf)
3025 {
3026 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3027 
3028 	return sysfs_emit(buf, "%u\n", target->cmd_sg_cnt);
3029 }
3030 
3031 static DEVICE_ATTR_RO(cmd_sg_entries);
3032 
3033 static ssize_t allow_ext_sg_show(struct device *dev,
3034 				 struct device_attribute *attr, char *buf)
3035 {
3036 	struct srp_target_port *target = host_to_target(class_to_shost(dev));
3037 
3038 	return sysfs_emit(buf, "%s\n", target->allow_ext_sg ? "true" : "false");
3039 }
3040 
3041 static DEVICE_ATTR_RO(allow_ext_sg);
3042 
3043 static struct attribute *srp_host_attrs[] = {
3044 	&dev_attr_id_ext.attr,
3045 	&dev_attr_ioc_guid.attr,
3046 	&dev_attr_service_id.attr,
3047 	&dev_attr_pkey.attr,
3048 	&dev_attr_sgid.attr,
3049 	&dev_attr_dgid.attr,
3050 	&dev_attr_orig_dgid.attr,
3051 	&dev_attr_req_lim.attr,
3052 	&dev_attr_zero_req_lim.attr,
3053 	&dev_attr_local_ib_port.attr,
3054 	&dev_attr_local_ib_device.attr,
3055 	&dev_attr_ch_count.attr,
3056 	&dev_attr_comp_vector.attr,
3057 	&dev_attr_tl_retry_count.attr,
3058 	&dev_attr_cmd_sg_entries.attr,
3059 	&dev_attr_allow_ext_sg.attr,
3060 	NULL
3061 };
3062 
3063 ATTRIBUTE_GROUPS(srp_host);
3064 
3065 static const struct scsi_host_template srp_template = {
3066 	.module				= THIS_MODULE,
3067 	.name				= "InfiniBand SRP initiator",
3068 	.proc_name			= DRV_NAME,
3069 	.target_alloc			= srp_target_alloc,
3070 	.slave_configure		= srp_slave_configure,
3071 	.info				= srp_target_info,
3072 	.init_cmd_priv			= srp_init_cmd_priv,
3073 	.exit_cmd_priv			= srp_exit_cmd_priv,
3074 	.queuecommand			= srp_queuecommand,
3075 	.change_queue_depth             = srp_change_queue_depth,
3076 	.eh_timed_out			= srp_timed_out,
3077 	.eh_abort_handler		= srp_abort,
3078 	.eh_device_reset_handler	= srp_reset_device,
3079 	.eh_host_reset_handler		= srp_reset_host,
3080 	.skip_settle_delay		= true,
3081 	.sg_tablesize			= SRP_DEF_SG_TABLESIZE,
3082 	.can_queue			= SRP_DEFAULT_CMD_SQ_SIZE,
3083 	.this_id			= -1,
3084 	.cmd_per_lun			= SRP_DEFAULT_CMD_SQ_SIZE,
3085 	.shost_groups			= srp_host_groups,
3086 	.track_queue_depth		= 1,
3087 	.cmd_size			= sizeof(struct srp_request),
3088 };
3089 
3090 static int srp_sdev_count(struct Scsi_Host *host)
3091 {
3092 	struct scsi_device *sdev;
3093 	int c = 0;
3094 
3095 	shost_for_each_device(sdev, host)
3096 		c++;
3097 
3098 	return c;
3099 }
3100 
3101 /*
3102  * Return values:
3103  * < 0 upon failure. Caller is responsible for SRP target port cleanup.
3104  * 0 and target->state == SRP_TARGET_REMOVED if asynchronous target port
3105  *    removal has been scheduled.
3106  * 0 and target->state != SRP_TARGET_REMOVED upon success.
3107  */
3108 static int srp_add_target(struct srp_host *host, struct srp_target_port *target)
3109 {
3110 	struct srp_rport_identifiers ids;
3111 	struct srp_rport *rport;
3112 
3113 	target->state = SRP_TARGET_SCANNING;
3114 	sprintf(target->target_name, "SRP.T10:%016llX",
3115 		be64_to_cpu(target->id_ext));
3116 
3117 	if (scsi_add_host(target->scsi_host, host->srp_dev->dev->dev.parent))
3118 		return -ENODEV;
3119 
3120 	memcpy(ids.port_id, &target->id_ext, 8);
3121 	memcpy(ids.port_id + 8, &target->ioc_guid, 8);
3122 	ids.roles = SRP_RPORT_ROLE_TARGET;
3123 	rport = srp_rport_add(target->scsi_host, &ids);
3124 	if (IS_ERR(rport)) {
3125 		scsi_remove_host(target->scsi_host);
3126 		return PTR_ERR(rport);
3127 	}
3128 
3129 	rport->lld_data = target;
3130 	target->rport = rport;
3131 
3132 	spin_lock(&host->target_lock);
3133 	list_add_tail(&target->list, &host->target_list);
3134 	spin_unlock(&host->target_lock);
3135 
3136 	scsi_scan_target(&target->scsi_host->shost_gendev,
3137 			 0, target->scsi_id, SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
3138 
3139 	if (srp_connected_ch(target) < target->ch_count ||
3140 	    target->qp_in_error) {
3141 		shost_printk(KERN_INFO, target->scsi_host,
3142 			     PFX "SCSI scan failed - removing SCSI host\n");
3143 		srp_queue_remove_work(target);
3144 		goto out;
3145 	}
3146 
3147 	pr_debug("%s: SCSI scan succeeded - detected %d LUNs\n",
3148 		 dev_name(&target->scsi_host->shost_gendev),
3149 		 srp_sdev_count(target->scsi_host));
3150 
3151 	spin_lock_irq(&target->lock);
3152 	if (target->state == SRP_TARGET_SCANNING)
3153 		target->state = SRP_TARGET_LIVE;
3154 	spin_unlock_irq(&target->lock);
3155 
3156 out:
3157 	return 0;
3158 }
3159 
3160 static void srp_release_dev(struct device *dev)
3161 {
3162 	struct srp_host *host =
3163 		container_of(dev, struct srp_host, dev);
3164 
3165 	kfree(host);
3166 }
3167 
3168 static struct attribute *srp_class_attrs[];
3169 
3170 ATTRIBUTE_GROUPS(srp_class);
3171 
3172 static struct class srp_class = {
3173 	.name    = "infiniband_srp",
3174 	.dev_groups = srp_class_groups,
3175 	.dev_release = srp_release_dev
3176 };
3177 
3178 /**
3179  * srp_conn_unique() - check whether the connection to a target is unique
3180  * @host:   SRP host.
3181  * @target: SRP target port.
3182  */
3183 static bool srp_conn_unique(struct srp_host *host,
3184 			    struct srp_target_port *target)
3185 {
3186 	struct srp_target_port *t;
3187 	bool ret = false;
3188 
3189 	if (target->state == SRP_TARGET_REMOVED)
3190 		goto out;
3191 
3192 	ret = true;
3193 
3194 	spin_lock(&host->target_lock);
3195 	list_for_each_entry(t, &host->target_list, list) {
3196 		if (t != target &&
3197 		    target->id_ext == t->id_ext &&
3198 		    target->ioc_guid == t->ioc_guid &&
3199 		    target->initiator_ext == t->initiator_ext) {
3200 			ret = false;
3201 			break;
3202 		}
3203 	}
3204 	spin_unlock(&host->target_lock);
3205 
3206 out:
3207 	return ret;
3208 }
3209 
3210 /*
3211  * Target ports are added by writing
3212  *
3213  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,dgid=<dest GID>,
3214  *     pkey=<P_Key>,service_id=<service ID>
3215  * or
3216  *     id_ext=<SRP ID ext>,ioc_guid=<SRP IOC GUID>,
3217  *     [src=<IPv4 address>,]dest=<IPv4 address>:<port number>
3218  *
3219  * to the add_target sysfs attribute.
3220  */
3221 enum {
3222 	SRP_OPT_ERR		= 0,
3223 	SRP_OPT_ID_EXT		= 1 << 0,
3224 	SRP_OPT_IOC_GUID	= 1 << 1,
3225 	SRP_OPT_DGID		= 1 << 2,
3226 	SRP_OPT_PKEY		= 1 << 3,
3227 	SRP_OPT_SERVICE_ID	= 1 << 4,
3228 	SRP_OPT_MAX_SECT	= 1 << 5,
3229 	SRP_OPT_MAX_CMD_PER_LUN	= 1 << 6,
3230 	SRP_OPT_IO_CLASS	= 1 << 7,
3231 	SRP_OPT_INITIATOR_EXT	= 1 << 8,
3232 	SRP_OPT_CMD_SG_ENTRIES	= 1 << 9,
3233 	SRP_OPT_ALLOW_EXT_SG	= 1 << 10,
3234 	SRP_OPT_SG_TABLESIZE	= 1 << 11,
3235 	SRP_OPT_COMP_VECTOR	= 1 << 12,
3236 	SRP_OPT_TL_RETRY_COUNT	= 1 << 13,
3237 	SRP_OPT_QUEUE_SIZE	= 1 << 14,
3238 	SRP_OPT_IP_SRC		= 1 << 15,
3239 	SRP_OPT_IP_DEST		= 1 << 16,
3240 	SRP_OPT_TARGET_CAN_QUEUE= 1 << 17,
3241 	SRP_OPT_MAX_IT_IU_SIZE  = 1 << 18,
3242 	SRP_OPT_CH_COUNT	= 1 << 19,
3243 };
3244 
3245 static unsigned int srp_opt_mandatory[] = {
3246 	SRP_OPT_ID_EXT		|
3247 	SRP_OPT_IOC_GUID	|
3248 	SRP_OPT_DGID		|
3249 	SRP_OPT_PKEY		|
3250 	SRP_OPT_SERVICE_ID,
3251 	SRP_OPT_ID_EXT		|
3252 	SRP_OPT_IOC_GUID	|
3253 	SRP_OPT_IP_DEST,
3254 };
3255 
3256 static const match_table_t srp_opt_tokens = {
3257 	{ SRP_OPT_ID_EXT,		"id_ext=%s" 		},
3258 	{ SRP_OPT_IOC_GUID,		"ioc_guid=%s" 		},
3259 	{ SRP_OPT_DGID,			"dgid=%s" 		},
3260 	{ SRP_OPT_PKEY,			"pkey=%x" 		},
3261 	{ SRP_OPT_SERVICE_ID,		"service_id=%s"		},
3262 	{ SRP_OPT_MAX_SECT,		"max_sect=%d" 		},
3263 	{ SRP_OPT_MAX_CMD_PER_LUN,	"max_cmd_per_lun=%d" 	},
3264 	{ SRP_OPT_TARGET_CAN_QUEUE,	"target_can_queue=%d"	},
3265 	{ SRP_OPT_IO_CLASS,		"io_class=%x"		},
3266 	{ SRP_OPT_INITIATOR_EXT,	"initiator_ext=%s"	},
3267 	{ SRP_OPT_CMD_SG_ENTRIES,	"cmd_sg_entries=%u"	},
3268 	{ SRP_OPT_ALLOW_EXT_SG,		"allow_ext_sg=%u"	},
3269 	{ SRP_OPT_SG_TABLESIZE,		"sg_tablesize=%u"	},
3270 	{ SRP_OPT_COMP_VECTOR,		"comp_vector=%u"	},
3271 	{ SRP_OPT_TL_RETRY_COUNT,	"tl_retry_count=%u"	},
3272 	{ SRP_OPT_QUEUE_SIZE,		"queue_size=%d"		},
3273 	{ SRP_OPT_IP_SRC,		"src=%s"		},
3274 	{ SRP_OPT_IP_DEST,		"dest=%s"		},
3275 	{ SRP_OPT_MAX_IT_IU_SIZE,	"max_it_iu_size=%d"	},
3276 	{ SRP_OPT_CH_COUNT,		"ch_count=%u",		},
3277 	{ SRP_OPT_ERR,			NULL 			}
3278 };
3279 
3280 /**
3281  * srp_parse_in - parse an IP address and port number combination
3282  * @net:	   [in]  Network namespace.
3283  * @sa:		   [out] Address family, IP address and port number.
3284  * @addr_port_str: [in]  IP address and port number.
3285  * @has_port:	   [out] Whether or not @addr_port_str includes a port number.
3286  *
3287  * Parse the following address formats:
3288  * - IPv4: <ip_address>:<port>, e.g. 1.2.3.4:5.
3289  * - IPv6: \[<ipv6_address>\]:<port>, e.g. [1::2:3%4]:5.
3290  */
3291 static int srp_parse_in(struct net *net, struct sockaddr_storage *sa,
3292 			const char *addr_port_str, bool *has_port)
3293 {
3294 	char *addr_end, *addr = kstrdup(addr_port_str, GFP_KERNEL);
3295 	char *port_str;
3296 	int ret;
3297 
3298 	if (!addr)
3299 		return -ENOMEM;
3300 	port_str = strrchr(addr, ':');
3301 	if (port_str && strchr(port_str, ']'))
3302 		port_str = NULL;
3303 	if (port_str)
3304 		*port_str++ = '\0';
3305 	if (has_port)
3306 		*has_port = port_str != NULL;
3307 	ret = inet_pton_with_scope(net, AF_INET, addr, port_str, sa);
3308 	if (ret && addr[0]) {
3309 		addr_end = addr + strlen(addr) - 1;
3310 		if (addr[0] == '[' && *addr_end == ']') {
3311 			*addr_end = '\0';
3312 			ret = inet_pton_with_scope(net, AF_INET6, addr + 1,
3313 						   port_str, sa);
3314 		}
3315 	}
3316 	kfree(addr);
3317 	pr_debug("%s -> %pISpfsc\n", addr_port_str, sa);
3318 	return ret;
3319 }
3320 
3321 static int srp_parse_options(struct net *net, const char *buf,
3322 			     struct srp_target_port *target)
3323 {
3324 	char *options, *sep_opt;
3325 	char *p;
3326 	substring_t args[MAX_OPT_ARGS];
3327 	unsigned long long ull;
3328 	bool has_port;
3329 	int opt_mask = 0;
3330 	int token;
3331 	int ret = -EINVAL;
3332 	int i;
3333 
3334 	options = kstrdup(buf, GFP_KERNEL);
3335 	if (!options)
3336 		return -ENOMEM;
3337 
3338 	sep_opt = options;
3339 	while ((p = strsep(&sep_opt, ",\n")) != NULL) {
3340 		if (!*p)
3341 			continue;
3342 
3343 		token = match_token(p, srp_opt_tokens, args);
3344 		opt_mask |= token;
3345 
3346 		switch (token) {
3347 		case SRP_OPT_ID_EXT:
3348 			p = match_strdup(args);
3349 			if (!p) {
3350 				ret = -ENOMEM;
3351 				goto out;
3352 			}
3353 			ret = kstrtoull(p, 16, &ull);
3354 			if (ret) {
3355 				pr_warn("invalid id_ext parameter '%s'\n", p);
3356 				kfree(p);
3357 				goto out;
3358 			}
3359 			target->id_ext = cpu_to_be64(ull);
3360 			kfree(p);
3361 			break;
3362 
3363 		case SRP_OPT_IOC_GUID:
3364 			p = match_strdup(args);
3365 			if (!p) {
3366 				ret = -ENOMEM;
3367 				goto out;
3368 			}
3369 			ret = kstrtoull(p, 16, &ull);
3370 			if (ret) {
3371 				pr_warn("invalid ioc_guid parameter '%s'\n", p);
3372 				kfree(p);
3373 				goto out;
3374 			}
3375 			target->ioc_guid = cpu_to_be64(ull);
3376 			kfree(p);
3377 			break;
3378 
3379 		case SRP_OPT_DGID:
3380 			p = match_strdup(args);
3381 			if (!p) {
3382 				ret = -ENOMEM;
3383 				goto out;
3384 			}
3385 			if (strlen(p) != 32) {
3386 				pr_warn("bad dest GID parameter '%s'\n", p);
3387 				kfree(p);
3388 				goto out;
3389 			}
3390 
3391 			ret = hex2bin(target->ib_cm.orig_dgid.raw, p, 16);
3392 			kfree(p);
3393 			if (ret < 0)
3394 				goto out;
3395 			break;
3396 
3397 		case SRP_OPT_PKEY:
3398 			ret = match_hex(args, &token);
3399 			if (ret) {
3400 				pr_warn("bad P_Key parameter '%s'\n", p);
3401 				goto out;
3402 			}
3403 			target->ib_cm.pkey = cpu_to_be16(token);
3404 			break;
3405 
3406 		case SRP_OPT_SERVICE_ID:
3407 			p = match_strdup(args);
3408 			if (!p) {
3409 				ret = -ENOMEM;
3410 				goto out;
3411 			}
3412 			ret = kstrtoull(p, 16, &ull);
3413 			if (ret) {
3414 				pr_warn("bad service_id parameter '%s'\n", p);
3415 				kfree(p);
3416 				goto out;
3417 			}
3418 			target->ib_cm.service_id = cpu_to_be64(ull);
3419 			kfree(p);
3420 			break;
3421 
3422 		case SRP_OPT_IP_SRC:
3423 			p = match_strdup(args);
3424 			if (!p) {
3425 				ret = -ENOMEM;
3426 				goto out;
3427 			}
3428 			ret = srp_parse_in(net, &target->rdma_cm.src.ss, p,
3429 					   NULL);
3430 			if (ret < 0) {
3431 				pr_warn("bad source parameter '%s'\n", p);
3432 				kfree(p);
3433 				goto out;
3434 			}
3435 			target->rdma_cm.src_specified = true;
3436 			kfree(p);
3437 			break;
3438 
3439 		case SRP_OPT_IP_DEST:
3440 			p = match_strdup(args);
3441 			if (!p) {
3442 				ret = -ENOMEM;
3443 				goto out;
3444 			}
3445 			ret = srp_parse_in(net, &target->rdma_cm.dst.ss, p,
3446 					   &has_port);
3447 			if (!has_port)
3448 				ret = -EINVAL;
3449 			if (ret < 0) {
3450 				pr_warn("bad dest parameter '%s'\n", p);
3451 				kfree(p);
3452 				goto out;
3453 			}
3454 			target->using_rdma_cm = true;
3455 			kfree(p);
3456 			break;
3457 
3458 		case SRP_OPT_MAX_SECT:
3459 			ret = match_int(args, &token);
3460 			if (ret) {
3461 				pr_warn("bad max sect parameter '%s'\n", p);
3462 				goto out;
3463 			}
3464 			target->scsi_host->max_sectors = token;
3465 			break;
3466 
3467 		case SRP_OPT_QUEUE_SIZE:
3468 			ret = match_int(args, &token);
3469 			if (ret) {
3470 				pr_warn("match_int() failed for queue_size parameter '%s', Error %d\n",
3471 					p, ret);
3472 				goto out;
3473 			}
3474 			if (token < 1) {
3475 				pr_warn("bad queue_size parameter '%s'\n", p);
3476 				ret = -EINVAL;
3477 				goto out;
3478 			}
3479 			target->scsi_host->can_queue = token;
3480 			target->queue_size = token + SRP_RSP_SQ_SIZE +
3481 					     SRP_TSK_MGMT_SQ_SIZE;
3482 			if (!(opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3483 				target->scsi_host->cmd_per_lun = token;
3484 			break;
3485 
3486 		case SRP_OPT_MAX_CMD_PER_LUN:
3487 			ret = match_int(args, &token);
3488 			if (ret) {
3489 				pr_warn("match_int() failed for max cmd_per_lun parameter '%s', Error %d\n",
3490 					p, ret);
3491 				goto out;
3492 			}
3493 			if (token < 1) {
3494 				pr_warn("bad max cmd_per_lun parameter '%s'\n",
3495 					p);
3496 				ret = -EINVAL;
3497 				goto out;
3498 			}
3499 			target->scsi_host->cmd_per_lun = token;
3500 			break;
3501 
3502 		case SRP_OPT_TARGET_CAN_QUEUE:
3503 			ret = match_int(args, &token);
3504 			if (ret) {
3505 				pr_warn("match_int() failed for max target_can_queue parameter '%s', Error %d\n",
3506 					p, ret);
3507 				goto out;
3508 			}
3509 			if (token < 1) {
3510 				pr_warn("bad max target_can_queue parameter '%s'\n",
3511 					p);
3512 				ret = -EINVAL;
3513 				goto out;
3514 			}
3515 			target->target_can_queue = token;
3516 			break;
3517 
3518 		case SRP_OPT_IO_CLASS:
3519 			ret = match_hex(args, &token);
3520 			if (ret) {
3521 				pr_warn("bad IO class parameter '%s'\n", p);
3522 				goto out;
3523 			}
3524 			if (token != SRP_REV10_IB_IO_CLASS &&
3525 			    token != SRP_REV16A_IB_IO_CLASS) {
3526 				pr_warn("unknown IO class parameter value %x specified (use %x or %x).\n",
3527 					token, SRP_REV10_IB_IO_CLASS,
3528 					SRP_REV16A_IB_IO_CLASS);
3529 				ret = -EINVAL;
3530 				goto out;
3531 			}
3532 			target->io_class = token;
3533 			break;
3534 
3535 		case SRP_OPT_INITIATOR_EXT:
3536 			p = match_strdup(args);
3537 			if (!p) {
3538 				ret = -ENOMEM;
3539 				goto out;
3540 			}
3541 			ret = kstrtoull(p, 16, &ull);
3542 			if (ret) {
3543 				pr_warn("bad initiator_ext value '%s'\n", p);
3544 				kfree(p);
3545 				goto out;
3546 			}
3547 			target->initiator_ext = cpu_to_be64(ull);
3548 			kfree(p);
3549 			break;
3550 
3551 		case SRP_OPT_CMD_SG_ENTRIES:
3552 			ret = match_int(args, &token);
3553 			if (ret) {
3554 				pr_warn("match_int() failed for max cmd_sg_entries parameter '%s', Error %d\n",
3555 					p, ret);
3556 				goto out;
3557 			}
3558 			if (token < 1 || token > 255) {
3559 				pr_warn("bad max cmd_sg_entries parameter '%s'\n",
3560 					p);
3561 				ret = -EINVAL;
3562 				goto out;
3563 			}
3564 			target->cmd_sg_cnt = token;
3565 			break;
3566 
3567 		case SRP_OPT_ALLOW_EXT_SG:
3568 			ret = match_int(args, &token);
3569 			if (ret) {
3570 				pr_warn("bad allow_ext_sg parameter '%s'\n", p);
3571 				goto out;
3572 			}
3573 			target->allow_ext_sg = !!token;
3574 			break;
3575 
3576 		case SRP_OPT_SG_TABLESIZE:
3577 			ret = match_int(args, &token);
3578 			if (ret) {
3579 				pr_warn("match_int() failed for max sg_tablesize parameter '%s', Error %d\n",
3580 					p, ret);
3581 				goto out;
3582 			}
3583 			if (token < 1 || token > SG_MAX_SEGMENTS) {
3584 				pr_warn("bad max sg_tablesize parameter '%s'\n",
3585 					p);
3586 				ret = -EINVAL;
3587 				goto out;
3588 			}
3589 			target->sg_tablesize = token;
3590 			break;
3591 
3592 		case SRP_OPT_COMP_VECTOR:
3593 			ret = match_int(args, &token);
3594 			if (ret) {
3595 				pr_warn("match_int() failed for comp_vector parameter '%s', Error %d\n",
3596 					p, ret);
3597 				goto out;
3598 			}
3599 			if (token < 0) {
3600 				pr_warn("bad comp_vector parameter '%s'\n", p);
3601 				ret = -EINVAL;
3602 				goto out;
3603 			}
3604 			target->comp_vector = token;
3605 			break;
3606 
3607 		case SRP_OPT_TL_RETRY_COUNT:
3608 			ret = match_int(args, &token);
3609 			if (ret) {
3610 				pr_warn("match_int() failed for tl_retry_count parameter '%s', Error %d\n",
3611 					p, ret);
3612 				goto out;
3613 			}
3614 			if (token < 2 || token > 7) {
3615 				pr_warn("bad tl_retry_count parameter '%s' (must be a number between 2 and 7)\n",
3616 					p);
3617 				ret = -EINVAL;
3618 				goto out;
3619 			}
3620 			target->tl_retry_count = token;
3621 			break;
3622 
3623 		case SRP_OPT_MAX_IT_IU_SIZE:
3624 			ret = match_int(args, &token);
3625 			if (ret) {
3626 				pr_warn("match_int() failed for max it_iu_size parameter '%s', Error %d\n",
3627 					p, ret);
3628 				goto out;
3629 			}
3630 			if (token < 0) {
3631 				pr_warn("bad maximum initiator to target IU size '%s'\n", p);
3632 				ret = -EINVAL;
3633 				goto out;
3634 			}
3635 			target->max_it_iu_size = token;
3636 			break;
3637 
3638 		case SRP_OPT_CH_COUNT:
3639 			ret = match_int(args, &token);
3640 			if (ret) {
3641 				pr_warn("match_int() failed for channel count parameter '%s', Error %d\n",
3642 					p, ret);
3643 				goto out;
3644 			}
3645 			if (token < 1) {
3646 				pr_warn("bad channel count %s\n", p);
3647 				ret = -EINVAL;
3648 				goto out;
3649 			}
3650 			target->ch_count = token;
3651 			break;
3652 
3653 		default:
3654 			pr_warn("unknown parameter or missing value '%s' in target creation request\n",
3655 				p);
3656 			ret = -EINVAL;
3657 			goto out;
3658 		}
3659 	}
3660 
3661 	for (i = 0; i < ARRAY_SIZE(srp_opt_mandatory); i++) {
3662 		if ((opt_mask & srp_opt_mandatory[i]) == srp_opt_mandatory[i]) {
3663 			ret = 0;
3664 			break;
3665 		}
3666 	}
3667 	if (ret)
3668 		pr_warn("target creation request is missing one or more parameters\n");
3669 
3670 	if (target->scsi_host->cmd_per_lun > target->scsi_host->can_queue
3671 	    && (opt_mask & SRP_OPT_MAX_CMD_PER_LUN))
3672 		pr_warn("cmd_per_lun = %d > queue_size = %d\n",
3673 			target->scsi_host->cmd_per_lun,
3674 			target->scsi_host->can_queue);
3675 
3676 out:
3677 	kfree(options);
3678 	return ret;
3679 }
3680 
3681 static ssize_t add_target_store(struct device *dev,
3682 				struct device_attribute *attr, const char *buf,
3683 				size_t count)
3684 {
3685 	struct srp_host *host =
3686 		container_of(dev, struct srp_host, dev);
3687 	struct Scsi_Host *target_host;
3688 	struct srp_target_port *target;
3689 	struct srp_rdma_ch *ch;
3690 	struct srp_device *srp_dev = host->srp_dev;
3691 	struct ib_device *ibdev = srp_dev->dev;
3692 	int ret, i, ch_idx;
3693 	unsigned int max_sectors_per_mr, mr_per_cmd = 0;
3694 	bool multich = false;
3695 	uint32_t max_iu_len;
3696 
3697 	target_host = scsi_host_alloc(&srp_template,
3698 				      sizeof (struct srp_target_port));
3699 	if (!target_host)
3700 		return -ENOMEM;
3701 
3702 	target_host->transportt  = ib_srp_transport_template;
3703 	target_host->max_channel = 0;
3704 	target_host->max_id      = 1;
3705 	target_host->max_lun     = -1LL;
3706 	target_host->max_cmd_len = sizeof ((struct srp_cmd *) (void *) 0L)->cdb;
3707 	target_host->max_segment_size = ib_dma_max_seg_size(ibdev);
3708 
3709 	if (!(ibdev->attrs.kernel_cap_flags & IBK_SG_GAPS_REG))
3710 		target_host->virt_boundary_mask = ~srp_dev->mr_page_mask;
3711 
3712 	target = host_to_target(target_host);
3713 
3714 	target->net		= kobj_ns_grab_current(KOBJ_NS_TYPE_NET);
3715 	target->io_class	= SRP_REV16A_IB_IO_CLASS;
3716 	target->scsi_host	= target_host;
3717 	target->srp_host	= host;
3718 	target->lkey		= host->srp_dev->pd->local_dma_lkey;
3719 	target->global_rkey	= host->srp_dev->global_rkey;
3720 	target->cmd_sg_cnt	= cmd_sg_entries;
3721 	target->sg_tablesize	= indirect_sg_entries ? : cmd_sg_entries;
3722 	target->allow_ext_sg	= allow_ext_sg;
3723 	target->tl_retry_count	= 7;
3724 	target->queue_size	= SRP_DEFAULT_QUEUE_SIZE;
3725 
3726 	/*
3727 	 * Avoid that the SCSI host can be removed by srp_remove_target()
3728 	 * before this function returns.
3729 	 */
3730 	scsi_host_get(target->scsi_host);
3731 
3732 	ret = mutex_lock_interruptible(&host->add_target_mutex);
3733 	if (ret < 0)
3734 		goto put;
3735 
3736 	ret = srp_parse_options(target->net, buf, target);
3737 	if (ret)
3738 		goto out;
3739 
3740 	if (!srp_conn_unique(target->srp_host, target)) {
3741 		if (target->using_rdma_cm) {
3742 			shost_printk(KERN_INFO, target->scsi_host,
3743 				     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;dest=%pIS\n",
3744 				     be64_to_cpu(target->id_ext),
3745 				     be64_to_cpu(target->ioc_guid),
3746 				     &target->rdma_cm.dst);
3747 		} else {
3748 			shost_printk(KERN_INFO, target->scsi_host,
3749 				     PFX "Already connected to target port with id_ext=%016llx;ioc_guid=%016llx;initiator_ext=%016llx\n",
3750 				     be64_to_cpu(target->id_ext),
3751 				     be64_to_cpu(target->ioc_guid),
3752 				     be64_to_cpu(target->initiator_ext));
3753 		}
3754 		ret = -EEXIST;
3755 		goto out;
3756 	}
3757 
3758 	if (!srp_dev->has_fr && !target->allow_ext_sg &&
3759 	    target->cmd_sg_cnt < target->sg_tablesize) {
3760 		pr_warn("No MR pool and no external indirect descriptors, limiting sg_tablesize to cmd_sg_cnt\n");
3761 		target->sg_tablesize = target->cmd_sg_cnt;
3762 	}
3763 
3764 	if (srp_dev->use_fast_reg) {
3765 		bool gaps_reg = ibdev->attrs.kernel_cap_flags &
3766 				 IBK_SG_GAPS_REG;
3767 
3768 		max_sectors_per_mr = srp_dev->max_pages_per_mr <<
3769 				  (ilog2(srp_dev->mr_page_size) - 9);
3770 		if (!gaps_reg) {
3771 			/*
3772 			 * FR can only map one HCA page per entry. If the start
3773 			 * address is not aligned on a HCA page boundary two
3774 			 * entries will be used for the head and the tail
3775 			 * although these two entries combined contain at most
3776 			 * one HCA page of data. Hence the "+ 1" in the
3777 			 * calculation below.
3778 			 *
3779 			 * The indirect data buffer descriptor is contiguous
3780 			 * so the memory for that buffer will only be
3781 			 * registered if register_always is true. Hence add
3782 			 * one to mr_per_cmd if register_always has been set.
3783 			 */
3784 			mr_per_cmd = register_always +
3785 				(target->scsi_host->max_sectors + 1 +
3786 				 max_sectors_per_mr - 1) / max_sectors_per_mr;
3787 		} else {
3788 			mr_per_cmd = register_always +
3789 				(target->sg_tablesize +
3790 				 srp_dev->max_pages_per_mr - 1) /
3791 				srp_dev->max_pages_per_mr;
3792 		}
3793 		pr_debug("max_sectors = %u; max_pages_per_mr = %u; mr_page_size = %u; max_sectors_per_mr = %u; mr_per_cmd = %u\n",
3794 			 target->scsi_host->max_sectors, srp_dev->max_pages_per_mr, srp_dev->mr_page_size,
3795 			 max_sectors_per_mr, mr_per_cmd);
3796 	}
3797 
3798 	target_host->sg_tablesize = target->sg_tablesize;
3799 	target->mr_pool_size = target->scsi_host->can_queue * mr_per_cmd;
3800 	target->mr_per_cmd = mr_per_cmd;
3801 	target->indirect_size = target->sg_tablesize *
3802 				sizeof (struct srp_direct_buf);
3803 	max_iu_len = srp_max_it_iu_len(target->cmd_sg_cnt,
3804 				       srp_use_imm_data,
3805 				       target->max_it_iu_size);
3806 
3807 	INIT_WORK(&target->tl_err_work, srp_tl_err_work);
3808 	INIT_WORK(&target->remove_work, srp_remove_work);
3809 	spin_lock_init(&target->lock);
3810 	ret = rdma_query_gid(ibdev, host->port, 0, &target->sgid);
3811 	if (ret)
3812 		goto out;
3813 
3814 	ret = -ENOMEM;
3815 	if (target->ch_count == 0) {
3816 		target->ch_count =
3817 			min(ch_count ?:
3818 				max(4 * num_online_nodes(),
3819 				    ibdev->num_comp_vectors),
3820 				num_online_cpus());
3821 	}
3822 
3823 	target->ch = kcalloc(target->ch_count, sizeof(*target->ch),
3824 			     GFP_KERNEL);
3825 	if (!target->ch)
3826 		goto out;
3827 
3828 	for (ch_idx = 0; ch_idx < target->ch_count; ++ch_idx) {
3829 		ch = &target->ch[ch_idx];
3830 		ch->target = target;
3831 		ch->comp_vector = ch_idx % ibdev->num_comp_vectors;
3832 		spin_lock_init(&ch->lock);
3833 		INIT_LIST_HEAD(&ch->free_tx);
3834 		ret = srp_new_cm_id(ch);
3835 		if (ret)
3836 			goto err_disconnect;
3837 
3838 		ret = srp_create_ch_ib(ch);
3839 		if (ret)
3840 			goto err_disconnect;
3841 
3842 		ret = srp_connect_ch(ch, max_iu_len, multich);
3843 		if (ret) {
3844 			char dst[64];
3845 
3846 			if (target->using_rdma_cm)
3847 				snprintf(dst, sizeof(dst), "%pIS",
3848 					&target->rdma_cm.dst);
3849 			else
3850 				snprintf(dst, sizeof(dst), "%pI6",
3851 					target->ib_cm.orig_dgid.raw);
3852 			shost_printk(KERN_ERR, target->scsi_host,
3853 				PFX "Connection %d/%d to %s failed\n",
3854 				ch_idx,
3855 				target->ch_count, dst);
3856 			if (ch_idx == 0) {
3857 				goto free_ch;
3858 			} else {
3859 				srp_free_ch_ib(target, ch);
3860 				target->ch_count = ch - target->ch;
3861 				goto connected;
3862 			}
3863 		}
3864 		multich = true;
3865 	}
3866 
3867 connected:
3868 	target->scsi_host->nr_hw_queues = target->ch_count;
3869 
3870 	ret = srp_add_target(host, target);
3871 	if (ret)
3872 		goto err_disconnect;
3873 
3874 	if (target->state != SRP_TARGET_REMOVED) {
3875 		if (target->using_rdma_cm) {
3876 			shost_printk(KERN_DEBUG, target->scsi_host, PFX
3877 				     "new target: id_ext %016llx ioc_guid %016llx sgid %pI6 dest %pIS\n",
3878 				     be64_to_cpu(target->id_ext),
3879 				     be64_to_cpu(target->ioc_guid),
3880 				     target->sgid.raw, &target->rdma_cm.dst);
3881 		} else {
3882 			shost_printk(KERN_DEBUG, target->scsi_host, PFX
3883 				     "new target: id_ext %016llx ioc_guid %016llx pkey %04x service_id %016llx sgid %pI6 dgid %pI6\n",
3884 				     be64_to_cpu(target->id_ext),
3885 				     be64_to_cpu(target->ioc_guid),
3886 				     be16_to_cpu(target->ib_cm.pkey),
3887 				     be64_to_cpu(target->ib_cm.service_id),
3888 				     target->sgid.raw,
3889 				     target->ib_cm.orig_dgid.raw);
3890 		}
3891 	}
3892 
3893 	ret = count;
3894 
3895 out:
3896 	mutex_unlock(&host->add_target_mutex);
3897 
3898 put:
3899 	scsi_host_put(target->scsi_host);
3900 	if (ret < 0) {
3901 		/*
3902 		 * If a call to srp_remove_target() has not been scheduled,
3903 		 * drop the network namespace reference now that was obtained
3904 		 * earlier in this function.
3905 		 */
3906 		if (target->state != SRP_TARGET_REMOVED)
3907 			kobj_ns_drop(KOBJ_NS_TYPE_NET, target->net);
3908 		scsi_host_put(target->scsi_host);
3909 	}
3910 
3911 	return ret;
3912 
3913 err_disconnect:
3914 	srp_disconnect_target(target);
3915 
3916 free_ch:
3917 	for (i = 0; i < target->ch_count; i++) {
3918 		ch = &target->ch[i];
3919 		srp_free_ch_ib(target, ch);
3920 	}
3921 
3922 	kfree(target->ch);
3923 	goto out;
3924 }
3925 
3926 static DEVICE_ATTR_WO(add_target);
3927 
3928 static ssize_t ibdev_show(struct device *dev, struct device_attribute *attr,
3929 			  char *buf)
3930 {
3931 	struct srp_host *host = container_of(dev, struct srp_host, dev);
3932 
3933 	return sysfs_emit(buf, "%s\n", dev_name(&host->srp_dev->dev->dev));
3934 }
3935 
3936 static DEVICE_ATTR_RO(ibdev);
3937 
3938 static ssize_t port_show(struct device *dev, struct device_attribute *attr,
3939 			 char *buf)
3940 {
3941 	struct srp_host *host = container_of(dev, struct srp_host, dev);
3942 
3943 	return sysfs_emit(buf, "%u\n", host->port);
3944 }
3945 
3946 static DEVICE_ATTR_RO(port);
3947 
3948 static struct attribute *srp_class_attrs[] = {
3949 	&dev_attr_add_target.attr,
3950 	&dev_attr_ibdev.attr,
3951 	&dev_attr_port.attr,
3952 	NULL
3953 };
3954 
3955 static struct srp_host *srp_add_port(struct srp_device *device, u32 port)
3956 {
3957 	struct srp_host *host;
3958 
3959 	host = kzalloc(sizeof *host, GFP_KERNEL);
3960 	if (!host)
3961 		return NULL;
3962 
3963 	INIT_LIST_HEAD(&host->target_list);
3964 	spin_lock_init(&host->target_lock);
3965 	mutex_init(&host->add_target_mutex);
3966 	host->srp_dev = device;
3967 	host->port = port;
3968 
3969 	device_initialize(&host->dev);
3970 	host->dev.class = &srp_class;
3971 	host->dev.parent = device->dev->dev.parent;
3972 	if (dev_set_name(&host->dev, "srp-%s-%u", dev_name(&device->dev->dev),
3973 			 port))
3974 		goto put_host;
3975 	if (device_add(&host->dev))
3976 		goto put_host;
3977 
3978 	return host;
3979 
3980 put_host:
3981 	device_del(&host->dev);
3982 	put_device(&host->dev);
3983 	return NULL;
3984 }
3985 
3986 static void srp_rename_dev(struct ib_device *device, void *client_data)
3987 {
3988 	struct srp_device *srp_dev = client_data;
3989 	struct srp_host *host, *tmp_host;
3990 
3991 	list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
3992 		char name[IB_DEVICE_NAME_MAX + 8];
3993 
3994 		snprintf(name, sizeof(name), "srp-%s-%u",
3995 			 dev_name(&device->dev), host->port);
3996 		device_rename(&host->dev, name);
3997 	}
3998 }
3999 
4000 static int srp_add_one(struct ib_device *device)
4001 {
4002 	struct srp_device *srp_dev;
4003 	struct ib_device_attr *attr = &device->attrs;
4004 	struct srp_host *host;
4005 	int mr_page_shift;
4006 	u32 p;
4007 	u64 max_pages_per_mr;
4008 	unsigned int flags = 0;
4009 
4010 	srp_dev = kzalloc(sizeof(*srp_dev), GFP_KERNEL);
4011 	if (!srp_dev)
4012 		return -ENOMEM;
4013 
4014 	/*
4015 	 * Use the smallest page size supported by the HCA, down to a
4016 	 * minimum of 4096 bytes. We're unlikely to build large sglists
4017 	 * out of smaller entries.
4018 	 */
4019 	mr_page_shift		= max(12, ffs(attr->page_size_cap) - 1);
4020 	srp_dev->mr_page_size	= 1 << mr_page_shift;
4021 	srp_dev->mr_page_mask	= ~((u64) srp_dev->mr_page_size - 1);
4022 	max_pages_per_mr	= attr->max_mr_size;
4023 	do_div(max_pages_per_mr, srp_dev->mr_page_size);
4024 	pr_debug("%s: %llu / %u = %llu <> %u\n", __func__,
4025 		 attr->max_mr_size, srp_dev->mr_page_size,
4026 		 max_pages_per_mr, SRP_MAX_PAGES_PER_MR);
4027 	srp_dev->max_pages_per_mr = min_t(u64, SRP_MAX_PAGES_PER_MR,
4028 					  max_pages_per_mr);
4029 
4030 	srp_dev->has_fr = (attr->device_cap_flags &
4031 			   IB_DEVICE_MEM_MGT_EXTENSIONS);
4032 	if (!never_register && !srp_dev->has_fr)
4033 		dev_warn(&device->dev, "FR is not supported\n");
4034 	else if (!never_register &&
4035 		 attr->max_mr_size >= 2 * srp_dev->mr_page_size)
4036 		srp_dev->use_fast_reg = srp_dev->has_fr;
4037 
4038 	if (never_register || !register_always || !srp_dev->has_fr)
4039 		flags |= IB_PD_UNSAFE_GLOBAL_RKEY;
4040 
4041 	if (srp_dev->use_fast_reg) {
4042 		srp_dev->max_pages_per_mr =
4043 			min_t(u32, srp_dev->max_pages_per_mr,
4044 			      attr->max_fast_reg_page_list_len);
4045 	}
4046 	srp_dev->mr_max_size	= srp_dev->mr_page_size *
4047 				   srp_dev->max_pages_per_mr;
4048 	pr_debug("%s: mr_page_shift = %d, device->max_mr_size = %#llx, device->max_fast_reg_page_list_len = %u, max_pages_per_mr = %d, mr_max_size = %#x\n",
4049 		 dev_name(&device->dev), mr_page_shift, attr->max_mr_size,
4050 		 attr->max_fast_reg_page_list_len,
4051 		 srp_dev->max_pages_per_mr, srp_dev->mr_max_size);
4052 
4053 	INIT_LIST_HEAD(&srp_dev->dev_list);
4054 
4055 	srp_dev->dev = device;
4056 	srp_dev->pd  = ib_alloc_pd(device, flags);
4057 	if (IS_ERR(srp_dev->pd)) {
4058 		int ret = PTR_ERR(srp_dev->pd);
4059 
4060 		kfree(srp_dev);
4061 		return ret;
4062 	}
4063 
4064 	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
4065 		srp_dev->global_rkey = srp_dev->pd->unsafe_global_rkey;
4066 		WARN_ON_ONCE(srp_dev->global_rkey == 0);
4067 	}
4068 
4069 	rdma_for_each_port (device, p) {
4070 		host = srp_add_port(srp_dev, p);
4071 		if (host)
4072 			list_add_tail(&host->list, &srp_dev->dev_list);
4073 	}
4074 
4075 	ib_set_client_data(device, &srp_client, srp_dev);
4076 	return 0;
4077 }
4078 
4079 static void srp_remove_one(struct ib_device *device, void *client_data)
4080 {
4081 	struct srp_device *srp_dev;
4082 	struct srp_host *host, *tmp_host;
4083 	struct srp_target_port *target;
4084 
4085 	srp_dev = client_data;
4086 
4087 	list_for_each_entry_safe(host, tmp_host, &srp_dev->dev_list, list) {
4088 		/*
4089 		 * Remove the add_target sysfs entry so that no new target ports
4090 		 * can be created.
4091 		 */
4092 		device_del(&host->dev);
4093 
4094 		/*
4095 		 * Remove all target ports.
4096 		 */
4097 		spin_lock(&host->target_lock);
4098 		list_for_each_entry(target, &host->target_list, list)
4099 			srp_queue_remove_work(target);
4100 		spin_unlock(&host->target_lock);
4101 
4102 		/*
4103 		 * srp_queue_remove_work() queues a call to
4104 		 * srp_remove_target(). The latter function cancels
4105 		 * target->tl_err_work so waiting for the remove works to
4106 		 * finish is sufficient.
4107 		 */
4108 		flush_workqueue(srp_remove_wq);
4109 
4110 		put_device(&host->dev);
4111 	}
4112 
4113 	ib_dealloc_pd(srp_dev->pd);
4114 
4115 	kfree(srp_dev);
4116 }
4117 
4118 static struct srp_function_template ib_srp_transport_functions = {
4119 	.has_rport_state	 = true,
4120 	.reset_timer_if_blocked	 = true,
4121 	.reconnect_delay	 = &srp_reconnect_delay,
4122 	.fast_io_fail_tmo	 = &srp_fast_io_fail_tmo,
4123 	.dev_loss_tmo		 = &srp_dev_loss_tmo,
4124 	.reconnect		 = srp_rport_reconnect,
4125 	.rport_delete		 = srp_rport_delete,
4126 	.terminate_rport_io	 = srp_terminate_io,
4127 };
4128 
4129 static int __init srp_init_module(void)
4130 {
4131 	int ret;
4132 
4133 	BUILD_BUG_ON(sizeof(struct srp_aer_req) != 36);
4134 	BUILD_BUG_ON(sizeof(struct srp_cmd) != 48);
4135 	BUILD_BUG_ON(sizeof(struct srp_imm_buf) != 4);
4136 	BUILD_BUG_ON(sizeof(struct srp_indirect_buf) != 20);
4137 	BUILD_BUG_ON(sizeof(struct srp_login_req) != 64);
4138 	BUILD_BUG_ON(sizeof(struct srp_login_req_rdma) != 56);
4139 	BUILD_BUG_ON(sizeof(struct srp_rsp) != 36);
4140 
4141 	if (srp_sg_tablesize) {
4142 		pr_warn("srp_sg_tablesize is deprecated, please use cmd_sg_entries\n");
4143 		if (!cmd_sg_entries)
4144 			cmd_sg_entries = srp_sg_tablesize;
4145 	}
4146 
4147 	if (!cmd_sg_entries)
4148 		cmd_sg_entries = SRP_DEF_SG_TABLESIZE;
4149 
4150 	if (cmd_sg_entries > 255) {
4151 		pr_warn("Clamping cmd_sg_entries to 255\n");
4152 		cmd_sg_entries = 255;
4153 	}
4154 
4155 	if (!indirect_sg_entries)
4156 		indirect_sg_entries = cmd_sg_entries;
4157 	else if (indirect_sg_entries < cmd_sg_entries) {
4158 		pr_warn("Bumping up indirect_sg_entries to match cmd_sg_entries (%u)\n",
4159 			cmd_sg_entries);
4160 		indirect_sg_entries = cmd_sg_entries;
4161 	}
4162 
4163 	if (indirect_sg_entries > SG_MAX_SEGMENTS) {
4164 		pr_warn("Clamping indirect_sg_entries to %u\n",
4165 			SG_MAX_SEGMENTS);
4166 		indirect_sg_entries = SG_MAX_SEGMENTS;
4167 	}
4168 
4169 	srp_remove_wq = create_workqueue("srp_remove");
4170 	if (!srp_remove_wq) {
4171 		ret = -ENOMEM;
4172 		goto out;
4173 	}
4174 
4175 	ret = -ENOMEM;
4176 	ib_srp_transport_template =
4177 		srp_attach_transport(&ib_srp_transport_functions);
4178 	if (!ib_srp_transport_template)
4179 		goto destroy_wq;
4180 
4181 	ret = class_register(&srp_class);
4182 	if (ret) {
4183 		pr_err("couldn't register class infiniband_srp\n");
4184 		goto release_tr;
4185 	}
4186 
4187 	ib_sa_register_client(&srp_sa_client);
4188 
4189 	ret = ib_register_client(&srp_client);
4190 	if (ret) {
4191 		pr_err("couldn't register IB client\n");
4192 		goto unreg_sa;
4193 	}
4194 
4195 out:
4196 	return ret;
4197 
4198 unreg_sa:
4199 	ib_sa_unregister_client(&srp_sa_client);
4200 	class_unregister(&srp_class);
4201 
4202 release_tr:
4203 	srp_release_transport(ib_srp_transport_template);
4204 
4205 destroy_wq:
4206 	destroy_workqueue(srp_remove_wq);
4207 	goto out;
4208 }
4209 
4210 static void __exit srp_cleanup_module(void)
4211 {
4212 	ib_unregister_client(&srp_client);
4213 	ib_sa_unregister_client(&srp_sa_client);
4214 	class_unregister(&srp_class);
4215 	srp_release_transport(ib_srp_transport_template);
4216 	destroy_workqueue(srp_remove_wq);
4217 }
4218 
4219 module_init(srp_init_module);
4220 module_exit(srp_cleanup_module);
4221