1 /*
2  * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
4  * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *	- Redistributions of source code must retain the above
17  *	  copyright notice, this list of conditions and the following
18  *	  disclaimer.
19  *
20  *	- Redistributions in binary form must reproduce the above
21  *	  copyright notice, this list of conditions and the following
22  *	  disclaimer in the documentation and/or other materials
23  *	  provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 #include <linux/kernel.h>
35 #include <linux/module.h>
36 #include <linux/slab.h>
37 #include <linux/delay.h>
38 
39 #include "iscsi_iser.h"
40 
41 #define ISCSI_ISER_MAX_CONN	8
42 #define ISER_MAX_RX_LEN		(ISER_QP_MAX_RECV_DTOS * ISCSI_ISER_MAX_CONN)
43 #define ISER_MAX_TX_LEN		(ISER_QP_MAX_REQ_DTOS  * ISCSI_ISER_MAX_CONN)
44 #define ISER_MAX_CQ_LEN		(ISER_MAX_RX_LEN + ISER_MAX_TX_LEN + \
45 				 ISCSI_ISER_MAX_CONN)
46 
47 static void iser_qp_event_callback(struct ib_event *cause, void *context)
48 {
49 	iser_err("qp event %s (%d)\n",
50 		 ib_event_msg(cause->event), cause->event);
51 }
52 
53 static void iser_event_handler(struct ib_event_handler *handler,
54 				struct ib_event *event)
55 {
56 	iser_err("async event %s (%d) on device %s port %d\n",
57 		 ib_event_msg(event->event), event->event,
58 		dev_name(&event->device->dev), event->element.port_num);
59 }
60 
61 /**
62  * iser_create_device_ib_res - creates Protection Domain (PD), Completion
63  * Queue (CQ), DMA Memory Region (DMA MR) with the device associated with
64  * the adapator.
65  *
66  * returns 0 on success, -1 on failure
67  */
68 static int iser_create_device_ib_res(struct iser_device *device)
69 {
70 	struct ib_device *ib_dev = device->ib_device;
71 	int ret, i, max_cqe;
72 
73 	ret = iser_assign_reg_ops(device);
74 	if (ret)
75 		return ret;
76 
77 	device->comps_used = min_t(int, num_online_cpus(),
78 				 ib_dev->num_comp_vectors);
79 
80 	device->comps = kcalloc(device->comps_used, sizeof(*device->comps),
81 				GFP_KERNEL);
82 	if (!device->comps)
83 		goto comps_err;
84 
85 	max_cqe = min(ISER_MAX_CQ_LEN, ib_dev->attrs.max_cqe);
86 
87 	iser_info("using %d CQs, device %s supports %d vectors max_cqe %d\n",
88 		  device->comps_used, dev_name(&ib_dev->dev),
89 		  ib_dev->num_comp_vectors, max_cqe);
90 
91 	device->pd = ib_alloc_pd(ib_dev,
92 		iser_always_reg ? 0 : IB_PD_UNSAFE_GLOBAL_RKEY);
93 	if (IS_ERR(device->pd))
94 		goto pd_err;
95 
96 	for (i = 0; i < device->comps_used; i++) {
97 		struct iser_comp *comp = &device->comps[i];
98 
99 		comp->cq = ib_alloc_cq(ib_dev, comp, max_cqe, i,
100 				       IB_POLL_SOFTIRQ);
101 		if (IS_ERR(comp->cq)) {
102 			comp->cq = NULL;
103 			goto cq_err;
104 		}
105 	}
106 
107 	INIT_IB_EVENT_HANDLER(&device->event_handler, ib_dev,
108 			      iser_event_handler);
109 	ib_register_event_handler(&device->event_handler);
110 	return 0;
111 
112 cq_err:
113 	for (i = 0; i < device->comps_used; i++) {
114 		struct iser_comp *comp = &device->comps[i];
115 
116 		if (comp->cq)
117 			ib_free_cq(comp->cq);
118 	}
119 	ib_dealloc_pd(device->pd);
120 pd_err:
121 	kfree(device->comps);
122 comps_err:
123 	iser_err("failed to allocate an IB resource\n");
124 	return -1;
125 }
126 
127 /**
128  * iser_free_device_ib_res - destroy/dealloc/dereg the DMA MR,
129  * CQ and PD created with the device associated with the adapator.
130  */
131 static void iser_free_device_ib_res(struct iser_device *device)
132 {
133 	int i;
134 
135 	for (i = 0; i < device->comps_used; i++) {
136 		struct iser_comp *comp = &device->comps[i];
137 
138 		ib_free_cq(comp->cq);
139 		comp->cq = NULL;
140 	}
141 
142 	ib_unregister_event_handler(&device->event_handler);
143 	ib_dealloc_pd(device->pd);
144 
145 	kfree(device->comps);
146 	device->comps = NULL;
147 	device->pd = NULL;
148 }
149 
150 /**
151  * iser_alloc_fmr_pool - Creates FMR pool and page_vector
152  *
153  * returns 0 on success, or errno code on failure
154  */
155 int iser_alloc_fmr_pool(struct ib_conn *ib_conn,
156 			unsigned cmds_max,
157 			unsigned int size)
158 {
159 	struct iser_device *device = ib_conn->device;
160 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
161 	struct iser_page_vec *page_vec;
162 	struct iser_fr_desc *desc;
163 	struct ib_fmr_pool *fmr_pool;
164 	struct ib_fmr_pool_param params;
165 	int ret;
166 
167 	INIT_LIST_HEAD(&fr_pool->list);
168 	spin_lock_init(&fr_pool->lock);
169 
170 	desc = kzalloc(sizeof(*desc), GFP_KERNEL);
171 	if (!desc)
172 		return -ENOMEM;
173 
174 	page_vec = kmalloc(sizeof(*page_vec) + (sizeof(u64) * size),
175 			   GFP_KERNEL);
176 	if (!page_vec) {
177 		ret = -ENOMEM;
178 		goto err_frpl;
179 	}
180 
181 	page_vec->pages = (u64 *)(page_vec + 1);
182 
183 	params.page_shift        = SHIFT_4K;
184 	params.max_pages_per_fmr = size;
185 	/* make the pool size twice the max number of SCSI commands *
186 	 * the ML is expected to queue, watermark for unmap at 50%  */
187 	params.pool_size	 = cmds_max * 2;
188 	params.dirty_watermark	 = cmds_max;
189 	params.cache		 = 0;
190 	params.flush_function	 = NULL;
191 	params.access		 = (IB_ACCESS_LOCAL_WRITE  |
192 				    IB_ACCESS_REMOTE_WRITE |
193 				    IB_ACCESS_REMOTE_READ);
194 
195 	fmr_pool = ib_create_fmr_pool(device->pd, &params);
196 	if (IS_ERR(fmr_pool)) {
197 		ret = PTR_ERR(fmr_pool);
198 		iser_err("FMR allocation failed, err %d\n", ret);
199 		goto err_fmr;
200 	}
201 
202 	desc->rsc.page_vec = page_vec;
203 	desc->rsc.fmr_pool = fmr_pool;
204 	list_add(&desc->list, &fr_pool->list);
205 
206 	return 0;
207 
208 err_fmr:
209 	kfree(page_vec);
210 err_frpl:
211 	kfree(desc);
212 
213 	return ret;
214 }
215 
216 /**
217  * iser_free_fmr_pool - releases the FMR pool and page vec
218  */
219 void iser_free_fmr_pool(struct ib_conn *ib_conn)
220 {
221 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
222 	struct iser_fr_desc *desc;
223 
224 	desc = list_first_entry(&fr_pool->list,
225 				struct iser_fr_desc, list);
226 	list_del(&desc->list);
227 
228 	iser_info("freeing conn %p fmr pool %p\n",
229 		  ib_conn, desc->rsc.fmr_pool);
230 
231 	ib_destroy_fmr_pool(desc->rsc.fmr_pool);
232 	kfree(desc->rsc.page_vec);
233 	kfree(desc);
234 }
235 
236 static int
237 iser_alloc_reg_res(struct iser_device *device,
238 		   struct ib_pd *pd,
239 		   struct iser_reg_resources *res,
240 		   unsigned int size)
241 {
242 	struct ib_device *ib_dev = device->ib_device;
243 	enum ib_mr_type mr_type;
244 	int ret;
245 
246 	if (ib_dev->attrs.device_cap_flags & IB_DEVICE_SG_GAPS_REG)
247 		mr_type = IB_MR_TYPE_SG_GAPS;
248 	else
249 		mr_type = IB_MR_TYPE_MEM_REG;
250 
251 	res->mr = ib_alloc_mr(pd, mr_type, size);
252 	if (IS_ERR(res->mr)) {
253 		ret = PTR_ERR(res->mr);
254 		iser_err("Failed to allocate ib_fast_reg_mr err=%d\n", ret);
255 		return ret;
256 	}
257 	res->mr_valid = 0;
258 
259 	return 0;
260 }
261 
262 static void
263 iser_free_reg_res(struct iser_reg_resources *rsc)
264 {
265 	ib_dereg_mr(rsc->mr);
266 }
267 
268 static int
269 iser_alloc_pi_ctx(struct iser_device *device,
270 		  struct ib_pd *pd,
271 		  struct iser_fr_desc *desc,
272 		  unsigned int size)
273 {
274 	struct iser_pi_context *pi_ctx = NULL;
275 	int ret;
276 
277 	desc->pi_ctx = kzalloc(sizeof(*desc->pi_ctx), GFP_KERNEL);
278 	if (!desc->pi_ctx)
279 		return -ENOMEM;
280 
281 	pi_ctx = desc->pi_ctx;
282 
283 	ret = iser_alloc_reg_res(device, pd, &pi_ctx->rsc, size);
284 	if (ret) {
285 		iser_err("failed to allocate reg_resources\n");
286 		goto alloc_reg_res_err;
287 	}
288 
289 	pi_ctx->sig_mr = ib_alloc_mr(pd, IB_MR_TYPE_SIGNATURE, 2);
290 	if (IS_ERR(pi_ctx->sig_mr)) {
291 		ret = PTR_ERR(pi_ctx->sig_mr);
292 		goto sig_mr_failure;
293 	}
294 	pi_ctx->sig_mr_valid = 0;
295 	desc->pi_ctx->sig_protected = 0;
296 
297 	return 0;
298 
299 sig_mr_failure:
300 	iser_free_reg_res(&pi_ctx->rsc);
301 alloc_reg_res_err:
302 	kfree(desc->pi_ctx);
303 
304 	return ret;
305 }
306 
307 static void
308 iser_free_pi_ctx(struct iser_pi_context *pi_ctx)
309 {
310 	iser_free_reg_res(&pi_ctx->rsc);
311 	ib_dereg_mr(pi_ctx->sig_mr);
312 	kfree(pi_ctx);
313 }
314 
315 static struct iser_fr_desc *
316 iser_create_fastreg_desc(struct iser_device *device,
317 			 struct ib_pd *pd,
318 			 bool pi_enable,
319 			 unsigned int size)
320 {
321 	struct iser_fr_desc *desc;
322 	int ret;
323 
324 	desc = kzalloc(sizeof(*desc), GFP_KERNEL);
325 	if (!desc)
326 		return ERR_PTR(-ENOMEM);
327 
328 	ret = iser_alloc_reg_res(device, pd, &desc->rsc, size);
329 	if (ret)
330 		goto reg_res_alloc_failure;
331 
332 	if (pi_enable) {
333 		ret = iser_alloc_pi_ctx(device, pd, desc, size);
334 		if (ret)
335 			goto pi_ctx_alloc_failure;
336 	}
337 
338 	return desc;
339 
340 pi_ctx_alloc_failure:
341 	iser_free_reg_res(&desc->rsc);
342 reg_res_alloc_failure:
343 	kfree(desc);
344 
345 	return ERR_PTR(ret);
346 }
347 
348 /**
349  * iser_alloc_fastreg_pool - Creates pool of fast_reg descriptors
350  * for fast registration work requests.
351  * returns 0 on success, or errno code on failure
352  */
353 int iser_alloc_fastreg_pool(struct ib_conn *ib_conn,
354 			    unsigned cmds_max,
355 			    unsigned int size)
356 {
357 	struct iser_device *device = ib_conn->device;
358 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
359 	struct iser_fr_desc *desc;
360 	int i, ret;
361 
362 	INIT_LIST_HEAD(&fr_pool->list);
363 	INIT_LIST_HEAD(&fr_pool->all_list);
364 	spin_lock_init(&fr_pool->lock);
365 	fr_pool->size = 0;
366 	for (i = 0; i < cmds_max; i++) {
367 		desc = iser_create_fastreg_desc(device, device->pd,
368 						ib_conn->pi_support, size);
369 		if (IS_ERR(desc)) {
370 			ret = PTR_ERR(desc);
371 			goto err;
372 		}
373 
374 		list_add_tail(&desc->list, &fr_pool->list);
375 		list_add_tail(&desc->all_list, &fr_pool->all_list);
376 		fr_pool->size++;
377 	}
378 
379 	return 0;
380 
381 err:
382 	iser_free_fastreg_pool(ib_conn);
383 	return ret;
384 }
385 
386 /**
387  * iser_free_fastreg_pool - releases the pool of fast_reg descriptors
388  */
389 void iser_free_fastreg_pool(struct ib_conn *ib_conn)
390 {
391 	struct iser_fr_pool *fr_pool = &ib_conn->fr_pool;
392 	struct iser_fr_desc *desc, *tmp;
393 	int i = 0;
394 
395 	if (list_empty(&fr_pool->all_list))
396 		return;
397 
398 	iser_info("freeing conn %p fr pool\n", ib_conn);
399 
400 	list_for_each_entry_safe(desc, tmp, &fr_pool->all_list, all_list) {
401 		list_del(&desc->all_list);
402 		iser_free_reg_res(&desc->rsc);
403 		if (desc->pi_ctx)
404 			iser_free_pi_ctx(desc->pi_ctx);
405 		kfree(desc);
406 		++i;
407 	}
408 
409 	if (i < fr_pool->size)
410 		iser_warn("pool still has %d regions registered\n",
411 			  fr_pool->size - i);
412 }
413 
414 /**
415  * iser_create_ib_conn_res - Queue-Pair (QP)
416  *
417  * returns 0 on success, -1 on failure
418  */
419 static int iser_create_ib_conn_res(struct ib_conn *ib_conn)
420 {
421 	struct iser_conn *iser_conn = to_iser_conn(ib_conn);
422 	struct iser_device	*device;
423 	struct ib_device	*ib_dev;
424 	struct ib_qp_init_attr	init_attr;
425 	int			ret = -ENOMEM;
426 	int index, min_index = 0;
427 
428 	BUG_ON(ib_conn->device == NULL);
429 
430 	device = ib_conn->device;
431 	ib_dev = device->ib_device;
432 
433 	memset(&init_attr, 0, sizeof init_attr);
434 
435 	mutex_lock(&ig.connlist_mutex);
436 	/* select the CQ with the minimal number of usages */
437 	for (index = 0; index < device->comps_used; index++) {
438 		if (device->comps[index].active_qps <
439 		    device->comps[min_index].active_qps)
440 			min_index = index;
441 	}
442 	ib_conn->comp = &device->comps[min_index];
443 	ib_conn->comp->active_qps++;
444 	mutex_unlock(&ig.connlist_mutex);
445 	iser_info("cq index %d used for ib_conn %p\n", min_index, ib_conn);
446 
447 	init_attr.event_handler = iser_qp_event_callback;
448 	init_attr.qp_context	= (void *)ib_conn;
449 	init_attr.send_cq	= ib_conn->comp->cq;
450 	init_attr.recv_cq	= ib_conn->comp->cq;
451 	init_attr.cap.max_recv_wr  = ISER_QP_MAX_RECV_DTOS;
452 	init_attr.cap.max_send_sge = 2;
453 	init_attr.cap.max_recv_sge = 1;
454 	init_attr.sq_sig_type	= IB_SIGNAL_REQ_WR;
455 	init_attr.qp_type	= IB_QPT_RC;
456 	if (ib_conn->pi_support) {
457 		init_attr.cap.max_send_wr = ISER_QP_SIG_MAX_REQ_DTOS + 1;
458 		init_attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
459 		iser_conn->max_cmds =
460 			ISER_GET_MAX_XMIT_CMDS(ISER_QP_SIG_MAX_REQ_DTOS);
461 	} else {
462 		if (ib_dev->attrs.max_qp_wr > ISER_QP_MAX_REQ_DTOS) {
463 			init_attr.cap.max_send_wr  = ISER_QP_MAX_REQ_DTOS + 1;
464 			iser_conn->max_cmds =
465 				ISER_GET_MAX_XMIT_CMDS(ISER_QP_MAX_REQ_DTOS);
466 		} else {
467 			init_attr.cap.max_send_wr = ib_dev->attrs.max_qp_wr;
468 			iser_conn->max_cmds =
469 				ISER_GET_MAX_XMIT_CMDS(ib_dev->attrs.max_qp_wr);
470 			iser_dbg("device %s supports max_send_wr %d\n",
471 				 dev_name(&device->ib_device->dev),
472 				 ib_dev->attrs.max_qp_wr);
473 		}
474 	}
475 
476 	ret = rdma_create_qp(ib_conn->cma_id, device->pd, &init_attr);
477 	if (ret)
478 		goto out_err;
479 
480 	ib_conn->qp = ib_conn->cma_id->qp;
481 	iser_info("setting conn %p cma_id %p qp %p\n",
482 		  ib_conn, ib_conn->cma_id,
483 		  ib_conn->cma_id->qp);
484 	return ret;
485 
486 out_err:
487 	mutex_lock(&ig.connlist_mutex);
488 	ib_conn->comp->active_qps--;
489 	mutex_unlock(&ig.connlist_mutex);
490 	iser_err("unable to alloc mem or create resource, err %d\n", ret);
491 
492 	return ret;
493 }
494 
495 /**
496  * based on the resolved device node GUID see if there already allocated
497  * device for this device. If there's no such, create one.
498  */
499 static
500 struct iser_device *iser_device_find_by_ib_device(struct rdma_cm_id *cma_id)
501 {
502 	struct iser_device *device;
503 
504 	mutex_lock(&ig.device_list_mutex);
505 
506 	list_for_each_entry(device, &ig.device_list, ig_list)
507 		/* find if there's a match using the node GUID */
508 		if (device->ib_device->node_guid == cma_id->device->node_guid)
509 			goto inc_refcnt;
510 
511 	device = kzalloc(sizeof *device, GFP_KERNEL);
512 	if (device == NULL)
513 		goto out;
514 
515 	/* assign this device to the device */
516 	device->ib_device = cma_id->device;
517 	/* init the device and link it into ig device list */
518 	if (iser_create_device_ib_res(device)) {
519 		kfree(device);
520 		device = NULL;
521 		goto out;
522 	}
523 	list_add(&device->ig_list, &ig.device_list);
524 
525 inc_refcnt:
526 	device->refcount++;
527 out:
528 	mutex_unlock(&ig.device_list_mutex);
529 	return device;
530 }
531 
532 /* if there's no demand for this device, release it */
533 static void iser_device_try_release(struct iser_device *device)
534 {
535 	mutex_lock(&ig.device_list_mutex);
536 	device->refcount--;
537 	iser_info("device %p refcount %d\n", device, device->refcount);
538 	if (!device->refcount) {
539 		iser_free_device_ib_res(device);
540 		list_del(&device->ig_list);
541 		kfree(device);
542 	}
543 	mutex_unlock(&ig.device_list_mutex);
544 }
545 
546 /**
547  * Called with state mutex held
548  **/
549 static int iser_conn_state_comp_exch(struct iser_conn *iser_conn,
550 				     enum iser_conn_state comp,
551 				     enum iser_conn_state exch)
552 {
553 	int ret;
554 
555 	ret = (iser_conn->state == comp);
556 	if (ret)
557 		iser_conn->state = exch;
558 
559 	return ret;
560 }
561 
562 void iser_release_work(struct work_struct *work)
563 {
564 	struct iser_conn *iser_conn;
565 
566 	iser_conn = container_of(work, struct iser_conn, release_work);
567 
568 	/* Wait for conn_stop to complete */
569 	wait_for_completion(&iser_conn->stop_completion);
570 	/* Wait for IB resouces cleanup to complete */
571 	wait_for_completion(&iser_conn->ib_completion);
572 
573 	mutex_lock(&iser_conn->state_mutex);
574 	iser_conn->state = ISER_CONN_DOWN;
575 	mutex_unlock(&iser_conn->state_mutex);
576 
577 	iser_conn_release(iser_conn);
578 }
579 
580 /**
581  * iser_free_ib_conn_res - release IB related resources
582  * @iser_conn: iser connection struct
583  * @destroy: indicator if we need to try to release the
584  *     iser device and memory regoins pool (only iscsi
585  *     shutdown and DEVICE_REMOVAL will use this).
586  *
587  * This routine is called with the iser state mutex held
588  * so the cm_id removal is out of here. It is Safe to
589  * be invoked multiple times.
590  */
591 static void iser_free_ib_conn_res(struct iser_conn *iser_conn,
592 				  bool destroy)
593 {
594 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
595 	struct iser_device *device = ib_conn->device;
596 
597 	iser_info("freeing conn %p cma_id %p qp %p\n",
598 		  iser_conn, ib_conn->cma_id, ib_conn->qp);
599 
600 	if (ib_conn->qp != NULL) {
601 		mutex_lock(&ig.connlist_mutex);
602 		ib_conn->comp->active_qps--;
603 		mutex_unlock(&ig.connlist_mutex);
604 		rdma_destroy_qp(ib_conn->cma_id);
605 		ib_conn->qp = NULL;
606 	}
607 
608 	if (destroy) {
609 		if (iser_conn->rx_descs)
610 			iser_free_rx_descriptors(iser_conn);
611 
612 		if (device != NULL) {
613 			iser_device_try_release(device);
614 			ib_conn->device = NULL;
615 		}
616 	}
617 }
618 
619 /**
620  * Frees all conn objects and deallocs conn descriptor
621  */
622 void iser_conn_release(struct iser_conn *iser_conn)
623 {
624 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
625 
626 	mutex_lock(&ig.connlist_mutex);
627 	list_del(&iser_conn->conn_list);
628 	mutex_unlock(&ig.connlist_mutex);
629 
630 	mutex_lock(&iser_conn->state_mutex);
631 	/* In case we endup here without ep_disconnect being invoked. */
632 	if (iser_conn->state != ISER_CONN_DOWN) {
633 		iser_warn("iser conn %p state %d, expected state down.\n",
634 			  iser_conn, iser_conn->state);
635 		iscsi_destroy_endpoint(iser_conn->ep);
636 		iser_conn->state = ISER_CONN_DOWN;
637 	}
638 	/*
639 	 * In case we never got to bind stage, we still need to
640 	 * release IB resources (which is safe to call more than once).
641 	 */
642 	iser_free_ib_conn_res(iser_conn, true);
643 	mutex_unlock(&iser_conn->state_mutex);
644 
645 	if (ib_conn->cma_id != NULL) {
646 		rdma_destroy_id(ib_conn->cma_id);
647 		ib_conn->cma_id = NULL;
648 	}
649 
650 	kfree(iser_conn);
651 }
652 
653 /**
654  * triggers start of the disconnect procedures and wait for them to be done
655  * Called with state mutex held
656  */
657 int iser_conn_terminate(struct iser_conn *iser_conn)
658 {
659 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
660 	int err = 0;
661 
662 	/* terminate the iser conn only if the conn state is UP */
663 	if (!iser_conn_state_comp_exch(iser_conn, ISER_CONN_UP,
664 				       ISER_CONN_TERMINATING))
665 		return 0;
666 
667 	iser_info("iser_conn %p state %d\n", iser_conn, iser_conn->state);
668 
669 	/* suspend queuing of new iscsi commands */
670 	if (iser_conn->iscsi_conn)
671 		iscsi_suspend_queue(iser_conn->iscsi_conn);
672 
673 	/*
674 	 * In case we didn't already clean up the cma_id (peer initiated
675 	 * a disconnection), we need to Cause the CMA to change the QP
676 	 * state to ERROR.
677 	 */
678 	if (ib_conn->cma_id) {
679 		err = rdma_disconnect(ib_conn->cma_id);
680 		if (err)
681 			iser_err("Failed to disconnect, conn: 0x%p err %d\n",
682 				 iser_conn, err);
683 
684 		/* block until all flush errors are consumed */
685 		ib_drain_sq(ib_conn->qp);
686 	}
687 
688 	return 1;
689 }
690 
691 /**
692  * Called with state mutex held
693  **/
694 static void iser_connect_error(struct rdma_cm_id *cma_id)
695 {
696 	struct iser_conn *iser_conn;
697 
698 	iser_conn = (struct iser_conn *)cma_id->context;
699 	iser_conn->state = ISER_CONN_TERMINATING;
700 }
701 
702 static void
703 iser_calc_scsi_params(struct iser_conn *iser_conn,
704 		      unsigned int max_sectors)
705 {
706 	struct iser_device *device = iser_conn->ib_conn.device;
707 	struct ib_device_attr *attr = &device->ib_device->attrs;
708 	unsigned short sg_tablesize, sup_sg_tablesize;
709 	unsigned short reserved_mr_pages;
710 
711 	/*
712 	 * FRs without SG_GAPS or FMRs can only map up to a (device) page per
713 	 * entry, but if the first entry is misaligned we'll end up using two
714 	 * entries (head and tail) for a single page worth data, so one
715 	 * additional entry is required.
716 	 */
717 	if ((attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS) &&
718 	    (attr->device_cap_flags & IB_DEVICE_SG_GAPS_REG))
719 		reserved_mr_pages = 0;
720 	else
721 		reserved_mr_pages = 1;
722 
723 	sg_tablesize = DIV_ROUND_UP(max_sectors * 512, SIZE_4K);
724 	if (attr->device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)
725 		sup_sg_tablesize =
726 			min_t(
727 			 uint, ISCSI_ISER_MAX_SG_TABLESIZE,
728 			 attr->max_fast_reg_page_list_len - reserved_mr_pages);
729 	else
730 		sup_sg_tablesize = ISCSI_ISER_MAX_SG_TABLESIZE;
731 
732 	iser_conn->scsi_sg_tablesize = min(sg_tablesize, sup_sg_tablesize);
733 	iser_conn->pages_per_mr =
734 		iser_conn->scsi_sg_tablesize + reserved_mr_pages;
735 }
736 
737 /**
738  * Called with state mutex held
739  **/
740 static void iser_addr_handler(struct rdma_cm_id *cma_id)
741 {
742 	struct iser_device *device;
743 	struct iser_conn   *iser_conn;
744 	struct ib_conn   *ib_conn;
745 	int    ret;
746 
747 	iser_conn = (struct iser_conn *)cma_id->context;
748 	if (iser_conn->state != ISER_CONN_PENDING)
749 		/* bailout */
750 		return;
751 
752 	ib_conn = &iser_conn->ib_conn;
753 	device = iser_device_find_by_ib_device(cma_id);
754 	if (!device) {
755 		iser_err("device lookup/creation failed\n");
756 		iser_connect_error(cma_id);
757 		return;
758 	}
759 
760 	ib_conn->device = device;
761 
762 	/* connection T10-PI support */
763 	if (iser_pi_enable) {
764 		if (!(device->ib_device->attrs.device_cap_flags &
765 		      IB_DEVICE_SIGNATURE_HANDOVER)) {
766 			iser_warn("T10-PI requested but not supported on %s, "
767 				  "continue without T10-PI\n",
768 				  dev_name(&ib_conn->device->ib_device->dev));
769 			ib_conn->pi_support = false;
770 		} else {
771 			ib_conn->pi_support = true;
772 		}
773 	}
774 
775 	iser_calc_scsi_params(iser_conn, iser_max_sectors);
776 
777 	ret = rdma_resolve_route(cma_id, 1000);
778 	if (ret) {
779 		iser_err("resolve route failed: %d\n", ret);
780 		iser_connect_error(cma_id);
781 		return;
782 	}
783 }
784 
785 /**
786  * Called with state mutex held
787  **/
788 static void iser_route_handler(struct rdma_cm_id *cma_id)
789 {
790 	struct rdma_conn_param conn_param;
791 	int    ret;
792 	struct iser_cm_hdr req_hdr;
793 	struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
794 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
795 	struct iser_device *device = ib_conn->device;
796 
797 	if (iser_conn->state != ISER_CONN_PENDING)
798 		/* bailout */
799 		return;
800 
801 	ret = iser_create_ib_conn_res(ib_conn);
802 	if (ret)
803 		goto failure;
804 
805 	memset(&conn_param, 0, sizeof conn_param);
806 	conn_param.responder_resources = device->ib_device->attrs.max_qp_rd_atom;
807 	conn_param.initiator_depth     = 1;
808 	conn_param.retry_count	       = 7;
809 	conn_param.rnr_retry_count     = 6;
810 
811 	memset(&req_hdr, 0, sizeof(req_hdr));
812 	req_hdr.flags = ISER_ZBVA_NOT_SUP;
813 	if (!device->remote_inv_sup)
814 		req_hdr.flags |= ISER_SEND_W_INV_NOT_SUP;
815 	conn_param.private_data	= (void *)&req_hdr;
816 	conn_param.private_data_len = sizeof(struct iser_cm_hdr);
817 
818 	ret = rdma_connect(cma_id, &conn_param);
819 	if (ret) {
820 		iser_err("failure connecting: %d\n", ret);
821 		goto failure;
822 	}
823 
824 	return;
825 failure:
826 	iser_connect_error(cma_id);
827 }
828 
829 static void iser_connected_handler(struct rdma_cm_id *cma_id,
830 				   const void *private_data)
831 {
832 	struct iser_conn *iser_conn;
833 	struct ib_qp_attr attr;
834 	struct ib_qp_init_attr init_attr;
835 
836 	iser_conn = (struct iser_conn *)cma_id->context;
837 	if (iser_conn->state != ISER_CONN_PENDING)
838 		/* bailout */
839 		return;
840 
841 	(void)ib_query_qp(cma_id->qp, &attr, ~0, &init_attr);
842 	iser_info("remote qpn:%x my qpn:%x\n", attr.dest_qp_num, cma_id->qp->qp_num);
843 
844 	if (private_data) {
845 		u8 flags = *(u8 *)private_data;
846 
847 		iser_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP);
848 	}
849 
850 	iser_info("conn %p: negotiated %s invalidation\n",
851 		  iser_conn, iser_conn->snd_w_inv ? "remote" : "local");
852 
853 	iser_conn->state = ISER_CONN_UP;
854 	complete(&iser_conn->up_completion);
855 }
856 
857 static void iser_disconnected_handler(struct rdma_cm_id *cma_id)
858 {
859 	struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
860 
861 	if (iser_conn_terminate(iser_conn)) {
862 		if (iser_conn->iscsi_conn)
863 			iscsi_conn_failure(iser_conn->iscsi_conn,
864 					   ISCSI_ERR_CONN_FAILED);
865 		else
866 			iser_err("iscsi_iser connection isn't bound\n");
867 	}
868 }
869 
870 static void iser_cleanup_handler(struct rdma_cm_id *cma_id,
871 				 bool destroy)
872 {
873 	struct iser_conn *iser_conn = (struct iser_conn *)cma_id->context;
874 
875 	/*
876 	 * We are not guaranteed that we visited disconnected_handler
877 	 * by now, call it here to be safe that we handle CM drep
878 	 * and flush errors.
879 	 */
880 	iser_disconnected_handler(cma_id);
881 	iser_free_ib_conn_res(iser_conn, destroy);
882 	complete(&iser_conn->ib_completion);
883 };
884 
885 static int iser_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
886 {
887 	struct iser_conn *iser_conn;
888 	int ret = 0;
889 
890 	iser_conn = (struct iser_conn *)cma_id->context;
891 	iser_info("%s (%d): status %d conn %p id %p\n",
892 		  rdma_event_msg(event->event), event->event,
893 		  event->status, cma_id->context, cma_id);
894 
895 	mutex_lock(&iser_conn->state_mutex);
896 	switch (event->event) {
897 	case RDMA_CM_EVENT_ADDR_RESOLVED:
898 		iser_addr_handler(cma_id);
899 		break;
900 	case RDMA_CM_EVENT_ROUTE_RESOLVED:
901 		iser_route_handler(cma_id);
902 		break;
903 	case RDMA_CM_EVENT_ESTABLISHED:
904 		iser_connected_handler(cma_id, event->param.conn.private_data);
905 		break;
906 	case RDMA_CM_EVENT_REJECTED:
907 		iser_info("Connection rejected: %s\n",
908 			 rdma_reject_msg(cma_id, event->status));
909 		/* FALLTHROUGH */
910 	case RDMA_CM_EVENT_ADDR_ERROR:
911 	case RDMA_CM_EVENT_ROUTE_ERROR:
912 	case RDMA_CM_EVENT_CONNECT_ERROR:
913 	case RDMA_CM_EVENT_UNREACHABLE:
914 		iser_connect_error(cma_id);
915 		break;
916 	case RDMA_CM_EVENT_DISCONNECTED:
917 	case RDMA_CM_EVENT_ADDR_CHANGE:
918 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:
919 		iser_cleanup_handler(cma_id, false);
920 		break;
921 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
922 		/*
923 		 * we *must* destroy the device as we cannot rely
924 		 * on iscsid to be around to initiate error handling.
925 		 * also if we are not in state DOWN implicitly destroy
926 		 * the cma_id.
927 		 */
928 		iser_cleanup_handler(cma_id, true);
929 		if (iser_conn->state != ISER_CONN_DOWN) {
930 			iser_conn->ib_conn.cma_id = NULL;
931 			ret = 1;
932 		}
933 		break;
934 	default:
935 		iser_err("Unexpected RDMA CM event: %s (%d)\n",
936 			 rdma_event_msg(event->event), event->event);
937 		break;
938 	}
939 	mutex_unlock(&iser_conn->state_mutex);
940 
941 	return ret;
942 }
943 
944 void iser_conn_init(struct iser_conn *iser_conn)
945 {
946 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
947 
948 	iser_conn->state = ISER_CONN_INIT;
949 	init_completion(&iser_conn->stop_completion);
950 	init_completion(&iser_conn->ib_completion);
951 	init_completion(&iser_conn->up_completion);
952 	INIT_LIST_HEAD(&iser_conn->conn_list);
953 	mutex_init(&iser_conn->state_mutex);
954 
955 	ib_conn->post_recv_buf_count = 0;
956 	ib_conn->reg_cqe.done = iser_reg_comp;
957 }
958 
959  /**
960  * starts the process of connecting to the target
961  * sleeps until the connection is established or rejected
962  */
963 int iser_connect(struct iser_conn   *iser_conn,
964 		 struct sockaddr    *src_addr,
965 		 struct sockaddr    *dst_addr,
966 		 int                 non_blocking)
967 {
968 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
969 	int err = 0;
970 
971 	mutex_lock(&iser_conn->state_mutex);
972 
973 	sprintf(iser_conn->name, "%pISp", dst_addr);
974 
975 	iser_info("connecting to: %s\n", iser_conn->name);
976 
977 	/* the device is known only --after-- address resolution */
978 	ib_conn->device = NULL;
979 
980 	iser_conn->state = ISER_CONN_PENDING;
981 
982 	ib_conn->cma_id = rdma_create_id(&init_net, iser_cma_handler,
983 					 (void *)iser_conn,
984 					 RDMA_PS_TCP, IB_QPT_RC);
985 	if (IS_ERR(ib_conn->cma_id)) {
986 		err = PTR_ERR(ib_conn->cma_id);
987 		iser_err("rdma_create_id failed: %d\n", err);
988 		goto id_failure;
989 	}
990 
991 	err = rdma_resolve_addr(ib_conn->cma_id, src_addr, dst_addr, 1000);
992 	if (err) {
993 		iser_err("rdma_resolve_addr failed: %d\n", err);
994 		goto addr_failure;
995 	}
996 
997 	if (!non_blocking) {
998 		wait_for_completion_interruptible(&iser_conn->up_completion);
999 
1000 		if (iser_conn->state != ISER_CONN_UP) {
1001 			err =  -EIO;
1002 			goto connect_failure;
1003 		}
1004 	}
1005 	mutex_unlock(&iser_conn->state_mutex);
1006 
1007 	mutex_lock(&ig.connlist_mutex);
1008 	list_add(&iser_conn->conn_list, &ig.connlist);
1009 	mutex_unlock(&ig.connlist_mutex);
1010 	return 0;
1011 
1012 id_failure:
1013 	ib_conn->cma_id = NULL;
1014 addr_failure:
1015 	iser_conn->state = ISER_CONN_DOWN;
1016 connect_failure:
1017 	mutex_unlock(&iser_conn->state_mutex);
1018 	iser_conn_release(iser_conn);
1019 	return err;
1020 }
1021 
1022 int iser_post_recvl(struct iser_conn *iser_conn)
1023 {
1024 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
1025 	struct iser_login_desc *desc = &iser_conn->login_desc;
1026 	struct ib_recv_wr wr;
1027 	int ib_ret;
1028 
1029 	desc->sge.addr = desc->rsp_dma;
1030 	desc->sge.length = ISER_RX_LOGIN_SIZE;
1031 	desc->sge.lkey = ib_conn->device->pd->local_dma_lkey;
1032 
1033 	desc->cqe.done = iser_login_rsp;
1034 	wr.wr_cqe = &desc->cqe;
1035 	wr.sg_list = &desc->sge;
1036 	wr.num_sge = 1;
1037 	wr.next = NULL;
1038 
1039 	ib_conn->post_recv_buf_count++;
1040 	ib_ret = ib_post_recv(ib_conn->qp, &wr, NULL);
1041 	if (ib_ret) {
1042 		iser_err("ib_post_recv failed ret=%d\n", ib_ret);
1043 		ib_conn->post_recv_buf_count--;
1044 	}
1045 
1046 	return ib_ret;
1047 }
1048 
1049 int iser_post_recvm(struct iser_conn *iser_conn, int count)
1050 {
1051 	struct ib_conn *ib_conn = &iser_conn->ib_conn;
1052 	unsigned int my_rx_head = iser_conn->rx_desc_head;
1053 	struct iser_rx_desc *rx_desc;
1054 	struct ib_recv_wr *wr;
1055 	int i, ib_ret;
1056 
1057 	for (wr = ib_conn->rx_wr, i = 0; i < count; i++, wr++) {
1058 		rx_desc = &iser_conn->rx_descs[my_rx_head];
1059 		rx_desc->cqe.done = iser_task_rsp;
1060 		wr->wr_cqe = &rx_desc->cqe;
1061 		wr->sg_list = &rx_desc->rx_sg;
1062 		wr->num_sge = 1;
1063 		wr->next = wr + 1;
1064 		my_rx_head = (my_rx_head + 1) & iser_conn->qp_max_recv_dtos_mask;
1065 	}
1066 
1067 	wr--;
1068 	wr->next = NULL; /* mark end of work requests list */
1069 
1070 	ib_conn->post_recv_buf_count += count;
1071 	ib_ret = ib_post_recv(ib_conn->qp, ib_conn->rx_wr, NULL);
1072 	if (ib_ret) {
1073 		iser_err("ib_post_recv failed ret=%d\n", ib_ret);
1074 		ib_conn->post_recv_buf_count -= count;
1075 	} else
1076 		iser_conn->rx_desc_head = my_rx_head;
1077 
1078 	return ib_ret;
1079 }
1080 
1081 
1082 /**
1083  * iser_start_send - Initiate a Send DTO operation
1084  *
1085  * returns 0 on success, -1 on failure
1086  */
1087 int iser_post_send(struct ib_conn *ib_conn, struct iser_tx_desc *tx_desc,
1088 		   bool signal)
1089 {
1090 	struct ib_send_wr *wr = iser_tx_next_wr(tx_desc);
1091 	int ib_ret;
1092 
1093 	ib_dma_sync_single_for_device(ib_conn->device->ib_device,
1094 				      tx_desc->dma_addr, ISER_HEADERS_LEN,
1095 				      DMA_TO_DEVICE);
1096 
1097 	wr->next = NULL;
1098 	wr->wr_cqe = &tx_desc->cqe;
1099 	wr->sg_list = tx_desc->tx_sg;
1100 	wr->num_sge = tx_desc->num_sge;
1101 	wr->opcode = IB_WR_SEND;
1102 	wr->send_flags = signal ? IB_SEND_SIGNALED : 0;
1103 
1104 	ib_ret = ib_post_send(ib_conn->qp, &tx_desc->wrs[0].send, NULL);
1105 	if (ib_ret)
1106 		iser_err("ib_post_send failed, ret:%d opcode:%d\n",
1107 			 ib_ret, wr->opcode);
1108 
1109 	return ib_ret;
1110 }
1111 
1112 u8 iser_check_task_pi_status(struct iscsi_iser_task *iser_task,
1113 			     enum iser_data_dir cmd_dir, sector_t *sector)
1114 {
1115 	struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir];
1116 	struct iser_fr_desc *desc = reg->mem_h;
1117 	unsigned long sector_size = iser_task->sc->device->sector_size;
1118 	struct ib_mr_status mr_status;
1119 	int ret;
1120 
1121 	if (desc && desc->pi_ctx->sig_protected) {
1122 		desc->pi_ctx->sig_protected = 0;
1123 		ret = ib_check_mr_status(desc->pi_ctx->sig_mr,
1124 					 IB_MR_CHECK_SIG_STATUS, &mr_status);
1125 		if (ret) {
1126 			pr_err("ib_check_mr_status failed, ret %d\n", ret);
1127 			/* Not a lot we can do, return ambiguous guard error */
1128 			*sector = 0;
1129 			return 0x1;
1130 		}
1131 
1132 		if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
1133 			sector_t sector_off = mr_status.sig_err.sig_err_offset;
1134 
1135 			sector_div(sector_off, sector_size + 8);
1136 			*sector = scsi_get_lba(iser_task->sc) + sector_off;
1137 
1138 			pr_err("PI error found type %d at sector %llx "
1139 			       "expected %x vs actual %x\n",
1140 			       mr_status.sig_err.err_type,
1141 			       (unsigned long long)*sector,
1142 			       mr_status.sig_err.expected,
1143 			       mr_status.sig_err.actual);
1144 
1145 			switch (mr_status.sig_err.err_type) {
1146 			case IB_SIG_BAD_GUARD:
1147 				return 0x1;
1148 			case IB_SIG_BAD_REFTAG:
1149 				return 0x3;
1150 			case IB_SIG_BAD_APPTAG:
1151 				return 0x2;
1152 			}
1153 		}
1154 	}
1155 
1156 	return 0;
1157 }
1158 
1159 void iser_err_comp(struct ib_wc *wc, const char *type)
1160 {
1161 	if (wc->status != IB_WC_WR_FLUSH_ERR) {
1162 		struct iser_conn *iser_conn = to_iser_conn(wc->qp->qp_context);
1163 
1164 		iser_err("%s failure: %s (%d) vend_err %#x\n", type,
1165 			 ib_wc_status_msg(wc->status), wc->status,
1166 			 wc->vendor_err);
1167 
1168 		if (iser_conn->iscsi_conn)
1169 			iscsi_conn_failure(iser_conn->iscsi_conn,
1170 					   ISCSI_ERR_CONN_FAILED);
1171 	} else {
1172 		iser_dbg("%s failure: %s (%d)\n", type,
1173 			 ib_wc_status_msg(wc->status), wc->status);
1174 	}
1175 }
1176