1 /*******************************************************************************
2  * This file contains iSCSI extentions for RDMA (iSER) Verbs
3  *
4  * (c) Copyright 2013 Datera, Inc.
5  *
6  * Nicholas A. Bellinger <nab@linux-iscsi.org>
7  *
8  * This program is free software; you can redistribute it and/or modify
9  * it under the terms of the GNU General Public License as published by
10  * the Free Software Foundation; either version 2 of the License, or
11  * (at your option) any later version.
12  *
13  * This program is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
16  * GNU General Public License for more details.
17  ****************************************************************************/
18 
19 #include <linux/string.h>
20 #include <linux/module.h>
21 #include <linux/scatterlist.h>
22 #include <linux/socket.h>
23 #include <linux/in.h>
24 #include <linux/in6.h>
25 #include <rdma/ib_verbs.h>
26 #include <rdma/rdma_cm.h>
27 #include <target/target_core_base.h>
28 #include <target/target_core_fabric.h>
29 #include <target/iscsi/iscsi_transport.h>
30 #include <linux/semaphore.h>
31 
32 #include "ib_isert.h"
33 
34 #define	ISERT_MAX_CONN		8
35 #define ISER_MAX_RX_CQ_LEN	(ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN)
36 #define ISER_MAX_TX_CQ_LEN \
37 	((ISERT_QP_MAX_REQ_DTOS + ISCSI_DEF_XMIT_CMDS_MAX) * ISERT_MAX_CONN)
38 #define ISER_MAX_CQ_LEN		(ISER_MAX_RX_CQ_LEN + ISER_MAX_TX_CQ_LEN + \
39 				 ISERT_MAX_CONN)
40 
41 static int isert_debug_level;
42 module_param_named(debug_level, isert_debug_level, int, 0644);
43 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0 (default:0)");
44 
45 static DEFINE_MUTEX(device_list_mutex);
46 static LIST_HEAD(device_list);
47 static struct workqueue_struct *isert_comp_wq;
48 static struct workqueue_struct *isert_release_wq;
49 
50 static int
51 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd);
52 static int
53 isert_login_post_recv(struct isert_conn *isert_conn);
54 static int
55 isert_rdma_accept(struct isert_conn *isert_conn);
56 struct rdma_cm_id *isert_setup_id(struct isert_np *isert_np);
57 
58 static void isert_release_work(struct work_struct *work);
59 static void isert_recv_done(struct ib_cq *cq, struct ib_wc *wc);
60 static void isert_send_done(struct ib_cq *cq, struct ib_wc *wc);
61 static void isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc);
62 static void isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc);
63 
64 static inline bool
65 isert_prot_cmd(struct isert_conn *conn, struct se_cmd *cmd)
66 {
67 	return (conn->pi_support &&
68 		cmd->prot_op != TARGET_PROT_NORMAL);
69 }
70 
71 
72 static void
73 isert_qp_event_callback(struct ib_event *e, void *context)
74 {
75 	struct isert_conn *isert_conn = context;
76 
77 	isert_err("%s (%d): conn %p\n",
78 		  ib_event_msg(e->event), e->event, isert_conn);
79 
80 	switch (e->event) {
81 	case IB_EVENT_COMM_EST:
82 		rdma_notify(isert_conn->cm_id, IB_EVENT_COMM_EST);
83 		break;
84 	case IB_EVENT_QP_LAST_WQE_REACHED:
85 		isert_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED\n");
86 		break;
87 	default:
88 		break;
89 	}
90 }
91 
92 static struct isert_comp *
93 isert_comp_get(struct isert_conn *isert_conn)
94 {
95 	struct isert_device *device = isert_conn->device;
96 	struct isert_comp *comp;
97 	int i, min = 0;
98 
99 	mutex_lock(&device_list_mutex);
100 	for (i = 0; i < device->comps_used; i++)
101 		if (device->comps[i].active_qps <
102 		    device->comps[min].active_qps)
103 			min = i;
104 	comp = &device->comps[min];
105 	comp->active_qps++;
106 	mutex_unlock(&device_list_mutex);
107 
108 	isert_info("conn %p, using comp %p min_index: %d\n",
109 		   isert_conn, comp, min);
110 
111 	return comp;
112 }
113 
114 static void
115 isert_comp_put(struct isert_comp *comp)
116 {
117 	mutex_lock(&device_list_mutex);
118 	comp->active_qps--;
119 	mutex_unlock(&device_list_mutex);
120 }
121 
122 static struct ib_qp *
123 isert_create_qp(struct isert_conn *isert_conn,
124 		struct isert_comp *comp,
125 		struct rdma_cm_id *cma_id)
126 {
127 	struct isert_device *device = isert_conn->device;
128 	struct ib_qp_init_attr attr;
129 	int ret;
130 
131 	memset(&attr, 0, sizeof(struct ib_qp_init_attr));
132 	attr.event_handler = isert_qp_event_callback;
133 	attr.qp_context = isert_conn;
134 	attr.send_cq = comp->cq;
135 	attr.recv_cq = comp->cq;
136 	attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS + 1;
137 	attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS + 1;
138 	attr.cap.max_rdma_ctxs = ISCSI_DEF_XMIT_CMDS_MAX;
139 	attr.cap.max_send_sge = device->ib_device->attrs.max_send_sge;
140 	attr.cap.max_recv_sge = 1;
141 	attr.sq_sig_type = IB_SIGNAL_REQ_WR;
142 	attr.qp_type = IB_QPT_RC;
143 	if (device->pi_capable)
144 		attr.create_flags |= IB_QP_CREATE_SIGNATURE_EN;
145 
146 	ret = rdma_create_qp(cma_id, device->pd, &attr);
147 	if (ret) {
148 		isert_err("rdma_create_qp failed for cma_id %d\n", ret);
149 		return ERR_PTR(ret);
150 	}
151 
152 	return cma_id->qp;
153 }
154 
155 static int
156 isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id)
157 {
158 	struct isert_comp *comp;
159 	int ret;
160 
161 	comp = isert_comp_get(isert_conn);
162 	isert_conn->qp = isert_create_qp(isert_conn, comp, cma_id);
163 	if (IS_ERR(isert_conn->qp)) {
164 		ret = PTR_ERR(isert_conn->qp);
165 		goto err;
166 	}
167 
168 	return 0;
169 err:
170 	isert_comp_put(comp);
171 	return ret;
172 }
173 
174 static int
175 isert_alloc_rx_descriptors(struct isert_conn *isert_conn)
176 {
177 	struct isert_device *device = isert_conn->device;
178 	struct ib_device *ib_dev = device->ib_device;
179 	struct iser_rx_desc *rx_desc;
180 	struct ib_sge *rx_sg;
181 	u64 dma_addr;
182 	int i, j;
183 
184 	isert_conn->rx_descs = kcalloc(ISERT_QP_MAX_RECV_DTOS,
185 				       sizeof(struct iser_rx_desc),
186 				       GFP_KERNEL);
187 	if (!isert_conn->rx_descs)
188 		return -ENOMEM;
189 
190 	rx_desc = isert_conn->rx_descs;
191 
192 	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
193 		dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc,
194 					ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
195 		if (ib_dma_mapping_error(ib_dev, dma_addr))
196 			goto dma_map_fail;
197 
198 		rx_desc->dma_addr = dma_addr;
199 
200 		rx_sg = &rx_desc->rx_sg;
201 		rx_sg->addr = rx_desc->dma_addr;
202 		rx_sg->length = ISER_RX_PAYLOAD_SIZE;
203 		rx_sg->lkey = device->pd->local_dma_lkey;
204 		rx_desc->rx_cqe.done = isert_recv_done;
205 	}
206 
207 	return 0;
208 
209 dma_map_fail:
210 	rx_desc = isert_conn->rx_descs;
211 	for (j = 0; j < i; j++, rx_desc++) {
212 		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
213 				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
214 	}
215 	kfree(isert_conn->rx_descs);
216 	isert_conn->rx_descs = NULL;
217 	isert_err("conn %p failed to allocate rx descriptors\n", isert_conn);
218 	return -ENOMEM;
219 }
220 
221 static void
222 isert_free_rx_descriptors(struct isert_conn *isert_conn)
223 {
224 	struct ib_device *ib_dev = isert_conn->device->ib_device;
225 	struct iser_rx_desc *rx_desc;
226 	int i;
227 
228 	if (!isert_conn->rx_descs)
229 		return;
230 
231 	rx_desc = isert_conn->rx_descs;
232 	for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++)  {
233 		ib_dma_unmap_single(ib_dev, rx_desc->dma_addr,
234 				    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
235 	}
236 
237 	kfree(isert_conn->rx_descs);
238 	isert_conn->rx_descs = NULL;
239 }
240 
241 static void
242 isert_free_comps(struct isert_device *device)
243 {
244 	int i;
245 
246 	for (i = 0; i < device->comps_used; i++) {
247 		struct isert_comp *comp = &device->comps[i];
248 
249 		if (comp->cq)
250 			ib_free_cq(comp->cq);
251 	}
252 	kfree(device->comps);
253 }
254 
255 static int
256 isert_alloc_comps(struct isert_device *device)
257 {
258 	int i, max_cqe, ret = 0;
259 
260 	device->comps_used = min(ISERT_MAX_CQ, min_t(int, num_online_cpus(),
261 				 device->ib_device->num_comp_vectors));
262 
263 	isert_info("Using %d CQs, %s supports %d vectors support "
264 		   "pi_capable %d\n",
265 		   device->comps_used, dev_name(&device->ib_device->dev),
266 		   device->ib_device->num_comp_vectors,
267 		   device->pi_capable);
268 
269 	device->comps = kcalloc(device->comps_used, sizeof(struct isert_comp),
270 				GFP_KERNEL);
271 	if (!device->comps)
272 		return -ENOMEM;
273 
274 	max_cqe = min(ISER_MAX_CQ_LEN, device->ib_device->attrs.max_cqe);
275 
276 	for (i = 0; i < device->comps_used; i++) {
277 		struct isert_comp *comp = &device->comps[i];
278 
279 		comp->device = device;
280 		comp->cq = ib_alloc_cq(device->ib_device, comp, max_cqe, i,
281 				IB_POLL_WORKQUEUE);
282 		if (IS_ERR(comp->cq)) {
283 			isert_err("Unable to allocate cq\n");
284 			ret = PTR_ERR(comp->cq);
285 			comp->cq = NULL;
286 			goto out_cq;
287 		}
288 	}
289 
290 	return 0;
291 out_cq:
292 	isert_free_comps(device);
293 	return ret;
294 }
295 
296 static int
297 isert_create_device_ib_res(struct isert_device *device)
298 {
299 	struct ib_device *ib_dev = device->ib_device;
300 	int ret;
301 
302 	isert_dbg("devattr->max_send_sge: %d devattr->max_recv_sge %d\n",
303 		  ib_dev->attrs.max_send_sge, ib_dev->attrs.max_recv_sge);
304 	isert_dbg("devattr->max_sge_rd: %d\n", ib_dev->attrs.max_sge_rd);
305 
306 	ret = isert_alloc_comps(device);
307 	if (ret)
308 		goto out;
309 
310 	device->pd = ib_alloc_pd(ib_dev, 0);
311 	if (IS_ERR(device->pd)) {
312 		ret = PTR_ERR(device->pd);
313 		isert_err("failed to allocate pd, device %p, ret=%d\n",
314 			  device, ret);
315 		goto out_cq;
316 	}
317 
318 	/* Check signature cap */
319 	device->pi_capable = ib_dev->attrs.device_cap_flags &
320 			     IB_DEVICE_SIGNATURE_HANDOVER ? true : false;
321 
322 	return 0;
323 
324 out_cq:
325 	isert_free_comps(device);
326 out:
327 	if (ret > 0)
328 		ret = -EINVAL;
329 	return ret;
330 }
331 
332 static void
333 isert_free_device_ib_res(struct isert_device *device)
334 {
335 	isert_info("device %p\n", device);
336 
337 	ib_dealloc_pd(device->pd);
338 	isert_free_comps(device);
339 }
340 
341 static void
342 isert_device_put(struct isert_device *device)
343 {
344 	mutex_lock(&device_list_mutex);
345 	device->refcount--;
346 	isert_info("device %p refcount %d\n", device, device->refcount);
347 	if (!device->refcount) {
348 		isert_free_device_ib_res(device);
349 		list_del(&device->dev_node);
350 		kfree(device);
351 	}
352 	mutex_unlock(&device_list_mutex);
353 }
354 
355 static struct isert_device *
356 isert_device_get(struct rdma_cm_id *cma_id)
357 {
358 	struct isert_device *device;
359 	int ret;
360 
361 	mutex_lock(&device_list_mutex);
362 	list_for_each_entry(device, &device_list, dev_node) {
363 		if (device->ib_device->node_guid == cma_id->device->node_guid) {
364 			device->refcount++;
365 			isert_info("Found iser device %p refcount %d\n",
366 				   device, device->refcount);
367 			mutex_unlock(&device_list_mutex);
368 			return device;
369 		}
370 	}
371 
372 	device = kzalloc(sizeof(struct isert_device), GFP_KERNEL);
373 	if (!device) {
374 		mutex_unlock(&device_list_mutex);
375 		return ERR_PTR(-ENOMEM);
376 	}
377 
378 	INIT_LIST_HEAD(&device->dev_node);
379 
380 	device->ib_device = cma_id->device;
381 	ret = isert_create_device_ib_res(device);
382 	if (ret) {
383 		kfree(device);
384 		mutex_unlock(&device_list_mutex);
385 		return ERR_PTR(ret);
386 	}
387 
388 	device->refcount++;
389 	list_add_tail(&device->dev_node, &device_list);
390 	isert_info("Created a new iser device %p refcount %d\n",
391 		   device, device->refcount);
392 	mutex_unlock(&device_list_mutex);
393 
394 	return device;
395 }
396 
397 static void
398 isert_init_conn(struct isert_conn *isert_conn)
399 {
400 	isert_conn->state = ISER_CONN_INIT;
401 	INIT_LIST_HEAD(&isert_conn->node);
402 	init_completion(&isert_conn->login_comp);
403 	init_completion(&isert_conn->login_req_comp);
404 	init_waitqueue_head(&isert_conn->rem_wait);
405 	kref_init(&isert_conn->kref);
406 	mutex_init(&isert_conn->mutex);
407 	INIT_WORK(&isert_conn->release_work, isert_release_work);
408 }
409 
410 static void
411 isert_free_login_buf(struct isert_conn *isert_conn)
412 {
413 	struct ib_device *ib_dev = isert_conn->device->ib_device;
414 
415 	ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma,
416 			    ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
417 	kfree(isert_conn->login_rsp_buf);
418 
419 	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
420 			    ISER_RX_PAYLOAD_SIZE,
421 			    DMA_FROM_DEVICE);
422 	kfree(isert_conn->login_req_buf);
423 }
424 
425 static int
426 isert_alloc_login_buf(struct isert_conn *isert_conn,
427 		      struct ib_device *ib_dev)
428 {
429 	int ret;
430 
431 	isert_conn->login_req_buf = kzalloc(sizeof(*isert_conn->login_req_buf),
432 			GFP_KERNEL);
433 	if (!isert_conn->login_req_buf)
434 		return -ENOMEM;
435 
436 	isert_conn->login_req_dma = ib_dma_map_single(ib_dev,
437 				isert_conn->login_req_buf,
438 				ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
439 	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma);
440 	if (ret) {
441 		isert_err("login_req_dma mapping error: %d\n", ret);
442 		isert_conn->login_req_dma = 0;
443 		goto out_free_login_req_buf;
444 	}
445 
446 	isert_conn->login_rsp_buf = kzalloc(ISER_RX_PAYLOAD_SIZE, GFP_KERNEL);
447 	if (!isert_conn->login_rsp_buf) {
448 		ret = -ENOMEM;
449 		goto out_unmap_login_req_buf;
450 	}
451 
452 	isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev,
453 					isert_conn->login_rsp_buf,
454 					ISER_RX_PAYLOAD_SIZE, DMA_TO_DEVICE);
455 	ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma);
456 	if (ret) {
457 		isert_err("login_rsp_dma mapping error: %d\n", ret);
458 		isert_conn->login_rsp_dma = 0;
459 		goto out_free_login_rsp_buf;
460 	}
461 
462 	return 0;
463 
464 out_free_login_rsp_buf:
465 	kfree(isert_conn->login_rsp_buf);
466 out_unmap_login_req_buf:
467 	ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma,
468 			    ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
469 out_free_login_req_buf:
470 	kfree(isert_conn->login_req_buf);
471 	return ret;
472 }
473 
474 static void
475 isert_set_nego_params(struct isert_conn *isert_conn,
476 		      struct rdma_conn_param *param)
477 {
478 	struct ib_device_attr *attr = &isert_conn->device->ib_device->attrs;
479 
480 	/* Set max inflight RDMA READ requests */
481 	isert_conn->initiator_depth = min_t(u8, param->initiator_depth,
482 				attr->max_qp_init_rd_atom);
483 	isert_dbg("Using initiator_depth: %u\n", isert_conn->initiator_depth);
484 
485 	if (param->private_data) {
486 		u8 flags = *(u8 *)param->private_data;
487 
488 		/*
489 		 * use remote invalidation if the both initiator
490 		 * and the HCA support it
491 		 */
492 		isert_conn->snd_w_inv = !(flags & ISER_SEND_W_INV_NOT_SUP) &&
493 					  (attr->device_cap_flags &
494 					   IB_DEVICE_MEM_MGT_EXTENSIONS);
495 		if (isert_conn->snd_w_inv)
496 			isert_info("Using remote invalidation\n");
497 	}
498 }
499 
500 static int
501 isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
502 {
503 	struct isert_np *isert_np = cma_id->context;
504 	struct iscsi_np *np = isert_np->np;
505 	struct isert_conn *isert_conn;
506 	struct isert_device *device;
507 	int ret = 0;
508 
509 	spin_lock_bh(&np->np_thread_lock);
510 	if (!np->enabled) {
511 		spin_unlock_bh(&np->np_thread_lock);
512 		isert_dbg("iscsi_np is not enabled, reject connect request\n");
513 		return rdma_reject(cma_id, NULL, 0);
514 	}
515 	spin_unlock_bh(&np->np_thread_lock);
516 
517 	isert_dbg("cma_id: %p, portal: %p\n",
518 		 cma_id, cma_id->context);
519 
520 	isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL);
521 	if (!isert_conn)
522 		return -ENOMEM;
523 
524 	isert_init_conn(isert_conn);
525 	isert_conn->cm_id = cma_id;
526 
527 	ret = isert_alloc_login_buf(isert_conn, cma_id->device);
528 	if (ret)
529 		goto out;
530 
531 	device = isert_device_get(cma_id);
532 	if (IS_ERR(device)) {
533 		ret = PTR_ERR(device);
534 		goto out_rsp_dma_map;
535 	}
536 	isert_conn->device = device;
537 
538 	isert_set_nego_params(isert_conn, &event->param.conn);
539 
540 	ret = isert_conn_setup_qp(isert_conn, cma_id);
541 	if (ret)
542 		goto out_conn_dev;
543 
544 	ret = isert_login_post_recv(isert_conn);
545 	if (ret)
546 		goto out_conn_dev;
547 
548 	ret = isert_rdma_accept(isert_conn);
549 	if (ret)
550 		goto out_conn_dev;
551 
552 	mutex_lock(&isert_np->mutex);
553 	list_add_tail(&isert_conn->node, &isert_np->accepted);
554 	mutex_unlock(&isert_np->mutex);
555 
556 	return 0;
557 
558 out_conn_dev:
559 	isert_device_put(device);
560 out_rsp_dma_map:
561 	isert_free_login_buf(isert_conn);
562 out:
563 	kfree(isert_conn);
564 	rdma_reject(cma_id, NULL, 0);
565 	return ret;
566 }
567 
568 static void
569 isert_connect_release(struct isert_conn *isert_conn)
570 {
571 	struct isert_device *device = isert_conn->device;
572 
573 	isert_dbg("conn %p\n", isert_conn);
574 
575 	BUG_ON(!device);
576 
577 	isert_free_rx_descriptors(isert_conn);
578 	if (isert_conn->cm_id &&
579 	    !isert_conn->dev_removed)
580 		rdma_destroy_id(isert_conn->cm_id);
581 
582 	if (isert_conn->qp) {
583 		struct isert_comp *comp = isert_conn->qp->recv_cq->cq_context;
584 
585 		isert_comp_put(comp);
586 		ib_destroy_qp(isert_conn->qp);
587 	}
588 
589 	if (isert_conn->login_req_buf)
590 		isert_free_login_buf(isert_conn);
591 
592 	isert_device_put(device);
593 
594 	if (isert_conn->dev_removed)
595 		wake_up_interruptible(&isert_conn->rem_wait);
596 	else
597 		kfree(isert_conn);
598 }
599 
600 static void
601 isert_connected_handler(struct rdma_cm_id *cma_id)
602 {
603 	struct isert_conn *isert_conn = cma_id->qp->qp_context;
604 	struct isert_np *isert_np = cma_id->context;
605 
606 	isert_info("conn %p\n", isert_conn);
607 
608 	mutex_lock(&isert_conn->mutex);
609 	isert_conn->state = ISER_CONN_UP;
610 	kref_get(&isert_conn->kref);
611 	mutex_unlock(&isert_conn->mutex);
612 
613 	mutex_lock(&isert_np->mutex);
614 	list_move_tail(&isert_conn->node, &isert_np->pending);
615 	mutex_unlock(&isert_np->mutex);
616 
617 	isert_info("np %p: Allow accept_np to continue\n", isert_np);
618 	up(&isert_np->sem);
619 }
620 
621 static void
622 isert_release_kref(struct kref *kref)
623 {
624 	struct isert_conn *isert_conn = container_of(kref,
625 				struct isert_conn, kref);
626 
627 	isert_info("conn %p final kref %s/%d\n", isert_conn, current->comm,
628 		   current->pid);
629 
630 	isert_connect_release(isert_conn);
631 }
632 
633 static void
634 isert_put_conn(struct isert_conn *isert_conn)
635 {
636 	kref_put(&isert_conn->kref, isert_release_kref);
637 }
638 
639 static void
640 isert_handle_unbound_conn(struct isert_conn *isert_conn)
641 {
642 	struct isert_np *isert_np = isert_conn->cm_id->context;
643 
644 	mutex_lock(&isert_np->mutex);
645 	if (!list_empty(&isert_conn->node)) {
646 		/*
647 		 * This means iscsi doesn't know this connection
648 		 * so schedule a cleanup ourselves
649 		 */
650 		list_del_init(&isert_conn->node);
651 		isert_put_conn(isert_conn);
652 		queue_work(isert_release_wq, &isert_conn->release_work);
653 	}
654 	mutex_unlock(&isert_np->mutex);
655 }
656 
657 /**
658  * isert_conn_terminate() - Initiate connection termination
659  * @isert_conn: isert connection struct
660  *
661  * Notes:
662  * In case the connection state is BOUND, move state
663  * to TEMINATING and start teardown sequence (rdma_disconnect).
664  * In case the connection state is UP, complete flush as well.
665  *
666  * This routine must be called with mutex held. Thus it is
667  * safe to call multiple times.
668  */
669 static void
670 isert_conn_terminate(struct isert_conn *isert_conn)
671 {
672 	int err;
673 
674 	if (isert_conn->state >= ISER_CONN_TERMINATING)
675 		return;
676 
677 	isert_info("Terminating conn %p state %d\n",
678 		   isert_conn, isert_conn->state);
679 	isert_conn->state = ISER_CONN_TERMINATING;
680 	err = rdma_disconnect(isert_conn->cm_id);
681 	if (err)
682 		isert_warn("Failed rdma_disconnect isert_conn %p\n",
683 			   isert_conn);
684 }
685 
686 static int
687 isert_np_cma_handler(struct isert_np *isert_np,
688 		     enum rdma_cm_event_type event)
689 {
690 	isert_dbg("%s (%d): isert np %p\n",
691 		  rdma_event_msg(event), event, isert_np);
692 
693 	switch (event) {
694 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
695 		isert_np->cm_id = NULL;
696 		break;
697 	case RDMA_CM_EVENT_ADDR_CHANGE:
698 		isert_np->cm_id = isert_setup_id(isert_np);
699 		if (IS_ERR(isert_np->cm_id)) {
700 			isert_err("isert np %p setup id failed: %ld\n",
701 				  isert_np, PTR_ERR(isert_np->cm_id));
702 			isert_np->cm_id = NULL;
703 		}
704 		break;
705 	default:
706 		isert_err("isert np %p Unexpected event %d\n",
707 			  isert_np, event);
708 	}
709 
710 	return -1;
711 }
712 
713 static int
714 isert_disconnected_handler(struct rdma_cm_id *cma_id,
715 			   enum rdma_cm_event_type event)
716 {
717 	struct isert_conn *isert_conn = cma_id->qp->qp_context;
718 
719 	mutex_lock(&isert_conn->mutex);
720 	switch (isert_conn->state) {
721 	case ISER_CONN_TERMINATING:
722 		break;
723 	case ISER_CONN_UP:
724 		isert_conn_terminate(isert_conn);
725 		ib_drain_qp(isert_conn->qp);
726 		isert_handle_unbound_conn(isert_conn);
727 		break;
728 	case ISER_CONN_BOUND:
729 	case ISER_CONN_FULL_FEATURE: /* FALLTHRU */
730 		iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
731 		break;
732 	default:
733 		isert_warn("conn %p terminating in state %d\n",
734 			   isert_conn, isert_conn->state);
735 	}
736 	mutex_unlock(&isert_conn->mutex);
737 
738 	return 0;
739 }
740 
741 static int
742 isert_connect_error(struct rdma_cm_id *cma_id)
743 {
744 	struct isert_conn *isert_conn = cma_id->qp->qp_context;
745 
746 	ib_drain_qp(isert_conn->qp);
747 	list_del_init(&isert_conn->node);
748 	isert_conn->cm_id = NULL;
749 	isert_put_conn(isert_conn);
750 
751 	return -1;
752 }
753 
754 static int
755 isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
756 {
757 	struct isert_np *isert_np = cma_id->context;
758 	struct isert_conn *isert_conn;
759 	int ret = 0;
760 
761 	isert_info("%s (%d): status %d id %p np %p\n",
762 		   rdma_event_msg(event->event), event->event,
763 		   event->status, cma_id, cma_id->context);
764 
765 	if (isert_np->cm_id == cma_id)
766 		return isert_np_cma_handler(cma_id->context, event->event);
767 
768 	switch (event->event) {
769 	case RDMA_CM_EVENT_CONNECT_REQUEST:
770 		ret = isert_connect_request(cma_id, event);
771 		if (ret)
772 			isert_err("failed handle connect request %d\n", ret);
773 		break;
774 	case RDMA_CM_EVENT_ESTABLISHED:
775 		isert_connected_handler(cma_id);
776 		break;
777 	case RDMA_CM_EVENT_ADDR_CHANGE:    /* FALLTHRU */
778 	case RDMA_CM_EVENT_DISCONNECTED:   /* FALLTHRU */
779 	case RDMA_CM_EVENT_TIMEWAIT_EXIT:  /* FALLTHRU */
780 		ret = isert_disconnected_handler(cma_id, event->event);
781 		break;
782 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
783 		isert_conn = cma_id->qp->qp_context;
784 		isert_conn->dev_removed = true;
785 		isert_disconnected_handler(cma_id, event->event);
786 		wait_event_interruptible(isert_conn->rem_wait,
787 					 isert_conn->state == ISER_CONN_DOWN);
788 		kfree(isert_conn);
789 		/*
790 		 * return non-zero from the callback to destroy
791 		 * the rdma cm id
792 		 */
793 		return 1;
794 	case RDMA_CM_EVENT_REJECTED:
795 		isert_info("Connection rejected: %s\n",
796 			   rdma_reject_msg(cma_id, event->status));
797 		/* fall through */
798 	case RDMA_CM_EVENT_UNREACHABLE:
799 	case RDMA_CM_EVENT_CONNECT_ERROR:
800 		ret = isert_connect_error(cma_id);
801 		break;
802 	default:
803 		isert_err("Unhandled RDMA CMA event: %d\n", event->event);
804 		break;
805 	}
806 
807 	return ret;
808 }
809 
810 static int
811 isert_post_recvm(struct isert_conn *isert_conn, u32 count)
812 {
813 	struct ib_recv_wr *rx_wr;
814 	int i, ret;
815 	struct iser_rx_desc *rx_desc;
816 
817 	for (rx_wr = isert_conn->rx_wr, i = 0; i < count; i++, rx_wr++) {
818 		rx_desc = &isert_conn->rx_descs[i];
819 
820 		rx_wr->wr_cqe = &rx_desc->rx_cqe;
821 		rx_wr->sg_list = &rx_desc->rx_sg;
822 		rx_wr->num_sge = 1;
823 		rx_wr->next = rx_wr + 1;
824 		rx_desc->in_use = false;
825 	}
826 	rx_wr--;
827 	rx_wr->next = NULL; /* mark end of work requests list */
828 
829 	ret = ib_post_recv(isert_conn->qp, isert_conn->rx_wr, NULL);
830 	if (ret)
831 		isert_err("ib_post_recv() failed with ret: %d\n", ret);
832 
833 	return ret;
834 }
835 
836 static int
837 isert_post_recv(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc)
838 {
839 	struct ib_recv_wr rx_wr;
840 	int ret;
841 
842 	if (!rx_desc->in_use) {
843 		/*
844 		 * if the descriptor is not in-use we already reposted it
845 		 * for recv, so just silently return
846 		 */
847 		return 0;
848 	}
849 
850 	rx_desc->in_use = false;
851 	rx_wr.wr_cqe = &rx_desc->rx_cqe;
852 	rx_wr.sg_list = &rx_desc->rx_sg;
853 	rx_wr.num_sge = 1;
854 	rx_wr.next = NULL;
855 
856 	ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
857 	if (ret)
858 		isert_err("ib_post_recv() failed with ret: %d\n", ret);
859 
860 	return ret;
861 }
862 
863 static int
864 isert_login_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc)
865 {
866 	struct ib_device *ib_dev = isert_conn->cm_id->device;
867 	struct ib_send_wr send_wr;
868 	int ret;
869 
870 	ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr,
871 				      ISER_HEADERS_LEN, DMA_TO_DEVICE);
872 
873 	tx_desc->tx_cqe.done = isert_login_send_done;
874 
875 	send_wr.next	= NULL;
876 	send_wr.wr_cqe	= &tx_desc->tx_cqe;
877 	send_wr.sg_list	= tx_desc->tx_sg;
878 	send_wr.num_sge	= tx_desc->num_sge;
879 	send_wr.opcode	= IB_WR_SEND;
880 	send_wr.send_flags = IB_SEND_SIGNALED;
881 
882 	ret = ib_post_send(isert_conn->qp, &send_wr, NULL);
883 	if (ret)
884 		isert_err("ib_post_send() failed, ret: %d\n", ret);
885 
886 	return ret;
887 }
888 
889 static void
890 __isert_create_send_desc(struct isert_device *device,
891 			 struct iser_tx_desc *tx_desc)
892 {
893 
894 	memset(&tx_desc->iser_header, 0, sizeof(struct iser_ctrl));
895 	tx_desc->iser_header.flags = ISCSI_CTRL;
896 
897 	tx_desc->num_sge = 1;
898 
899 	if (tx_desc->tx_sg[0].lkey != device->pd->local_dma_lkey) {
900 		tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
901 		isert_dbg("tx_desc %p lkey mismatch, fixing\n", tx_desc);
902 	}
903 }
904 
905 static void
906 isert_create_send_desc(struct isert_conn *isert_conn,
907 		       struct isert_cmd *isert_cmd,
908 		       struct iser_tx_desc *tx_desc)
909 {
910 	struct isert_device *device = isert_conn->device;
911 	struct ib_device *ib_dev = device->ib_device;
912 
913 	ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr,
914 				   ISER_HEADERS_LEN, DMA_TO_DEVICE);
915 
916 	__isert_create_send_desc(device, tx_desc);
917 }
918 
919 static int
920 isert_init_tx_hdrs(struct isert_conn *isert_conn,
921 		   struct iser_tx_desc *tx_desc)
922 {
923 	struct isert_device *device = isert_conn->device;
924 	struct ib_device *ib_dev = device->ib_device;
925 	u64 dma_addr;
926 
927 	dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc,
928 			ISER_HEADERS_LEN, DMA_TO_DEVICE);
929 	if (ib_dma_mapping_error(ib_dev, dma_addr)) {
930 		isert_err("ib_dma_mapping_error() failed\n");
931 		return -ENOMEM;
932 	}
933 
934 	tx_desc->dma_addr = dma_addr;
935 	tx_desc->tx_sg[0].addr	= tx_desc->dma_addr;
936 	tx_desc->tx_sg[0].length = ISER_HEADERS_LEN;
937 	tx_desc->tx_sg[0].lkey = device->pd->local_dma_lkey;
938 
939 	isert_dbg("Setup tx_sg[0].addr: 0x%llx length: %u lkey: 0x%x\n",
940 		  tx_desc->tx_sg[0].addr, tx_desc->tx_sg[0].length,
941 		  tx_desc->tx_sg[0].lkey);
942 
943 	return 0;
944 }
945 
946 static void
947 isert_init_send_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
948 		   struct ib_send_wr *send_wr)
949 {
950 	struct iser_tx_desc *tx_desc = &isert_cmd->tx_desc;
951 
952 	tx_desc->tx_cqe.done = isert_send_done;
953 	send_wr->wr_cqe = &tx_desc->tx_cqe;
954 
955 	if (isert_conn->snd_w_inv && isert_cmd->inv_rkey) {
956 		send_wr->opcode  = IB_WR_SEND_WITH_INV;
957 		send_wr->ex.invalidate_rkey = isert_cmd->inv_rkey;
958 	} else {
959 		send_wr->opcode = IB_WR_SEND;
960 	}
961 
962 	send_wr->sg_list = &tx_desc->tx_sg[0];
963 	send_wr->num_sge = isert_cmd->tx_desc.num_sge;
964 	send_wr->send_flags = IB_SEND_SIGNALED;
965 }
966 
967 static int
968 isert_login_post_recv(struct isert_conn *isert_conn)
969 {
970 	struct ib_recv_wr rx_wr;
971 	struct ib_sge sge;
972 	int ret;
973 
974 	memset(&sge, 0, sizeof(struct ib_sge));
975 	sge.addr = isert_conn->login_req_dma;
976 	sge.length = ISER_RX_PAYLOAD_SIZE;
977 	sge.lkey = isert_conn->device->pd->local_dma_lkey;
978 
979 	isert_dbg("Setup sge: addr: %llx length: %d 0x%08x\n",
980 		sge.addr, sge.length, sge.lkey);
981 
982 	isert_conn->login_req_buf->rx_cqe.done = isert_login_recv_done;
983 
984 	memset(&rx_wr, 0, sizeof(struct ib_recv_wr));
985 	rx_wr.wr_cqe = &isert_conn->login_req_buf->rx_cqe;
986 	rx_wr.sg_list = &sge;
987 	rx_wr.num_sge = 1;
988 
989 	ret = ib_post_recv(isert_conn->qp, &rx_wr, NULL);
990 	if (ret)
991 		isert_err("ib_post_recv() failed: %d\n", ret);
992 
993 	return ret;
994 }
995 
996 static int
997 isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login,
998 		   u32 length)
999 {
1000 	struct isert_conn *isert_conn = conn->context;
1001 	struct isert_device *device = isert_conn->device;
1002 	struct ib_device *ib_dev = device->ib_device;
1003 	struct iser_tx_desc *tx_desc = &isert_conn->login_tx_desc;
1004 	int ret;
1005 
1006 	__isert_create_send_desc(device, tx_desc);
1007 
1008 	memcpy(&tx_desc->iscsi_header, &login->rsp[0],
1009 	       sizeof(struct iscsi_hdr));
1010 
1011 	isert_init_tx_hdrs(isert_conn, tx_desc);
1012 
1013 	if (length > 0) {
1014 		struct ib_sge *tx_dsg = &tx_desc->tx_sg[1];
1015 
1016 		ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma,
1017 					   length, DMA_TO_DEVICE);
1018 
1019 		memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length);
1020 
1021 		ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma,
1022 					      length, DMA_TO_DEVICE);
1023 
1024 		tx_dsg->addr	= isert_conn->login_rsp_dma;
1025 		tx_dsg->length	= length;
1026 		tx_dsg->lkey	= isert_conn->device->pd->local_dma_lkey;
1027 		tx_desc->num_sge = 2;
1028 	}
1029 	if (!login->login_failed) {
1030 		if (login->login_complete) {
1031 			ret = isert_alloc_rx_descriptors(isert_conn);
1032 			if (ret)
1033 				return ret;
1034 
1035 			ret = isert_post_recvm(isert_conn,
1036 					       ISERT_QP_MAX_RECV_DTOS);
1037 			if (ret)
1038 				return ret;
1039 
1040 			/* Now we are in FULL_FEATURE phase */
1041 			mutex_lock(&isert_conn->mutex);
1042 			isert_conn->state = ISER_CONN_FULL_FEATURE;
1043 			mutex_unlock(&isert_conn->mutex);
1044 			goto post_send;
1045 		}
1046 
1047 		ret = isert_login_post_recv(isert_conn);
1048 		if (ret)
1049 			return ret;
1050 	}
1051 post_send:
1052 	ret = isert_login_post_send(isert_conn, tx_desc);
1053 	if (ret)
1054 		return ret;
1055 
1056 	return 0;
1057 }
1058 
1059 static void
1060 isert_rx_login_req(struct isert_conn *isert_conn)
1061 {
1062 	struct iser_rx_desc *rx_desc = isert_conn->login_req_buf;
1063 	int rx_buflen = isert_conn->login_req_len;
1064 	struct iscsi_conn *conn = isert_conn->conn;
1065 	struct iscsi_login *login = conn->conn_login;
1066 	int size;
1067 
1068 	isert_info("conn %p\n", isert_conn);
1069 
1070 	WARN_ON_ONCE(!login);
1071 
1072 	if (login->first_request) {
1073 		struct iscsi_login_req *login_req =
1074 			(struct iscsi_login_req *)&rx_desc->iscsi_header;
1075 		/*
1076 		 * Setup the initial iscsi_login values from the leading
1077 		 * login request PDU.
1078 		 */
1079 		login->leading_connection = (!login_req->tsih) ? 1 : 0;
1080 		login->current_stage =
1081 			(login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK)
1082 			 >> 2;
1083 		login->version_min	= login_req->min_version;
1084 		login->version_max	= login_req->max_version;
1085 		memcpy(login->isid, login_req->isid, 6);
1086 		login->cmd_sn		= be32_to_cpu(login_req->cmdsn);
1087 		login->init_task_tag	= login_req->itt;
1088 		login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn);
1089 		login->cid		= be16_to_cpu(login_req->cid);
1090 		login->tsih		= be16_to_cpu(login_req->tsih);
1091 	}
1092 
1093 	memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN);
1094 
1095 	size = min(rx_buflen, MAX_KEY_VALUE_PAIRS);
1096 	isert_dbg("Using login payload size: %d, rx_buflen: %d "
1097 		  "MAX_KEY_VALUE_PAIRS: %d\n", size, rx_buflen,
1098 		  MAX_KEY_VALUE_PAIRS);
1099 	memcpy(login->req_buf, &rx_desc->data[0], size);
1100 
1101 	if (login->first_request) {
1102 		complete(&isert_conn->login_comp);
1103 		return;
1104 	}
1105 	schedule_delayed_work(&conn->login_work, 0);
1106 }
1107 
1108 static struct iscsi_cmd
1109 *isert_allocate_cmd(struct iscsi_conn *conn, struct iser_rx_desc *rx_desc)
1110 {
1111 	struct isert_conn *isert_conn = conn->context;
1112 	struct isert_cmd *isert_cmd;
1113 	struct iscsi_cmd *cmd;
1114 
1115 	cmd = iscsit_allocate_cmd(conn, TASK_INTERRUPTIBLE);
1116 	if (!cmd) {
1117 		isert_err("Unable to allocate iscsi_cmd + isert_cmd\n");
1118 		return NULL;
1119 	}
1120 	isert_cmd = iscsit_priv_cmd(cmd);
1121 	isert_cmd->conn = isert_conn;
1122 	isert_cmd->iscsi_cmd = cmd;
1123 	isert_cmd->rx_desc = rx_desc;
1124 
1125 	return cmd;
1126 }
1127 
1128 static int
1129 isert_handle_scsi_cmd(struct isert_conn *isert_conn,
1130 		      struct isert_cmd *isert_cmd, struct iscsi_cmd *cmd,
1131 		      struct iser_rx_desc *rx_desc, unsigned char *buf)
1132 {
1133 	struct iscsi_conn *conn = isert_conn->conn;
1134 	struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf;
1135 	int imm_data, imm_data_len, unsol_data, sg_nents, rc;
1136 	bool dump_payload = false;
1137 	unsigned int data_len;
1138 
1139 	rc = iscsit_setup_scsi_cmd(conn, cmd, buf);
1140 	if (rc < 0)
1141 		return rc;
1142 
1143 	imm_data = cmd->immediate_data;
1144 	imm_data_len = cmd->first_burst_len;
1145 	unsol_data = cmd->unsolicited_data;
1146 	data_len = cmd->se_cmd.data_length;
1147 
1148 	if (imm_data && imm_data_len == data_len)
1149 		cmd->se_cmd.se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
1150 	rc = iscsit_process_scsi_cmd(conn, cmd, hdr);
1151 	if (rc < 0) {
1152 		return 0;
1153 	} else if (rc > 0) {
1154 		dump_payload = true;
1155 		goto sequence_cmd;
1156 	}
1157 
1158 	if (!imm_data)
1159 		return 0;
1160 
1161 	if (imm_data_len != data_len) {
1162 		sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE));
1163 		sg_copy_from_buffer(cmd->se_cmd.t_data_sg, sg_nents,
1164 				    &rx_desc->data[0], imm_data_len);
1165 		isert_dbg("Copy Immediate sg_nents: %u imm_data_len: %d\n",
1166 			  sg_nents, imm_data_len);
1167 	} else {
1168 		sg_init_table(&isert_cmd->sg, 1);
1169 		cmd->se_cmd.t_data_sg = &isert_cmd->sg;
1170 		cmd->se_cmd.t_data_nents = 1;
1171 		sg_set_buf(&isert_cmd->sg, &rx_desc->data[0], imm_data_len);
1172 		isert_dbg("Transfer Immediate imm_data_len: %d\n",
1173 			  imm_data_len);
1174 	}
1175 
1176 	cmd->write_data_done += imm_data_len;
1177 
1178 	if (cmd->write_data_done == cmd->se_cmd.data_length) {
1179 		spin_lock_bh(&cmd->istate_lock);
1180 		cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1181 		cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1182 		spin_unlock_bh(&cmd->istate_lock);
1183 	}
1184 
1185 sequence_cmd:
1186 	rc = iscsit_sequence_cmd(conn, cmd, buf, hdr->cmdsn);
1187 
1188 	if (!rc && dump_payload == false && unsol_data)
1189 		iscsit_set_unsolicited_dataout(cmd);
1190 	else if (dump_payload && imm_data)
1191 		target_put_sess_cmd(&cmd->se_cmd);
1192 
1193 	return 0;
1194 }
1195 
1196 static int
1197 isert_handle_iscsi_dataout(struct isert_conn *isert_conn,
1198 			   struct iser_rx_desc *rx_desc, unsigned char *buf)
1199 {
1200 	struct scatterlist *sg_start;
1201 	struct iscsi_conn *conn = isert_conn->conn;
1202 	struct iscsi_cmd *cmd = NULL;
1203 	struct iscsi_data *hdr = (struct iscsi_data *)buf;
1204 	u32 unsol_data_len = ntoh24(hdr->dlength);
1205 	int rc, sg_nents, sg_off, page_off;
1206 
1207 	rc = iscsit_check_dataout_hdr(conn, buf, &cmd);
1208 	if (rc < 0)
1209 		return rc;
1210 	else if (!cmd)
1211 		return 0;
1212 	/*
1213 	 * FIXME: Unexpected unsolicited_data out
1214 	 */
1215 	if (!cmd->unsolicited_data) {
1216 		isert_err("Received unexpected solicited data payload\n");
1217 		dump_stack();
1218 		return -1;
1219 	}
1220 
1221 	isert_dbg("Unsolicited DataOut unsol_data_len: %u, "
1222 		  "write_data_done: %u, data_length: %u\n",
1223 		  unsol_data_len,  cmd->write_data_done,
1224 		  cmd->se_cmd.data_length);
1225 
1226 	sg_off = cmd->write_data_done / PAGE_SIZE;
1227 	sg_start = &cmd->se_cmd.t_data_sg[sg_off];
1228 	sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE));
1229 	page_off = cmd->write_data_done % PAGE_SIZE;
1230 	/*
1231 	 * FIXME: Non page-aligned unsolicited_data out
1232 	 */
1233 	if (page_off) {
1234 		isert_err("unexpected non-page aligned data payload\n");
1235 		dump_stack();
1236 		return -1;
1237 	}
1238 	isert_dbg("Copying DataOut: sg_start: %p, sg_off: %u "
1239 		  "sg_nents: %u from %p %u\n", sg_start, sg_off,
1240 		  sg_nents, &rx_desc->data[0], unsol_data_len);
1241 
1242 	sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0],
1243 			    unsol_data_len);
1244 
1245 	rc = iscsit_check_dataout_payload(cmd, hdr, false);
1246 	if (rc < 0)
1247 		return rc;
1248 
1249 	/*
1250 	 * multiple data-outs on the same command can arrive -
1251 	 * so post the buffer before hand
1252 	 */
1253 	rc = isert_post_recv(isert_conn, rx_desc);
1254 	if (rc) {
1255 		isert_err("ib_post_recv failed with %d\n", rc);
1256 		return rc;
1257 	}
1258 	return 0;
1259 }
1260 
1261 static int
1262 isert_handle_nop_out(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1263 		     struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1264 		     unsigned char *buf)
1265 {
1266 	struct iscsi_conn *conn = isert_conn->conn;
1267 	struct iscsi_nopout *hdr = (struct iscsi_nopout *)buf;
1268 	int rc;
1269 
1270 	rc = iscsit_setup_nop_out(conn, cmd, hdr);
1271 	if (rc < 0)
1272 		return rc;
1273 	/*
1274 	 * FIXME: Add support for NOPOUT payload using unsolicited RDMA payload
1275 	 */
1276 
1277 	return iscsit_process_nop_out(conn, cmd, hdr);
1278 }
1279 
1280 static int
1281 isert_handle_text_cmd(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd,
1282 		      struct iscsi_cmd *cmd, struct iser_rx_desc *rx_desc,
1283 		      struct iscsi_text *hdr)
1284 {
1285 	struct iscsi_conn *conn = isert_conn->conn;
1286 	u32 payload_length = ntoh24(hdr->dlength);
1287 	int rc;
1288 	unsigned char *text_in = NULL;
1289 
1290 	rc = iscsit_setup_text_cmd(conn, cmd, hdr);
1291 	if (rc < 0)
1292 		return rc;
1293 
1294 	if (payload_length) {
1295 		text_in = kzalloc(payload_length, GFP_KERNEL);
1296 		if (!text_in)
1297 			return -ENOMEM;
1298 	}
1299 	cmd->text_in_ptr = text_in;
1300 
1301 	memcpy(cmd->text_in_ptr, &rx_desc->data[0], payload_length);
1302 
1303 	return iscsit_process_text_cmd(conn, cmd, hdr);
1304 }
1305 
1306 static int
1307 isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc,
1308 		uint32_t read_stag, uint64_t read_va,
1309 		uint32_t write_stag, uint64_t write_va)
1310 {
1311 	struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1312 	struct iscsi_conn *conn = isert_conn->conn;
1313 	struct iscsi_cmd *cmd;
1314 	struct isert_cmd *isert_cmd;
1315 	int ret = -EINVAL;
1316 	u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK);
1317 
1318 	if (conn->sess->sess_ops->SessionType &&
1319 	   (!(opcode & ISCSI_OP_TEXT) || !(opcode & ISCSI_OP_LOGOUT))) {
1320 		isert_err("Got illegal opcode: 0x%02x in SessionType=Discovery,"
1321 			  " ignoring\n", opcode);
1322 		return 0;
1323 	}
1324 
1325 	switch (opcode) {
1326 	case ISCSI_OP_SCSI_CMD:
1327 		cmd = isert_allocate_cmd(conn, rx_desc);
1328 		if (!cmd)
1329 			break;
1330 
1331 		isert_cmd = iscsit_priv_cmd(cmd);
1332 		isert_cmd->read_stag = read_stag;
1333 		isert_cmd->read_va = read_va;
1334 		isert_cmd->write_stag = write_stag;
1335 		isert_cmd->write_va = write_va;
1336 		isert_cmd->inv_rkey = read_stag ? read_stag : write_stag;
1337 
1338 		ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, cmd,
1339 					rx_desc, (unsigned char *)hdr);
1340 		break;
1341 	case ISCSI_OP_NOOP_OUT:
1342 		cmd = isert_allocate_cmd(conn, rx_desc);
1343 		if (!cmd)
1344 			break;
1345 
1346 		isert_cmd = iscsit_priv_cmd(cmd);
1347 		ret = isert_handle_nop_out(isert_conn, isert_cmd, cmd,
1348 					   rx_desc, (unsigned char *)hdr);
1349 		break;
1350 	case ISCSI_OP_SCSI_DATA_OUT:
1351 		ret = isert_handle_iscsi_dataout(isert_conn, rx_desc,
1352 						(unsigned char *)hdr);
1353 		break;
1354 	case ISCSI_OP_SCSI_TMFUNC:
1355 		cmd = isert_allocate_cmd(conn, rx_desc);
1356 		if (!cmd)
1357 			break;
1358 
1359 		ret = iscsit_handle_task_mgt_cmd(conn, cmd,
1360 						(unsigned char *)hdr);
1361 		break;
1362 	case ISCSI_OP_LOGOUT:
1363 		cmd = isert_allocate_cmd(conn, rx_desc);
1364 		if (!cmd)
1365 			break;
1366 
1367 		ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr);
1368 		break;
1369 	case ISCSI_OP_TEXT:
1370 		if (be32_to_cpu(hdr->ttt) != 0xFFFFFFFF)
1371 			cmd = iscsit_find_cmd_from_itt(conn, hdr->itt);
1372 		else
1373 			cmd = isert_allocate_cmd(conn, rx_desc);
1374 
1375 		if (!cmd)
1376 			break;
1377 
1378 		isert_cmd = iscsit_priv_cmd(cmd);
1379 		ret = isert_handle_text_cmd(isert_conn, isert_cmd, cmd,
1380 					    rx_desc, (struct iscsi_text *)hdr);
1381 		break;
1382 	default:
1383 		isert_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode);
1384 		dump_stack();
1385 		break;
1386 	}
1387 
1388 	return ret;
1389 }
1390 
1391 static void
1392 isert_print_wc(struct ib_wc *wc, const char *type)
1393 {
1394 	if (wc->status != IB_WC_WR_FLUSH_ERR)
1395 		isert_err("%s failure: %s (%d) vend_err %x\n", type,
1396 			  ib_wc_status_msg(wc->status), wc->status,
1397 			  wc->vendor_err);
1398 	else
1399 		isert_dbg("%s failure: %s (%d)\n", type,
1400 			  ib_wc_status_msg(wc->status), wc->status);
1401 }
1402 
1403 static void
1404 isert_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1405 {
1406 	struct isert_conn *isert_conn = wc->qp->qp_context;
1407 	struct ib_device *ib_dev = isert_conn->cm_id->device;
1408 	struct iser_rx_desc *rx_desc = cqe_to_rx_desc(wc->wr_cqe);
1409 	struct iscsi_hdr *hdr = &rx_desc->iscsi_header;
1410 	struct iser_ctrl *iser_ctrl = &rx_desc->iser_header;
1411 	uint64_t read_va = 0, write_va = 0;
1412 	uint32_t read_stag = 0, write_stag = 0;
1413 
1414 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1415 		isert_print_wc(wc, "recv");
1416 		if (wc->status != IB_WC_WR_FLUSH_ERR)
1417 			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1418 		return;
1419 	}
1420 
1421 	rx_desc->in_use = true;
1422 
1423 	ib_dma_sync_single_for_cpu(ib_dev, rx_desc->dma_addr,
1424 			ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1425 
1426 	isert_dbg("DMA: 0x%llx, iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n",
1427 		 rx_desc->dma_addr, hdr->opcode, hdr->itt, hdr->flags,
1428 		 (int)(wc->byte_len - ISER_HEADERS_LEN));
1429 
1430 	switch (iser_ctrl->flags & 0xF0) {
1431 	case ISCSI_CTRL:
1432 		if (iser_ctrl->flags & ISER_RSV) {
1433 			read_stag = be32_to_cpu(iser_ctrl->read_stag);
1434 			read_va = be64_to_cpu(iser_ctrl->read_va);
1435 			isert_dbg("ISER_RSV: read_stag: 0x%x read_va: 0x%llx\n",
1436 				  read_stag, (unsigned long long)read_va);
1437 		}
1438 		if (iser_ctrl->flags & ISER_WSV) {
1439 			write_stag = be32_to_cpu(iser_ctrl->write_stag);
1440 			write_va = be64_to_cpu(iser_ctrl->write_va);
1441 			isert_dbg("ISER_WSV: write_stag: 0x%x write_va: 0x%llx\n",
1442 				  write_stag, (unsigned long long)write_va);
1443 		}
1444 
1445 		isert_dbg("ISER ISCSI_CTRL PDU\n");
1446 		break;
1447 	case ISER_HELLO:
1448 		isert_err("iSER Hello message\n");
1449 		break;
1450 	default:
1451 		isert_warn("Unknown iSER hdr flags: 0x%02x\n", iser_ctrl->flags);
1452 		break;
1453 	}
1454 
1455 	isert_rx_opcode(isert_conn, rx_desc,
1456 			read_stag, read_va, write_stag, write_va);
1457 
1458 	ib_dma_sync_single_for_device(ib_dev, rx_desc->dma_addr,
1459 			ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1460 }
1461 
1462 static void
1463 isert_login_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1464 {
1465 	struct isert_conn *isert_conn = wc->qp->qp_context;
1466 	struct ib_device *ib_dev = isert_conn->device->ib_device;
1467 
1468 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1469 		isert_print_wc(wc, "login recv");
1470 		return;
1471 	}
1472 
1473 	ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_req_dma,
1474 			ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1475 
1476 	isert_conn->login_req_len = wc->byte_len - ISER_HEADERS_LEN;
1477 
1478 	if (isert_conn->conn) {
1479 		struct iscsi_login *login = isert_conn->conn->conn_login;
1480 
1481 		if (login && !login->first_request)
1482 			isert_rx_login_req(isert_conn);
1483 	}
1484 
1485 	mutex_lock(&isert_conn->mutex);
1486 	complete(&isert_conn->login_req_comp);
1487 	mutex_unlock(&isert_conn->mutex);
1488 
1489 	ib_dma_sync_single_for_device(ib_dev, isert_conn->login_req_dma,
1490 				ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE);
1491 }
1492 
1493 static void
1494 isert_rdma_rw_ctx_destroy(struct isert_cmd *cmd, struct isert_conn *conn)
1495 {
1496 	struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
1497 	enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
1498 
1499 	if (!cmd->rw.nr_ops)
1500 		return;
1501 
1502 	if (isert_prot_cmd(conn, se_cmd)) {
1503 		rdma_rw_ctx_destroy_signature(&cmd->rw, conn->qp,
1504 				conn->cm_id->port_num, se_cmd->t_data_sg,
1505 				se_cmd->t_data_nents, se_cmd->t_prot_sg,
1506 				se_cmd->t_prot_nents, dir);
1507 	} else {
1508 		rdma_rw_ctx_destroy(&cmd->rw, conn->qp, conn->cm_id->port_num,
1509 				se_cmd->t_data_sg, se_cmd->t_data_nents, dir);
1510 	}
1511 
1512 	cmd->rw.nr_ops = 0;
1513 }
1514 
1515 static void
1516 isert_put_cmd(struct isert_cmd *isert_cmd, bool comp_err)
1517 {
1518 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1519 	struct isert_conn *isert_conn = isert_cmd->conn;
1520 	struct iscsi_conn *conn = isert_conn->conn;
1521 	struct iscsi_text_rsp *hdr;
1522 
1523 	isert_dbg("Cmd %p\n", isert_cmd);
1524 
1525 	switch (cmd->iscsi_opcode) {
1526 	case ISCSI_OP_SCSI_CMD:
1527 		spin_lock_bh(&conn->cmd_lock);
1528 		if (!list_empty(&cmd->i_conn_node))
1529 			list_del_init(&cmd->i_conn_node);
1530 		spin_unlock_bh(&conn->cmd_lock);
1531 
1532 		if (cmd->data_direction == DMA_TO_DEVICE) {
1533 			iscsit_stop_dataout_timer(cmd);
1534 			/*
1535 			 * Check for special case during comp_err where
1536 			 * WRITE_PENDING has been handed off from core,
1537 			 * but requires an extra target_put_sess_cmd()
1538 			 * before transport_generic_free_cmd() below.
1539 			 */
1540 			if (comp_err &&
1541 			    cmd->se_cmd.t_state == TRANSPORT_WRITE_PENDING) {
1542 				struct se_cmd *se_cmd = &cmd->se_cmd;
1543 
1544 				target_put_sess_cmd(se_cmd);
1545 			}
1546 		}
1547 
1548 		isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1549 		transport_generic_free_cmd(&cmd->se_cmd, 0);
1550 		break;
1551 	case ISCSI_OP_SCSI_TMFUNC:
1552 		spin_lock_bh(&conn->cmd_lock);
1553 		if (!list_empty(&cmd->i_conn_node))
1554 			list_del_init(&cmd->i_conn_node);
1555 		spin_unlock_bh(&conn->cmd_lock);
1556 
1557 		transport_generic_free_cmd(&cmd->se_cmd, 0);
1558 		break;
1559 	case ISCSI_OP_REJECT:
1560 	case ISCSI_OP_NOOP_OUT:
1561 	case ISCSI_OP_TEXT:
1562 		hdr = (struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
1563 		/* If the continue bit is on, keep the command alive */
1564 		if (hdr->flags & ISCSI_FLAG_TEXT_CONTINUE)
1565 			break;
1566 
1567 		spin_lock_bh(&conn->cmd_lock);
1568 		if (!list_empty(&cmd->i_conn_node))
1569 			list_del_init(&cmd->i_conn_node);
1570 		spin_unlock_bh(&conn->cmd_lock);
1571 
1572 		/*
1573 		 * Handle special case for REJECT when iscsi_add_reject*() has
1574 		 * overwritten the original iscsi_opcode assignment, and the
1575 		 * associated cmd->se_cmd needs to be released.
1576 		 */
1577 		if (cmd->se_cmd.se_tfo != NULL) {
1578 			isert_dbg("Calling transport_generic_free_cmd for 0x%02x\n",
1579 				 cmd->iscsi_opcode);
1580 			transport_generic_free_cmd(&cmd->se_cmd, 0);
1581 			break;
1582 		}
1583 		/* fall through */
1584 	default:
1585 		iscsit_release_cmd(cmd);
1586 		break;
1587 	}
1588 }
1589 
1590 static void
1591 isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev)
1592 {
1593 	if (tx_desc->dma_addr != 0) {
1594 		isert_dbg("unmap single for tx_desc->dma_addr\n");
1595 		ib_dma_unmap_single(ib_dev, tx_desc->dma_addr,
1596 				    ISER_HEADERS_LEN, DMA_TO_DEVICE);
1597 		tx_desc->dma_addr = 0;
1598 	}
1599 }
1600 
1601 static void
1602 isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd,
1603 		     struct ib_device *ib_dev, bool comp_err)
1604 {
1605 	if (isert_cmd->pdu_buf_dma != 0) {
1606 		isert_dbg("unmap single for isert_cmd->pdu_buf_dma\n");
1607 		ib_dma_unmap_single(ib_dev, isert_cmd->pdu_buf_dma,
1608 				    isert_cmd->pdu_buf_len, DMA_TO_DEVICE);
1609 		isert_cmd->pdu_buf_dma = 0;
1610 	}
1611 
1612 	isert_unmap_tx_desc(tx_desc, ib_dev);
1613 	isert_put_cmd(isert_cmd, comp_err);
1614 }
1615 
1616 static int
1617 isert_check_pi_status(struct se_cmd *se_cmd, struct ib_mr *sig_mr)
1618 {
1619 	struct ib_mr_status mr_status;
1620 	int ret;
1621 
1622 	ret = ib_check_mr_status(sig_mr, IB_MR_CHECK_SIG_STATUS, &mr_status);
1623 	if (ret) {
1624 		isert_err("ib_check_mr_status failed, ret %d\n", ret);
1625 		goto fail_mr_status;
1626 	}
1627 
1628 	if (mr_status.fail_status & IB_MR_CHECK_SIG_STATUS) {
1629 		u64 sec_offset_err;
1630 		u32 block_size = se_cmd->se_dev->dev_attrib.block_size + 8;
1631 
1632 		switch (mr_status.sig_err.err_type) {
1633 		case IB_SIG_BAD_GUARD:
1634 			se_cmd->pi_err = TCM_LOGICAL_BLOCK_GUARD_CHECK_FAILED;
1635 			break;
1636 		case IB_SIG_BAD_REFTAG:
1637 			se_cmd->pi_err = TCM_LOGICAL_BLOCK_REF_TAG_CHECK_FAILED;
1638 			break;
1639 		case IB_SIG_BAD_APPTAG:
1640 			se_cmd->pi_err = TCM_LOGICAL_BLOCK_APP_TAG_CHECK_FAILED;
1641 			break;
1642 		}
1643 		sec_offset_err = mr_status.sig_err.sig_err_offset;
1644 		do_div(sec_offset_err, block_size);
1645 		se_cmd->bad_sector = sec_offset_err + se_cmd->t_task_lba;
1646 
1647 		isert_err("PI error found type %d at sector 0x%llx "
1648 			  "expected 0x%x vs actual 0x%x\n",
1649 			  mr_status.sig_err.err_type,
1650 			  (unsigned long long)se_cmd->bad_sector,
1651 			  mr_status.sig_err.expected,
1652 			  mr_status.sig_err.actual);
1653 		ret = 1;
1654 	}
1655 
1656 fail_mr_status:
1657 	return ret;
1658 }
1659 
1660 static void
1661 isert_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc)
1662 {
1663 	struct isert_conn *isert_conn = wc->qp->qp_context;
1664 	struct isert_device *device = isert_conn->device;
1665 	struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1666 	struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
1667 	struct se_cmd *cmd = &isert_cmd->iscsi_cmd->se_cmd;
1668 	int ret = 0;
1669 
1670 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1671 		isert_print_wc(wc, "rdma write");
1672 		if (wc->status != IB_WC_WR_FLUSH_ERR)
1673 			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1674 		isert_completion_put(desc, isert_cmd, device->ib_device, true);
1675 		return;
1676 	}
1677 
1678 	isert_dbg("Cmd %p\n", isert_cmd);
1679 
1680 	ret = isert_check_pi_status(cmd, isert_cmd->rw.sig->sig_mr);
1681 	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1682 
1683 	if (ret) {
1684 		/*
1685 		 * transport_generic_request_failure() expects to have
1686 		 * plus two references to handle queue-full, so re-add
1687 		 * one here as target-core will have already dropped
1688 		 * it after the first isert_put_datain() callback.
1689 		 */
1690 		kref_get(&cmd->cmd_kref);
1691 		transport_generic_request_failure(cmd, cmd->pi_err);
1692 	} else {
1693 		/*
1694 		 * XXX: isert_put_response() failure is not retried.
1695 		 */
1696 		ret = isert_put_response(isert_conn->conn, isert_cmd->iscsi_cmd);
1697 		if (ret)
1698 			pr_warn_ratelimited("isert_put_response() ret: %d\n", ret);
1699 	}
1700 }
1701 
1702 static void
1703 isert_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1704 {
1705 	struct isert_conn *isert_conn = wc->qp->qp_context;
1706 	struct isert_device *device = isert_conn->device;
1707 	struct iser_tx_desc *desc = cqe_to_tx_desc(wc->wr_cqe);
1708 	struct isert_cmd *isert_cmd = tx_desc_to_cmd(desc);
1709 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1710 	struct se_cmd *se_cmd = &cmd->se_cmd;
1711 	int ret = 0;
1712 
1713 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1714 		isert_print_wc(wc, "rdma read");
1715 		if (wc->status != IB_WC_WR_FLUSH_ERR)
1716 			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1717 		isert_completion_put(desc, isert_cmd, device->ib_device, true);
1718 		return;
1719 	}
1720 
1721 	isert_dbg("Cmd %p\n", isert_cmd);
1722 
1723 	iscsit_stop_dataout_timer(cmd);
1724 
1725 	if (isert_prot_cmd(isert_conn, se_cmd))
1726 		ret = isert_check_pi_status(se_cmd, isert_cmd->rw.sig->sig_mr);
1727 	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1728 	cmd->write_data_done = 0;
1729 
1730 	isert_dbg("Cmd: %p RDMA_READ comp calling execute_cmd\n", isert_cmd);
1731 	spin_lock_bh(&cmd->istate_lock);
1732 	cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT;
1733 	cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT;
1734 	spin_unlock_bh(&cmd->istate_lock);
1735 
1736 	/*
1737 	 * transport_generic_request_failure() will drop the extra
1738 	 * se_cmd->cmd_kref reference after T10-PI error, and handle
1739 	 * any non-zero ->queue_status() callback error retries.
1740 	 */
1741 	if (ret)
1742 		transport_generic_request_failure(se_cmd, se_cmd->pi_err);
1743 	else
1744 		target_execute_cmd(se_cmd);
1745 }
1746 
1747 static void
1748 isert_do_control_comp(struct work_struct *work)
1749 {
1750 	struct isert_cmd *isert_cmd = container_of(work,
1751 			struct isert_cmd, comp_work);
1752 	struct isert_conn *isert_conn = isert_cmd->conn;
1753 	struct ib_device *ib_dev = isert_conn->cm_id->device;
1754 	struct iscsi_cmd *cmd = isert_cmd->iscsi_cmd;
1755 
1756 	isert_dbg("Cmd %p i_state %d\n", isert_cmd, cmd->i_state);
1757 
1758 	switch (cmd->i_state) {
1759 	case ISTATE_SEND_TASKMGTRSP:
1760 		iscsit_tmr_post_handler(cmd, cmd->conn);
1761 		/* fall through */
1762 	case ISTATE_SEND_REJECT:
1763 	case ISTATE_SEND_TEXTRSP:
1764 		cmd->i_state = ISTATE_SENT_STATUS;
1765 		isert_completion_put(&isert_cmd->tx_desc, isert_cmd,
1766 				     ib_dev, false);
1767 		break;
1768 	case ISTATE_SEND_LOGOUTRSP:
1769 		iscsit_logout_post_handler(cmd, cmd->conn);
1770 		break;
1771 	default:
1772 		isert_err("Unknown i_state %d\n", cmd->i_state);
1773 		dump_stack();
1774 		break;
1775 	}
1776 }
1777 
1778 static void
1779 isert_login_send_done(struct ib_cq *cq, struct ib_wc *wc)
1780 {
1781 	struct isert_conn *isert_conn = wc->qp->qp_context;
1782 	struct ib_device *ib_dev = isert_conn->cm_id->device;
1783 	struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1784 
1785 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1786 		isert_print_wc(wc, "login send");
1787 		if (wc->status != IB_WC_WR_FLUSH_ERR)
1788 			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1789 	}
1790 
1791 	isert_unmap_tx_desc(tx_desc, ib_dev);
1792 }
1793 
1794 static void
1795 isert_send_done(struct ib_cq *cq, struct ib_wc *wc)
1796 {
1797 	struct isert_conn *isert_conn = wc->qp->qp_context;
1798 	struct ib_device *ib_dev = isert_conn->cm_id->device;
1799 	struct iser_tx_desc *tx_desc = cqe_to_tx_desc(wc->wr_cqe);
1800 	struct isert_cmd *isert_cmd = tx_desc_to_cmd(tx_desc);
1801 
1802 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1803 		isert_print_wc(wc, "send");
1804 		if (wc->status != IB_WC_WR_FLUSH_ERR)
1805 			iscsit_cause_connection_reinstatement(isert_conn->conn, 0);
1806 		isert_completion_put(tx_desc, isert_cmd, ib_dev, true);
1807 		return;
1808 	}
1809 
1810 	isert_dbg("Cmd %p\n", isert_cmd);
1811 
1812 	switch (isert_cmd->iscsi_cmd->i_state) {
1813 	case ISTATE_SEND_TASKMGTRSP:
1814 	case ISTATE_SEND_LOGOUTRSP:
1815 	case ISTATE_SEND_REJECT:
1816 	case ISTATE_SEND_TEXTRSP:
1817 		isert_unmap_tx_desc(tx_desc, ib_dev);
1818 
1819 		INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp);
1820 		queue_work(isert_comp_wq, &isert_cmd->comp_work);
1821 		return;
1822 	default:
1823 		isert_cmd->iscsi_cmd->i_state = ISTATE_SENT_STATUS;
1824 		isert_completion_put(tx_desc, isert_cmd, ib_dev, false);
1825 		break;
1826 	}
1827 }
1828 
1829 static int
1830 isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd)
1831 {
1832 	int ret;
1833 
1834 	ret = isert_post_recv(isert_conn, isert_cmd->rx_desc);
1835 	if (ret) {
1836 		isert_err("ib_post_recv failed with %d\n", ret);
1837 		return ret;
1838 	}
1839 
1840 	ret = ib_post_send(isert_conn->qp, &isert_cmd->tx_desc.send_wr, NULL);
1841 	if (ret) {
1842 		isert_err("ib_post_send failed with %d\n", ret);
1843 		return ret;
1844 	}
1845 	return ret;
1846 }
1847 
1848 static int
1849 isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1850 {
1851 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1852 	struct isert_conn *isert_conn = conn->context;
1853 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1854 	struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)
1855 				&isert_cmd->tx_desc.iscsi_header;
1856 
1857 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1858 	iscsit_build_rsp_pdu(cmd, conn, true, hdr);
1859 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1860 	/*
1861 	 * Attach SENSE DATA payload to iSCSI Response PDU
1862 	 */
1863 	if (cmd->se_cmd.sense_buffer &&
1864 	    ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) ||
1865 	    (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) {
1866 		struct isert_device *device = isert_conn->device;
1867 		struct ib_device *ib_dev = device->ib_device;
1868 		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
1869 		u32 padding, pdu_len;
1870 
1871 		put_unaligned_be16(cmd->se_cmd.scsi_sense_length,
1872 				   cmd->sense_buffer);
1873 		cmd->se_cmd.scsi_sense_length += sizeof(__be16);
1874 
1875 		padding = -(cmd->se_cmd.scsi_sense_length) & 3;
1876 		hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length);
1877 		pdu_len = cmd->se_cmd.scsi_sense_length + padding;
1878 
1879 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
1880 				(void *)cmd->sense_buffer, pdu_len,
1881 				DMA_TO_DEVICE);
1882 		if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
1883 			return -ENOMEM;
1884 
1885 		isert_cmd->pdu_buf_len = pdu_len;
1886 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;
1887 		tx_dsg->length	= pdu_len;
1888 		tx_dsg->lkey	= device->pd->local_dma_lkey;
1889 		isert_cmd->tx_desc.num_sge = 2;
1890 	}
1891 
1892 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1893 
1894 	isert_dbg("Posting SCSI Response\n");
1895 
1896 	return isert_post_response(isert_conn, isert_cmd);
1897 }
1898 
1899 static void
1900 isert_aborted_task(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
1901 {
1902 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1903 	struct isert_conn *isert_conn = conn->context;
1904 
1905 	spin_lock_bh(&conn->cmd_lock);
1906 	if (!list_empty(&cmd->i_conn_node))
1907 		list_del_init(&cmd->i_conn_node);
1908 	spin_unlock_bh(&conn->cmd_lock);
1909 
1910 	if (cmd->data_direction == DMA_TO_DEVICE)
1911 		iscsit_stop_dataout_timer(cmd);
1912 	isert_rdma_rw_ctx_destroy(isert_cmd, isert_conn);
1913 }
1914 
1915 static enum target_prot_op
1916 isert_get_sup_prot_ops(struct iscsi_conn *conn)
1917 {
1918 	struct isert_conn *isert_conn = conn->context;
1919 	struct isert_device *device = isert_conn->device;
1920 
1921 	if (conn->tpg->tpg_attrib.t10_pi) {
1922 		if (device->pi_capable) {
1923 			isert_info("conn %p PI offload enabled\n", isert_conn);
1924 			isert_conn->pi_support = true;
1925 			return TARGET_PROT_ALL;
1926 		}
1927 	}
1928 
1929 	isert_info("conn %p PI offload disabled\n", isert_conn);
1930 	isert_conn->pi_support = false;
1931 
1932 	return TARGET_PROT_NORMAL;
1933 }
1934 
1935 static int
1936 isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn,
1937 		bool nopout_response)
1938 {
1939 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1940 	struct isert_conn *isert_conn = conn->context;
1941 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1942 
1943 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1944 	iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *)
1945 			       &isert_cmd->tx_desc.iscsi_header,
1946 			       nopout_response);
1947 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1948 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1949 
1950 	isert_dbg("conn %p Posting NOPIN Response\n", isert_conn);
1951 
1952 	return isert_post_response(isert_conn, isert_cmd);
1953 }
1954 
1955 static int
1956 isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1957 {
1958 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1959 	struct isert_conn *isert_conn = conn->context;
1960 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1961 
1962 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1963 	iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *)
1964 				&isert_cmd->tx_desc.iscsi_header);
1965 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1966 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1967 
1968 	isert_dbg("conn %p Posting Logout Response\n", isert_conn);
1969 
1970 	return isert_post_response(isert_conn, isert_cmd);
1971 }
1972 
1973 static int
1974 isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1975 {
1976 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1977 	struct isert_conn *isert_conn = conn->context;
1978 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1979 
1980 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
1981 	iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *)
1982 				  &isert_cmd->tx_desc.iscsi_header);
1983 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
1984 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
1985 
1986 	isert_dbg("conn %p Posting Task Management Response\n", isert_conn);
1987 
1988 	return isert_post_response(isert_conn, isert_cmd);
1989 }
1990 
1991 static int
1992 isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
1993 {
1994 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
1995 	struct isert_conn *isert_conn = conn->context;
1996 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
1997 	struct isert_device *device = isert_conn->device;
1998 	struct ib_device *ib_dev = device->ib_device;
1999 	struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2000 	struct iscsi_reject *hdr =
2001 		(struct iscsi_reject *)&isert_cmd->tx_desc.iscsi_header;
2002 
2003 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2004 	iscsit_build_reject(cmd, conn, hdr);
2005 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2006 
2007 	hton24(hdr->dlength, ISCSI_HDR_LEN);
2008 	isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2009 			(void *)cmd->buf_ptr, ISCSI_HDR_LEN,
2010 			DMA_TO_DEVICE);
2011 	if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
2012 		return -ENOMEM;
2013 	isert_cmd->pdu_buf_len = ISCSI_HDR_LEN;
2014 	tx_dsg->addr	= isert_cmd->pdu_buf_dma;
2015 	tx_dsg->length	= ISCSI_HDR_LEN;
2016 	tx_dsg->lkey	= device->pd->local_dma_lkey;
2017 	isert_cmd->tx_desc.num_sge = 2;
2018 
2019 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2020 
2021 	isert_dbg("conn %p Posting Reject\n", isert_conn);
2022 
2023 	return isert_post_response(isert_conn, isert_cmd);
2024 }
2025 
2026 static int
2027 isert_put_text_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn)
2028 {
2029 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2030 	struct isert_conn *isert_conn = conn->context;
2031 	struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr;
2032 	struct iscsi_text_rsp *hdr =
2033 		(struct iscsi_text_rsp *)&isert_cmd->tx_desc.iscsi_header;
2034 	u32 txt_rsp_len;
2035 	int rc;
2036 
2037 	isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc);
2038 	rc = iscsit_build_text_rsp(cmd, conn, hdr, ISCSI_INFINIBAND);
2039 	if (rc < 0)
2040 		return rc;
2041 
2042 	txt_rsp_len = rc;
2043 	isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2044 
2045 	if (txt_rsp_len) {
2046 		struct isert_device *device = isert_conn->device;
2047 		struct ib_device *ib_dev = device->ib_device;
2048 		struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1];
2049 		void *txt_rsp_buf = cmd->buf_ptr;
2050 
2051 		isert_cmd->pdu_buf_dma = ib_dma_map_single(ib_dev,
2052 				txt_rsp_buf, txt_rsp_len, DMA_TO_DEVICE);
2053 		if (ib_dma_mapping_error(ib_dev, isert_cmd->pdu_buf_dma))
2054 			return -ENOMEM;
2055 
2056 		isert_cmd->pdu_buf_len = txt_rsp_len;
2057 		tx_dsg->addr	= isert_cmd->pdu_buf_dma;
2058 		tx_dsg->length	= txt_rsp_len;
2059 		tx_dsg->lkey	= device->pd->local_dma_lkey;
2060 		isert_cmd->tx_desc.num_sge = 2;
2061 	}
2062 	isert_init_send_wr(isert_conn, isert_cmd, send_wr);
2063 
2064 	isert_dbg("conn %p Text Response\n", isert_conn);
2065 
2066 	return isert_post_response(isert_conn, isert_cmd);
2067 }
2068 
2069 static inline void
2070 isert_set_dif_domain(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs,
2071 		     struct ib_sig_domain *domain)
2072 {
2073 	domain->sig_type = IB_SIG_TYPE_T10_DIF;
2074 	domain->sig.dif.bg_type = IB_T10DIF_CRC;
2075 	domain->sig.dif.pi_interval = se_cmd->se_dev->dev_attrib.block_size;
2076 	domain->sig.dif.ref_tag = se_cmd->reftag_seed;
2077 	/*
2078 	 * At the moment we hard code those, but if in the future
2079 	 * the target core would like to use it, we will take it
2080 	 * from se_cmd.
2081 	 */
2082 	domain->sig.dif.apptag_check_mask = 0xffff;
2083 	domain->sig.dif.app_escape = true;
2084 	domain->sig.dif.ref_escape = true;
2085 	if (se_cmd->prot_type == TARGET_DIF_TYPE1_PROT ||
2086 	    se_cmd->prot_type == TARGET_DIF_TYPE2_PROT)
2087 		domain->sig.dif.ref_remap = true;
2088 };
2089 
2090 static int
2091 isert_set_sig_attrs(struct se_cmd *se_cmd, struct ib_sig_attrs *sig_attrs)
2092 {
2093 	memset(sig_attrs, 0, sizeof(*sig_attrs));
2094 
2095 	switch (se_cmd->prot_op) {
2096 	case TARGET_PROT_DIN_INSERT:
2097 	case TARGET_PROT_DOUT_STRIP:
2098 		sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE;
2099 		isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2100 		break;
2101 	case TARGET_PROT_DOUT_INSERT:
2102 	case TARGET_PROT_DIN_STRIP:
2103 		sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE;
2104 		isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
2105 		break;
2106 	case TARGET_PROT_DIN_PASS:
2107 	case TARGET_PROT_DOUT_PASS:
2108 		isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->wire);
2109 		isert_set_dif_domain(se_cmd, sig_attrs, &sig_attrs->mem);
2110 		break;
2111 	default:
2112 		isert_err("Unsupported PI operation %d\n", se_cmd->prot_op);
2113 		return -EINVAL;
2114 	}
2115 
2116 	if (se_cmd->prot_checks & TARGET_DIF_CHECK_GUARD)
2117 		sig_attrs->check_mask |= IB_SIG_CHECK_GUARD;
2118 	if (se_cmd->prot_checks & TARGET_DIF_CHECK_APPTAG)
2119 		sig_attrs->check_mask |= IB_SIG_CHECK_APPTAG;
2120 	if (se_cmd->prot_checks & TARGET_DIF_CHECK_REFTAG)
2121 		sig_attrs->check_mask |= IB_SIG_CHECK_REFTAG;
2122 
2123 	return 0;
2124 }
2125 
2126 static int
2127 isert_rdma_rw_ctx_post(struct isert_cmd *cmd, struct isert_conn *conn,
2128 		struct ib_cqe *cqe, struct ib_send_wr *chain_wr)
2129 {
2130 	struct se_cmd *se_cmd = &cmd->iscsi_cmd->se_cmd;
2131 	enum dma_data_direction dir = target_reverse_dma_direction(se_cmd);
2132 	u8 port_num = conn->cm_id->port_num;
2133 	u64 addr;
2134 	u32 rkey, offset;
2135 	int ret;
2136 
2137 	if (cmd->ctx_init_done)
2138 		goto rdma_ctx_post;
2139 
2140 	if (dir == DMA_FROM_DEVICE) {
2141 		addr = cmd->write_va;
2142 		rkey = cmd->write_stag;
2143 		offset = cmd->iscsi_cmd->write_data_done;
2144 	} else {
2145 		addr = cmd->read_va;
2146 		rkey = cmd->read_stag;
2147 		offset = 0;
2148 	}
2149 
2150 	if (isert_prot_cmd(conn, se_cmd)) {
2151 		struct ib_sig_attrs sig_attrs;
2152 
2153 		ret = isert_set_sig_attrs(se_cmd, &sig_attrs);
2154 		if (ret)
2155 			return ret;
2156 
2157 		WARN_ON_ONCE(offset);
2158 		ret = rdma_rw_ctx_signature_init(&cmd->rw, conn->qp, port_num,
2159 				se_cmd->t_data_sg, se_cmd->t_data_nents,
2160 				se_cmd->t_prot_sg, se_cmd->t_prot_nents,
2161 				&sig_attrs, addr, rkey, dir);
2162 	} else {
2163 		ret = rdma_rw_ctx_init(&cmd->rw, conn->qp, port_num,
2164 				se_cmd->t_data_sg, se_cmd->t_data_nents,
2165 				offset, addr, rkey, dir);
2166 	}
2167 
2168 	if (ret < 0) {
2169 		isert_err("Cmd: %p failed to prepare RDMA res\n", cmd);
2170 		return ret;
2171 	}
2172 
2173 	cmd->ctx_init_done = true;
2174 
2175 rdma_ctx_post:
2176 	ret = rdma_rw_ctx_post(&cmd->rw, conn->qp, port_num, cqe, chain_wr);
2177 	if (ret < 0)
2178 		isert_err("Cmd: %p failed to post RDMA res\n", cmd);
2179 	return ret;
2180 }
2181 
2182 static int
2183 isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd)
2184 {
2185 	struct se_cmd *se_cmd = &cmd->se_cmd;
2186 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2187 	struct isert_conn *isert_conn = conn->context;
2188 	struct ib_cqe *cqe = NULL;
2189 	struct ib_send_wr *chain_wr = NULL;
2190 	int rc;
2191 
2192 	isert_dbg("Cmd: %p RDMA_WRITE data_length: %u\n",
2193 		 isert_cmd, se_cmd->data_length);
2194 
2195 	if (isert_prot_cmd(isert_conn, se_cmd)) {
2196 		isert_cmd->tx_desc.tx_cqe.done = isert_rdma_write_done;
2197 		cqe = &isert_cmd->tx_desc.tx_cqe;
2198 	} else {
2199 		/*
2200 		 * Build isert_conn->tx_desc for iSCSI response PDU and attach
2201 		 */
2202 		isert_create_send_desc(isert_conn, isert_cmd,
2203 				       &isert_cmd->tx_desc);
2204 		iscsit_build_rsp_pdu(cmd, conn, true, (struct iscsi_scsi_rsp *)
2205 				     &isert_cmd->tx_desc.iscsi_header);
2206 		isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc);
2207 		isert_init_send_wr(isert_conn, isert_cmd,
2208 				   &isert_cmd->tx_desc.send_wr);
2209 
2210 		rc = isert_post_recv(isert_conn, isert_cmd->rx_desc);
2211 		if (rc) {
2212 			isert_err("ib_post_recv failed with %d\n", rc);
2213 			return rc;
2214 		}
2215 
2216 		chain_wr = &isert_cmd->tx_desc.send_wr;
2217 	}
2218 
2219 	rc = isert_rdma_rw_ctx_post(isert_cmd, isert_conn, cqe, chain_wr);
2220 	isert_dbg("Cmd: %p posted RDMA_WRITE for iSER Data READ rc: %d\n",
2221 		  isert_cmd, rc);
2222 	return rc;
2223 }
2224 
2225 static int
2226 isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery)
2227 {
2228 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2229 	int ret;
2230 
2231 	isert_dbg("Cmd: %p RDMA_READ data_length: %u write_data_done: %u\n",
2232 		 isert_cmd, cmd->se_cmd.data_length, cmd->write_data_done);
2233 
2234 	isert_cmd->tx_desc.tx_cqe.done = isert_rdma_read_done;
2235 	ret = isert_rdma_rw_ctx_post(isert_cmd, conn->context,
2236 				     &isert_cmd->tx_desc.tx_cqe, NULL);
2237 
2238 	isert_dbg("Cmd: %p posted RDMA_READ memory for ISER Data WRITE rc: %d\n",
2239 		 isert_cmd, ret);
2240 	return ret;
2241 }
2242 
2243 static int
2244 isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2245 {
2246 	struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2247 	int ret = 0;
2248 
2249 	switch (state) {
2250 	case ISTATE_REMOVE:
2251 		spin_lock_bh(&conn->cmd_lock);
2252 		list_del_init(&cmd->i_conn_node);
2253 		spin_unlock_bh(&conn->cmd_lock);
2254 		isert_put_cmd(isert_cmd, true);
2255 		break;
2256 	case ISTATE_SEND_NOPIN_WANT_RESPONSE:
2257 		ret = isert_put_nopin(cmd, conn, false);
2258 		break;
2259 	default:
2260 		isert_err("Unknown immediate state: 0x%02x\n", state);
2261 		ret = -EINVAL;
2262 		break;
2263 	}
2264 
2265 	return ret;
2266 }
2267 
2268 static int
2269 isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state)
2270 {
2271 	struct isert_conn *isert_conn = conn->context;
2272 	int ret;
2273 
2274 	switch (state) {
2275 	case ISTATE_SEND_LOGOUTRSP:
2276 		ret = isert_put_logout_rsp(cmd, conn);
2277 		if (!ret)
2278 			isert_conn->logout_posted = true;
2279 		break;
2280 	case ISTATE_SEND_NOPIN:
2281 		ret = isert_put_nopin(cmd, conn, true);
2282 		break;
2283 	case ISTATE_SEND_TASKMGTRSP:
2284 		ret = isert_put_tm_rsp(cmd, conn);
2285 		break;
2286 	case ISTATE_SEND_REJECT:
2287 		ret = isert_put_reject(cmd, conn);
2288 		break;
2289 	case ISTATE_SEND_TEXTRSP:
2290 		ret = isert_put_text_rsp(cmd, conn);
2291 		break;
2292 	case ISTATE_SEND_STATUS:
2293 		/*
2294 		 * Special case for sending non GOOD SCSI status from TX thread
2295 		 * context during pre se_cmd excecution failure.
2296 		 */
2297 		ret = isert_put_response(conn, cmd);
2298 		break;
2299 	default:
2300 		isert_err("Unknown response state: 0x%02x\n", state);
2301 		ret = -EINVAL;
2302 		break;
2303 	}
2304 
2305 	return ret;
2306 }
2307 
2308 struct rdma_cm_id *
2309 isert_setup_id(struct isert_np *isert_np)
2310 {
2311 	struct iscsi_np *np = isert_np->np;
2312 	struct rdma_cm_id *id;
2313 	struct sockaddr *sa;
2314 	int ret;
2315 
2316 	sa = (struct sockaddr *)&np->np_sockaddr;
2317 	isert_dbg("ksockaddr: %p, sa: %p\n", &np->np_sockaddr, sa);
2318 
2319 	id = rdma_create_id(&init_net, isert_cma_handler, isert_np,
2320 			    RDMA_PS_TCP, IB_QPT_RC);
2321 	if (IS_ERR(id)) {
2322 		isert_err("rdma_create_id() failed: %ld\n", PTR_ERR(id));
2323 		ret = PTR_ERR(id);
2324 		goto out;
2325 	}
2326 	isert_dbg("id %p context %p\n", id, id->context);
2327 
2328 	ret = rdma_bind_addr(id, sa);
2329 	if (ret) {
2330 		isert_err("rdma_bind_addr() failed: %d\n", ret);
2331 		goto out_id;
2332 	}
2333 
2334 	ret = rdma_listen(id, 0);
2335 	if (ret) {
2336 		isert_err("rdma_listen() failed: %d\n", ret);
2337 		goto out_id;
2338 	}
2339 
2340 	return id;
2341 out_id:
2342 	rdma_destroy_id(id);
2343 out:
2344 	return ERR_PTR(ret);
2345 }
2346 
2347 static int
2348 isert_setup_np(struct iscsi_np *np,
2349 	       struct sockaddr_storage *ksockaddr)
2350 {
2351 	struct isert_np *isert_np;
2352 	struct rdma_cm_id *isert_lid;
2353 	int ret;
2354 
2355 	isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL);
2356 	if (!isert_np)
2357 		return -ENOMEM;
2358 
2359 	sema_init(&isert_np->sem, 0);
2360 	mutex_init(&isert_np->mutex);
2361 	INIT_LIST_HEAD(&isert_np->accepted);
2362 	INIT_LIST_HEAD(&isert_np->pending);
2363 	isert_np->np = np;
2364 
2365 	/*
2366 	 * Setup the np->np_sockaddr from the passed sockaddr setup
2367 	 * in iscsi_target_configfs.c code..
2368 	 */
2369 	memcpy(&np->np_sockaddr, ksockaddr,
2370 	       sizeof(struct sockaddr_storage));
2371 
2372 	isert_lid = isert_setup_id(isert_np);
2373 	if (IS_ERR(isert_lid)) {
2374 		ret = PTR_ERR(isert_lid);
2375 		goto out;
2376 	}
2377 
2378 	isert_np->cm_id = isert_lid;
2379 	np->np_context = isert_np;
2380 
2381 	return 0;
2382 
2383 out:
2384 	kfree(isert_np);
2385 
2386 	return ret;
2387 }
2388 
2389 static int
2390 isert_rdma_accept(struct isert_conn *isert_conn)
2391 {
2392 	struct rdma_cm_id *cm_id = isert_conn->cm_id;
2393 	struct rdma_conn_param cp;
2394 	int ret;
2395 	struct iser_cm_hdr rsp_hdr;
2396 
2397 	memset(&cp, 0, sizeof(struct rdma_conn_param));
2398 	cp.initiator_depth = isert_conn->initiator_depth;
2399 	cp.retry_count = 7;
2400 	cp.rnr_retry_count = 7;
2401 
2402 	memset(&rsp_hdr, 0, sizeof(rsp_hdr));
2403 	rsp_hdr.flags = ISERT_ZBVA_NOT_USED;
2404 	if (!isert_conn->snd_w_inv)
2405 		rsp_hdr.flags = rsp_hdr.flags | ISERT_SEND_W_INV_NOT_USED;
2406 	cp.private_data = (void *)&rsp_hdr;
2407 	cp.private_data_len = sizeof(rsp_hdr);
2408 
2409 	ret = rdma_accept(cm_id, &cp);
2410 	if (ret) {
2411 		isert_err("rdma_accept() failed with: %d\n", ret);
2412 		return ret;
2413 	}
2414 
2415 	return 0;
2416 }
2417 
2418 static int
2419 isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login)
2420 {
2421 	struct isert_conn *isert_conn = conn->context;
2422 	int ret;
2423 
2424 	isert_info("before login_req comp conn: %p\n", isert_conn);
2425 	ret = wait_for_completion_interruptible(&isert_conn->login_req_comp);
2426 	if (ret) {
2427 		isert_err("isert_conn %p interrupted before got login req\n",
2428 			  isert_conn);
2429 		return ret;
2430 	}
2431 	reinit_completion(&isert_conn->login_req_comp);
2432 
2433 	/*
2434 	 * For login requests after the first PDU, isert_rx_login_req() will
2435 	 * kick schedule_delayed_work(&conn->login_work) as the packet is
2436 	 * received, which turns this callback from iscsi_target_do_login_rx()
2437 	 * into a NOP.
2438 	 */
2439 	if (!login->first_request)
2440 		return 0;
2441 
2442 	isert_rx_login_req(isert_conn);
2443 
2444 	isert_info("before login_comp conn: %p\n", conn);
2445 	ret = wait_for_completion_interruptible(&isert_conn->login_comp);
2446 	if (ret)
2447 		return ret;
2448 
2449 	isert_info("processing login->req: %p\n", login->req);
2450 
2451 	return 0;
2452 }
2453 
2454 static void
2455 isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn,
2456 		    struct isert_conn *isert_conn)
2457 {
2458 	struct rdma_cm_id *cm_id = isert_conn->cm_id;
2459 	struct rdma_route *cm_route = &cm_id->route;
2460 
2461 	conn->login_family = np->np_sockaddr.ss_family;
2462 
2463 	conn->login_sockaddr = cm_route->addr.dst_addr;
2464 	conn->local_sockaddr = cm_route->addr.src_addr;
2465 }
2466 
2467 static int
2468 isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn)
2469 {
2470 	struct isert_np *isert_np = np->np_context;
2471 	struct isert_conn *isert_conn;
2472 	int ret;
2473 
2474 accept_wait:
2475 	ret = down_interruptible(&isert_np->sem);
2476 	if (ret)
2477 		return -ENODEV;
2478 
2479 	spin_lock_bh(&np->np_thread_lock);
2480 	if (np->np_thread_state >= ISCSI_NP_THREAD_RESET) {
2481 		spin_unlock_bh(&np->np_thread_lock);
2482 		isert_dbg("np_thread_state %d\n",
2483 			 np->np_thread_state);
2484 		/**
2485 		 * No point in stalling here when np_thread
2486 		 * is in state RESET/SHUTDOWN/EXIT - bail
2487 		 **/
2488 		return -ENODEV;
2489 	}
2490 	spin_unlock_bh(&np->np_thread_lock);
2491 
2492 	mutex_lock(&isert_np->mutex);
2493 	if (list_empty(&isert_np->pending)) {
2494 		mutex_unlock(&isert_np->mutex);
2495 		goto accept_wait;
2496 	}
2497 	isert_conn = list_first_entry(&isert_np->pending,
2498 			struct isert_conn, node);
2499 	list_del_init(&isert_conn->node);
2500 	mutex_unlock(&isert_np->mutex);
2501 
2502 	conn->context = isert_conn;
2503 	isert_conn->conn = conn;
2504 	isert_conn->state = ISER_CONN_BOUND;
2505 
2506 	isert_set_conn_info(np, conn, isert_conn);
2507 
2508 	isert_dbg("Processing isert_conn: %p\n", isert_conn);
2509 
2510 	return 0;
2511 }
2512 
2513 static void
2514 isert_free_np(struct iscsi_np *np)
2515 {
2516 	struct isert_np *isert_np = np->np_context;
2517 	struct isert_conn *isert_conn, *n;
2518 
2519 	if (isert_np->cm_id)
2520 		rdma_destroy_id(isert_np->cm_id);
2521 
2522 	/*
2523 	 * FIXME: At this point we don't have a good way to insure
2524 	 * that at this point we don't have hanging connections that
2525 	 * completed RDMA establishment but didn't start iscsi login
2526 	 * process. So work-around this by cleaning up what ever piled
2527 	 * up in accepted and pending lists.
2528 	 */
2529 	mutex_lock(&isert_np->mutex);
2530 	if (!list_empty(&isert_np->pending)) {
2531 		isert_info("Still have isert pending connections\n");
2532 		list_for_each_entry_safe(isert_conn, n,
2533 					 &isert_np->pending,
2534 					 node) {
2535 			isert_info("cleaning isert_conn %p state (%d)\n",
2536 				   isert_conn, isert_conn->state);
2537 			isert_connect_release(isert_conn);
2538 		}
2539 	}
2540 
2541 	if (!list_empty(&isert_np->accepted)) {
2542 		isert_info("Still have isert accepted connections\n");
2543 		list_for_each_entry_safe(isert_conn, n,
2544 					 &isert_np->accepted,
2545 					 node) {
2546 			isert_info("cleaning isert_conn %p state (%d)\n",
2547 				   isert_conn, isert_conn->state);
2548 			isert_connect_release(isert_conn);
2549 		}
2550 	}
2551 	mutex_unlock(&isert_np->mutex);
2552 
2553 	np->np_context = NULL;
2554 	kfree(isert_np);
2555 }
2556 
2557 static void isert_release_work(struct work_struct *work)
2558 {
2559 	struct isert_conn *isert_conn = container_of(work,
2560 						     struct isert_conn,
2561 						     release_work);
2562 
2563 	isert_info("Starting release conn %p\n", isert_conn);
2564 
2565 	mutex_lock(&isert_conn->mutex);
2566 	isert_conn->state = ISER_CONN_DOWN;
2567 	mutex_unlock(&isert_conn->mutex);
2568 
2569 	isert_info("Destroying conn %p\n", isert_conn);
2570 	isert_put_conn(isert_conn);
2571 }
2572 
2573 static void
2574 isert_wait4logout(struct isert_conn *isert_conn)
2575 {
2576 	struct iscsi_conn *conn = isert_conn->conn;
2577 
2578 	isert_info("conn %p\n", isert_conn);
2579 
2580 	if (isert_conn->logout_posted) {
2581 		isert_info("conn %p wait for conn_logout_comp\n", isert_conn);
2582 		wait_for_completion_timeout(&conn->conn_logout_comp,
2583 					    SECONDS_FOR_LOGOUT_COMP * HZ);
2584 	}
2585 }
2586 
2587 static void
2588 isert_wait4cmds(struct iscsi_conn *conn)
2589 {
2590 	isert_info("iscsi_conn %p\n", conn);
2591 
2592 	if (conn->sess) {
2593 		target_sess_cmd_list_set_waiting(conn->sess->se_sess);
2594 		target_wait_for_sess_cmds(conn->sess->se_sess);
2595 	}
2596 }
2597 
2598 /**
2599  * isert_put_unsol_pending_cmds() - Drop commands waiting for
2600  *     unsolicitate dataout
2601  * @conn:    iscsi connection
2602  *
2603  * We might still have commands that are waiting for unsolicited
2604  * dataouts messages. We must put the extra reference on those
2605  * before blocking on the target_wait_for_session_cmds
2606  */
2607 static void
2608 isert_put_unsol_pending_cmds(struct iscsi_conn *conn)
2609 {
2610 	struct iscsi_cmd *cmd, *tmp;
2611 	static LIST_HEAD(drop_cmd_list);
2612 
2613 	spin_lock_bh(&conn->cmd_lock);
2614 	list_for_each_entry_safe(cmd, tmp, &conn->conn_cmd_list, i_conn_node) {
2615 		if ((cmd->cmd_flags & ICF_NON_IMMEDIATE_UNSOLICITED_DATA) &&
2616 		    (cmd->write_data_done < conn->sess->sess_ops->FirstBurstLength) &&
2617 		    (cmd->write_data_done < cmd->se_cmd.data_length))
2618 			list_move_tail(&cmd->i_conn_node, &drop_cmd_list);
2619 	}
2620 	spin_unlock_bh(&conn->cmd_lock);
2621 
2622 	list_for_each_entry_safe(cmd, tmp, &drop_cmd_list, i_conn_node) {
2623 		list_del_init(&cmd->i_conn_node);
2624 		if (cmd->i_state != ISTATE_REMOVE) {
2625 			struct isert_cmd *isert_cmd = iscsit_priv_cmd(cmd);
2626 
2627 			isert_info("conn %p dropping cmd %p\n", conn, cmd);
2628 			isert_put_cmd(isert_cmd, true);
2629 		}
2630 	}
2631 }
2632 
2633 static void isert_wait_conn(struct iscsi_conn *conn)
2634 {
2635 	struct isert_conn *isert_conn = conn->context;
2636 
2637 	isert_info("Starting conn %p\n", isert_conn);
2638 
2639 	mutex_lock(&isert_conn->mutex);
2640 	isert_conn_terminate(isert_conn);
2641 	mutex_unlock(&isert_conn->mutex);
2642 
2643 	ib_drain_qp(isert_conn->qp);
2644 	isert_put_unsol_pending_cmds(conn);
2645 	isert_wait4cmds(conn);
2646 	isert_wait4logout(isert_conn);
2647 
2648 	queue_work(isert_release_wq, &isert_conn->release_work);
2649 }
2650 
2651 static void isert_free_conn(struct iscsi_conn *conn)
2652 {
2653 	struct isert_conn *isert_conn = conn->context;
2654 
2655 	ib_drain_qp(isert_conn->qp);
2656 	isert_put_conn(isert_conn);
2657 }
2658 
2659 static void isert_get_rx_pdu(struct iscsi_conn *conn)
2660 {
2661 	struct completion comp;
2662 
2663 	init_completion(&comp);
2664 
2665 	wait_for_completion_interruptible(&comp);
2666 }
2667 
2668 static struct iscsit_transport iser_target_transport = {
2669 	.name			= "IB/iSER",
2670 	.transport_type		= ISCSI_INFINIBAND,
2671 	.rdma_shutdown		= true,
2672 	.priv_size		= sizeof(struct isert_cmd),
2673 	.owner			= THIS_MODULE,
2674 	.iscsit_setup_np	= isert_setup_np,
2675 	.iscsit_accept_np	= isert_accept_np,
2676 	.iscsit_free_np		= isert_free_np,
2677 	.iscsit_wait_conn	= isert_wait_conn,
2678 	.iscsit_free_conn	= isert_free_conn,
2679 	.iscsit_get_login_rx	= isert_get_login_rx,
2680 	.iscsit_put_login_tx	= isert_put_login_tx,
2681 	.iscsit_immediate_queue	= isert_immediate_queue,
2682 	.iscsit_response_queue	= isert_response_queue,
2683 	.iscsit_get_dataout	= isert_get_dataout,
2684 	.iscsit_queue_data_in	= isert_put_datain,
2685 	.iscsit_queue_status	= isert_put_response,
2686 	.iscsit_aborted_task	= isert_aborted_task,
2687 	.iscsit_get_rx_pdu	= isert_get_rx_pdu,
2688 	.iscsit_get_sup_prot_ops = isert_get_sup_prot_ops,
2689 };
2690 
2691 static int __init isert_init(void)
2692 {
2693 	int ret;
2694 
2695 	isert_comp_wq = alloc_workqueue("isert_comp_wq",
2696 					WQ_UNBOUND | WQ_HIGHPRI, 0);
2697 	if (!isert_comp_wq) {
2698 		isert_err("Unable to allocate isert_comp_wq\n");
2699 		return -ENOMEM;
2700 	}
2701 
2702 	isert_release_wq = alloc_workqueue("isert_release_wq", WQ_UNBOUND,
2703 					WQ_UNBOUND_MAX_ACTIVE);
2704 	if (!isert_release_wq) {
2705 		isert_err("Unable to allocate isert_release_wq\n");
2706 		ret = -ENOMEM;
2707 		goto destroy_comp_wq;
2708 	}
2709 
2710 	iscsit_register_transport(&iser_target_transport);
2711 	isert_info("iSER_TARGET[0] - Loaded iser_target_transport\n");
2712 
2713 	return 0;
2714 
2715 destroy_comp_wq:
2716 	destroy_workqueue(isert_comp_wq);
2717 
2718 	return ret;
2719 }
2720 
2721 static void __exit isert_exit(void)
2722 {
2723 	flush_scheduled_work();
2724 	destroy_workqueue(isert_release_wq);
2725 	destroy_workqueue(isert_comp_wq);
2726 	iscsit_unregister_transport(&iser_target_transport);
2727 	isert_info("iSER_TARGET[0] - Released iser_target_transport\n");
2728 }
2729 
2730 MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure");
2731 MODULE_AUTHOR("nab@Linux-iSCSI.org");
2732 MODULE_LICENSE("GPL");
2733 
2734 module_init(isert_init);
2735 module_exit(isert_exit);
2736