1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  *
33  */
34 
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/err.h>
39 #include <linux/ctype.h>
40 #include <linux/kthread.h>
41 #include <linux/string.h>
42 #include <linux/delay.h>
43 #include <linux/atomic.h>
44 #include <scsi/scsi_proto.h>
45 #include <scsi/scsi_tcq.h>
46 #include <target/target_core_base.h>
47 #include <target/target_core_fabric.h>
48 #include "ib_srpt.h"
49 
50 /* Name of this kernel module. */
51 #define DRV_NAME		"ib_srpt"
52 #define DRV_VERSION		"2.0.0"
53 #define DRV_RELDATE		"2011-02-14"
54 
55 #define SRPT_ID_STRING	"Linux SRP target"
56 
57 #undef pr_fmt
58 #define pr_fmt(fmt) DRV_NAME " " fmt
59 
60 MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
62 		   "v" DRV_VERSION " (" DRV_RELDATE ")");
63 MODULE_LICENSE("Dual BSD/GPL");
64 
65 /*
66  * Global Variables
67  */
68 
69 static u64 srpt_service_guid;
70 static DEFINE_SPINLOCK(srpt_dev_lock);	/* Protects srpt_dev_list. */
71 static LIST_HEAD(srpt_dev_list);	/* List of srpt_device structures. */
72 
73 static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
74 module_param(srp_max_req_size, int, 0444);
75 MODULE_PARM_DESC(srp_max_req_size,
76 		 "Maximum size of SRP request messages in bytes.");
77 
78 static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
79 module_param(srpt_srq_size, int, 0444);
80 MODULE_PARM_DESC(srpt_srq_size,
81 		 "Shared receive queue (SRQ) size.");
82 
83 static int srpt_get_u64_x(char *buffer, struct kernel_param *kp)
84 {
85 	return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
86 }
87 module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
88 		  0444);
89 MODULE_PARM_DESC(srpt_service_guid,
90 		 "Using this value for ioc_guid, id_ext, and cm_listen_id"
91 		 " instead of using the node_guid of the first HCA.");
92 
93 static struct ib_client srpt_client;
94 static void srpt_release_cmd(struct se_cmd *se_cmd);
95 static void srpt_free_ch(struct kref *kref);
96 static int srpt_queue_status(struct se_cmd *cmd);
97 static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
98 static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
99 static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
100 
101 /*
102  * The only allowed channel state changes are those that change the channel
103  * state into a state with a higher numerical value. Hence the new > prev test.
104  */
105 static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
106 {
107 	unsigned long flags;
108 	enum rdma_ch_state prev;
109 	bool changed = false;
110 
111 	spin_lock_irqsave(&ch->spinlock, flags);
112 	prev = ch->state;
113 	if (new > prev) {
114 		ch->state = new;
115 		changed = true;
116 	}
117 	spin_unlock_irqrestore(&ch->spinlock, flags);
118 
119 	return changed;
120 }
121 
122 /**
123  * srpt_event_handler() - Asynchronous IB event callback function.
124  *
125  * Callback function called by the InfiniBand core when an asynchronous IB
126  * event occurs. This callback may occur in interrupt context. See also
127  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
128  * Architecture Specification.
129  */
130 static void srpt_event_handler(struct ib_event_handler *handler,
131 			       struct ib_event *event)
132 {
133 	struct srpt_device *sdev;
134 	struct srpt_port *sport;
135 
136 	sdev = ib_get_client_data(event->device, &srpt_client);
137 	if (!sdev || sdev->device != event->device)
138 		return;
139 
140 	pr_debug("ASYNC event= %d on device= %s\n", event->event,
141 		 sdev->device->name);
142 
143 	switch (event->event) {
144 	case IB_EVENT_PORT_ERR:
145 		if (event->element.port_num <= sdev->device->phys_port_cnt) {
146 			sport = &sdev->port[event->element.port_num - 1];
147 			sport->lid = 0;
148 			sport->sm_lid = 0;
149 		}
150 		break;
151 	case IB_EVENT_PORT_ACTIVE:
152 	case IB_EVENT_LID_CHANGE:
153 	case IB_EVENT_PKEY_CHANGE:
154 	case IB_EVENT_SM_CHANGE:
155 	case IB_EVENT_CLIENT_REREGISTER:
156 	case IB_EVENT_GID_CHANGE:
157 		/* Refresh port data asynchronously. */
158 		if (event->element.port_num <= sdev->device->phys_port_cnt) {
159 			sport = &sdev->port[event->element.port_num - 1];
160 			if (!sport->lid && !sport->sm_lid)
161 				schedule_work(&sport->work);
162 		}
163 		break;
164 	default:
165 		pr_err("received unrecognized IB event %d\n",
166 		       event->event);
167 		break;
168 	}
169 }
170 
171 /**
172  * srpt_srq_event() - SRQ event callback function.
173  */
174 static void srpt_srq_event(struct ib_event *event, void *ctx)
175 {
176 	pr_info("SRQ event %d\n", event->event);
177 }
178 
179 static const char *get_ch_state_name(enum rdma_ch_state s)
180 {
181 	switch (s) {
182 	case CH_CONNECTING:
183 		return "connecting";
184 	case CH_LIVE:
185 		return "live";
186 	case CH_DISCONNECTING:
187 		return "disconnecting";
188 	case CH_DRAINING:
189 		return "draining";
190 	case CH_DISCONNECTED:
191 		return "disconnected";
192 	}
193 	return "???";
194 }
195 
196 /**
197  * srpt_qp_event() - QP event callback function.
198  */
199 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
200 {
201 	pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
202 		 event->event, ch->cm_id, ch->sess_name, ch->state);
203 
204 	switch (event->event) {
205 	case IB_EVENT_COMM_EST:
206 		ib_cm_notify(ch->cm_id, event->event);
207 		break;
208 	case IB_EVENT_QP_LAST_WQE_REACHED:
209 		pr_debug("%s-%d, state %s: received Last WQE event.\n",
210 			 ch->sess_name, ch->qp->qp_num,
211 			 get_ch_state_name(ch->state));
212 		break;
213 	default:
214 		pr_err("received unrecognized IB QP event %d\n", event->event);
215 		break;
216 	}
217 }
218 
219 /**
220  * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure.
221  *
222  * @slot: one-based slot number.
223  * @value: four-bit value.
224  *
225  * Copies the lowest four bits of value in element slot of the array of four
226  * bit elements called c_list (controller list). The index slot is one-based.
227  */
228 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
229 {
230 	u16 id;
231 	u8 tmp;
232 
233 	id = (slot - 1) / 2;
234 	if (slot & 0x1) {
235 		tmp = c_list[id] & 0xf;
236 		c_list[id] = (value << 4) | tmp;
237 	} else {
238 		tmp = c_list[id] & 0xf0;
239 		c_list[id] = (value & 0xf) | tmp;
240 	}
241 }
242 
243 /**
244  * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram.
245  *
246  * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
247  * Specification.
248  */
249 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
250 {
251 	struct ib_class_port_info *cif;
252 
253 	cif = (struct ib_class_port_info *)mad->data;
254 	memset(cif, 0, sizeof(*cif));
255 	cif->base_version = 1;
256 	cif->class_version = 1;
257 
258 	ib_set_cpi_resp_time(cif, 20);
259 	mad->mad_hdr.status = 0;
260 }
261 
262 /**
263  * srpt_get_iou() - Write IOUnitInfo to a management datagram.
264  *
265  * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
266  * Specification. See also section B.7, table B.6 in the SRP r16a document.
267  */
268 static void srpt_get_iou(struct ib_dm_mad *mad)
269 {
270 	struct ib_dm_iou_info *ioui;
271 	u8 slot;
272 	int i;
273 
274 	ioui = (struct ib_dm_iou_info *)mad->data;
275 	ioui->change_id = cpu_to_be16(1);
276 	ioui->max_controllers = 16;
277 
278 	/* set present for slot 1 and empty for the rest */
279 	srpt_set_ioc(ioui->controller_list, 1, 1);
280 	for (i = 1, slot = 2; i < 16; i++, slot++)
281 		srpt_set_ioc(ioui->controller_list, slot, 0);
282 
283 	mad->mad_hdr.status = 0;
284 }
285 
286 /**
287  * srpt_get_ioc() - Write IOControllerprofile to a management datagram.
288  *
289  * See also section 16.3.3.4 IOControllerProfile in the InfiniBand
290  * Architecture Specification. See also section B.7, table B.7 in the SRP
291  * r16a document.
292  */
293 static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
294 			 struct ib_dm_mad *mad)
295 {
296 	struct srpt_device *sdev = sport->sdev;
297 	struct ib_dm_ioc_profile *iocp;
298 
299 	iocp = (struct ib_dm_ioc_profile *)mad->data;
300 
301 	if (!slot || slot > 16) {
302 		mad->mad_hdr.status
303 			= cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
304 		return;
305 	}
306 
307 	if (slot > 2) {
308 		mad->mad_hdr.status
309 			= cpu_to_be16(DM_MAD_STATUS_NO_IOC);
310 		return;
311 	}
312 
313 	memset(iocp, 0, sizeof(*iocp));
314 	strcpy(iocp->id_string, SRPT_ID_STRING);
315 	iocp->guid = cpu_to_be64(srpt_service_guid);
316 	iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
317 	iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
318 	iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
319 	iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
320 	iocp->subsys_device_id = 0x0;
321 	iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
322 	iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
323 	iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
324 	iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
325 	iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);
326 	iocp->rdma_read_depth = 4;
327 	iocp->send_size = cpu_to_be32(srp_max_req_size);
328 	iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
329 					  1U << 24));
330 	iocp->num_svc_entries = 1;
331 	iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
332 		SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
333 
334 	mad->mad_hdr.status = 0;
335 }
336 
337 /**
338  * srpt_get_svc_entries() - Write ServiceEntries to a management datagram.
339  *
340  * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
341  * Specification. See also section B.7, table B.8 in the SRP r16a document.
342  */
343 static void srpt_get_svc_entries(u64 ioc_guid,
344 				 u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
345 {
346 	struct ib_dm_svc_entries *svc_entries;
347 
348 	WARN_ON(!ioc_guid);
349 
350 	if (!slot || slot > 16) {
351 		mad->mad_hdr.status
352 			= cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
353 		return;
354 	}
355 
356 	if (slot > 2 || lo > hi || hi > 1) {
357 		mad->mad_hdr.status
358 			= cpu_to_be16(DM_MAD_STATUS_NO_IOC);
359 		return;
360 	}
361 
362 	svc_entries = (struct ib_dm_svc_entries *)mad->data;
363 	memset(svc_entries, 0, sizeof(*svc_entries));
364 	svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
365 	snprintf(svc_entries->service_entries[0].name,
366 		 sizeof(svc_entries->service_entries[0].name),
367 		 "%s%016llx",
368 		 SRP_SERVICE_NAME_PREFIX,
369 		 ioc_guid);
370 
371 	mad->mad_hdr.status = 0;
372 }
373 
374 /**
375  * srpt_mgmt_method_get() - Process a received management datagram.
376  * @sp:      source port through which the MAD has been received.
377  * @rq_mad:  received MAD.
378  * @rsp_mad: response MAD.
379  */
380 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
381 				 struct ib_dm_mad *rsp_mad)
382 {
383 	u16 attr_id;
384 	u32 slot;
385 	u8 hi, lo;
386 
387 	attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
388 	switch (attr_id) {
389 	case DM_ATTR_CLASS_PORT_INFO:
390 		srpt_get_class_port_info(rsp_mad);
391 		break;
392 	case DM_ATTR_IOU_INFO:
393 		srpt_get_iou(rsp_mad);
394 		break;
395 	case DM_ATTR_IOC_PROFILE:
396 		slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
397 		srpt_get_ioc(sp, slot, rsp_mad);
398 		break;
399 	case DM_ATTR_SVC_ENTRIES:
400 		slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
401 		hi = (u8) ((slot >> 8) & 0xff);
402 		lo = (u8) (slot & 0xff);
403 		slot = (u16) ((slot >> 16) & 0xffff);
404 		srpt_get_svc_entries(srpt_service_guid,
405 				     slot, hi, lo, rsp_mad);
406 		break;
407 	default:
408 		rsp_mad->mad_hdr.status =
409 		    cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
410 		break;
411 	}
412 }
413 
414 /**
415  * srpt_mad_send_handler() - Post MAD-send callback function.
416  */
417 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
418 				  struct ib_mad_send_wc *mad_wc)
419 {
420 	ib_destroy_ah(mad_wc->send_buf->ah);
421 	ib_free_send_mad(mad_wc->send_buf);
422 }
423 
424 /**
425  * srpt_mad_recv_handler() - MAD reception callback function.
426  */
427 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
428 				  struct ib_mad_send_buf *send_buf,
429 				  struct ib_mad_recv_wc *mad_wc)
430 {
431 	struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
432 	struct ib_ah *ah;
433 	struct ib_mad_send_buf *rsp;
434 	struct ib_dm_mad *dm_mad;
435 
436 	if (!mad_wc || !mad_wc->recv_buf.mad)
437 		return;
438 
439 	ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
440 				  mad_wc->recv_buf.grh, mad_agent->port_num);
441 	if (IS_ERR(ah))
442 		goto err;
443 
444 	BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
445 
446 	rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
447 				 mad_wc->wc->pkey_index, 0,
448 				 IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
449 				 GFP_KERNEL,
450 				 IB_MGMT_BASE_VERSION);
451 	if (IS_ERR(rsp))
452 		goto err_rsp;
453 
454 	rsp->ah = ah;
455 
456 	dm_mad = rsp->mad;
457 	memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
458 	dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
459 	dm_mad->mad_hdr.status = 0;
460 
461 	switch (mad_wc->recv_buf.mad->mad_hdr.method) {
462 	case IB_MGMT_METHOD_GET:
463 		srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
464 		break;
465 	case IB_MGMT_METHOD_SET:
466 		dm_mad->mad_hdr.status =
467 		    cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
468 		break;
469 	default:
470 		dm_mad->mad_hdr.status =
471 		    cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
472 		break;
473 	}
474 
475 	if (!ib_post_send_mad(rsp, NULL)) {
476 		ib_free_recv_mad(mad_wc);
477 		/* will destroy_ah & free_send_mad in send completion */
478 		return;
479 	}
480 
481 	ib_free_send_mad(rsp);
482 
483 err_rsp:
484 	ib_destroy_ah(ah);
485 err:
486 	ib_free_recv_mad(mad_wc);
487 }
488 
489 /**
490  * srpt_refresh_port() - Configure a HCA port.
491  *
492  * Enable InfiniBand management datagram processing, update the cached sm_lid,
493  * lid and gid values, and register a callback function for processing MADs
494  * on the specified port.
495  *
496  * Note: It is safe to call this function more than once for the same port.
497  */
498 static int srpt_refresh_port(struct srpt_port *sport)
499 {
500 	struct ib_mad_reg_req reg_req;
501 	struct ib_port_modify port_modify;
502 	struct ib_port_attr port_attr;
503 	int ret;
504 
505 	memset(&port_modify, 0, sizeof(port_modify));
506 	port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
507 	port_modify.clr_port_cap_mask = 0;
508 
509 	ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
510 	if (ret)
511 		goto err_mod_port;
512 
513 	ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
514 	if (ret)
515 		goto err_query_port;
516 
517 	sport->sm_lid = port_attr.sm_lid;
518 	sport->lid = port_attr.lid;
519 
520 	ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid,
521 			   NULL);
522 	if (ret)
523 		goto err_query_port;
524 
525 	if (!sport->mad_agent) {
526 		memset(&reg_req, 0, sizeof(reg_req));
527 		reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
528 		reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
529 		set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
530 		set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
531 
532 		sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
533 							 sport->port,
534 							 IB_QPT_GSI,
535 							 &reg_req, 0,
536 							 srpt_mad_send_handler,
537 							 srpt_mad_recv_handler,
538 							 sport, 0);
539 		if (IS_ERR(sport->mad_agent)) {
540 			ret = PTR_ERR(sport->mad_agent);
541 			sport->mad_agent = NULL;
542 			goto err_query_port;
543 		}
544 	}
545 
546 	return 0;
547 
548 err_query_port:
549 
550 	port_modify.set_port_cap_mask = 0;
551 	port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
552 	ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
553 
554 err_mod_port:
555 
556 	return ret;
557 }
558 
559 /**
560  * srpt_unregister_mad_agent() - Unregister MAD callback functions.
561  *
562  * Note: It is safe to call this function more than once for the same device.
563  */
564 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
565 {
566 	struct ib_port_modify port_modify = {
567 		.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
568 	};
569 	struct srpt_port *sport;
570 	int i;
571 
572 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
573 		sport = &sdev->port[i - 1];
574 		WARN_ON(sport->port != i);
575 		if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
576 			pr_err("disabling MAD processing failed.\n");
577 		if (sport->mad_agent) {
578 			ib_unregister_mad_agent(sport->mad_agent);
579 			sport->mad_agent = NULL;
580 		}
581 	}
582 }
583 
584 /**
585  * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure.
586  */
587 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
588 					   int ioctx_size, int dma_size,
589 					   enum dma_data_direction dir)
590 {
591 	struct srpt_ioctx *ioctx;
592 
593 	ioctx = kmalloc(ioctx_size, GFP_KERNEL);
594 	if (!ioctx)
595 		goto err;
596 
597 	ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
598 	if (!ioctx->buf)
599 		goto err_free_ioctx;
600 
601 	ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
602 	if (ib_dma_mapping_error(sdev->device, ioctx->dma))
603 		goto err_free_buf;
604 
605 	return ioctx;
606 
607 err_free_buf:
608 	kfree(ioctx->buf);
609 err_free_ioctx:
610 	kfree(ioctx);
611 err:
612 	return NULL;
613 }
614 
615 /**
616  * srpt_free_ioctx() - Free an SRPT I/O context structure.
617  */
618 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
619 			    int dma_size, enum dma_data_direction dir)
620 {
621 	if (!ioctx)
622 		return;
623 
624 	ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
625 	kfree(ioctx->buf);
626 	kfree(ioctx);
627 }
628 
629 /**
630  * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures.
631  * @sdev:       Device to allocate the I/O context ring for.
632  * @ring_size:  Number of elements in the I/O context ring.
633  * @ioctx_size: I/O context size.
634  * @dma_size:   DMA buffer size.
635  * @dir:        DMA data direction.
636  */
637 static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
638 				int ring_size, int ioctx_size,
639 				int dma_size, enum dma_data_direction dir)
640 {
641 	struct srpt_ioctx **ring;
642 	int i;
643 
644 	WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
645 		&& ioctx_size != sizeof(struct srpt_send_ioctx));
646 
647 	ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL);
648 	if (!ring)
649 		goto out;
650 	for (i = 0; i < ring_size; ++i) {
651 		ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
652 		if (!ring[i])
653 			goto err;
654 		ring[i]->index = i;
655 	}
656 	goto out;
657 
658 err:
659 	while (--i >= 0)
660 		srpt_free_ioctx(sdev, ring[i], dma_size, dir);
661 	kfree(ring);
662 	ring = NULL;
663 out:
664 	return ring;
665 }
666 
667 /**
668  * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures.
669  */
670 static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
671 				 struct srpt_device *sdev, int ring_size,
672 				 int dma_size, enum dma_data_direction dir)
673 {
674 	int i;
675 
676 	for (i = 0; i < ring_size; ++i)
677 		srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
678 	kfree(ioctx_ring);
679 }
680 
681 /**
682  * srpt_get_cmd_state() - Get the state of a SCSI command.
683  */
684 static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
685 {
686 	enum srpt_command_state state;
687 	unsigned long flags;
688 
689 	BUG_ON(!ioctx);
690 
691 	spin_lock_irqsave(&ioctx->spinlock, flags);
692 	state = ioctx->state;
693 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
694 	return state;
695 }
696 
697 /**
698  * srpt_set_cmd_state() - Set the state of a SCSI command.
699  *
700  * Does not modify the state of aborted commands. Returns the previous command
701  * state.
702  */
703 static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
704 						  enum srpt_command_state new)
705 {
706 	enum srpt_command_state previous;
707 	unsigned long flags;
708 
709 	BUG_ON(!ioctx);
710 
711 	spin_lock_irqsave(&ioctx->spinlock, flags);
712 	previous = ioctx->state;
713 	if (previous != SRPT_STATE_DONE)
714 		ioctx->state = new;
715 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
716 
717 	return previous;
718 }
719 
720 /**
721  * srpt_test_and_set_cmd_state() - Test and set the state of a command.
722  *
723  * Returns true if and only if the previous command state was equal to 'old'.
724  */
725 static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
726 					enum srpt_command_state old,
727 					enum srpt_command_state new)
728 {
729 	enum srpt_command_state previous;
730 	unsigned long flags;
731 
732 	WARN_ON(!ioctx);
733 	WARN_ON(old == SRPT_STATE_DONE);
734 	WARN_ON(new == SRPT_STATE_NEW);
735 
736 	spin_lock_irqsave(&ioctx->spinlock, flags);
737 	previous = ioctx->state;
738 	if (previous == old)
739 		ioctx->state = new;
740 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
741 	return previous == old;
742 }
743 
744 /**
745  * srpt_post_recv() - Post an IB receive request.
746  */
747 static int srpt_post_recv(struct srpt_device *sdev,
748 			  struct srpt_recv_ioctx *ioctx)
749 {
750 	struct ib_sge list;
751 	struct ib_recv_wr wr, *bad_wr;
752 
753 	BUG_ON(!sdev);
754 	list.addr = ioctx->ioctx.dma;
755 	list.length = srp_max_req_size;
756 	list.lkey = sdev->pd->local_dma_lkey;
757 
758 	ioctx->ioctx.cqe.done = srpt_recv_done;
759 	wr.wr_cqe = &ioctx->ioctx.cqe;
760 	wr.next = NULL;
761 	wr.sg_list = &list;
762 	wr.num_sge = 1;
763 
764 	return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
765 }
766 
767 /**
768  * srpt_zerolength_write() - Perform a zero-length RDMA write.
769  *
770  * A quote from the InfiniBand specification: C9-88: For an HCA responder
771  * using Reliable Connection service, for each zero-length RDMA READ or WRITE
772  * request, the R_Key shall not be validated, even if the request includes
773  * Immediate data.
774  */
775 static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
776 {
777 	struct ib_send_wr wr, *bad_wr;
778 
779 	memset(&wr, 0, sizeof(wr));
780 	wr.opcode = IB_WR_RDMA_WRITE;
781 	wr.wr_cqe = &ch->zw_cqe;
782 	wr.send_flags = IB_SEND_SIGNALED;
783 	return ib_post_send(ch->qp, &wr, &bad_wr);
784 }
785 
786 static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
787 {
788 	struct srpt_rdma_ch *ch = cq->cq_context;
789 
790 	if (wc->status == IB_WC_SUCCESS) {
791 		srpt_process_wait_list(ch);
792 	} else {
793 		if (srpt_set_ch_state(ch, CH_DISCONNECTED))
794 			schedule_work(&ch->release_work);
795 		else
796 			WARN_ONCE(1, "%s-%d\n", ch->sess_name, ch->qp->qp_num);
797 	}
798 }
799 
800 static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,
801 		struct srp_direct_buf *db, int nbufs, struct scatterlist **sg,
802 		unsigned *sg_cnt)
803 {
804 	enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
805 	struct srpt_rdma_ch *ch = ioctx->ch;
806 	struct scatterlist *prev = NULL;
807 	unsigned prev_nents;
808 	int ret, i;
809 
810 	if (nbufs == 1) {
811 		ioctx->rw_ctxs = &ioctx->s_rw_ctx;
812 	} else {
813 		ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),
814 			GFP_KERNEL);
815 		if (!ioctx->rw_ctxs)
816 			return -ENOMEM;
817 	}
818 
819 	for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {
820 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
821 		u64 remote_addr = be64_to_cpu(db->va);
822 		u32 size = be32_to_cpu(db->len);
823 		u32 rkey = be32_to_cpu(db->key);
824 
825 		ret = target_alloc_sgl(&ctx->sg, &ctx->nents, size, false,
826 				i < nbufs - 1);
827 		if (ret)
828 			goto unwind;
829 
830 		ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port,
831 				ctx->sg, ctx->nents, 0, remote_addr, rkey, dir);
832 		if (ret < 0) {
833 			target_free_sgl(ctx->sg, ctx->nents);
834 			goto unwind;
835 		}
836 
837 		ioctx->n_rdma += ret;
838 		ioctx->n_rw_ctx++;
839 
840 		if (prev) {
841 			sg_unmark_end(&prev[prev_nents - 1]);
842 			sg_chain(prev, prev_nents + 1, ctx->sg);
843 		} else {
844 			*sg = ctx->sg;
845 		}
846 
847 		prev = ctx->sg;
848 		prev_nents = ctx->nents;
849 
850 		*sg_cnt += ctx->nents;
851 	}
852 
853 	return 0;
854 
855 unwind:
856 	while (--i >= 0) {
857 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
858 
859 		rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
860 				ctx->sg, ctx->nents, dir);
861 		target_free_sgl(ctx->sg, ctx->nents);
862 	}
863 	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
864 		kfree(ioctx->rw_ctxs);
865 	return ret;
866 }
867 
868 static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch,
869 				    struct srpt_send_ioctx *ioctx)
870 {
871 	enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
872 	int i;
873 
874 	for (i = 0; i < ioctx->n_rw_ctx; i++) {
875 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
876 
877 		rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
878 				ctx->sg, ctx->nents, dir);
879 		target_free_sgl(ctx->sg, ctx->nents);
880 	}
881 
882 	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
883 		kfree(ioctx->rw_ctxs);
884 }
885 
886 static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
887 {
888 	/*
889 	 * The pointer computations below will only be compiled correctly
890 	 * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
891 	 * whether srp_cmd::add_data has been declared as a byte pointer.
892 	 */
893 	BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) &&
894 		     !__same_type(srp_cmd->add_data[0], (u8)0));
895 
896 	/*
897 	 * According to the SRP spec, the lower two bits of the 'ADDITIONAL
898 	 * CDB LENGTH' field are reserved and the size in bytes of this field
899 	 * is four times the value specified in bits 3..7. Hence the "& ~3".
900 	 */
901 	return srp_cmd->add_data + (srp_cmd->add_cdb_len & ~3);
902 }
903 
904 /**
905  * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
906  * @ioctx: Pointer to the I/O context associated with the request.
907  * @srp_cmd: Pointer to the SRP_CMD request data.
908  * @dir: Pointer to the variable to which the transfer direction will be
909  *   written.
910  * @data_len: Pointer to the variable to which the total data length of all
911  *   descriptors in the SRP_CMD request will be written.
912  *
913  * This function initializes ioctx->nrbuf and ioctx->r_bufs.
914  *
915  * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
916  * -ENOMEM when memory allocation fails and zero upon success.
917  */
918 static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
919 		struct srp_cmd *srp_cmd, enum dma_data_direction *dir,
920 		struct scatterlist **sg, unsigned *sg_cnt, u64 *data_len)
921 {
922 	BUG_ON(!dir);
923 	BUG_ON(!data_len);
924 
925 	/*
926 	 * The lower four bits of the buffer format field contain the DATA-IN
927 	 * buffer descriptor format, and the highest four bits contain the
928 	 * DATA-OUT buffer descriptor format.
929 	 */
930 	if (srp_cmd->buf_fmt & 0xf)
931 		/* DATA-IN: transfer data from target to initiator (read). */
932 		*dir = DMA_FROM_DEVICE;
933 	else if (srp_cmd->buf_fmt >> 4)
934 		/* DATA-OUT: transfer data from initiator to target (write). */
935 		*dir = DMA_TO_DEVICE;
936 	else
937 		*dir = DMA_NONE;
938 
939 	/* initialize data_direction early as srpt_alloc_rw_ctxs needs it */
940 	ioctx->cmd.data_direction = *dir;
941 
942 	if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
943 	    ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
944 	    	struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
945 
946 		*data_len = be32_to_cpu(db->len);
947 		return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);
948 	} else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
949 		   ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
950 		struct srp_indirect_buf *idb = srpt_get_desc_buf(srp_cmd);
951 		int nbufs = be32_to_cpu(idb->table_desc.len) /
952 				sizeof(struct srp_direct_buf);
953 
954 		if (nbufs >
955 		    (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
956 			pr_err("received unsupported SRP_CMD request"
957 			       " type (%u out + %u in != %u / %zu)\n",
958 			       srp_cmd->data_out_desc_cnt,
959 			       srp_cmd->data_in_desc_cnt,
960 			       be32_to_cpu(idb->table_desc.len),
961 			       sizeof(struct srp_direct_buf));
962 			return -EINVAL;
963 		}
964 
965 		*data_len = be32_to_cpu(idb->len);
966 		return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,
967 				sg, sg_cnt);
968 	} else {
969 		*data_len = 0;
970 		return 0;
971 	}
972 }
973 
974 /**
975  * srpt_init_ch_qp() - Initialize queue pair attributes.
976  *
977  * Initialized the attributes of queue pair 'qp' by allowing local write,
978  * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
979  */
980 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
981 {
982 	struct ib_qp_attr *attr;
983 	int ret;
984 
985 	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
986 	if (!attr)
987 		return -ENOMEM;
988 
989 	attr->qp_state = IB_QPS_INIT;
990 	attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
991 	    IB_ACCESS_REMOTE_WRITE;
992 	attr->port_num = ch->sport->port;
993 	attr->pkey_index = 0;
994 
995 	ret = ib_modify_qp(qp, attr,
996 			   IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
997 			   IB_QP_PKEY_INDEX);
998 
999 	kfree(attr);
1000 	return ret;
1001 }
1002 
1003 /**
1004  * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR).
1005  * @ch: channel of the queue pair.
1006  * @qp: queue pair to change the state of.
1007  *
1008  * Returns zero upon success and a negative value upon failure.
1009  *
1010  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1011  * If this structure ever becomes larger, it might be necessary to allocate
1012  * it dynamically instead of on the stack.
1013  */
1014 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1015 {
1016 	struct ib_qp_attr qp_attr;
1017 	int attr_mask;
1018 	int ret;
1019 
1020 	qp_attr.qp_state = IB_QPS_RTR;
1021 	ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1022 	if (ret)
1023 		goto out;
1024 
1025 	qp_attr.max_dest_rd_atomic = 4;
1026 
1027 	ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1028 
1029 out:
1030 	return ret;
1031 }
1032 
1033 /**
1034  * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS).
1035  * @ch: channel of the queue pair.
1036  * @qp: queue pair to change the state of.
1037  *
1038  * Returns zero upon success and a negative value upon failure.
1039  *
1040  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1041  * If this structure ever becomes larger, it might be necessary to allocate
1042  * it dynamically instead of on the stack.
1043  */
1044 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1045 {
1046 	struct ib_qp_attr qp_attr;
1047 	int attr_mask;
1048 	int ret;
1049 
1050 	qp_attr.qp_state = IB_QPS_RTS;
1051 	ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1052 	if (ret)
1053 		goto out;
1054 
1055 	qp_attr.max_rd_atomic = 4;
1056 
1057 	ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1058 
1059 out:
1060 	return ret;
1061 }
1062 
1063 /**
1064  * srpt_ch_qp_err() - Set the channel queue pair state to 'error'.
1065  */
1066 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1067 {
1068 	struct ib_qp_attr qp_attr;
1069 
1070 	qp_attr.qp_state = IB_QPS_ERR;
1071 	return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1072 }
1073 
1074 /**
1075  * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator.
1076  */
1077 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1078 {
1079 	struct srpt_send_ioctx *ioctx;
1080 	unsigned long flags;
1081 
1082 	BUG_ON(!ch);
1083 
1084 	ioctx = NULL;
1085 	spin_lock_irqsave(&ch->spinlock, flags);
1086 	if (!list_empty(&ch->free_list)) {
1087 		ioctx = list_first_entry(&ch->free_list,
1088 					 struct srpt_send_ioctx, free_list);
1089 		list_del(&ioctx->free_list);
1090 	}
1091 	spin_unlock_irqrestore(&ch->spinlock, flags);
1092 
1093 	if (!ioctx)
1094 		return ioctx;
1095 
1096 	BUG_ON(ioctx->ch != ch);
1097 	spin_lock_init(&ioctx->spinlock);
1098 	ioctx->state = SRPT_STATE_NEW;
1099 	ioctx->n_rdma = 0;
1100 	ioctx->n_rw_ctx = 0;
1101 	init_completion(&ioctx->tx_done);
1102 	ioctx->queue_status_only = false;
1103 	/*
1104 	 * transport_init_se_cmd() does not initialize all fields, so do it
1105 	 * here.
1106 	 */
1107 	memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1108 	memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
1109 
1110 	return ioctx;
1111 }
1112 
1113 /**
1114  * srpt_abort_cmd() - Abort a SCSI command.
1115  * @ioctx:   I/O context associated with the SCSI command.
1116  * @context: Preferred execution context.
1117  */
1118 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1119 {
1120 	enum srpt_command_state state;
1121 	unsigned long flags;
1122 
1123 	BUG_ON(!ioctx);
1124 
1125 	/*
1126 	 * If the command is in a state where the target core is waiting for
1127 	 * the ib_srpt driver, change the state to the next state.
1128 	 */
1129 
1130 	spin_lock_irqsave(&ioctx->spinlock, flags);
1131 	state = ioctx->state;
1132 	switch (state) {
1133 	case SRPT_STATE_NEED_DATA:
1134 		ioctx->state = SRPT_STATE_DATA_IN;
1135 		break;
1136 	case SRPT_STATE_CMD_RSP_SENT:
1137 	case SRPT_STATE_MGMT_RSP_SENT:
1138 		ioctx->state = SRPT_STATE_DONE;
1139 		break;
1140 	default:
1141 		WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1142 			  __func__, state);
1143 		break;
1144 	}
1145 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
1146 
1147 	pr_debug("Aborting cmd with state %d and tag %lld\n", state,
1148 		 ioctx->cmd.tag);
1149 
1150 	switch (state) {
1151 	case SRPT_STATE_NEW:
1152 	case SRPT_STATE_DATA_IN:
1153 	case SRPT_STATE_MGMT:
1154 	case SRPT_STATE_DONE:
1155 		/*
1156 		 * Do nothing - defer abort processing until
1157 		 * srpt_queue_response() is invoked.
1158 		 */
1159 		break;
1160 	case SRPT_STATE_NEED_DATA:
1161 		pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1162 		transport_generic_request_failure(&ioctx->cmd,
1163 					TCM_CHECK_CONDITION_ABORT_CMD);
1164 		break;
1165 	case SRPT_STATE_CMD_RSP_SENT:
1166 		/*
1167 		 * SRP_RSP sending failed or the SRP_RSP send completion has
1168 		 * not been received in time.
1169 		 */
1170 		transport_generic_free_cmd(&ioctx->cmd, 0);
1171 		break;
1172 	case SRPT_STATE_MGMT_RSP_SENT:
1173 		transport_generic_free_cmd(&ioctx->cmd, 0);
1174 		break;
1175 	default:
1176 		WARN(1, "Unexpected command state (%d)", state);
1177 		break;
1178 	}
1179 
1180 	return state;
1181 }
1182 
1183 /**
1184  * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
1185  * the data that has been transferred via IB RDMA had to be postponed until the
1186  * check_stop_free() callback.  None of this is necessary anymore and needs to
1187  * be cleaned up.
1188  */
1189 static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1190 {
1191 	struct srpt_rdma_ch *ch = cq->cq_context;
1192 	struct srpt_send_ioctx *ioctx =
1193 		container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
1194 
1195 	WARN_ON(ioctx->n_rdma <= 0);
1196 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
1197 	ioctx->n_rdma = 0;
1198 
1199 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1200 		pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1201 			ioctx, wc->status);
1202 		srpt_abort_cmd(ioctx);
1203 		return;
1204 	}
1205 
1206 	if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1207 					SRPT_STATE_DATA_IN))
1208 		target_execute_cmd(&ioctx->cmd);
1209 	else
1210 		pr_err("%s[%d]: wrong state = %d\n", __func__,
1211 		       __LINE__, srpt_get_cmd_state(ioctx));
1212 }
1213 
1214 /**
1215  * srpt_build_cmd_rsp() - Build an SRP_RSP response.
1216  * @ch: RDMA channel through which the request has been received.
1217  * @ioctx: I/O context associated with the SRP_CMD request. The response will
1218  *   be built in the buffer ioctx->buf points at and hence this function will
1219  *   overwrite the request data.
1220  * @tag: tag of the request for which this response is being generated.
1221  * @status: value for the STATUS field of the SRP_RSP information unit.
1222  *
1223  * Returns the size in bytes of the SRP_RSP response.
1224  *
1225  * An SRP_RSP response contains a SCSI status or service response. See also
1226  * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1227  * response. See also SPC-2 for more information about sense data.
1228  */
1229 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1230 			      struct srpt_send_ioctx *ioctx, u64 tag,
1231 			      int status)
1232 {
1233 	struct srp_rsp *srp_rsp;
1234 	const u8 *sense_data;
1235 	int sense_data_len, max_sense_len;
1236 
1237 	/*
1238 	 * The lowest bit of all SAM-3 status codes is zero (see also
1239 	 * paragraph 5.3 in SAM-3).
1240 	 */
1241 	WARN_ON(status & 1);
1242 
1243 	srp_rsp = ioctx->ioctx.buf;
1244 	BUG_ON(!srp_rsp);
1245 
1246 	sense_data = ioctx->sense_data;
1247 	sense_data_len = ioctx->cmd.scsi_sense_length;
1248 	WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1249 
1250 	memset(srp_rsp, 0, sizeof(*srp_rsp));
1251 	srp_rsp->opcode = SRP_RSP;
1252 	srp_rsp->req_lim_delta =
1253 		cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1254 	srp_rsp->tag = tag;
1255 	srp_rsp->status = status;
1256 
1257 	if (sense_data_len) {
1258 		BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1259 		max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1260 		if (sense_data_len > max_sense_len) {
1261 			pr_warn("truncated sense data from %d to %d"
1262 				" bytes\n", sense_data_len, max_sense_len);
1263 			sense_data_len = max_sense_len;
1264 		}
1265 
1266 		srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1267 		srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1268 		memcpy(srp_rsp + 1, sense_data, sense_data_len);
1269 	}
1270 
1271 	return sizeof(*srp_rsp) + sense_data_len;
1272 }
1273 
1274 /**
1275  * srpt_build_tskmgmt_rsp() - Build a task management response.
1276  * @ch:       RDMA channel through which the request has been received.
1277  * @ioctx:    I/O context in which the SRP_RSP response will be built.
1278  * @rsp_code: RSP_CODE that will be stored in the response.
1279  * @tag:      Tag of the request for which this response is being generated.
1280  *
1281  * Returns the size in bytes of the SRP_RSP response.
1282  *
1283  * An SRP_RSP response contains a SCSI status or service response. See also
1284  * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1285  * response.
1286  */
1287 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1288 				  struct srpt_send_ioctx *ioctx,
1289 				  u8 rsp_code, u64 tag)
1290 {
1291 	struct srp_rsp *srp_rsp;
1292 	int resp_data_len;
1293 	int resp_len;
1294 
1295 	resp_data_len = 4;
1296 	resp_len = sizeof(*srp_rsp) + resp_data_len;
1297 
1298 	srp_rsp = ioctx->ioctx.buf;
1299 	BUG_ON(!srp_rsp);
1300 	memset(srp_rsp, 0, sizeof(*srp_rsp));
1301 
1302 	srp_rsp->opcode = SRP_RSP;
1303 	srp_rsp->req_lim_delta =
1304 		cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1305 	srp_rsp->tag = tag;
1306 
1307 	srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1308 	srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1309 	srp_rsp->data[3] = rsp_code;
1310 
1311 	return resp_len;
1312 }
1313 
1314 static int srpt_check_stop_free(struct se_cmd *cmd)
1315 {
1316 	struct srpt_send_ioctx *ioctx = container_of(cmd,
1317 				struct srpt_send_ioctx, cmd);
1318 
1319 	return target_put_sess_cmd(&ioctx->cmd);
1320 }
1321 
1322 /**
1323  * srpt_handle_cmd() - Process SRP_CMD.
1324  */
1325 static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1326 			    struct srpt_recv_ioctx *recv_ioctx,
1327 			    struct srpt_send_ioctx *send_ioctx)
1328 {
1329 	struct se_cmd *cmd;
1330 	struct srp_cmd *srp_cmd;
1331 	struct scatterlist *sg = NULL;
1332 	unsigned sg_cnt = 0;
1333 	u64 data_len;
1334 	enum dma_data_direction dir;
1335 	int rc;
1336 
1337 	BUG_ON(!send_ioctx);
1338 
1339 	srp_cmd = recv_ioctx->ioctx.buf;
1340 	cmd = &send_ioctx->cmd;
1341 	cmd->tag = srp_cmd->tag;
1342 
1343 	switch (srp_cmd->task_attr) {
1344 	case SRP_CMD_SIMPLE_Q:
1345 		cmd->sam_task_attr = TCM_SIMPLE_TAG;
1346 		break;
1347 	case SRP_CMD_ORDERED_Q:
1348 	default:
1349 		cmd->sam_task_attr = TCM_ORDERED_TAG;
1350 		break;
1351 	case SRP_CMD_HEAD_OF_Q:
1352 		cmd->sam_task_attr = TCM_HEAD_TAG;
1353 		break;
1354 	case SRP_CMD_ACA:
1355 		cmd->sam_task_attr = TCM_ACA_TAG;
1356 		break;
1357 	}
1358 
1359 	rc = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &sg, &sg_cnt,
1360 			&data_len);
1361 	if (rc) {
1362 		if (rc != -EAGAIN) {
1363 			pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1364 			       srp_cmd->tag);
1365 		}
1366 		goto release_ioctx;
1367 	}
1368 
1369 	rc = target_submit_cmd_map_sgls(cmd, ch->sess, srp_cmd->cdb,
1370 			       &send_ioctx->sense_data[0],
1371 			       scsilun_to_int(&srp_cmd->lun), data_len,
1372 			       TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF,
1373 			       sg, sg_cnt, NULL, 0, NULL, 0);
1374 	if (rc != 0) {
1375 		pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1376 			 srp_cmd->tag);
1377 		goto release_ioctx;
1378 	}
1379 	return;
1380 
1381 release_ioctx:
1382 	send_ioctx->state = SRPT_STATE_DONE;
1383 	srpt_release_cmd(cmd);
1384 }
1385 
1386 static int srp_tmr_to_tcm(int fn)
1387 {
1388 	switch (fn) {
1389 	case SRP_TSK_ABORT_TASK:
1390 		return TMR_ABORT_TASK;
1391 	case SRP_TSK_ABORT_TASK_SET:
1392 		return TMR_ABORT_TASK_SET;
1393 	case SRP_TSK_CLEAR_TASK_SET:
1394 		return TMR_CLEAR_TASK_SET;
1395 	case SRP_TSK_LUN_RESET:
1396 		return TMR_LUN_RESET;
1397 	case SRP_TSK_CLEAR_ACA:
1398 		return TMR_CLEAR_ACA;
1399 	default:
1400 		return -1;
1401 	}
1402 }
1403 
1404 /**
1405  * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
1406  *
1407  * Returns 0 if and only if the request will be processed by the target core.
1408  *
1409  * For more information about SRP_TSK_MGMT information units, see also section
1410  * 6.7 in the SRP r16a document.
1411  */
1412 static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1413 				 struct srpt_recv_ioctx *recv_ioctx,
1414 				 struct srpt_send_ioctx *send_ioctx)
1415 {
1416 	struct srp_tsk_mgmt *srp_tsk;
1417 	struct se_cmd *cmd;
1418 	struct se_session *sess = ch->sess;
1419 	int tcm_tmr;
1420 	int rc;
1421 
1422 	BUG_ON(!send_ioctx);
1423 
1424 	srp_tsk = recv_ioctx->ioctx.buf;
1425 	cmd = &send_ioctx->cmd;
1426 
1427 	pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
1428 		 " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
1429 		 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
1430 
1431 	srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
1432 	send_ioctx->cmd.tag = srp_tsk->tag;
1433 	tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
1434 	rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL,
1435 			       scsilun_to_int(&srp_tsk->lun), srp_tsk, tcm_tmr,
1436 			       GFP_KERNEL, srp_tsk->task_tag,
1437 			       TARGET_SCF_ACK_KREF);
1438 	if (rc != 0) {
1439 		send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
1440 		goto fail;
1441 	}
1442 	return;
1443 fail:
1444 	transport_send_check_condition_and_sense(cmd, 0, 0); // XXX:
1445 }
1446 
1447 /**
1448  * srpt_handle_new_iu() - Process a newly received information unit.
1449  * @ch:    RDMA channel through which the information unit has been received.
1450  * @ioctx: SRPT I/O context associated with the information unit.
1451  */
1452 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1453 			       struct srpt_recv_ioctx *recv_ioctx,
1454 			       struct srpt_send_ioctx *send_ioctx)
1455 {
1456 	struct srp_cmd *srp_cmd;
1457 
1458 	BUG_ON(!ch);
1459 	BUG_ON(!recv_ioctx);
1460 
1461 	ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1462 				   recv_ioctx->ioctx.dma, srp_max_req_size,
1463 				   DMA_FROM_DEVICE);
1464 
1465 	if (unlikely(ch->state == CH_CONNECTING))
1466 		goto out_wait;
1467 
1468 	if (unlikely(ch->state != CH_LIVE))
1469 		return;
1470 
1471 	srp_cmd = recv_ioctx->ioctx.buf;
1472 	if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
1473 		if (!send_ioctx) {
1474 			if (!list_empty(&ch->cmd_wait_list))
1475 				goto out_wait;
1476 			send_ioctx = srpt_get_send_ioctx(ch);
1477 		}
1478 		if (unlikely(!send_ioctx))
1479 			goto out_wait;
1480 	}
1481 
1482 	switch (srp_cmd->opcode) {
1483 	case SRP_CMD:
1484 		srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1485 		break;
1486 	case SRP_TSK_MGMT:
1487 		srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1488 		break;
1489 	case SRP_I_LOGOUT:
1490 		pr_err("Not yet implemented: SRP_I_LOGOUT\n");
1491 		break;
1492 	case SRP_CRED_RSP:
1493 		pr_debug("received SRP_CRED_RSP\n");
1494 		break;
1495 	case SRP_AER_RSP:
1496 		pr_debug("received SRP_AER_RSP\n");
1497 		break;
1498 	case SRP_RSP:
1499 		pr_err("Received SRP_RSP\n");
1500 		break;
1501 	default:
1502 		pr_err("received IU with unknown opcode 0x%x\n",
1503 		       srp_cmd->opcode);
1504 		break;
1505 	}
1506 
1507 	srpt_post_recv(ch->sport->sdev, recv_ioctx);
1508 	return;
1509 
1510 out_wait:
1511 	list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
1512 }
1513 
1514 static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1515 {
1516 	struct srpt_rdma_ch *ch = cq->cq_context;
1517 	struct srpt_recv_ioctx *ioctx =
1518 		container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
1519 
1520 	if (wc->status == IB_WC_SUCCESS) {
1521 		int req_lim;
1522 
1523 		req_lim = atomic_dec_return(&ch->req_lim);
1524 		if (unlikely(req_lim < 0))
1525 			pr_err("req_lim = %d < 0\n", req_lim);
1526 		srpt_handle_new_iu(ch, ioctx, NULL);
1527 	} else {
1528 		pr_info("receiving failed for ioctx %p with status %d\n",
1529 			ioctx, wc->status);
1530 	}
1531 }
1532 
1533 /*
1534  * This function must be called from the context in which RDMA completions are
1535  * processed because it accesses the wait list without protection against
1536  * access from other threads.
1537  */
1538 static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
1539 {
1540 	struct srpt_send_ioctx *ioctx;
1541 
1542 	while (!list_empty(&ch->cmd_wait_list) &&
1543 	       ch->state >= CH_LIVE &&
1544 	       (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
1545 		struct srpt_recv_ioctx *recv_ioctx;
1546 
1547 		recv_ioctx = list_first_entry(&ch->cmd_wait_list,
1548 					      struct srpt_recv_ioctx,
1549 					      wait_list);
1550 		list_del(&recv_ioctx->wait_list);
1551 		srpt_handle_new_iu(ch, recv_ioctx, ioctx);
1552 	}
1553 }
1554 
1555 /**
1556  * Note: Although this has not yet been observed during tests, at least in
1557  * theory it is possible that the srpt_get_send_ioctx() call invoked by
1558  * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
1559  * value in each response is set to one, and it is possible that this response
1560  * makes the initiator send a new request before the send completion for that
1561  * response has been processed. This could e.g. happen if the call to
1562  * srpt_put_send_iotcx() is delayed because of a higher priority interrupt or
1563  * if IB retransmission causes generation of the send completion to be
1564  * delayed. Incoming information units for which srpt_get_send_ioctx() fails
1565  * are queued on cmd_wait_list. The code below processes these delayed
1566  * requests one at a time.
1567  */
1568 static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
1569 {
1570 	struct srpt_rdma_ch *ch = cq->cq_context;
1571 	struct srpt_send_ioctx *ioctx =
1572 		container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1573 	enum srpt_command_state state;
1574 
1575 	state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1576 
1577 	WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1578 		state != SRPT_STATE_MGMT_RSP_SENT);
1579 
1580 	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
1581 
1582 	if (wc->status != IB_WC_SUCCESS)
1583 		pr_info("sending response for ioctx 0x%p failed"
1584 			" with status %d\n", ioctx, wc->status);
1585 
1586 	if (state != SRPT_STATE_DONE) {
1587 		transport_generic_free_cmd(&ioctx->cmd, 0);
1588 	} else {
1589 		pr_err("IB completion has been received too late for"
1590 		       " wr_id = %u.\n", ioctx->ioctx.index);
1591 	}
1592 
1593 	srpt_process_wait_list(ch);
1594 }
1595 
1596 /**
1597  * srpt_create_ch_ib() - Create receive and send completion queues.
1598  */
1599 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1600 {
1601 	struct ib_qp_init_attr *qp_init;
1602 	struct srpt_port *sport = ch->sport;
1603 	struct srpt_device *sdev = sport->sdev;
1604 	u32 srp_sq_size = sport->port_attrib.srp_sq_size;
1605 	int ret;
1606 
1607 	WARN_ON(ch->rq_size < 1);
1608 
1609 	ret = -ENOMEM;
1610 	qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
1611 	if (!qp_init)
1612 		goto out;
1613 
1614 retry:
1615 	ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
1616 			0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
1617 	if (IS_ERR(ch->cq)) {
1618 		ret = PTR_ERR(ch->cq);
1619 		pr_err("failed to create CQ cqe= %d ret= %d\n",
1620 		       ch->rq_size + srp_sq_size, ret);
1621 		goto out;
1622 	}
1623 
1624 	qp_init->qp_context = (void *)ch;
1625 	qp_init->event_handler
1626 		= (void(*)(struct ib_event *, void*))srpt_qp_event;
1627 	qp_init->send_cq = ch->cq;
1628 	qp_init->recv_cq = ch->cq;
1629 	qp_init->srq = sdev->srq;
1630 	qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1631 	qp_init->qp_type = IB_QPT_RC;
1632 	/*
1633 	 * We divide up our send queue size into half SEND WRs to send the
1634 	 * completions, and half R/W contexts to actually do the RDMA
1635 	 * READ/WRITE transfers.  Note that we need to allocate CQ slots for
1636 	 * both both, as RDMA contexts will also post completions for the
1637 	 * RDMA READ case.
1638 	 */
1639 	qp_init->cap.max_send_wr = srp_sq_size / 2;
1640 	qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
1641 	qp_init->cap.max_send_sge = SRPT_DEF_SG_PER_WQE;
1642 	qp_init->port_num = ch->sport->port;
1643 
1644 	ch->qp = ib_create_qp(sdev->pd, qp_init);
1645 	if (IS_ERR(ch->qp)) {
1646 		ret = PTR_ERR(ch->qp);
1647 		if (ret == -ENOMEM) {
1648 			srp_sq_size /= 2;
1649 			if (srp_sq_size >= MIN_SRPT_SQ_SIZE) {
1650 				ib_destroy_cq(ch->cq);
1651 				goto retry;
1652 			}
1653 		}
1654 		pr_err("failed to create_qp ret= %d\n", ret);
1655 		goto err_destroy_cq;
1656 	}
1657 
1658 	atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
1659 
1660 	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1661 		 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1662 		 qp_init->cap.max_send_wr, ch->cm_id);
1663 
1664 	ret = srpt_init_ch_qp(ch, ch->qp);
1665 	if (ret)
1666 		goto err_destroy_qp;
1667 
1668 out:
1669 	kfree(qp_init);
1670 	return ret;
1671 
1672 err_destroy_qp:
1673 	ib_destroy_qp(ch->qp);
1674 err_destroy_cq:
1675 	ib_free_cq(ch->cq);
1676 	goto out;
1677 }
1678 
1679 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1680 {
1681 	ib_destroy_qp(ch->qp);
1682 	ib_free_cq(ch->cq);
1683 }
1684 
1685 /**
1686  * srpt_close_ch() - Close an RDMA channel.
1687  *
1688  * Make sure all resources associated with the channel will be deallocated at
1689  * an appropriate time.
1690  *
1691  * Returns true if and only if the channel state has been modified into
1692  * CH_DRAINING.
1693  */
1694 static bool srpt_close_ch(struct srpt_rdma_ch *ch)
1695 {
1696 	int ret;
1697 
1698 	if (!srpt_set_ch_state(ch, CH_DRAINING)) {
1699 		pr_debug("%s-%d: already closed\n", ch->sess_name,
1700 			 ch->qp->qp_num);
1701 		return false;
1702 	}
1703 
1704 	kref_get(&ch->kref);
1705 
1706 	ret = srpt_ch_qp_err(ch);
1707 	if (ret < 0)
1708 		pr_err("%s-%d: changing queue pair into error state failed: %d\n",
1709 		       ch->sess_name, ch->qp->qp_num, ret);
1710 
1711 	pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
1712 		 ch->qp->qp_num);
1713 	ret = srpt_zerolength_write(ch);
1714 	if (ret < 0) {
1715 		pr_err("%s-%d: queuing zero-length write failed: %d\n",
1716 		       ch->sess_name, ch->qp->qp_num, ret);
1717 		if (srpt_set_ch_state(ch, CH_DISCONNECTED))
1718 			schedule_work(&ch->release_work);
1719 		else
1720 			WARN_ON_ONCE(true);
1721 	}
1722 
1723 	kref_put(&ch->kref, srpt_free_ch);
1724 
1725 	return true;
1726 }
1727 
1728 /*
1729  * Change the channel state into CH_DISCONNECTING. If a channel has not yet
1730  * reached the connected state, close it. If a channel is in the connected
1731  * state, send a DREQ. If a DREQ has been received, send a DREP. Note: it is
1732  * the responsibility of the caller to ensure that this function is not
1733  * invoked concurrently with the code that accepts a connection. This means
1734  * that this function must either be invoked from inside a CM callback
1735  * function or that it must be invoked with the srpt_port.mutex held.
1736  */
1737 static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
1738 {
1739 	int ret;
1740 
1741 	if (!srpt_set_ch_state(ch, CH_DISCONNECTING))
1742 		return -ENOTCONN;
1743 
1744 	ret = ib_send_cm_dreq(ch->cm_id, NULL, 0);
1745 	if (ret < 0)
1746 		ret = ib_send_cm_drep(ch->cm_id, NULL, 0);
1747 
1748 	if (ret < 0 && srpt_close_ch(ch))
1749 		ret = 0;
1750 
1751 	return ret;
1752 }
1753 
1754 static void __srpt_close_all_ch(struct srpt_device *sdev)
1755 {
1756 	struct srpt_rdma_ch *ch;
1757 
1758 	lockdep_assert_held(&sdev->mutex);
1759 
1760 	list_for_each_entry(ch, &sdev->rch_list, list) {
1761 		if (srpt_disconnect_ch(ch) >= 0)
1762 			pr_info("Closing channel %s-%d because target %s has been disabled\n",
1763 				ch->sess_name, ch->qp->qp_num,
1764 				sdev->device->name);
1765 		srpt_close_ch(ch);
1766 	}
1767 }
1768 
1769 static void srpt_free_ch(struct kref *kref)
1770 {
1771 	struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
1772 
1773 	kfree(ch);
1774 }
1775 
1776 static void srpt_release_channel_work(struct work_struct *w)
1777 {
1778 	struct srpt_rdma_ch *ch;
1779 	struct srpt_device *sdev;
1780 	struct se_session *se_sess;
1781 
1782 	ch = container_of(w, struct srpt_rdma_ch, release_work);
1783 	pr_debug("%s: %s-%d; release_done = %p\n", __func__, ch->sess_name,
1784 		 ch->qp->qp_num, ch->release_done);
1785 
1786 	sdev = ch->sport->sdev;
1787 	BUG_ON(!sdev);
1788 
1789 	se_sess = ch->sess;
1790 	BUG_ON(!se_sess);
1791 
1792 	target_sess_cmd_list_set_waiting(se_sess);
1793 	target_wait_for_sess_cmds(se_sess);
1794 
1795 	transport_deregister_session_configfs(se_sess);
1796 	transport_deregister_session(se_sess);
1797 	ch->sess = NULL;
1798 
1799 	ib_destroy_cm_id(ch->cm_id);
1800 
1801 	srpt_destroy_ch_ib(ch);
1802 
1803 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
1804 			     ch->sport->sdev, ch->rq_size,
1805 			     ch->rsp_size, DMA_TO_DEVICE);
1806 
1807 	mutex_lock(&sdev->mutex);
1808 	list_del_init(&ch->list);
1809 	if (ch->release_done)
1810 		complete(ch->release_done);
1811 	mutex_unlock(&sdev->mutex);
1812 
1813 	wake_up(&sdev->ch_releaseQ);
1814 
1815 	kref_put(&ch->kref, srpt_free_ch);
1816 }
1817 
1818 /**
1819  * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
1820  *
1821  * Ownership of the cm_id is transferred to the target session if this
1822  * functions returns zero. Otherwise the caller remains the owner of cm_id.
1823  */
1824 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1825 			    struct ib_cm_req_event_param *param,
1826 			    void *private_data)
1827 {
1828 	struct srpt_device *sdev = cm_id->context;
1829 	struct srpt_port *sport = &sdev->port[param->port - 1];
1830 	struct srp_login_req *req;
1831 	struct srp_login_rsp *rsp;
1832 	struct srp_login_rej *rej;
1833 	struct ib_cm_rep_param *rep_param;
1834 	struct srpt_rdma_ch *ch, *tmp_ch;
1835 	u32 it_iu_len;
1836 	int i, ret = 0;
1837 	unsigned char *p;
1838 
1839 	WARN_ON_ONCE(irqs_disabled());
1840 
1841 	if (WARN_ON(!sdev || !private_data))
1842 		return -EINVAL;
1843 
1844 	req = (struct srp_login_req *)private_data;
1845 
1846 	it_iu_len = be32_to_cpu(req->req_it_iu_len);
1847 
1848 	pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
1849 		" t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
1850 		" (guid=0x%llx:0x%llx)\n",
1851 		be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
1852 		be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
1853 		be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
1854 		be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
1855 		it_iu_len,
1856 		param->port,
1857 		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
1858 		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
1859 
1860 	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
1861 	rej = kzalloc(sizeof(*rej), GFP_KERNEL);
1862 	rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
1863 
1864 	if (!rsp || !rej || !rep_param) {
1865 		ret = -ENOMEM;
1866 		goto out;
1867 	}
1868 
1869 	if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
1870 		rej->reason = cpu_to_be32(
1871 			      SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1872 		ret = -EINVAL;
1873 		pr_err("rejected SRP_LOGIN_REQ because its"
1874 		       " length (%d bytes) is out of range (%d .. %d)\n",
1875 		       it_iu_len, 64, srp_max_req_size);
1876 		goto reject;
1877 	}
1878 
1879 	if (!sport->enabled) {
1880 		rej->reason = cpu_to_be32(
1881 			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1882 		ret = -EINVAL;
1883 		pr_err("rejected SRP_LOGIN_REQ because the target port"
1884 		       " has not yet been enabled\n");
1885 		goto reject;
1886 	}
1887 
1888 	if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
1889 		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1890 
1891 		mutex_lock(&sdev->mutex);
1892 
1893 		list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1894 			if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1895 			    && !memcmp(ch->t_port_id, req->target_port_id, 16)
1896 			    && param->port == ch->sport->port
1897 			    && param->listen_id == ch->sport->sdev->cm_id
1898 			    && ch->cm_id) {
1899 				if (srpt_disconnect_ch(ch) < 0)
1900 					continue;
1901 				pr_info("Relogin - closed existing channel %s\n",
1902 					ch->sess_name);
1903 				rsp->rsp_flags =
1904 					SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1905 			}
1906 		}
1907 
1908 		mutex_unlock(&sdev->mutex);
1909 
1910 	} else
1911 		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1912 
1913 	if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
1914 	    || *(__be64 *)(req->target_port_id + 8) !=
1915 	       cpu_to_be64(srpt_service_guid)) {
1916 		rej->reason = cpu_to_be32(
1917 			      SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1918 		ret = -ENOMEM;
1919 		pr_err("rejected SRP_LOGIN_REQ because it"
1920 		       " has an invalid target port identifier.\n");
1921 		goto reject;
1922 	}
1923 
1924 	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1925 	if (!ch) {
1926 		rej->reason = cpu_to_be32(
1927 			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1928 		pr_err("rejected SRP_LOGIN_REQ because no memory.\n");
1929 		ret = -ENOMEM;
1930 		goto reject;
1931 	}
1932 
1933 	kref_init(&ch->kref);
1934 	ch->zw_cqe.done = srpt_zerolength_write_done;
1935 	INIT_WORK(&ch->release_work, srpt_release_channel_work);
1936 	memcpy(ch->i_port_id, req->initiator_port_id, 16);
1937 	memcpy(ch->t_port_id, req->target_port_id, 16);
1938 	ch->sport = &sdev->port[param->port - 1];
1939 	ch->cm_id = cm_id;
1940 	cm_id->context = ch;
1941 	/*
1942 	 * Avoid QUEUE_FULL conditions by limiting the number of buffers used
1943 	 * for the SRP protocol to the command queue size.
1944 	 */
1945 	ch->rq_size = SRPT_RQ_SIZE;
1946 	spin_lock_init(&ch->spinlock);
1947 	ch->state = CH_CONNECTING;
1948 	INIT_LIST_HEAD(&ch->cmd_wait_list);
1949 	ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
1950 
1951 	ch->ioctx_ring = (struct srpt_send_ioctx **)
1952 		srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
1953 				      sizeof(*ch->ioctx_ring[0]),
1954 				      ch->rsp_size, DMA_TO_DEVICE);
1955 	if (!ch->ioctx_ring)
1956 		goto free_ch;
1957 
1958 	INIT_LIST_HEAD(&ch->free_list);
1959 	for (i = 0; i < ch->rq_size; i++) {
1960 		ch->ioctx_ring[i]->ch = ch;
1961 		list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
1962 	}
1963 
1964 	ret = srpt_create_ch_ib(ch);
1965 	if (ret) {
1966 		rej->reason = cpu_to_be32(
1967 			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1968 		pr_err("rejected SRP_LOGIN_REQ because creating"
1969 		       " a new RDMA channel failed.\n");
1970 		goto free_ring;
1971 	}
1972 
1973 	ret = srpt_ch_qp_rtr(ch, ch->qp);
1974 	if (ret) {
1975 		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1976 		pr_err("rejected SRP_LOGIN_REQ because enabling"
1977 		       " RTR failed (error code = %d)\n", ret);
1978 		goto destroy_ib;
1979 	}
1980 
1981 	/*
1982 	 * Use the initator port identifier as the session name, when
1983 	 * checking against se_node_acl->initiatorname[] this can be
1984 	 * with or without preceeding '0x'.
1985 	 */
1986 	snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
1987 			be64_to_cpu(*(__be64 *)ch->i_port_id),
1988 			be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
1989 
1990 	pr_debug("registering session %s\n", ch->sess_name);
1991 	p = &ch->sess_name[0];
1992 
1993 try_again:
1994 	ch->sess = target_alloc_session(&sport->port_tpg_1, 0, 0,
1995 					TARGET_PROT_NORMAL, p, ch, NULL);
1996 	if (IS_ERR(ch->sess)) {
1997 		pr_info("Rejected login because no ACL has been"
1998 			" configured yet for initiator %s.\n", p);
1999 		/*
2000 		 * XXX: Hack to retry of ch->i_port_id without leading '0x'
2001 		 */
2002 		if (p == &ch->sess_name[0]) {
2003 			p += 2;
2004 			goto try_again;
2005 		}
2006 		rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ?
2007 				SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
2008 				SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
2009 		goto destroy_ib;
2010 	}
2011 
2012 	pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
2013 		 ch->sess_name, ch->cm_id);
2014 
2015 	/* create srp_login_response */
2016 	rsp->opcode = SRP_LOGIN_RSP;
2017 	rsp->tag = req->tag;
2018 	rsp->max_it_iu_len = req->req_it_iu_len;
2019 	rsp->max_ti_iu_len = req->req_it_iu_len;
2020 	ch->max_ti_iu_len = it_iu_len;
2021 	rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2022 				   | SRP_BUF_FORMAT_INDIRECT);
2023 	rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2024 	atomic_set(&ch->req_lim, ch->rq_size);
2025 	atomic_set(&ch->req_lim_delta, 0);
2026 
2027 	/* create cm reply */
2028 	rep_param->qp_num = ch->qp->qp_num;
2029 	rep_param->private_data = (void *)rsp;
2030 	rep_param->private_data_len = sizeof(*rsp);
2031 	rep_param->rnr_retry_count = 7;
2032 	rep_param->flow_control = 1;
2033 	rep_param->failover_accepted = 0;
2034 	rep_param->srq = 1;
2035 	rep_param->responder_resources = 4;
2036 	rep_param->initiator_depth = 4;
2037 
2038 	ret = ib_send_cm_rep(cm_id, rep_param);
2039 	if (ret) {
2040 		pr_err("sending SRP_LOGIN_REQ response failed"
2041 		       " (error code = %d)\n", ret);
2042 		goto release_channel;
2043 	}
2044 
2045 	mutex_lock(&sdev->mutex);
2046 	list_add_tail(&ch->list, &sdev->rch_list);
2047 	mutex_unlock(&sdev->mutex);
2048 
2049 	goto out;
2050 
2051 release_channel:
2052 	srpt_disconnect_ch(ch);
2053 	transport_deregister_session_configfs(ch->sess);
2054 	transport_deregister_session(ch->sess);
2055 	ch->sess = NULL;
2056 
2057 destroy_ib:
2058 	srpt_destroy_ch_ib(ch);
2059 
2060 free_ring:
2061 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2062 			     ch->sport->sdev, ch->rq_size,
2063 			     ch->rsp_size, DMA_TO_DEVICE);
2064 free_ch:
2065 	kfree(ch);
2066 
2067 reject:
2068 	rej->opcode = SRP_LOGIN_REJ;
2069 	rej->tag = req->tag;
2070 	rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2071 				   | SRP_BUF_FORMAT_INDIRECT);
2072 
2073 	ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2074 			     (void *)rej, sizeof(*rej));
2075 
2076 out:
2077 	kfree(rep_param);
2078 	kfree(rsp);
2079 	kfree(rej);
2080 
2081 	return ret;
2082 }
2083 
2084 static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2085 			     enum ib_cm_rej_reason reason,
2086 			     const u8 *private_data,
2087 			     u8 private_data_len)
2088 {
2089 	char *priv = NULL;
2090 	int i;
2091 
2092 	if (private_data_len && (priv = kmalloc(private_data_len * 3 + 1,
2093 						GFP_KERNEL))) {
2094 		for (i = 0; i < private_data_len; i++)
2095 			sprintf(priv + 3 * i, " %02x", private_data[i]);
2096 	}
2097 	pr_info("Received CM REJ for ch %s-%d; reason %d%s%s.\n",
2098 		ch->sess_name, ch->qp->qp_num, reason, private_data_len ?
2099 		"; private data" : "", priv ? priv : " (?)");
2100 	kfree(priv);
2101 }
2102 
2103 /**
2104  * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event.
2105  *
2106  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2107  * and that the recipient may begin transmitting (RTU = ready to use).
2108  */
2109 static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
2110 {
2111 	int ret;
2112 
2113 	if (srpt_set_ch_state(ch, CH_LIVE)) {
2114 		ret = srpt_ch_qp_rts(ch, ch->qp);
2115 
2116 		if (ret == 0) {
2117 			/* Trigger wait list processing. */
2118 			ret = srpt_zerolength_write(ch);
2119 			WARN_ONCE(ret < 0, "%d\n", ret);
2120 		} else {
2121 			srpt_close_ch(ch);
2122 		}
2123 	}
2124 }
2125 
2126 /**
2127  * srpt_cm_handler() - IB connection manager callback function.
2128  *
2129  * A non-zero return value will cause the caller destroy the CM ID.
2130  *
2131  * Note: srpt_cm_handler() must only return a non-zero value when transferring
2132  * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2133  * a non-zero value in any other case will trigger a race with the
2134  * ib_destroy_cm_id() call in srpt_release_channel().
2135  */
2136 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2137 {
2138 	struct srpt_rdma_ch *ch = cm_id->context;
2139 	int ret;
2140 
2141 	ret = 0;
2142 	switch (event->event) {
2143 	case IB_CM_REQ_RECEIVED:
2144 		ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2145 				       event->private_data);
2146 		break;
2147 	case IB_CM_REJ_RECEIVED:
2148 		srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
2149 				 event->private_data,
2150 				 IB_CM_REJ_PRIVATE_DATA_SIZE);
2151 		break;
2152 	case IB_CM_RTU_RECEIVED:
2153 	case IB_CM_USER_ESTABLISHED:
2154 		srpt_cm_rtu_recv(ch);
2155 		break;
2156 	case IB_CM_DREQ_RECEIVED:
2157 		srpt_disconnect_ch(ch);
2158 		break;
2159 	case IB_CM_DREP_RECEIVED:
2160 		pr_info("Received CM DREP message for ch %s-%d.\n",
2161 			ch->sess_name, ch->qp->qp_num);
2162 		srpt_close_ch(ch);
2163 		break;
2164 	case IB_CM_TIMEWAIT_EXIT:
2165 		pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2166 			ch->sess_name, ch->qp->qp_num);
2167 		srpt_close_ch(ch);
2168 		break;
2169 	case IB_CM_REP_ERROR:
2170 		pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2171 			ch->qp->qp_num);
2172 		break;
2173 	case IB_CM_DREQ_ERROR:
2174 		pr_info("Received CM DREQ ERROR event.\n");
2175 		break;
2176 	case IB_CM_MRA_RECEIVED:
2177 		pr_info("Received CM MRA event\n");
2178 		break;
2179 	default:
2180 		pr_err("received unrecognized CM event %d\n", event->event);
2181 		break;
2182 	}
2183 
2184 	return ret;
2185 }
2186 
2187 static int srpt_write_pending_status(struct se_cmd *se_cmd)
2188 {
2189 	struct srpt_send_ioctx *ioctx;
2190 
2191 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2192 	return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
2193 }
2194 
2195 /*
2196  * srpt_write_pending() - Start data transfer from initiator to target (write).
2197  */
2198 static int srpt_write_pending(struct se_cmd *se_cmd)
2199 {
2200 	struct srpt_send_ioctx *ioctx =
2201 		container_of(se_cmd, struct srpt_send_ioctx, cmd);
2202 	struct srpt_rdma_ch *ch = ioctx->ch;
2203 	struct ib_send_wr *first_wr = NULL, *bad_wr;
2204 	struct ib_cqe *cqe = &ioctx->rdma_cqe;
2205 	enum srpt_command_state new_state;
2206 	int ret, i;
2207 
2208 	new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2209 	WARN_ON(new_state == SRPT_STATE_DONE);
2210 
2211 	if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) {
2212 		pr_warn("%s: IB send queue full (needed %d)\n",
2213 				__func__, ioctx->n_rdma);
2214 		ret = -ENOMEM;
2215 		goto out_undo;
2216 	}
2217 
2218 	cqe->done = srpt_rdma_read_done;
2219 	for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2220 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2221 
2222 		first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port,
2223 				cqe, first_wr);
2224 		cqe = NULL;
2225 	}
2226 
2227 	ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2228 	if (ret) {
2229 		pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
2230 			 __func__, ret, ioctx->n_rdma,
2231 			 atomic_read(&ch->sq_wr_avail));
2232 		goto out_undo;
2233 	}
2234 
2235 	return 0;
2236 out_undo:
2237 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
2238 	return ret;
2239 }
2240 
2241 static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2242 {
2243 	switch (tcm_mgmt_status) {
2244 	case TMR_FUNCTION_COMPLETE:
2245 		return SRP_TSK_MGMT_SUCCESS;
2246 	case TMR_FUNCTION_REJECTED:
2247 		return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2248 	}
2249 	return SRP_TSK_MGMT_FAILED;
2250 }
2251 
2252 /**
2253  * srpt_queue_response() - Transmits the response to a SCSI command.
2254  *
2255  * Callback function called by the TCM core. Must not block since it can be
2256  * invoked on the context of the IB completion handler.
2257  */
2258 static void srpt_queue_response(struct se_cmd *cmd)
2259 {
2260 	struct srpt_send_ioctx *ioctx =
2261 		container_of(cmd, struct srpt_send_ioctx, cmd);
2262 	struct srpt_rdma_ch *ch = ioctx->ch;
2263 	struct srpt_device *sdev = ch->sport->sdev;
2264 	struct ib_send_wr send_wr, *first_wr = NULL, *bad_wr;
2265 	struct ib_sge sge;
2266 	enum srpt_command_state state;
2267 	unsigned long flags;
2268 	int resp_len, ret, i;
2269 	u8 srp_tm_status;
2270 
2271 	BUG_ON(!ch);
2272 
2273 	spin_lock_irqsave(&ioctx->spinlock, flags);
2274 	state = ioctx->state;
2275 	switch (state) {
2276 	case SRPT_STATE_NEW:
2277 	case SRPT_STATE_DATA_IN:
2278 		ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2279 		break;
2280 	case SRPT_STATE_MGMT:
2281 		ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2282 		break;
2283 	default:
2284 		WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2285 			ch, ioctx->ioctx.index, ioctx->state);
2286 		break;
2287 	}
2288 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
2289 
2290 	if (unlikely(transport_check_aborted_status(&ioctx->cmd, false)
2291 		     || WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))) {
2292 		atomic_inc(&ch->req_lim_delta);
2293 		srpt_abort_cmd(ioctx);
2294 		return;
2295 	}
2296 
2297 	/* For read commands, transfer the data to the initiator. */
2298 	if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
2299 	    ioctx->cmd.data_length &&
2300 	    !ioctx->queue_status_only) {
2301 		for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2302 			struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2303 
2304 			first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
2305 					ch->sport->port, NULL,
2306 					first_wr ? first_wr : &send_wr);
2307 		}
2308 	} else {
2309 		first_wr = &send_wr;
2310 	}
2311 
2312 	if (state != SRPT_STATE_MGMT)
2313 		resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,
2314 					      cmd->scsi_status);
2315 	else {
2316 		srp_tm_status
2317 			= tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
2318 		resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
2319 						 ioctx->cmd.tag);
2320 	}
2321 
2322 	atomic_inc(&ch->req_lim);
2323 
2324 	if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,
2325 			&ch->sq_wr_avail) < 0)) {
2326 		pr_warn("%s: IB send queue full (needed %d)\n",
2327 				__func__, ioctx->n_rdma);
2328 		ret = -ENOMEM;
2329 		goto out;
2330 	}
2331 
2332 	ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, resp_len,
2333 				      DMA_TO_DEVICE);
2334 
2335 	sge.addr = ioctx->ioctx.dma;
2336 	sge.length = resp_len;
2337 	sge.lkey = sdev->pd->local_dma_lkey;
2338 
2339 	ioctx->ioctx.cqe.done = srpt_send_done;
2340 	send_wr.next = NULL;
2341 	send_wr.wr_cqe = &ioctx->ioctx.cqe;
2342 	send_wr.sg_list = &sge;
2343 	send_wr.num_sge = 1;
2344 	send_wr.opcode = IB_WR_SEND;
2345 	send_wr.send_flags = IB_SEND_SIGNALED;
2346 
2347 	ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2348 	if (ret < 0) {
2349 		pr_err("%s: sending cmd response failed for tag %llu (%d)\n",
2350 			__func__, ioctx->cmd.tag, ret);
2351 		goto out;
2352 	}
2353 
2354 	return;
2355 
2356 out:
2357 	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
2358 	atomic_dec(&ch->req_lim);
2359 	srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
2360 	target_put_sess_cmd(&ioctx->cmd);
2361 }
2362 
2363 static int srpt_queue_data_in(struct se_cmd *cmd)
2364 {
2365 	srpt_queue_response(cmd);
2366 	return 0;
2367 }
2368 
2369 static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2370 {
2371 	srpt_queue_response(cmd);
2372 }
2373 
2374 static void srpt_aborted_task(struct se_cmd *cmd)
2375 {
2376 }
2377 
2378 static int srpt_queue_status(struct se_cmd *cmd)
2379 {
2380 	struct srpt_send_ioctx *ioctx;
2381 
2382 	ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2383 	BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2384 	if (cmd->se_cmd_flags &
2385 	    (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2386 		WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2387 	ioctx->queue_status_only = true;
2388 	srpt_queue_response(cmd);
2389 	return 0;
2390 }
2391 
2392 static void srpt_refresh_port_work(struct work_struct *work)
2393 {
2394 	struct srpt_port *sport = container_of(work, struct srpt_port, work);
2395 
2396 	srpt_refresh_port(sport);
2397 }
2398 
2399 /**
2400  * srpt_release_sdev() - Free the channel resources associated with a target.
2401  */
2402 static int srpt_release_sdev(struct srpt_device *sdev)
2403 {
2404 	int i, res;
2405 
2406 	WARN_ON_ONCE(irqs_disabled());
2407 
2408 	BUG_ON(!sdev);
2409 
2410 	mutex_lock(&sdev->mutex);
2411 	for (i = 0; i < ARRAY_SIZE(sdev->port); i++)
2412 		sdev->port[i].enabled = false;
2413 	__srpt_close_all_ch(sdev);
2414 	mutex_unlock(&sdev->mutex);
2415 
2416 	res = wait_event_interruptible(sdev->ch_releaseQ,
2417 				       list_empty_careful(&sdev->rch_list));
2418 	if (res)
2419 		pr_err("%s: interrupted.\n", __func__);
2420 
2421 	return 0;
2422 }
2423 
2424 static struct srpt_port *__srpt_lookup_port(const char *name)
2425 {
2426 	struct ib_device *dev;
2427 	struct srpt_device *sdev;
2428 	struct srpt_port *sport;
2429 	int i;
2430 
2431 	list_for_each_entry(sdev, &srpt_dev_list, list) {
2432 		dev = sdev->device;
2433 		if (!dev)
2434 			continue;
2435 
2436 		for (i = 0; i < dev->phys_port_cnt; i++) {
2437 			sport = &sdev->port[i];
2438 
2439 			if (!strcmp(sport->port_guid, name))
2440 				return sport;
2441 		}
2442 	}
2443 
2444 	return NULL;
2445 }
2446 
2447 static struct srpt_port *srpt_lookup_port(const char *name)
2448 {
2449 	struct srpt_port *sport;
2450 
2451 	spin_lock(&srpt_dev_lock);
2452 	sport = __srpt_lookup_port(name);
2453 	spin_unlock(&srpt_dev_lock);
2454 
2455 	return sport;
2456 }
2457 
2458 /**
2459  * srpt_add_one() - Infiniband device addition callback function.
2460  */
2461 static void srpt_add_one(struct ib_device *device)
2462 {
2463 	struct srpt_device *sdev;
2464 	struct srpt_port *sport;
2465 	struct ib_srq_init_attr srq_attr;
2466 	int i;
2467 
2468 	pr_debug("device = %p, device->dma_ops = %p\n", device,
2469 		 device->dma_ops);
2470 
2471 	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
2472 	if (!sdev)
2473 		goto err;
2474 
2475 	sdev->device = device;
2476 	INIT_LIST_HEAD(&sdev->rch_list);
2477 	init_waitqueue_head(&sdev->ch_releaseQ);
2478 	mutex_init(&sdev->mutex);
2479 
2480 	sdev->pd = ib_alloc_pd(device);
2481 	if (IS_ERR(sdev->pd))
2482 		goto free_dev;
2483 
2484 	sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
2485 
2486 	srq_attr.event_handler = srpt_srq_event;
2487 	srq_attr.srq_context = (void *)sdev;
2488 	srq_attr.attr.max_wr = sdev->srq_size;
2489 	srq_attr.attr.max_sge = 1;
2490 	srq_attr.attr.srq_limit = 0;
2491 	srq_attr.srq_type = IB_SRQT_BASIC;
2492 
2493 	sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2494 	if (IS_ERR(sdev->srq))
2495 		goto err_pd;
2496 
2497 	pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
2498 		 __func__, sdev->srq_size, sdev->device->attrs.max_srq_wr,
2499 		 device->name);
2500 
2501 	if (!srpt_service_guid)
2502 		srpt_service_guid = be64_to_cpu(device->node_guid);
2503 
2504 	sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2505 	if (IS_ERR(sdev->cm_id))
2506 		goto err_srq;
2507 
2508 	/* print out target login information */
2509 	pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
2510 		 "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
2511 		 srpt_service_guid, srpt_service_guid);
2512 
2513 	/*
2514 	 * We do not have a consistent service_id (ie. also id_ext of target_id)
2515 	 * to identify this target. We currently use the guid of the first HCA
2516 	 * in the system as service_id; therefore, the target_id will change
2517 	 * if this HCA is gone bad and replaced by different HCA
2518 	 */
2519 	if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0))
2520 		goto err_cm;
2521 
2522 	INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2523 			      srpt_event_handler);
2524 	if (ib_register_event_handler(&sdev->event_handler))
2525 		goto err_cm;
2526 
2527 	sdev->ioctx_ring = (struct srpt_recv_ioctx **)
2528 		srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
2529 				      sizeof(*sdev->ioctx_ring[0]),
2530 				      srp_max_req_size, DMA_FROM_DEVICE);
2531 	if (!sdev->ioctx_ring)
2532 		goto err_event;
2533 
2534 	for (i = 0; i < sdev->srq_size; ++i)
2535 		srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2536 
2537 	WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));
2538 
2539 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2540 		sport = &sdev->port[i - 1];
2541 		sport->sdev = sdev;
2542 		sport->port = i;
2543 		sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
2544 		sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
2545 		sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
2546 		INIT_WORK(&sport->work, srpt_refresh_port_work);
2547 
2548 		if (srpt_refresh_port(sport)) {
2549 			pr_err("MAD registration failed for %s-%d.\n",
2550 			       sdev->device->name, i);
2551 			goto err_ring;
2552 		}
2553 		snprintf(sport->port_guid, sizeof(sport->port_guid),
2554 			"0x%016llx%016llx",
2555 			be64_to_cpu(sport->gid.global.subnet_prefix),
2556 			be64_to_cpu(sport->gid.global.interface_id));
2557 	}
2558 
2559 	spin_lock(&srpt_dev_lock);
2560 	list_add_tail(&sdev->list, &srpt_dev_list);
2561 	spin_unlock(&srpt_dev_lock);
2562 
2563 out:
2564 	ib_set_client_data(device, &srpt_client, sdev);
2565 	pr_debug("added %s.\n", device->name);
2566 	return;
2567 
2568 err_ring:
2569 	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2570 			     sdev->srq_size, srp_max_req_size,
2571 			     DMA_FROM_DEVICE);
2572 err_event:
2573 	ib_unregister_event_handler(&sdev->event_handler);
2574 err_cm:
2575 	ib_destroy_cm_id(sdev->cm_id);
2576 err_srq:
2577 	ib_destroy_srq(sdev->srq);
2578 err_pd:
2579 	ib_dealloc_pd(sdev->pd);
2580 free_dev:
2581 	kfree(sdev);
2582 err:
2583 	sdev = NULL;
2584 	pr_info("%s(%s) failed.\n", __func__, device->name);
2585 	goto out;
2586 }
2587 
2588 /**
2589  * srpt_remove_one() - InfiniBand device removal callback function.
2590  */
2591 static void srpt_remove_one(struct ib_device *device, void *client_data)
2592 {
2593 	struct srpt_device *sdev = client_data;
2594 	int i;
2595 
2596 	if (!sdev) {
2597 		pr_info("%s(%s): nothing to do.\n", __func__, device->name);
2598 		return;
2599 	}
2600 
2601 	srpt_unregister_mad_agent(sdev);
2602 
2603 	ib_unregister_event_handler(&sdev->event_handler);
2604 
2605 	/* Cancel any work queued by the just unregistered IB event handler. */
2606 	for (i = 0; i < sdev->device->phys_port_cnt; i++)
2607 		cancel_work_sync(&sdev->port[i].work);
2608 
2609 	ib_destroy_cm_id(sdev->cm_id);
2610 
2611 	/*
2612 	 * Unregistering a target must happen after destroying sdev->cm_id
2613 	 * such that no new SRP_LOGIN_REQ information units can arrive while
2614 	 * destroying the target.
2615 	 */
2616 	spin_lock(&srpt_dev_lock);
2617 	list_del(&sdev->list);
2618 	spin_unlock(&srpt_dev_lock);
2619 	srpt_release_sdev(sdev);
2620 
2621 	ib_destroy_srq(sdev->srq);
2622 	ib_dealloc_pd(sdev->pd);
2623 
2624 	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2625 			     sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
2626 	sdev->ioctx_ring = NULL;
2627 	kfree(sdev);
2628 }
2629 
2630 static struct ib_client srpt_client = {
2631 	.name = DRV_NAME,
2632 	.add = srpt_add_one,
2633 	.remove = srpt_remove_one
2634 };
2635 
2636 static int srpt_check_true(struct se_portal_group *se_tpg)
2637 {
2638 	return 1;
2639 }
2640 
2641 static int srpt_check_false(struct se_portal_group *se_tpg)
2642 {
2643 	return 0;
2644 }
2645 
2646 static char *srpt_get_fabric_name(void)
2647 {
2648 	return "srpt";
2649 }
2650 
2651 static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
2652 {
2653 	struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
2654 
2655 	return sport->port_guid;
2656 }
2657 
2658 static u16 srpt_get_tag(struct se_portal_group *tpg)
2659 {
2660 	return 1;
2661 }
2662 
2663 static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
2664 {
2665 	return 1;
2666 }
2667 
2668 static void srpt_release_cmd(struct se_cmd *se_cmd)
2669 {
2670 	struct srpt_send_ioctx *ioctx = container_of(se_cmd,
2671 				struct srpt_send_ioctx, cmd);
2672 	struct srpt_rdma_ch *ch = ioctx->ch;
2673 	unsigned long flags;
2674 
2675 	WARN_ON(ioctx->state != SRPT_STATE_DONE);
2676 
2677 	if (ioctx->n_rw_ctx) {
2678 		srpt_free_rw_ctxs(ch, ioctx);
2679 		ioctx->n_rw_ctx = 0;
2680 	}
2681 
2682 	spin_lock_irqsave(&ch->spinlock, flags);
2683 	list_add(&ioctx->free_list, &ch->free_list);
2684 	spin_unlock_irqrestore(&ch->spinlock, flags);
2685 }
2686 
2687 /**
2688  * srpt_close_session() - Forcibly close a session.
2689  *
2690  * Callback function invoked by the TCM core to clean up sessions associated
2691  * with a node ACL when the user invokes
2692  * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2693  */
2694 static void srpt_close_session(struct se_session *se_sess)
2695 {
2696 	DECLARE_COMPLETION_ONSTACK(release_done);
2697 	struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
2698 	struct srpt_device *sdev = ch->sport->sdev;
2699 	bool wait;
2700 
2701 	pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2702 		 ch->state);
2703 
2704 	mutex_lock(&sdev->mutex);
2705 	BUG_ON(ch->release_done);
2706 	ch->release_done = &release_done;
2707 	wait = !list_empty(&ch->list);
2708 	srpt_disconnect_ch(ch);
2709 	mutex_unlock(&sdev->mutex);
2710 
2711 	if (!wait)
2712 		return;
2713 
2714 	while (wait_for_completion_timeout(&release_done, 180 * HZ) == 0)
2715 		pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
2716 			ch->sess_name, ch->qp->qp_num, ch->state);
2717 }
2718 
2719 /**
2720  * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB).
2721  *
2722  * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
2723  * This object represents an arbitrary integer used to uniquely identify a
2724  * particular attached remote initiator port to a particular SCSI target port
2725  * within a particular SCSI target device within a particular SCSI instance.
2726  */
2727 static u32 srpt_sess_get_index(struct se_session *se_sess)
2728 {
2729 	return 0;
2730 }
2731 
2732 static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
2733 {
2734 }
2735 
2736 /* Note: only used from inside debug printk's by the TCM core. */
2737 static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
2738 {
2739 	struct srpt_send_ioctx *ioctx;
2740 
2741 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2742 	return srpt_get_cmd_state(ioctx);
2743 }
2744 
2745 /**
2746  * srpt_parse_i_port_id() - Parse an initiator port ID.
2747  * @name: ASCII representation of a 128-bit initiator port ID.
2748  * @i_port_id: Binary 128-bit port ID.
2749  */
2750 static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
2751 {
2752 	const char *p;
2753 	unsigned len, count, leading_zero_bytes;
2754 	int ret, rc;
2755 
2756 	p = name;
2757 	if (strncasecmp(p, "0x", 2) == 0)
2758 		p += 2;
2759 	ret = -EINVAL;
2760 	len = strlen(p);
2761 	if (len % 2)
2762 		goto out;
2763 	count = min(len / 2, 16U);
2764 	leading_zero_bytes = 16 - count;
2765 	memset(i_port_id, 0, leading_zero_bytes);
2766 	rc = hex2bin(i_port_id + leading_zero_bytes, p, count);
2767 	if (rc < 0)
2768 		pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", rc);
2769 	ret = 0;
2770 out:
2771 	return ret;
2772 }
2773 
2774 /*
2775  * configfs callback function invoked for
2776  * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2777  */
2778 static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
2779 {
2780 	u8 i_port_id[16];
2781 
2782 	if (srpt_parse_i_port_id(i_port_id, name) < 0) {
2783 		pr_err("invalid initiator port ID %s\n", name);
2784 		return -EINVAL;
2785 	}
2786 	return 0;
2787 }
2788 
2789 static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
2790 		char *page)
2791 {
2792 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2793 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2794 
2795 	return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
2796 }
2797 
2798 static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
2799 		const char *page, size_t count)
2800 {
2801 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2802 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2803 	unsigned long val;
2804 	int ret;
2805 
2806 	ret = kstrtoul(page, 0, &val);
2807 	if (ret < 0) {
2808 		pr_err("kstrtoul() failed with ret: %d\n", ret);
2809 		return -EINVAL;
2810 	}
2811 	if (val > MAX_SRPT_RDMA_SIZE) {
2812 		pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
2813 			MAX_SRPT_RDMA_SIZE);
2814 		return -EINVAL;
2815 	}
2816 	if (val < DEFAULT_MAX_RDMA_SIZE) {
2817 		pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
2818 			val, DEFAULT_MAX_RDMA_SIZE);
2819 		return -EINVAL;
2820 	}
2821 	sport->port_attrib.srp_max_rdma_size = val;
2822 
2823 	return count;
2824 }
2825 
2826 static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
2827 		char *page)
2828 {
2829 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2830 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2831 
2832 	return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
2833 }
2834 
2835 static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
2836 		const char *page, size_t count)
2837 {
2838 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2839 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2840 	unsigned long val;
2841 	int ret;
2842 
2843 	ret = kstrtoul(page, 0, &val);
2844 	if (ret < 0) {
2845 		pr_err("kstrtoul() failed with ret: %d\n", ret);
2846 		return -EINVAL;
2847 	}
2848 	if (val > MAX_SRPT_RSP_SIZE) {
2849 		pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
2850 			MAX_SRPT_RSP_SIZE);
2851 		return -EINVAL;
2852 	}
2853 	if (val < MIN_MAX_RSP_SIZE) {
2854 		pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
2855 			MIN_MAX_RSP_SIZE);
2856 		return -EINVAL;
2857 	}
2858 	sport->port_attrib.srp_max_rsp_size = val;
2859 
2860 	return count;
2861 }
2862 
2863 static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
2864 		char *page)
2865 {
2866 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2867 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2868 
2869 	return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
2870 }
2871 
2872 static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
2873 		const char *page, size_t count)
2874 {
2875 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2876 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2877 	unsigned long val;
2878 	int ret;
2879 
2880 	ret = kstrtoul(page, 0, &val);
2881 	if (ret < 0) {
2882 		pr_err("kstrtoul() failed with ret: %d\n", ret);
2883 		return -EINVAL;
2884 	}
2885 	if (val > MAX_SRPT_SRQ_SIZE) {
2886 		pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
2887 			MAX_SRPT_SRQ_SIZE);
2888 		return -EINVAL;
2889 	}
2890 	if (val < MIN_SRPT_SRQ_SIZE) {
2891 		pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
2892 			MIN_SRPT_SRQ_SIZE);
2893 		return -EINVAL;
2894 	}
2895 	sport->port_attrib.srp_sq_size = val;
2896 
2897 	return count;
2898 }
2899 
2900 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_max_rdma_size);
2901 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_max_rsp_size);
2902 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_sq_size);
2903 
2904 static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
2905 	&srpt_tpg_attrib_attr_srp_max_rdma_size,
2906 	&srpt_tpg_attrib_attr_srp_max_rsp_size,
2907 	&srpt_tpg_attrib_attr_srp_sq_size,
2908 	NULL,
2909 };
2910 
2911 static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page)
2912 {
2913 	struct se_portal_group *se_tpg = to_tpg(item);
2914 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2915 
2916 	return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
2917 }
2918 
2919 static ssize_t srpt_tpg_enable_store(struct config_item *item,
2920 		const char *page, size_t count)
2921 {
2922 	struct se_portal_group *se_tpg = to_tpg(item);
2923 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2924 	struct srpt_device *sdev = sport->sdev;
2925 	struct srpt_rdma_ch *ch;
2926 	unsigned long tmp;
2927         int ret;
2928 
2929 	ret = kstrtoul(page, 0, &tmp);
2930 	if (ret < 0) {
2931 		pr_err("Unable to extract srpt_tpg_store_enable\n");
2932 		return -EINVAL;
2933 	}
2934 
2935 	if ((tmp != 0) && (tmp != 1)) {
2936 		pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
2937 		return -EINVAL;
2938 	}
2939 	if (sport->enabled == tmp)
2940 		goto out;
2941 	sport->enabled = tmp;
2942 	if (sport->enabled)
2943 		goto out;
2944 
2945 	mutex_lock(&sdev->mutex);
2946 	list_for_each_entry(ch, &sdev->rch_list, list) {
2947 		if (ch->sport == sport) {
2948 			pr_debug("%s: ch %p %s-%d\n", __func__, ch,
2949 				 ch->sess_name, ch->qp->qp_num);
2950 			srpt_disconnect_ch(ch);
2951 			srpt_close_ch(ch);
2952 		}
2953 	}
2954 	mutex_unlock(&sdev->mutex);
2955 
2956 out:
2957 	return count;
2958 }
2959 
2960 CONFIGFS_ATTR(srpt_tpg_, enable);
2961 
2962 static struct configfs_attribute *srpt_tpg_attrs[] = {
2963 	&srpt_tpg_attr_enable,
2964 	NULL,
2965 };
2966 
2967 /**
2968  * configfs callback invoked for
2969  * mkdir /sys/kernel/config/target/$driver/$port/$tpg
2970  */
2971 static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
2972 					     struct config_group *group,
2973 					     const char *name)
2974 {
2975 	struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
2976 	int res;
2977 
2978 	/* Initialize sport->port_wwn and sport->port_tpg_1 */
2979 	res = core_tpg_register(&sport->port_wwn, &sport->port_tpg_1, SCSI_PROTOCOL_SRP);
2980 	if (res)
2981 		return ERR_PTR(res);
2982 
2983 	return &sport->port_tpg_1;
2984 }
2985 
2986 /**
2987  * configfs callback invoked for
2988  * rmdir /sys/kernel/config/target/$driver/$port/$tpg
2989  */
2990 static void srpt_drop_tpg(struct se_portal_group *tpg)
2991 {
2992 	struct srpt_port *sport = container_of(tpg,
2993 				struct srpt_port, port_tpg_1);
2994 
2995 	sport->enabled = false;
2996 	core_tpg_deregister(&sport->port_tpg_1);
2997 }
2998 
2999 /**
3000  * configfs callback invoked for
3001  * mkdir /sys/kernel/config/target/$driver/$port
3002  */
3003 static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3004 				      struct config_group *group,
3005 				      const char *name)
3006 {
3007 	struct srpt_port *sport;
3008 	int ret;
3009 
3010 	sport = srpt_lookup_port(name);
3011 	pr_debug("make_tport(%s)\n", name);
3012 	ret = -EINVAL;
3013 	if (!sport)
3014 		goto err;
3015 
3016 	return &sport->port_wwn;
3017 
3018 err:
3019 	return ERR_PTR(ret);
3020 }
3021 
3022 /**
3023  * configfs callback invoked for
3024  * rmdir /sys/kernel/config/target/$driver/$port
3025  */
3026 static void srpt_drop_tport(struct se_wwn *wwn)
3027 {
3028 	struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3029 
3030 	pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item));
3031 }
3032 
3033 static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
3034 {
3035 	return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
3036 }
3037 
3038 CONFIGFS_ATTR_RO(srpt_wwn_, version);
3039 
3040 static struct configfs_attribute *srpt_wwn_attrs[] = {
3041 	&srpt_wwn_attr_version,
3042 	NULL,
3043 };
3044 
3045 static const struct target_core_fabric_ops srpt_template = {
3046 	.module				= THIS_MODULE,
3047 	.name				= "srpt",
3048 	.get_fabric_name		= srpt_get_fabric_name,
3049 	.tpg_get_wwn			= srpt_get_fabric_wwn,
3050 	.tpg_get_tag			= srpt_get_tag,
3051 	.tpg_check_demo_mode		= srpt_check_false,
3052 	.tpg_check_demo_mode_cache	= srpt_check_true,
3053 	.tpg_check_demo_mode_write_protect = srpt_check_true,
3054 	.tpg_check_prod_mode_write_protect = srpt_check_false,
3055 	.tpg_get_inst_index		= srpt_tpg_get_inst_index,
3056 	.release_cmd			= srpt_release_cmd,
3057 	.check_stop_free		= srpt_check_stop_free,
3058 	.close_session			= srpt_close_session,
3059 	.sess_get_index			= srpt_sess_get_index,
3060 	.sess_get_initiator_sid		= NULL,
3061 	.write_pending			= srpt_write_pending,
3062 	.write_pending_status		= srpt_write_pending_status,
3063 	.set_default_node_attributes	= srpt_set_default_node_attrs,
3064 	.get_cmd_state			= srpt_get_tcm_cmd_state,
3065 	.queue_data_in			= srpt_queue_data_in,
3066 	.queue_status			= srpt_queue_status,
3067 	.queue_tm_rsp			= srpt_queue_tm_rsp,
3068 	.aborted_task			= srpt_aborted_task,
3069 	/*
3070 	 * Setup function pointers for generic logic in
3071 	 * target_core_fabric_configfs.c
3072 	 */
3073 	.fabric_make_wwn		= srpt_make_tport,
3074 	.fabric_drop_wwn		= srpt_drop_tport,
3075 	.fabric_make_tpg		= srpt_make_tpg,
3076 	.fabric_drop_tpg		= srpt_drop_tpg,
3077 	.fabric_init_nodeacl		= srpt_init_nodeacl,
3078 
3079 	.tfc_wwn_attrs			= srpt_wwn_attrs,
3080 	.tfc_tpg_base_attrs		= srpt_tpg_attrs,
3081 	.tfc_tpg_attrib_attrs		= srpt_tpg_attrib_attrs,
3082 };
3083 
3084 /**
3085  * srpt_init_module() - Kernel module initialization.
3086  *
3087  * Note: Since ib_register_client() registers callback functions, and since at
3088  * least one of these callback functions (srpt_add_one()) calls target core
3089  * functions, this driver must be registered with the target core before
3090  * ib_register_client() is called.
3091  */
3092 static int __init srpt_init_module(void)
3093 {
3094 	int ret;
3095 
3096 	ret = -EINVAL;
3097 	if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
3098 		pr_err("invalid value %d for kernel module parameter"
3099 		       " srp_max_req_size -- must be at least %d.\n",
3100 		       srp_max_req_size, MIN_MAX_REQ_SIZE);
3101 		goto out;
3102 	}
3103 
3104 	if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3105 	    || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
3106 		pr_err("invalid value %d for kernel module parameter"
3107 		       " srpt_srq_size -- must be in the range [%d..%d].\n",
3108 		       srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3109 		goto out;
3110 	}
3111 
3112 	ret = target_register_template(&srpt_template);
3113 	if (ret)
3114 		goto out;
3115 
3116 	ret = ib_register_client(&srpt_client);
3117 	if (ret) {
3118 		pr_err("couldn't register IB client\n");
3119 		goto out_unregister_target;
3120 	}
3121 
3122 	return 0;
3123 
3124 out_unregister_target:
3125 	target_unregister_template(&srpt_template);
3126 out:
3127 	return ret;
3128 }
3129 
3130 static void __exit srpt_cleanup_module(void)
3131 {
3132 	ib_unregister_client(&srpt_client);
3133 	target_unregister_template(&srpt_template);
3134 }
3135 
3136 module_init(srpt_init_module);
3137 module_exit(srpt_cleanup_module);
3138