1 /*
2  * Copyright (c) 2006 - 2009 Mellanox Technology Inc.  All rights reserved.
3  * Copyright (C) 2008 - 2011 Bart Van Assche <bvanassche@acm.org>.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  *
33  */
34 
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/slab.h>
38 #include <linux/err.h>
39 #include <linux/ctype.h>
40 #include <linux/kthread.h>
41 #include <linux/string.h>
42 #include <linux/delay.h>
43 #include <linux/atomic.h>
44 #include <scsi/scsi_proto.h>
45 #include <scsi/scsi_tcq.h>
46 #include <target/target_core_base.h>
47 #include <target/target_core_fabric.h>
48 #include "ib_srpt.h"
49 
50 /* Name of this kernel module. */
51 #define DRV_NAME		"ib_srpt"
52 #define DRV_VERSION		"2.0.0"
53 #define DRV_RELDATE		"2011-02-14"
54 
55 #define SRPT_ID_STRING	"Linux SRP target"
56 
57 #undef pr_fmt
58 #define pr_fmt(fmt) DRV_NAME " " fmt
59 
60 MODULE_AUTHOR("Vu Pham and Bart Van Assche");
61 MODULE_DESCRIPTION("InfiniBand SCSI RDMA Protocol target "
62 		   "v" DRV_VERSION " (" DRV_RELDATE ")");
63 MODULE_LICENSE("Dual BSD/GPL");
64 
65 /*
66  * Global Variables
67  */
68 
69 static u64 srpt_service_guid;
70 static DEFINE_SPINLOCK(srpt_dev_lock);	/* Protects srpt_dev_list. */
71 static LIST_HEAD(srpt_dev_list);	/* List of srpt_device structures. */
72 
73 static unsigned srp_max_req_size = DEFAULT_MAX_REQ_SIZE;
74 module_param(srp_max_req_size, int, 0444);
75 MODULE_PARM_DESC(srp_max_req_size,
76 		 "Maximum size of SRP request messages in bytes.");
77 
78 static int srpt_srq_size = DEFAULT_SRPT_SRQ_SIZE;
79 module_param(srpt_srq_size, int, 0444);
80 MODULE_PARM_DESC(srpt_srq_size,
81 		 "Shared receive queue (SRQ) size.");
82 
83 static int srpt_get_u64_x(char *buffer, struct kernel_param *kp)
84 {
85 	return sprintf(buffer, "0x%016llx", *(u64 *)kp->arg);
86 }
87 module_param_call(srpt_service_guid, NULL, srpt_get_u64_x, &srpt_service_guid,
88 		  0444);
89 MODULE_PARM_DESC(srpt_service_guid,
90 		 "Using this value for ioc_guid, id_ext, and cm_listen_id"
91 		 " instead of using the node_guid of the first HCA.");
92 
93 static struct ib_client srpt_client;
94 static void srpt_release_cmd(struct se_cmd *se_cmd);
95 static void srpt_free_ch(struct kref *kref);
96 static int srpt_queue_status(struct se_cmd *cmd);
97 static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc);
98 static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc);
99 static void srpt_process_wait_list(struct srpt_rdma_ch *ch);
100 
101 /*
102  * The only allowed channel state changes are those that change the channel
103  * state into a state with a higher numerical value. Hence the new > prev test.
104  */
105 static bool srpt_set_ch_state(struct srpt_rdma_ch *ch, enum rdma_ch_state new)
106 {
107 	unsigned long flags;
108 	enum rdma_ch_state prev;
109 	bool changed = false;
110 
111 	spin_lock_irqsave(&ch->spinlock, flags);
112 	prev = ch->state;
113 	if (new > prev) {
114 		ch->state = new;
115 		changed = true;
116 	}
117 	spin_unlock_irqrestore(&ch->spinlock, flags);
118 
119 	return changed;
120 }
121 
122 /**
123  * srpt_event_handler() - Asynchronous IB event callback function.
124  *
125  * Callback function called by the InfiniBand core when an asynchronous IB
126  * event occurs. This callback may occur in interrupt context. See also
127  * section 11.5.2, Set Asynchronous Event Handler in the InfiniBand
128  * Architecture Specification.
129  */
130 static void srpt_event_handler(struct ib_event_handler *handler,
131 			       struct ib_event *event)
132 {
133 	struct srpt_device *sdev;
134 	struct srpt_port *sport;
135 
136 	sdev = ib_get_client_data(event->device, &srpt_client);
137 	if (!sdev || sdev->device != event->device)
138 		return;
139 
140 	pr_debug("ASYNC event= %d on device= %s\n", event->event,
141 		 sdev->device->name);
142 
143 	switch (event->event) {
144 	case IB_EVENT_PORT_ERR:
145 		if (event->element.port_num <= sdev->device->phys_port_cnt) {
146 			sport = &sdev->port[event->element.port_num - 1];
147 			sport->lid = 0;
148 			sport->sm_lid = 0;
149 		}
150 		break;
151 	case IB_EVENT_PORT_ACTIVE:
152 	case IB_EVENT_LID_CHANGE:
153 	case IB_EVENT_PKEY_CHANGE:
154 	case IB_EVENT_SM_CHANGE:
155 	case IB_EVENT_CLIENT_REREGISTER:
156 	case IB_EVENT_GID_CHANGE:
157 		/* Refresh port data asynchronously. */
158 		if (event->element.port_num <= sdev->device->phys_port_cnt) {
159 			sport = &sdev->port[event->element.port_num - 1];
160 			if (!sport->lid && !sport->sm_lid)
161 				schedule_work(&sport->work);
162 		}
163 		break;
164 	default:
165 		pr_err("received unrecognized IB event %d\n",
166 		       event->event);
167 		break;
168 	}
169 }
170 
171 /**
172  * srpt_srq_event() - SRQ event callback function.
173  */
174 static void srpt_srq_event(struct ib_event *event, void *ctx)
175 {
176 	pr_info("SRQ event %d\n", event->event);
177 }
178 
179 static const char *get_ch_state_name(enum rdma_ch_state s)
180 {
181 	switch (s) {
182 	case CH_CONNECTING:
183 		return "connecting";
184 	case CH_LIVE:
185 		return "live";
186 	case CH_DISCONNECTING:
187 		return "disconnecting";
188 	case CH_DRAINING:
189 		return "draining";
190 	case CH_DISCONNECTED:
191 		return "disconnected";
192 	}
193 	return "???";
194 }
195 
196 /**
197  * srpt_qp_event() - QP event callback function.
198  */
199 static void srpt_qp_event(struct ib_event *event, struct srpt_rdma_ch *ch)
200 {
201 	pr_debug("QP event %d on cm_id=%p sess_name=%s state=%d\n",
202 		 event->event, ch->cm_id, ch->sess_name, ch->state);
203 
204 	switch (event->event) {
205 	case IB_EVENT_COMM_EST:
206 		ib_cm_notify(ch->cm_id, event->event);
207 		break;
208 	case IB_EVENT_QP_LAST_WQE_REACHED:
209 		pr_debug("%s-%d, state %s: received Last WQE event.\n",
210 			 ch->sess_name, ch->qp->qp_num,
211 			 get_ch_state_name(ch->state));
212 		break;
213 	default:
214 		pr_err("received unrecognized IB QP event %d\n", event->event);
215 		break;
216 	}
217 }
218 
219 /**
220  * srpt_set_ioc() - Helper function for initializing an IOUnitInfo structure.
221  *
222  * @slot: one-based slot number.
223  * @value: four-bit value.
224  *
225  * Copies the lowest four bits of value in element slot of the array of four
226  * bit elements called c_list (controller list). The index slot is one-based.
227  */
228 static void srpt_set_ioc(u8 *c_list, u32 slot, u8 value)
229 {
230 	u16 id;
231 	u8 tmp;
232 
233 	id = (slot - 1) / 2;
234 	if (slot & 0x1) {
235 		tmp = c_list[id] & 0xf;
236 		c_list[id] = (value << 4) | tmp;
237 	} else {
238 		tmp = c_list[id] & 0xf0;
239 		c_list[id] = (value & 0xf) | tmp;
240 	}
241 }
242 
243 /**
244  * srpt_get_class_port_info() - Copy ClassPortInfo to a management datagram.
245  *
246  * See also section 16.3.3.1 ClassPortInfo in the InfiniBand Architecture
247  * Specification.
248  */
249 static void srpt_get_class_port_info(struct ib_dm_mad *mad)
250 {
251 	struct ib_class_port_info *cif;
252 
253 	cif = (struct ib_class_port_info *)mad->data;
254 	memset(cif, 0, sizeof(*cif));
255 	cif->base_version = 1;
256 	cif->class_version = 1;
257 
258 	ib_set_cpi_resp_time(cif, 20);
259 	mad->mad_hdr.status = 0;
260 }
261 
262 /**
263  * srpt_get_iou() - Write IOUnitInfo to a management datagram.
264  *
265  * See also section 16.3.3.3 IOUnitInfo in the InfiniBand Architecture
266  * Specification. See also section B.7, table B.6 in the SRP r16a document.
267  */
268 static void srpt_get_iou(struct ib_dm_mad *mad)
269 {
270 	struct ib_dm_iou_info *ioui;
271 	u8 slot;
272 	int i;
273 
274 	ioui = (struct ib_dm_iou_info *)mad->data;
275 	ioui->change_id = cpu_to_be16(1);
276 	ioui->max_controllers = 16;
277 
278 	/* set present for slot 1 and empty for the rest */
279 	srpt_set_ioc(ioui->controller_list, 1, 1);
280 	for (i = 1, slot = 2; i < 16; i++, slot++)
281 		srpt_set_ioc(ioui->controller_list, slot, 0);
282 
283 	mad->mad_hdr.status = 0;
284 }
285 
286 /**
287  * srpt_get_ioc() - Write IOControllerprofile to a management datagram.
288  *
289  * See also section 16.3.3.4 IOControllerProfile in the InfiniBand
290  * Architecture Specification. See also section B.7, table B.7 in the SRP
291  * r16a document.
292  */
293 static void srpt_get_ioc(struct srpt_port *sport, u32 slot,
294 			 struct ib_dm_mad *mad)
295 {
296 	struct srpt_device *sdev = sport->sdev;
297 	struct ib_dm_ioc_profile *iocp;
298 
299 	iocp = (struct ib_dm_ioc_profile *)mad->data;
300 
301 	if (!slot || slot > 16) {
302 		mad->mad_hdr.status
303 			= cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
304 		return;
305 	}
306 
307 	if (slot > 2) {
308 		mad->mad_hdr.status
309 			= cpu_to_be16(DM_MAD_STATUS_NO_IOC);
310 		return;
311 	}
312 
313 	memset(iocp, 0, sizeof(*iocp));
314 	strcpy(iocp->id_string, SRPT_ID_STRING);
315 	iocp->guid = cpu_to_be64(srpt_service_guid);
316 	iocp->vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
317 	iocp->device_id = cpu_to_be32(sdev->device->attrs.vendor_part_id);
318 	iocp->device_version = cpu_to_be16(sdev->device->attrs.hw_ver);
319 	iocp->subsys_vendor_id = cpu_to_be32(sdev->device->attrs.vendor_id);
320 	iocp->subsys_device_id = 0x0;
321 	iocp->io_class = cpu_to_be16(SRP_REV16A_IB_IO_CLASS);
322 	iocp->io_subclass = cpu_to_be16(SRP_IO_SUBCLASS);
323 	iocp->protocol = cpu_to_be16(SRP_PROTOCOL);
324 	iocp->protocol_version = cpu_to_be16(SRP_PROTOCOL_VERSION);
325 	iocp->send_queue_depth = cpu_to_be16(sdev->srq_size);
326 	iocp->rdma_read_depth = 4;
327 	iocp->send_size = cpu_to_be32(srp_max_req_size);
328 	iocp->rdma_size = cpu_to_be32(min(sport->port_attrib.srp_max_rdma_size,
329 					  1U << 24));
330 	iocp->num_svc_entries = 1;
331 	iocp->op_cap_mask = SRP_SEND_TO_IOC | SRP_SEND_FROM_IOC |
332 		SRP_RDMA_READ_FROM_IOC | SRP_RDMA_WRITE_FROM_IOC;
333 
334 	mad->mad_hdr.status = 0;
335 }
336 
337 /**
338  * srpt_get_svc_entries() - Write ServiceEntries to a management datagram.
339  *
340  * See also section 16.3.3.5 ServiceEntries in the InfiniBand Architecture
341  * Specification. See also section B.7, table B.8 in the SRP r16a document.
342  */
343 static void srpt_get_svc_entries(u64 ioc_guid,
344 				 u16 slot, u8 hi, u8 lo, struct ib_dm_mad *mad)
345 {
346 	struct ib_dm_svc_entries *svc_entries;
347 
348 	WARN_ON(!ioc_guid);
349 
350 	if (!slot || slot > 16) {
351 		mad->mad_hdr.status
352 			= cpu_to_be16(DM_MAD_STATUS_INVALID_FIELD);
353 		return;
354 	}
355 
356 	if (slot > 2 || lo > hi || hi > 1) {
357 		mad->mad_hdr.status
358 			= cpu_to_be16(DM_MAD_STATUS_NO_IOC);
359 		return;
360 	}
361 
362 	svc_entries = (struct ib_dm_svc_entries *)mad->data;
363 	memset(svc_entries, 0, sizeof(*svc_entries));
364 	svc_entries->service_entries[0].id = cpu_to_be64(ioc_guid);
365 	snprintf(svc_entries->service_entries[0].name,
366 		 sizeof(svc_entries->service_entries[0].name),
367 		 "%s%016llx",
368 		 SRP_SERVICE_NAME_PREFIX,
369 		 ioc_guid);
370 
371 	mad->mad_hdr.status = 0;
372 }
373 
374 /**
375  * srpt_mgmt_method_get() - Process a received management datagram.
376  * @sp:      source port through which the MAD has been received.
377  * @rq_mad:  received MAD.
378  * @rsp_mad: response MAD.
379  */
380 static void srpt_mgmt_method_get(struct srpt_port *sp, struct ib_mad *rq_mad,
381 				 struct ib_dm_mad *rsp_mad)
382 {
383 	u16 attr_id;
384 	u32 slot;
385 	u8 hi, lo;
386 
387 	attr_id = be16_to_cpu(rq_mad->mad_hdr.attr_id);
388 	switch (attr_id) {
389 	case DM_ATTR_CLASS_PORT_INFO:
390 		srpt_get_class_port_info(rsp_mad);
391 		break;
392 	case DM_ATTR_IOU_INFO:
393 		srpt_get_iou(rsp_mad);
394 		break;
395 	case DM_ATTR_IOC_PROFILE:
396 		slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
397 		srpt_get_ioc(sp, slot, rsp_mad);
398 		break;
399 	case DM_ATTR_SVC_ENTRIES:
400 		slot = be32_to_cpu(rq_mad->mad_hdr.attr_mod);
401 		hi = (u8) ((slot >> 8) & 0xff);
402 		lo = (u8) (slot & 0xff);
403 		slot = (u16) ((slot >> 16) & 0xffff);
404 		srpt_get_svc_entries(srpt_service_guid,
405 				     slot, hi, lo, rsp_mad);
406 		break;
407 	default:
408 		rsp_mad->mad_hdr.status =
409 		    cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
410 		break;
411 	}
412 }
413 
414 /**
415  * srpt_mad_send_handler() - Post MAD-send callback function.
416  */
417 static void srpt_mad_send_handler(struct ib_mad_agent *mad_agent,
418 				  struct ib_mad_send_wc *mad_wc)
419 {
420 	ib_destroy_ah(mad_wc->send_buf->ah);
421 	ib_free_send_mad(mad_wc->send_buf);
422 }
423 
424 /**
425  * srpt_mad_recv_handler() - MAD reception callback function.
426  */
427 static void srpt_mad_recv_handler(struct ib_mad_agent *mad_agent,
428 				  struct ib_mad_send_buf *send_buf,
429 				  struct ib_mad_recv_wc *mad_wc)
430 {
431 	struct srpt_port *sport = (struct srpt_port *)mad_agent->context;
432 	struct ib_ah *ah;
433 	struct ib_mad_send_buf *rsp;
434 	struct ib_dm_mad *dm_mad;
435 
436 	if (!mad_wc || !mad_wc->recv_buf.mad)
437 		return;
438 
439 	ah = ib_create_ah_from_wc(mad_agent->qp->pd, mad_wc->wc,
440 				  mad_wc->recv_buf.grh, mad_agent->port_num);
441 	if (IS_ERR(ah))
442 		goto err;
443 
444 	BUILD_BUG_ON(offsetof(struct ib_dm_mad, data) != IB_MGMT_DEVICE_HDR);
445 
446 	rsp = ib_create_send_mad(mad_agent, mad_wc->wc->src_qp,
447 				 mad_wc->wc->pkey_index, 0,
448 				 IB_MGMT_DEVICE_HDR, IB_MGMT_DEVICE_DATA,
449 				 GFP_KERNEL,
450 				 IB_MGMT_BASE_VERSION);
451 	if (IS_ERR(rsp))
452 		goto err_rsp;
453 
454 	rsp->ah = ah;
455 
456 	dm_mad = rsp->mad;
457 	memcpy(dm_mad, mad_wc->recv_buf.mad, sizeof(*dm_mad));
458 	dm_mad->mad_hdr.method = IB_MGMT_METHOD_GET_RESP;
459 	dm_mad->mad_hdr.status = 0;
460 
461 	switch (mad_wc->recv_buf.mad->mad_hdr.method) {
462 	case IB_MGMT_METHOD_GET:
463 		srpt_mgmt_method_get(sport, mad_wc->recv_buf.mad, dm_mad);
464 		break;
465 	case IB_MGMT_METHOD_SET:
466 		dm_mad->mad_hdr.status =
467 		    cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD_ATTR);
468 		break;
469 	default:
470 		dm_mad->mad_hdr.status =
471 		    cpu_to_be16(DM_MAD_STATUS_UNSUP_METHOD);
472 		break;
473 	}
474 
475 	if (!ib_post_send_mad(rsp, NULL)) {
476 		ib_free_recv_mad(mad_wc);
477 		/* will destroy_ah & free_send_mad in send completion */
478 		return;
479 	}
480 
481 	ib_free_send_mad(rsp);
482 
483 err_rsp:
484 	ib_destroy_ah(ah);
485 err:
486 	ib_free_recv_mad(mad_wc);
487 }
488 
489 /**
490  * srpt_refresh_port() - Configure a HCA port.
491  *
492  * Enable InfiniBand management datagram processing, update the cached sm_lid,
493  * lid and gid values, and register a callback function for processing MADs
494  * on the specified port.
495  *
496  * Note: It is safe to call this function more than once for the same port.
497  */
498 static int srpt_refresh_port(struct srpt_port *sport)
499 {
500 	struct ib_mad_reg_req reg_req;
501 	struct ib_port_modify port_modify;
502 	struct ib_port_attr port_attr;
503 	int ret;
504 
505 	memset(&port_modify, 0, sizeof(port_modify));
506 	port_modify.set_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
507 	port_modify.clr_port_cap_mask = 0;
508 
509 	ret = ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
510 	if (ret)
511 		goto err_mod_port;
512 
513 	ret = ib_query_port(sport->sdev->device, sport->port, &port_attr);
514 	if (ret)
515 		goto err_query_port;
516 
517 	sport->sm_lid = port_attr.sm_lid;
518 	sport->lid = port_attr.lid;
519 
520 	ret = ib_query_gid(sport->sdev->device, sport->port, 0, &sport->gid,
521 			   NULL);
522 	if (ret)
523 		goto err_query_port;
524 
525 	if (!sport->mad_agent) {
526 		memset(&reg_req, 0, sizeof(reg_req));
527 		reg_req.mgmt_class = IB_MGMT_CLASS_DEVICE_MGMT;
528 		reg_req.mgmt_class_version = IB_MGMT_BASE_VERSION;
529 		set_bit(IB_MGMT_METHOD_GET, reg_req.method_mask);
530 		set_bit(IB_MGMT_METHOD_SET, reg_req.method_mask);
531 
532 		sport->mad_agent = ib_register_mad_agent(sport->sdev->device,
533 							 sport->port,
534 							 IB_QPT_GSI,
535 							 &reg_req, 0,
536 							 srpt_mad_send_handler,
537 							 srpt_mad_recv_handler,
538 							 sport, 0);
539 		if (IS_ERR(sport->mad_agent)) {
540 			ret = PTR_ERR(sport->mad_agent);
541 			sport->mad_agent = NULL;
542 			goto err_query_port;
543 		}
544 	}
545 
546 	return 0;
547 
548 err_query_port:
549 
550 	port_modify.set_port_cap_mask = 0;
551 	port_modify.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP;
552 	ib_modify_port(sport->sdev->device, sport->port, 0, &port_modify);
553 
554 err_mod_port:
555 
556 	return ret;
557 }
558 
559 /**
560  * srpt_unregister_mad_agent() - Unregister MAD callback functions.
561  *
562  * Note: It is safe to call this function more than once for the same device.
563  */
564 static void srpt_unregister_mad_agent(struct srpt_device *sdev)
565 {
566 	struct ib_port_modify port_modify = {
567 		.clr_port_cap_mask = IB_PORT_DEVICE_MGMT_SUP,
568 	};
569 	struct srpt_port *sport;
570 	int i;
571 
572 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
573 		sport = &sdev->port[i - 1];
574 		WARN_ON(sport->port != i);
575 		if (ib_modify_port(sdev->device, i, 0, &port_modify) < 0)
576 			pr_err("disabling MAD processing failed.\n");
577 		if (sport->mad_agent) {
578 			ib_unregister_mad_agent(sport->mad_agent);
579 			sport->mad_agent = NULL;
580 		}
581 	}
582 }
583 
584 /**
585  * srpt_alloc_ioctx() - Allocate an SRPT I/O context structure.
586  */
587 static struct srpt_ioctx *srpt_alloc_ioctx(struct srpt_device *sdev,
588 					   int ioctx_size, int dma_size,
589 					   enum dma_data_direction dir)
590 {
591 	struct srpt_ioctx *ioctx;
592 
593 	ioctx = kmalloc(ioctx_size, GFP_KERNEL);
594 	if (!ioctx)
595 		goto err;
596 
597 	ioctx->buf = kmalloc(dma_size, GFP_KERNEL);
598 	if (!ioctx->buf)
599 		goto err_free_ioctx;
600 
601 	ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf, dma_size, dir);
602 	if (ib_dma_mapping_error(sdev->device, ioctx->dma))
603 		goto err_free_buf;
604 
605 	return ioctx;
606 
607 err_free_buf:
608 	kfree(ioctx->buf);
609 err_free_ioctx:
610 	kfree(ioctx);
611 err:
612 	return NULL;
613 }
614 
615 /**
616  * srpt_free_ioctx() - Free an SRPT I/O context structure.
617  */
618 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,
619 			    int dma_size, enum dma_data_direction dir)
620 {
621 	if (!ioctx)
622 		return;
623 
624 	ib_dma_unmap_single(sdev->device, ioctx->dma, dma_size, dir);
625 	kfree(ioctx->buf);
626 	kfree(ioctx);
627 }
628 
629 /**
630  * srpt_alloc_ioctx_ring() - Allocate a ring of SRPT I/O context structures.
631  * @sdev:       Device to allocate the I/O context ring for.
632  * @ring_size:  Number of elements in the I/O context ring.
633  * @ioctx_size: I/O context size.
634  * @dma_size:   DMA buffer size.
635  * @dir:        DMA data direction.
636  */
637 static struct srpt_ioctx **srpt_alloc_ioctx_ring(struct srpt_device *sdev,
638 				int ring_size, int ioctx_size,
639 				int dma_size, enum dma_data_direction dir)
640 {
641 	struct srpt_ioctx **ring;
642 	int i;
643 
644 	WARN_ON(ioctx_size != sizeof(struct srpt_recv_ioctx)
645 		&& ioctx_size != sizeof(struct srpt_send_ioctx));
646 
647 	ring = kmalloc(ring_size * sizeof(ring[0]), GFP_KERNEL);
648 	if (!ring)
649 		goto out;
650 	for (i = 0; i < ring_size; ++i) {
651 		ring[i] = srpt_alloc_ioctx(sdev, ioctx_size, dma_size, dir);
652 		if (!ring[i])
653 			goto err;
654 		ring[i]->index = i;
655 	}
656 	goto out;
657 
658 err:
659 	while (--i >= 0)
660 		srpt_free_ioctx(sdev, ring[i], dma_size, dir);
661 	kfree(ring);
662 	ring = NULL;
663 out:
664 	return ring;
665 }
666 
667 /**
668  * srpt_free_ioctx_ring() - Free the ring of SRPT I/O context structures.
669  */
670 static void srpt_free_ioctx_ring(struct srpt_ioctx **ioctx_ring,
671 				 struct srpt_device *sdev, int ring_size,
672 				 int dma_size, enum dma_data_direction dir)
673 {
674 	int i;
675 
676 	for (i = 0; i < ring_size; ++i)
677 		srpt_free_ioctx(sdev, ioctx_ring[i], dma_size, dir);
678 	kfree(ioctx_ring);
679 }
680 
681 /**
682  * srpt_get_cmd_state() - Get the state of a SCSI command.
683  */
684 static enum srpt_command_state srpt_get_cmd_state(struct srpt_send_ioctx *ioctx)
685 {
686 	enum srpt_command_state state;
687 	unsigned long flags;
688 
689 	BUG_ON(!ioctx);
690 
691 	spin_lock_irqsave(&ioctx->spinlock, flags);
692 	state = ioctx->state;
693 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
694 	return state;
695 }
696 
697 /**
698  * srpt_set_cmd_state() - Set the state of a SCSI command.
699  *
700  * Does not modify the state of aborted commands. Returns the previous command
701  * state.
702  */
703 static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,
704 						  enum srpt_command_state new)
705 {
706 	enum srpt_command_state previous;
707 	unsigned long flags;
708 
709 	BUG_ON(!ioctx);
710 
711 	spin_lock_irqsave(&ioctx->spinlock, flags);
712 	previous = ioctx->state;
713 	if (previous != SRPT_STATE_DONE)
714 		ioctx->state = new;
715 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
716 
717 	return previous;
718 }
719 
720 /**
721  * srpt_test_and_set_cmd_state() - Test and set the state of a command.
722  *
723  * Returns true if and only if the previous command state was equal to 'old'.
724  */
725 static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,
726 					enum srpt_command_state old,
727 					enum srpt_command_state new)
728 {
729 	enum srpt_command_state previous;
730 	unsigned long flags;
731 
732 	WARN_ON(!ioctx);
733 	WARN_ON(old == SRPT_STATE_DONE);
734 	WARN_ON(new == SRPT_STATE_NEW);
735 
736 	spin_lock_irqsave(&ioctx->spinlock, flags);
737 	previous = ioctx->state;
738 	if (previous == old)
739 		ioctx->state = new;
740 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
741 	return previous == old;
742 }
743 
744 /**
745  * srpt_post_recv() - Post an IB receive request.
746  */
747 static int srpt_post_recv(struct srpt_device *sdev,
748 			  struct srpt_recv_ioctx *ioctx)
749 {
750 	struct ib_sge list;
751 	struct ib_recv_wr wr, *bad_wr;
752 
753 	BUG_ON(!sdev);
754 	list.addr = ioctx->ioctx.dma;
755 	list.length = srp_max_req_size;
756 	list.lkey = sdev->pd->local_dma_lkey;
757 
758 	ioctx->ioctx.cqe.done = srpt_recv_done;
759 	wr.wr_cqe = &ioctx->ioctx.cqe;
760 	wr.next = NULL;
761 	wr.sg_list = &list;
762 	wr.num_sge = 1;
763 
764 	return ib_post_srq_recv(sdev->srq, &wr, &bad_wr);
765 }
766 
767 /**
768  * srpt_zerolength_write() - Perform a zero-length RDMA write.
769  *
770  * A quote from the InfiniBand specification: C9-88: For an HCA responder
771  * using Reliable Connection service, for each zero-length RDMA READ or WRITE
772  * request, the R_Key shall not be validated, even if the request includes
773  * Immediate data.
774  */
775 static int srpt_zerolength_write(struct srpt_rdma_ch *ch)
776 {
777 	struct ib_send_wr wr, *bad_wr;
778 
779 	memset(&wr, 0, sizeof(wr));
780 	wr.opcode = IB_WR_RDMA_WRITE;
781 	wr.wr_cqe = &ch->zw_cqe;
782 	wr.send_flags = IB_SEND_SIGNALED;
783 	return ib_post_send(ch->qp, &wr, &bad_wr);
784 }
785 
786 static void srpt_zerolength_write_done(struct ib_cq *cq, struct ib_wc *wc)
787 {
788 	struct srpt_rdma_ch *ch = cq->cq_context;
789 
790 	if (wc->status == IB_WC_SUCCESS) {
791 		srpt_process_wait_list(ch);
792 	} else {
793 		if (srpt_set_ch_state(ch, CH_DISCONNECTED))
794 			schedule_work(&ch->release_work);
795 		else
796 			WARN_ONCE(1, "%s-%d\n", ch->sess_name, ch->qp->qp_num);
797 	}
798 }
799 
800 static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,
801 		struct srp_direct_buf *db, int nbufs, struct scatterlist **sg,
802 		unsigned *sg_cnt)
803 {
804 	enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
805 	struct srpt_rdma_ch *ch = ioctx->ch;
806 	struct scatterlist *prev = NULL;
807 	unsigned prev_nents;
808 	int ret, i;
809 
810 	if (nbufs == 1) {
811 		ioctx->rw_ctxs = &ioctx->s_rw_ctx;
812 	} else {
813 		ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),
814 			GFP_KERNEL);
815 		if (!ioctx->rw_ctxs)
816 			return -ENOMEM;
817 	}
818 
819 	for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {
820 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
821 		u64 remote_addr = be64_to_cpu(db->va);
822 		u32 size = be32_to_cpu(db->len);
823 		u32 rkey = be32_to_cpu(db->key);
824 
825 		ret = target_alloc_sgl(&ctx->sg, &ctx->nents, size, false,
826 				i < nbufs - 1);
827 		if (ret)
828 			goto unwind;
829 
830 		ret = rdma_rw_ctx_init(&ctx->rw, ch->qp, ch->sport->port,
831 				ctx->sg, ctx->nents, 0, remote_addr, rkey, dir);
832 		if (ret < 0) {
833 			target_free_sgl(ctx->sg, ctx->nents);
834 			goto unwind;
835 		}
836 
837 		ioctx->n_rdma += ret;
838 		ioctx->n_rw_ctx++;
839 
840 		if (prev) {
841 			sg_unmark_end(&prev[prev_nents - 1]);
842 			sg_chain(prev, prev_nents + 1, ctx->sg);
843 		} else {
844 			*sg = ctx->sg;
845 		}
846 
847 		prev = ctx->sg;
848 		prev_nents = ctx->nents;
849 
850 		*sg_cnt += ctx->nents;
851 	}
852 
853 	return 0;
854 
855 unwind:
856 	while (--i >= 0) {
857 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
858 
859 		rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
860 				ctx->sg, ctx->nents, dir);
861 		target_free_sgl(ctx->sg, ctx->nents);
862 	}
863 	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
864 		kfree(ioctx->rw_ctxs);
865 	return ret;
866 }
867 
868 static void srpt_free_rw_ctxs(struct srpt_rdma_ch *ch,
869 				    struct srpt_send_ioctx *ioctx)
870 {
871 	enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);
872 	int i;
873 
874 	for (i = 0; i < ioctx->n_rw_ctx; i++) {
875 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
876 
877 		rdma_rw_ctx_destroy(&ctx->rw, ch->qp, ch->sport->port,
878 				ctx->sg, ctx->nents, dir);
879 		target_free_sgl(ctx->sg, ctx->nents);
880 	}
881 
882 	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)
883 		kfree(ioctx->rw_ctxs);
884 }
885 
886 static inline void *srpt_get_desc_buf(struct srp_cmd *srp_cmd)
887 {
888 	/*
889 	 * The pointer computations below will only be compiled correctly
890 	 * if srp_cmd::add_data is declared as s8*, u8*, s8[] or u8[], so check
891 	 * whether srp_cmd::add_data has been declared as a byte pointer.
892 	 */
893 	BUILD_BUG_ON(!__same_type(srp_cmd->add_data[0], (s8)0) &&
894 		     !__same_type(srp_cmd->add_data[0], (u8)0));
895 
896 	/*
897 	 * According to the SRP spec, the lower two bits of the 'ADDITIONAL
898 	 * CDB LENGTH' field are reserved and the size in bytes of this field
899 	 * is four times the value specified in bits 3..7. Hence the "& ~3".
900 	 */
901 	return srp_cmd->add_data + (srp_cmd->add_cdb_len & ~3);
902 }
903 
904 /**
905  * srpt_get_desc_tbl() - Parse the data descriptors of an SRP_CMD request.
906  * @ioctx: Pointer to the I/O context associated with the request.
907  * @srp_cmd: Pointer to the SRP_CMD request data.
908  * @dir: Pointer to the variable to which the transfer direction will be
909  *   written.
910  * @data_len: Pointer to the variable to which the total data length of all
911  *   descriptors in the SRP_CMD request will be written.
912  *
913  * This function initializes ioctx->nrbuf and ioctx->r_bufs.
914  *
915  * Returns -EINVAL when the SRP_CMD request contains inconsistent descriptors;
916  * -ENOMEM when memory allocation fails and zero upon success.
917  */
918 static int srpt_get_desc_tbl(struct srpt_send_ioctx *ioctx,
919 		struct srp_cmd *srp_cmd, enum dma_data_direction *dir,
920 		struct scatterlist **sg, unsigned *sg_cnt, u64 *data_len)
921 {
922 	BUG_ON(!dir);
923 	BUG_ON(!data_len);
924 
925 	/*
926 	 * The lower four bits of the buffer format field contain the DATA-IN
927 	 * buffer descriptor format, and the highest four bits contain the
928 	 * DATA-OUT buffer descriptor format.
929 	 */
930 	if (srp_cmd->buf_fmt & 0xf)
931 		/* DATA-IN: transfer data from target to initiator (read). */
932 		*dir = DMA_FROM_DEVICE;
933 	else if (srp_cmd->buf_fmt >> 4)
934 		/* DATA-OUT: transfer data from initiator to target (write). */
935 		*dir = DMA_TO_DEVICE;
936 	else
937 		*dir = DMA_NONE;
938 
939 	/* initialize data_direction early as srpt_alloc_rw_ctxs needs it */
940 	ioctx->cmd.data_direction = *dir;
941 
942 	if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_DIRECT) ||
943 	    ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_DIRECT)) {
944 	    	struct srp_direct_buf *db = srpt_get_desc_buf(srp_cmd);
945 
946 		*data_len = be32_to_cpu(db->len);
947 		return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);
948 	} else if (((srp_cmd->buf_fmt & 0xf) == SRP_DATA_DESC_INDIRECT) ||
949 		   ((srp_cmd->buf_fmt >> 4) == SRP_DATA_DESC_INDIRECT)) {
950 		struct srp_indirect_buf *idb = srpt_get_desc_buf(srp_cmd);
951 		int nbufs = be32_to_cpu(idb->table_desc.len) /
952 				sizeof(struct srp_direct_buf);
953 
954 		if (nbufs >
955 		    (srp_cmd->data_out_desc_cnt + srp_cmd->data_in_desc_cnt)) {
956 			pr_err("received unsupported SRP_CMD request"
957 			       " type (%u out + %u in != %u / %zu)\n",
958 			       srp_cmd->data_out_desc_cnt,
959 			       srp_cmd->data_in_desc_cnt,
960 			       be32_to_cpu(idb->table_desc.len),
961 			       sizeof(struct srp_direct_buf));
962 			return -EINVAL;
963 		}
964 
965 		*data_len = be32_to_cpu(idb->len);
966 		return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,
967 				sg, sg_cnt);
968 	} else {
969 		*data_len = 0;
970 		return 0;
971 	}
972 }
973 
974 /**
975  * srpt_init_ch_qp() - Initialize queue pair attributes.
976  *
977  * Initialized the attributes of queue pair 'qp' by allowing local write,
978  * remote read and remote write. Also transitions 'qp' to state IB_QPS_INIT.
979  */
980 static int srpt_init_ch_qp(struct srpt_rdma_ch *ch, struct ib_qp *qp)
981 {
982 	struct ib_qp_attr *attr;
983 	int ret;
984 
985 	attr = kzalloc(sizeof(*attr), GFP_KERNEL);
986 	if (!attr)
987 		return -ENOMEM;
988 
989 	attr->qp_state = IB_QPS_INIT;
990 	attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_READ |
991 	    IB_ACCESS_REMOTE_WRITE;
992 	attr->port_num = ch->sport->port;
993 	attr->pkey_index = 0;
994 
995 	ret = ib_modify_qp(qp, attr,
996 			   IB_QP_STATE | IB_QP_ACCESS_FLAGS | IB_QP_PORT |
997 			   IB_QP_PKEY_INDEX);
998 
999 	kfree(attr);
1000 	return ret;
1001 }
1002 
1003 /**
1004  * srpt_ch_qp_rtr() - Change the state of a channel to 'ready to receive' (RTR).
1005  * @ch: channel of the queue pair.
1006  * @qp: queue pair to change the state of.
1007  *
1008  * Returns zero upon success and a negative value upon failure.
1009  *
1010  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1011  * If this structure ever becomes larger, it might be necessary to allocate
1012  * it dynamically instead of on the stack.
1013  */
1014 static int srpt_ch_qp_rtr(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1015 {
1016 	struct ib_qp_attr qp_attr;
1017 	int attr_mask;
1018 	int ret;
1019 
1020 	qp_attr.qp_state = IB_QPS_RTR;
1021 	ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1022 	if (ret)
1023 		goto out;
1024 
1025 	qp_attr.max_dest_rd_atomic = 4;
1026 
1027 	ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1028 
1029 out:
1030 	return ret;
1031 }
1032 
1033 /**
1034  * srpt_ch_qp_rts() - Change the state of a channel to 'ready to send' (RTS).
1035  * @ch: channel of the queue pair.
1036  * @qp: queue pair to change the state of.
1037  *
1038  * Returns zero upon success and a negative value upon failure.
1039  *
1040  * Note: currently a struct ib_qp_attr takes 136 bytes on a 64-bit system.
1041  * If this structure ever becomes larger, it might be necessary to allocate
1042  * it dynamically instead of on the stack.
1043  */
1044 static int srpt_ch_qp_rts(struct srpt_rdma_ch *ch, struct ib_qp *qp)
1045 {
1046 	struct ib_qp_attr qp_attr;
1047 	int attr_mask;
1048 	int ret;
1049 
1050 	qp_attr.qp_state = IB_QPS_RTS;
1051 	ret = ib_cm_init_qp_attr(ch->cm_id, &qp_attr, &attr_mask);
1052 	if (ret)
1053 		goto out;
1054 
1055 	qp_attr.max_rd_atomic = 4;
1056 
1057 	ret = ib_modify_qp(qp, &qp_attr, attr_mask);
1058 
1059 out:
1060 	return ret;
1061 }
1062 
1063 /**
1064  * srpt_ch_qp_err() - Set the channel queue pair state to 'error'.
1065  */
1066 static int srpt_ch_qp_err(struct srpt_rdma_ch *ch)
1067 {
1068 	struct ib_qp_attr qp_attr;
1069 
1070 	qp_attr.qp_state = IB_QPS_ERR;
1071 	return ib_modify_qp(ch->qp, &qp_attr, IB_QP_STATE);
1072 }
1073 
1074 /**
1075  * srpt_get_send_ioctx() - Obtain an I/O context for sending to the initiator.
1076  */
1077 static struct srpt_send_ioctx *srpt_get_send_ioctx(struct srpt_rdma_ch *ch)
1078 {
1079 	struct srpt_send_ioctx *ioctx;
1080 	unsigned long flags;
1081 
1082 	BUG_ON(!ch);
1083 
1084 	ioctx = NULL;
1085 	spin_lock_irqsave(&ch->spinlock, flags);
1086 	if (!list_empty(&ch->free_list)) {
1087 		ioctx = list_first_entry(&ch->free_list,
1088 					 struct srpt_send_ioctx, free_list);
1089 		list_del(&ioctx->free_list);
1090 	}
1091 	spin_unlock_irqrestore(&ch->spinlock, flags);
1092 
1093 	if (!ioctx)
1094 		return ioctx;
1095 
1096 	BUG_ON(ioctx->ch != ch);
1097 	spin_lock_init(&ioctx->spinlock);
1098 	ioctx->state = SRPT_STATE_NEW;
1099 	ioctx->n_rdma = 0;
1100 	ioctx->n_rw_ctx = 0;
1101 	init_completion(&ioctx->tx_done);
1102 	ioctx->queue_status_only = false;
1103 	/*
1104 	 * transport_init_se_cmd() does not initialize all fields, so do it
1105 	 * here.
1106 	 */
1107 	memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));
1108 	memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));
1109 
1110 	return ioctx;
1111 }
1112 
1113 /**
1114  * srpt_abort_cmd() - Abort a SCSI command.
1115  * @ioctx:   I/O context associated with the SCSI command.
1116  * @context: Preferred execution context.
1117  */
1118 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)
1119 {
1120 	enum srpt_command_state state;
1121 	unsigned long flags;
1122 
1123 	BUG_ON(!ioctx);
1124 
1125 	/*
1126 	 * If the command is in a state where the target core is waiting for
1127 	 * the ib_srpt driver, change the state to the next state.
1128 	 */
1129 
1130 	spin_lock_irqsave(&ioctx->spinlock, flags);
1131 	state = ioctx->state;
1132 	switch (state) {
1133 	case SRPT_STATE_NEED_DATA:
1134 		ioctx->state = SRPT_STATE_DATA_IN;
1135 		break;
1136 	case SRPT_STATE_CMD_RSP_SENT:
1137 	case SRPT_STATE_MGMT_RSP_SENT:
1138 		ioctx->state = SRPT_STATE_DONE;
1139 		break;
1140 	default:
1141 		WARN_ONCE(true, "%s: unexpected I/O context state %d\n",
1142 			  __func__, state);
1143 		break;
1144 	}
1145 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
1146 
1147 	pr_debug("Aborting cmd with state %d and tag %lld\n", state,
1148 		 ioctx->cmd.tag);
1149 
1150 	switch (state) {
1151 	case SRPT_STATE_NEW:
1152 	case SRPT_STATE_DATA_IN:
1153 	case SRPT_STATE_MGMT:
1154 	case SRPT_STATE_DONE:
1155 		/*
1156 		 * Do nothing - defer abort processing until
1157 		 * srpt_queue_response() is invoked.
1158 		 */
1159 		break;
1160 	case SRPT_STATE_NEED_DATA:
1161 		pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);
1162 		transport_generic_request_failure(&ioctx->cmd,
1163 					TCM_CHECK_CONDITION_ABORT_CMD);
1164 		break;
1165 	case SRPT_STATE_CMD_RSP_SENT:
1166 		/*
1167 		 * SRP_RSP sending failed or the SRP_RSP send completion has
1168 		 * not been received in time.
1169 		 */
1170 		transport_generic_free_cmd(&ioctx->cmd, 0);
1171 		break;
1172 	case SRPT_STATE_MGMT_RSP_SENT:
1173 		transport_generic_free_cmd(&ioctx->cmd, 0);
1174 		break;
1175 	default:
1176 		WARN(1, "Unexpected command state (%d)", state);
1177 		break;
1178 	}
1179 
1180 	return state;
1181 }
1182 
1183 /**
1184  * XXX: what is now target_execute_cmd used to be asynchronous, and unmapping
1185  * the data that has been transferred via IB RDMA had to be postponed until the
1186  * check_stop_free() callback.  None of this is necessary anymore and needs to
1187  * be cleaned up.
1188  */
1189 static void srpt_rdma_read_done(struct ib_cq *cq, struct ib_wc *wc)
1190 {
1191 	struct srpt_rdma_ch *ch = cq->cq_context;
1192 	struct srpt_send_ioctx *ioctx =
1193 		container_of(wc->wr_cqe, struct srpt_send_ioctx, rdma_cqe);
1194 
1195 	WARN_ON(ioctx->n_rdma <= 0);
1196 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
1197 	ioctx->n_rdma = 0;
1198 
1199 	if (unlikely(wc->status != IB_WC_SUCCESS)) {
1200 		pr_info("RDMA_READ for ioctx 0x%p failed with status %d\n",
1201 			ioctx, wc->status);
1202 		srpt_abort_cmd(ioctx);
1203 		return;
1204 	}
1205 
1206 	if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,
1207 					SRPT_STATE_DATA_IN))
1208 		target_execute_cmd(&ioctx->cmd);
1209 	else
1210 		pr_err("%s[%d]: wrong state = %d\n", __func__,
1211 		       __LINE__, srpt_get_cmd_state(ioctx));
1212 }
1213 
1214 /**
1215  * srpt_build_cmd_rsp() - Build an SRP_RSP response.
1216  * @ch: RDMA channel through which the request has been received.
1217  * @ioctx: I/O context associated with the SRP_CMD request. The response will
1218  *   be built in the buffer ioctx->buf points at and hence this function will
1219  *   overwrite the request data.
1220  * @tag: tag of the request for which this response is being generated.
1221  * @status: value for the STATUS field of the SRP_RSP information unit.
1222  *
1223  * Returns the size in bytes of the SRP_RSP response.
1224  *
1225  * An SRP_RSP response contains a SCSI status or service response. See also
1226  * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1227  * response. See also SPC-2 for more information about sense data.
1228  */
1229 static int srpt_build_cmd_rsp(struct srpt_rdma_ch *ch,
1230 			      struct srpt_send_ioctx *ioctx, u64 tag,
1231 			      int status)
1232 {
1233 	struct srp_rsp *srp_rsp;
1234 	const u8 *sense_data;
1235 	int sense_data_len, max_sense_len;
1236 
1237 	/*
1238 	 * The lowest bit of all SAM-3 status codes is zero (see also
1239 	 * paragraph 5.3 in SAM-3).
1240 	 */
1241 	WARN_ON(status & 1);
1242 
1243 	srp_rsp = ioctx->ioctx.buf;
1244 	BUG_ON(!srp_rsp);
1245 
1246 	sense_data = ioctx->sense_data;
1247 	sense_data_len = ioctx->cmd.scsi_sense_length;
1248 	WARN_ON(sense_data_len > sizeof(ioctx->sense_data));
1249 
1250 	memset(srp_rsp, 0, sizeof(*srp_rsp));
1251 	srp_rsp->opcode = SRP_RSP;
1252 	srp_rsp->req_lim_delta =
1253 		cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1254 	srp_rsp->tag = tag;
1255 	srp_rsp->status = status;
1256 
1257 	if (sense_data_len) {
1258 		BUILD_BUG_ON(MIN_MAX_RSP_SIZE <= sizeof(*srp_rsp));
1259 		max_sense_len = ch->max_ti_iu_len - sizeof(*srp_rsp);
1260 		if (sense_data_len > max_sense_len) {
1261 			pr_warn("truncated sense data from %d to %d"
1262 				" bytes\n", sense_data_len, max_sense_len);
1263 			sense_data_len = max_sense_len;
1264 		}
1265 
1266 		srp_rsp->flags |= SRP_RSP_FLAG_SNSVALID;
1267 		srp_rsp->sense_data_len = cpu_to_be32(sense_data_len);
1268 		memcpy(srp_rsp + 1, sense_data, sense_data_len);
1269 	}
1270 
1271 	return sizeof(*srp_rsp) + sense_data_len;
1272 }
1273 
1274 /**
1275  * srpt_build_tskmgmt_rsp() - Build a task management response.
1276  * @ch:       RDMA channel through which the request has been received.
1277  * @ioctx:    I/O context in which the SRP_RSP response will be built.
1278  * @rsp_code: RSP_CODE that will be stored in the response.
1279  * @tag:      Tag of the request for which this response is being generated.
1280  *
1281  * Returns the size in bytes of the SRP_RSP response.
1282  *
1283  * An SRP_RSP response contains a SCSI status or service response. See also
1284  * section 6.9 in the SRP r16a document for the format of an SRP_RSP
1285  * response.
1286  */
1287 static int srpt_build_tskmgmt_rsp(struct srpt_rdma_ch *ch,
1288 				  struct srpt_send_ioctx *ioctx,
1289 				  u8 rsp_code, u64 tag)
1290 {
1291 	struct srp_rsp *srp_rsp;
1292 	int resp_data_len;
1293 	int resp_len;
1294 
1295 	resp_data_len = 4;
1296 	resp_len = sizeof(*srp_rsp) + resp_data_len;
1297 
1298 	srp_rsp = ioctx->ioctx.buf;
1299 	BUG_ON(!srp_rsp);
1300 	memset(srp_rsp, 0, sizeof(*srp_rsp));
1301 
1302 	srp_rsp->opcode = SRP_RSP;
1303 	srp_rsp->req_lim_delta =
1304 		cpu_to_be32(1 + atomic_xchg(&ch->req_lim_delta, 0));
1305 	srp_rsp->tag = tag;
1306 
1307 	srp_rsp->flags |= SRP_RSP_FLAG_RSPVALID;
1308 	srp_rsp->resp_data_len = cpu_to_be32(resp_data_len);
1309 	srp_rsp->data[3] = rsp_code;
1310 
1311 	return resp_len;
1312 }
1313 
1314 static int srpt_check_stop_free(struct se_cmd *cmd)
1315 {
1316 	struct srpt_send_ioctx *ioctx = container_of(cmd,
1317 				struct srpt_send_ioctx, cmd);
1318 
1319 	return target_put_sess_cmd(&ioctx->cmd);
1320 }
1321 
1322 /**
1323  * srpt_handle_cmd() - Process SRP_CMD.
1324  */
1325 static void srpt_handle_cmd(struct srpt_rdma_ch *ch,
1326 			    struct srpt_recv_ioctx *recv_ioctx,
1327 			    struct srpt_send_ioctx *send_ioctx)
1328 {
1329 	struct se_cmd *cmd;
1330 	struct srp_cmd *srp_cmd;
1331 	struct scatterlist *sg = NULL;
1332 	unsigned sg_cnt = 0;
1333 	u64 data_len;
1334 	enum dma_data_direction dir;
1335 	int rc;
1336 
1337 	BUG_ON(!send_ioctx);
1338 
1339 	srp_cmd = recv_ioctx->ioctx.buf;
1340 	cmd = &send_ioctx->cmd;
1341 	cmd->tag = srp_cmd->tag;
1342 
1343 	switch (srp_cmd->task_attr) {
1344 	case SRP_CMD_SIMPLE_Q:
1345 		cmd->sam_task_attr = TCM_SIMPLE_TAG;
1346 		break;
1347 	case SRP_CMD_ORDERED_Q:
1348 	default:
1349 		cmd->sam_task_attr = TCM_ORDERED_TAG;
1350 		break;
1351 	case SRP_CMD_HEAD_OF_Q:
1352 		cmd->sam_task_attr = TCM_HEAD_TAG;
1353 		break;
1354 	case SRP_CMD_ACA:
1355 		cmd->sam_task_attr = TCM_ACA_TAG;
1356 		break;
1357 	}
1358 
1359 	rc = srpt_get_desc_tbl(send_ioctx, srp_cmd, &dir, &sg, &sg_cnt,
1360 			&data_len);
1361 	if (rc) {
1362 		if (rc != -EAGAIN) {
1363 			pr_err("0x%llx: parsing SRP descriptor table failed.\n",
1364 			       srp_cmd->tag);
1365 		}
1366 		goto release_ioctx;
1367 	}
1368 
1369 	rc = target_submit_cmd_map_sgls(cmd, ch->sess, srp_cmd->cdb,
1370 			       &send_ioctx->sense_data[0],
1371 			       scsilun_to_int(&srp_cmd->lun), data_len,
1372 			       TCM_SIMPLE_TAG, dir, TARGET_SCF_ACK_KREF,
1373 			       sg, sg_cnt, NULL, 0, NULL, 0);
1374 	if (rc != 0) {
1375 		pr_debug("target_submit_cmd() returned %d for tag %#llx\n", rc,
1376 			 srp_cmd->tag);
1377 		goto release_ioctx;
1378 	}
1379 	return;
1380 
1381 release_ioctx:
1382 	send_ioctx->state = SRPT_STATE_DONE;
1383 	srpt_release_cmd(cmd);
1384 }
1385 
1386 static int srp_tmr_to_tcm(int fn)
1387 {
1388 	switch (fn) {
1389 	case SRP_TSK_ABORT_TASK:
1390 		return TMR_ABORT_TASK;
1391 	case SRP_TSK_ABORT_TASK_SET:
1392 		return TMR_ABORT_TASK_SET;
1393 	case SRP_TSK_CLEAR_TASK_SET:
1394 		return TMR_CLEAR_TASK_SET;
1395 	case SRP_TSK_LUN_RESET:
1396 		return TMR_LUN_RESET;
1397 	case SRP_TSK_CLEAR_ACA:
1398 		return TMR_CLEAR_ACA;
1399 	default:
1400 		return -1;
1401 	}
1402 }
1403 
1404 /**
1405  * srpt_handle_tsk_mgmt() - Process an SRP_TSK_MGMT information unit.
1406  *
1407  * Returns 0 if and only if the request will be processed by the target core.
1408  *
1409  * For more information about SRP_TSK_MGMT information units, see also section
1410  * 6.7 in the SRP r16a document.
1411  */
1412 static void srpt_handle_tsk_mgmt(struct srpt_rdma_ch *ch,
1413 				 struct srpt_recv_ioctx *recv_ioctx,
1414 				 struct srpt_send_ioctx *send_ioctx)
1415 {
1416 	struct srp_tsk_mgmt *srp_tsk;
1417 	struct se_cmd *cmd;
1418 	struct se_session *sess = ch->sess;
1419 	int tcm_tmr;
1420 	int rc;
1421 
1422 	BUG_ON(!send_ioctx);
1423 
1424 	srp_tsk = recv_ioctx->ioctx.buf;
1425 	cmd = &send_ioctx->cmd;
1426 
1427 	pr_debug("recv tsk_mgmt fn %d for task_tag %lld and cmd tag %lld"
1428 		 " cm_id %p sess %p\n", srp_tsk->tsk_mgmt_func,
1429 		 srp_tsk->task_tag, srp_tsk->tag, ch->cm_id, ch->sess);
1430 
1431 	srpt_set_cmd_state(send_ioctx, SRPT_STATE_MGMT);
1432 	send_ioctx->cmd.tag = srp_tsk->tag;
1433 	tcm_tmr = srp_tmr_to_tcm(srp_tsk->tsk_mgmt_func);
1434 	rc = target_submit_tmr(&send_ioctx->cmd, sess, NULL,
1435 			       scsilun_to_int(&srp_tsk->lun), srp_tsk, tcm_tmr,
1436 			       GFP_KERNEL, srp_tsk->task_tag,
1437 			       TARGET_SCF_ACK_KREF);
1438 	if (rc != 0) {
1439 		send_ioctx->cmd.se_tmr_req->response = TMR_FUNCTION_REJECTED;
1440 		goto fail;
1441 	}
1442 	return;
1443 fail:
1444 	transport_send_check_condition_and_sense(cmd, 0, 0); // XXX:
1445 }
1446 
1447 /**
1448  * srpt_handle_new_iu() - Process a newly received information unit.
1449  * @ch:    RDMA channel through which the information unit has been received.
1450  * @ioctx: SRPT I/O context associated with the information unit.
1451  */
1452 static void srpt_handle_new_iu(struct srpt_rdma_ch *ch,
1453 			       struct srpt_recv_ioctx *recv_ioctx,
1454 			       struct srpt_send_ioctx *send_ioctx)
1455 {
1456 	struct srp_cmd *srp_cmd;
1457 
1458 	BUG_ON(!ch);
1459 	BUG_ON(!recv_ioctx);
1460 
1461 	ib_dma_sync_single_for_cpu(ch->sport->sdev->device,
1462 				   recv_ioctx->ioctx.dma, srp_max_req_size,
1463 				   DMA_FROM_DEVICE);
1464 
1465 	if (unlikely(ch->state == CH_CONNECTING))
1466 		goto out_wait;
1467 
1468 	if (unlikely(ch->state != CH_LIVE))
1469 		return;
1470 
1471 	srp_cmd = recv_ioctx->ioctx.buf;
1472 	if (srp_cmd->opcode == SRP_CMD || srp_cmd->opcode == SRP_TSK_MGMT) {
1473 		if (!send_ioctx) {
1474 			if (!list_empty(&ch->cmd_wait_list))
1475 				goto out_wait;
1476 			send_ioctx = srpt_get_send_ioctx(ch);
1477 		}
1478 		if (unlikely(!send_ioctx))
1479 			goto out_wait;
1480 	}
1481 
1482 	switch (srp_cmd->opcode) {
1483 	case SRP_CMD:
1484 		srpt_handle_cmd(ch, recv_ioctx, send_ioctx);
1485 		break;
1486 	case SRP_TSK_MGMT:
1487 		srpt_handle_tsk_mgmt(ch, recv_ioctx, send_ioctx);
1488 		break;
1489 	case SRP_I_LOGOUT:
1490 		pr_err("Not yet implemented: SRP_I_LOGOUT\n");
1491 		break;
1492 	case SRP_CRED_RSP:
1493 		pr_debug("received SRP_CRED_RSP\n");
1494 		break;
1495 	case SRP_AER_RSP:
1496 		pr_debug("received SRP_AER_RSP\n");
1497 		break;
1498 	case SRP_RSP:
1499 		pr_err("Received SRP_RSP\n");
1500 		break;
1501 	default:
1502 		pr_err("received IU with unknown opcode 0x%x\n",
1503 		       srp_cmd->opcode);
1504 		break;
1505 	}
1506 
1507 	srpt_post_recv(ch->sport->sdev, recv_ioctx);
1508 	return;
1509 
1510 out_wait:
1511 	list_add_tail(&recv_ioctx->wait_list, &ch->cmd_wait_list);
1512 }
1513 
1514 static void srpt_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1515 {
1516 	struct srpt_rdma_ch *ch = cq->cq_context;
1517 	struct srpt_recv_ioctx *ioctx =
1518 		container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);
1519 
1520 	if (wc->status == IB_WC_SUCCESS) {
1521 		int req_lim;
1522 
1523 		req_lim = atomic_dec_return(&ch->req_lim);
1524 		if (unlikely(req_lim < 0))
1525 			pr_err("req_lim = %d < 0\n", req_lim);
1526 		srpt_handle_new_iu(ch, ioctx, NULL);
1527 	} else {
1528 		pr_info("receiving failed for ioctx %p with status %d\n",
1529 			ioctx, wc->status);
1530 	}
1531 }
1532 
1533 /*
1534  * This function must be called from the context in which RDMA completions are
1535  * processed because it accesses the wait list without protection against
1536  * access from other threads.
1537  */
1538 static void srpt_process_wait_list(struct srpt_rdma_ch *ch)
1539 {
1540 	struct srpt_send_ioctx *ioctx;
1541 
1542 	while (!list_empty(&ch->cmd_wait_list) &&
1543 	       ch->state >= CH_LIVE &&
1544 	       (ioctx = srpt_get_send_ioctx(ch)) != NULL) {
1545 		struct srpt_recv_ioctx *recv_ioctx;
1546 
1547 		recv_ioctx = list_first_entry(&ch->cmd_wait_list,
1548 					      struct srpt_recv_ioctx,
1549 					      wait_list);
1550 		list_del(&recv_ioctx->wait_list);
1551 		srpt_handle_new_iu(ch, recv_ioctx, ioctx);
1552 	}
1553 }
1554 
1555 /**
1556  * Note: Although this has not yet been observed during tests, at least in
1557  * theory it is possible that the srpt_get_send_ioctx() call invoked by
1558  * srpt_handle_new_iu() fails. This is possible because the req_lim_delta
1559  * value in each response is set to one, and it is possible that this response
1560  * makes the initiator send a new request before the send completion for that
1561  * response has been processed. This could e.g. happen if the call to
1562  * srpt_put_send_iotcx() is delayed because of a higher priority interrupt or
1563  * if IB retransmission causes generation of the send completion to be
1564  * delayed. Incoming information units for which srpt_get_send_ioctx() fails
1565  * are queued on cmd_wait_list. The code below processes these delayed
1566  * requests one at a time.
1567  */
1568 static void srpt_send_done(struct ib_cq *cq, struct ib_wc *wc)
1569 {
1570 	struct srpt_rdma_ch *ch = cq->cq_context;
1571 	struct srpt_send_ioctx *ioctx =
1572 		container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);
1573 	enum srpt_command_state state;
1574 
1575 	state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
1576 
1577 	WARN_ON(state != SRPT_STATE_CMD_RSP_SENT &&
1578 		state != SRPT_STATE_MGMT_RSP_SENT);
1579 
1580 	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
1581 
1582 	if (wc->status != IB_WC_SUCCESS)
1583 		pr_info("sending response for ioctx 0x%p failed"
1584 			" with status %d\n", ioctx, wc->status);
1585 
1586 	if (state != SRPT_STATE_DONE) {
1587 		transport_generic_free_cmd(&ioctx->cmd, 0);
1588 	} else {
1589 		pr_err("IB completion has been received too late for"
1590 		       " wr_id = %u.\n", ioctx->ioctx.index);
1591 	}
1592 
1593 	srpt_process_wait_list(ch);
1594 }
1595 
1596 /**
1597  * srpt_create_ch_ib() - Create receive and send completion queues.
1598  */
1599 static int srpt_create_ch_ib(struct srpt_rdma_ch *ch)
1600 {
1601 	struct ib_qp_init_attr *qp_init;
1602 	struct srpt_port *sport = ch->sport;
1603 	struct srpt_device *sdev = sport->sdev;
1604 	const struct ib_device_attr *attrs = &sdev->device->attrs;
1605 	u32 srp_sq_size = sport->port_attrib.srp_sq_size;
1606 	int ret;
1607 
1608 	WARN_ON(ch->rq_size < 1);
1609 
1610 	ret = -ENOMEM;
1611 	qp_init = kzalloc(sizeof(*qp_init), GFP_KERNEL);
1612 	if (!qp_init)
1613 		goto out;
1614 
1615 retry:
1616 	ch->cq = ib_alloc_cq(sdev->device, ch, ch->rq_size + srp_sq_size,
1617 			0 /* XXX: spread CQs */, IB_POLL_WORKQUEUE);
1618 	if (IS_ERR(ch->cq)) {
1619 		ret = PTR_ERR(ch->cq);
1620 		pr_err("failed to create CQ cqe= %d ret= %d\n",
1621 		       ch->rq_size + srp_sq_size, ret);
1622 		goto out;
1623 	}
1624 
1625 	qp_init->qp_context = (void *)ch;
1626 	qp_init->event_handler
1627 		= (void(*)(struct ib_event *, void*))srpt_qp_event;
1628 	qp_init->send_cq = ch->cq;
1629 	qp_init->recv_cq = ch->cq;
1630 	qp_init->srq = sdev->srq;
1631 	qp_init->sq_sig_type = IB_SIGNAL_REQ_WR;
1632 	qp_init->qp_type = IB_QPT_RC;
1633 	/*
1634 	 * We divide up our send queue size into half SEND WRs to send the
1635 	 * completions, and half R/W contexts to actually do the RDMA
1636 	 * READ/WRITE transfers.  Note that we need to allocate CQ slots for
1637 	 * both both, as RDMA contexts will also post completions for the
1638 	 * RDMA READ case.
1639 	 */
1640 	qp_init->cap.max_send_wr = srp_sq_size / 2;
1641 	qp_init->cap.max_rdma_ctxs = srp_sq_size / 2;
1642 	qp_init->cap.max_send_sge = min(attrs->max_sge, SRPT_MAX_SG_PER_WQE);
1643 	qp_init->port_num = ch->sport->port;
1644 
1645 	ch->qp = ib_create_qp(sdev->pd, qp_init);
1646 	if (IS_ERR(ch->qp)) {
1647 		ret = PTR_ERR(ch->qp);
1648 		if (ret == -ENOMEM) {
1649 			srp_sq_size /= 2;
1650 			if (srp_sq_size >= MIN_SRPT_SQ_SIZE) {
1651 				ib_destroy_cq(ch->cq);
1652 				goto retry;
1653 			}
1654 		}
1655 		pr_err("failed to create_qp ret= %d\n", ret);
1656 		goto err_destroy_cq;
1657 	}
1658 
1659 	atomic_set(&ch->sq_wr_avail, qp_init->cap.max_send_wr);
1660 
1661 	pr_debug("%s: max_cqe= %d max_sge= %d sq_size = %d cm_id= %p\n",
1662 		 __func__, ch->cq->cqe, qp_init->cap.max_send_sge,
1663 		 qp_init->cap.max_send_wr, ch->cm_id);
1664 
1665 	ret = srpt_init_ch_qp(ch, ch->qp);
1666 	if (ret)
1667 		goto err_destroy_qp;
1668 
1669 out:
1670 	kfree(qp_init);
1671 	return ret;
1672 
1673 err_destroy_qp:
1674 	ib_destroy_qp(ch->qp);
1675 err_destroy_cq:
1676 	ib_free_cq(ch->cq);
1677 	goto out;
1678 }
1679 
1680 static void srpt_destroy_ch_ib(struct srpt_rdma_ch *ch)
1681 {
1682 	ib_destroy_qp(ch->qp);
1683 	ib_free_cq(ch->cq);
1684 }
1685 
1686 /**
1687  * srpt_close_ch() - Close an RDMA channel.
1688  *
1689  * Make sure all resources associated with the channel will be deallocated at
1690  * an appropriate time.
1691  *
1692  * Returns true if and only if the channel state has been modified into
1693  * CH_DRAINING.
1694  */
1695 static bool srpt_close_ch(struct srpt_rdma_ch *ch)
1696 {
1697 	int ret;
1698 
1699 	if (!srpt_set_ch_state(ch, CH_DRAINING)) {
1700 		pr_debug("%s-%d: already closed\n", ch->sess_name,
1701 			 ch->qp->qp_num);
1702 		return false;
1703 	}
1704 
1705 	kref_get(&ch->kref);
1706 
1707 	ret = srpt_ch_qp_err(ch);
1708 	if (ret < 0)
1709 		pr_err("%s-%d: changing queue pair into error state failed: %d\n",
1710 		       ch->sess_name, ch->qp->qp_num, ret);
1711 
1712 	pr_debug("%s-%d: queued zerolength write\n", ch->sess_name,
1713 		 ch->qp->qp_num);
1714 	ret = srpt_zerolength_write(ch);
1715 	if (ret < 0) {
1716 		pr_err("%s-%d: queuing zero-length write failed: %d\n",
1717 		       ch->sess_name, ch->qp->qp_num, ret);
1718 		if (srpt_set_ch_state(ch, CH_DISCONNECTED))
1719 			schedule_work(&ch->release_work);
1720 		else
1721 			WARN_ON_ONCE(true);
1722 	}
1723 
1724 	kref_put(&ch->kref, srpt_free_ch);
1725 
1726 	return true;
1727 }
1728 
1729 /*
1730  * Change the channel state into CH_DISCONNECTING. If a channel has not yet
1731  * reached the connected state, close it. If a channel is in the connected
1732  * state, send a DREQ. If a DREQ has been received, send a DREP. Note: it is
1733  * the responsibility of the caller to ensure that this function is not
1734  * invoked concurrently with the code that accepts a connection. This means
1735  * that this function must either be invoked from inside a CM callback
1736  * function or that it must be invoked with the srpt_port.mutex held.
1737  */
1738 static int srpt_disconnect_ch(struct srpt_rdma_ch *ch)
1739 {
1740 	int ret;
1741 
1742 	if (!srpt_set_ch_state(ch, CH_DISCONNECTING))
1743 		return -ENOTCONN;
1744 
1745 	ret = ib_send_cm_dreq(ch->cm_id, NULL, 0);
1746 	if (ret < 0)
1747 		ret = ib_send_cm_drep(ch->cm_id, NULL, 0);
1748 
1749 	if (ret < 0 && srpt_close_ch(ch))
1750 		ret = 0;
1751 
1752 	return ret;
1753 }
1754 
1755 static void __srpt_close_all_ch(struct srpt_device *sdev)
1756 {
1757 	struct srpt_rdma_ch *ch;
1758 
1759 	lockdep_assert_held(&sdev->mutex);
1760 
1761 	list_for_each_entry(ch, &sdev->rch_list, list) {
1762 		if (srpt_disconnect_ch(ch) >= 0)
1763 			pr_info("Closing channel %s-%d because target %s has been disabled\n",
1764 				ch->sess_name, ch->qp->qp_num,
1765 				sdev->device->name);
1766 		srpt_close_ch(ch);
1767 	}
1768 }
1769 
1770 static void srpt_free_ch(struct kref *kref)
1771 {
1772 	struct srpt_rdma_ch *ch = container_of(kref, struct srpt_rdma_ch, kref);
1773 
1774 	kfree(ch);
1775 }
1776 
1777 static void srpt_release_channel_work(struct work_struct *w)
1778 {
1779 	struct srpt_rdma_ch *ch;
1780 	struct srpt_device *sdev;
1781 	struct se_session *se_sess;
1782 
1783 	ch = container_of(w, struct srpt_rdma_ch, release_work);
1784 	pr_debug("%s: %s-%d; release_done = %p\n", __func__, ch->sess_name,
1785 		 ch->qp->qp_num, ch->release_done);
1786 
1787 	sdev = ch->sport->sdev;
1788 	BUG_ON(!sdev);
1789 
1790 	se_sess = ch->sess;
1791 	BUG_ON(!se_sess);
1792 
1793 	target_sess_cmd_list_set_waiting(se_sess);
1794 	target_wait_for_sess_cmds(se_sess);
1795 
1796 	transport_deregister_session_configfs(se_sess);
1797 	transport_deregister_session(se_sess);
1798 	ch->sess = NULL;
1799 
1800 	ib_destroy_cm_id(ch->cm_id);
1801 
1802 	srpt_destroy_ch_ib(ch);
1803 
1804 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
1805 			     ch->sport->sdev, ch->rq_size,
1806 			     ch->rsp_size, DMA_TO_DEVICE);
1807 
1808 	mutex_lock(&sdev->mutex);
1809 	list_del_init(&ch->list);
1810 	if (ch->release_done)
1811 		complete(ch->release_done);
1812 	mutex_unlock(&sdev->mutex);
1813 
1814 	wake_up(&sdev->ch_releaseQ);
1815 
1816 	kref_put(&ch->kref, srpt_free_ch);
1817 }
1818 
1819 /**
1820  * srpt_cm_req_recv() - Process the event IB_CM_REQ_RECEIVED.
1821  *
1822  * Ownership of the cm_id is transferred to the target session if this
1823  * functions returns zero. Otherwise the caller remains the owner of cm_id.
1824  */
1825 static int srpt_cm_req_recv(struct ib_cm_id *cm_id,
1826 			    struct ib_cm_req_event_param *param,
1827 			    void *private_data)
1828 {
1829 	struct srpt_device *sdev = cm_id->context;
1830 	struct srpt_port *sport = &sdev->port[param->port - 1];
1831 	struct srp_login_req *req;
1832 	struct srp_login_rsp *rsp;
1833 	struct srp_login_rej *rej;
1834 	struct ib_cm_rep_param *rep_param;
1835 	struct srpt_rdma_ch *ch, *tmp_ch;
1836 	u32 it_iu_len;
1837 	int i, ret = 0;
1838 	unsigned char *p;
1839 
1840 	WARN_ON_ONCE(irqs_disabled());
1841 
1842 	if (WARN_ON(!sdev || !private_data))
1843 		return -EINVAL;
1844 
1845 	req = (struct srp_login_req *)private_data;
1846 
1847 	it_iu_len = be32_to_cpu(req->req_it_iu_len);
1848 
1849 	pr_info("Received SRP_LOGIN_REQ with i_port_id 0x%llx:0x%llx,"
1850 		" t_port_id 0x%llx:0x%llx and it_iu_len %d on port %d"
1851 		" (guid=0x%llx:0x%llx)\n",
1852 		be64_to_cpu(*(__be64 *)&req->initiator_port_id[0]),
1853 		be64_to_cpu(*(__be64 *)&req->initiator_port_id[8]),
1854 		be64_to_cpu(*(__be64 *)&req->target_port_id[0]),
1855 		be64_to_cpu(*(__be64 *)&req->target_port_id[8]),
1856 		it_iu_len,
1857 		param->port,
1858 		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[0]),
1859 		be64_to_cpu(*(__be64 *)&sdev->port[param->port - 1].gid.raw[8]));
1860 
1861 	rsp = kzalloc(sizeof(*rsp), GFP_KERNEL);
1862 	rej = kzalloc(sizeof(*rej), GFP_KERNEL);
1863 	rep_param = kzalloc(sizeof(*rep_param), GFP_KERNEL);
1864 
1865 	if (!rsp || !rej || !rep_param) {
1866 		ret = -ENOMEM;
1867 		goto out;
1868 	}
1869 
1870 	if (it_iu_len > srp_max_req_size || it_iu_len < 64) {
1871 		rej->reason = cpu_to_be32(
1872 			      SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE);
1873 		ret = -EINVAL;
1874 		pr_err("rejected SRP_LOGIN_REQ because its"
1875 		       " length (%d bytes) is out of range (%d .. %d)\n",
1876 		       it_iu_len, 64, srp_max_req_size);
1877 		goto reject;
1878 	}
1879 
1880 	if (!sport->enabled) {
1881 		rej->reason = cpu_to_be32(
1882 			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1883 		ret = -EINVAL;
1884 		pr_err("rejected SRP_LOGIN_REQ because the target port"
1885 		       " has not yet been enabled\n");
1886 		goto reject;
1887 	}
1888 
1889 	if ((req->req_flags & SRP_MTCH_ACTION) == SRP_MULTICHAN_SINGLE) {
1890 		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_NO_CHAN;
1891 
1892 		mutex_lock(&sdev->mutex);
1893 
1894 		list_for_each_entry_safe(ch, tmp_ch, &sdev->rch_list, list) {
1895 			if (!memcmp(ch->i_port_id, req->initiator_port_id, 16)
1896 			    && !memcmp(ch->t_port_id, req->target_port_id, 16)
1897 			    && param->port == ch->sport->port
1898 			    && param->listen_id == ch->sport->sdev->cm_id
1899 			    && ch->cm_id) {
1900 				if (srpt_disconnect_ch(ch) < 0)
1901 					continue;
1902 				pr_info("Relogin - closed existing channel %s\n",
1903 					ch->sess_name);
1904 				rsp->rsp_flags =
1905 					SRP_LOGIN_RSP_MULTICHAN_TERMINATED;
1906 			}
1907 		}
1908 
1909 		mutex_unlock(&sdev->mutex);
1910 
1911 	} else
1912 		rsp->rsp_flags = SRP_LOGIN_RSP_MULTICHAN_MAINTAINED;
1913 
1914 	if (*(__be64 *)req->target_port_id != cpu_to_be64(srpt_service_guid)
1915 	    || *(__be64 *)(req->target_port_id + 8) !=
1916 	       cpu_to_be64(srpt_service_guid)) {
1917 		rej->reason = cpu_to_be32(
1918 			      SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL);
1919 		ret = -ENOMEM;
1920 		pr_err("rejected SRP_LOGIN_REQ because it"
1921 		       " has an invalid target port identifier.\n");
1922 		goto reject;
1923 	}
1924 
1925 	ch = kzalloc(sizeof(*ch), GFP_KERNEL);
1926 	if (!ch) {
1927 		rej->reason = cpu_to_be32(
1928 			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1929 		pr_err("rejected SRP_LOGIN_REQ because no memory.\n");
1930 		ret = -ENOMEM;
1931 		goto reject;
1932 	}
1933 
1934 	kref_init(&ch->kref);
1935 	ch->zw_cqe.done = srpt_zerolength_write_done;
1936 	INIT_WORK(&ch->release_work, srpt_release_channel_work);
1937 	memcpy(ch->i_port_id, req->initiator_port_id, 16);
1938 	memcpy(ch->t_port_id, req->target_port_id, 16);
1939 	ch->sport = &sdev->port[param->port - 1];
1940 	ch->cm_id = cm_id;
1941 	cm_id->context = ch;
1942 	/*
1943 	 * Avoid QUEUE_FULL conditions by limiting the number of buffers used
1944 	 * for the SRP protocol to the command queue size.
1945 	 */
1946 	ch->rq_size = SRPT_RQ_SIZE;
1947 	spin_lock_init(&ch->spinlock);
1948 	ch->state = CH_CONNECTING;
1949 	INIT_LIST_HEAD(&ch->cmd_wait_list);
1950 	ch->rsp_size = ch->sport->port_attrib.srp_max_rsp_size;
1951 
1952 	ch->ioctx_ring = (struct srpt_send_ioctx **)
1953 		srpt_alloc_ioctx_ring(ch->sport->sdev, ch->rq_size,
1954 				      sizeof(*ch->ioctx_ring[0]),
1955 				      ch->rsp_size, DMA_TO_DEVICE);
1956 	if (!ch->ioctx_ring)
1957 		goto free_ch;
1958 
1959 	INIT_LIST_HEAD(&ch->free_list);
1960 	for (i = 0; i < ch->rq_size; i++) {
1961 		ch->ioctx_ring[i]->ch = ch;
1962 		list_add_tail(&ch->ioctx_ring[i]->free_list, &ch->free_list);
1963 	}
1964 
1965 	ret = srpt_create_ch_ib(ch);
1966 	if (ret) {
1967 		rej->reason = cpu_to_be32(
1968 			      SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1969 		pr_err("rejected SRP_LOGIN_REQ because creating"
1970 		       " a new RDMA channel failed.\n");
1971 		goto free_ring;
1972 	}
1973 
1974 	ret = srpt_ch_qp_rtr(ch, ch->qp);
1975 	if (ret) {
1976 		rej->reason = cpu_to_be32(SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES);
1977 		pr_err("rejected SRP_LOGIN_REQ because enabling"
1978 		       " RTR failed (error code = %d)\n", ret);
1979 		goto destroy_ib;
1980 	}
1981 
1982 	/*
1983 	 * Use the initator port identifier as the session name, when
1984 	 * checking against se_node_acl->initiatorname[] this can be
1985 	 * with or without preceeding '0x'.
1986 	 */
1987 	snprintf(ch->sess_name, sizeof(ch->sess_name), "0x%016llx%016llx",
1988 			be64_to_cpu(*(__be64 *)ch->i_port_id),
1989 			be64_to_cpu(*(__be64 *)(ch->i_port_id + 8)));
1990 
1991 	pr_debug("registering session %s\n", ch->sess_name);
1992 	p = &ch->sess_name[0];
1993 
1994 try_again:
1995 	ch->sess = target_alloc_session(&sport->port_tpg_1, 0, 0,
1996 					TARGET_PROT_NORMAL, p, ch, NULL);
1997 	if (IS_ERR(ch->sess)) {
1998 		pr_info("Rejected login because no ACL has been"
1999 			" configured yet for initiator %s.\n", p);
2000 		/*
2001 		 * XXX: Hack to retry of ch->i_port_id without leading '0x'
2002 		 */
2003 		if (p == &ch->sess_name[0]) {
2004 			p += 2;
2005 			goto try_again;
2006 		}
2007 		rej->reason = cpu_to_be32((PTR_ERR(ch->sess) == -ENOMEM) ?
2008 				SRP_LOGIN_REJ_INSUFFICIENT_RESOURCES :
2009 				SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED);
2010 		goto destroy_ib;
2011 	}
2012 
2013 	pr_debug("Establish connection sess=%p name=%s cm_id=%p\n", ch->sess,
2014 		 ch->sess_name, ch->cm_id);
2015 
2016 	/* create srp_login_response */
2017 	rsp->opcode = SRP_LOGIN_RSP;
2018 	rsp->tag = req->tag;
2019 	rsp->max_it_iu_len = req->req_it_iu_len;
2020 	rsp->max_ti_iu_len = req->req_it_iu_len;
2021 	ch->max_ti_iu_len = it_iu_len;
2022 	rsp->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2023 				   | SRP_BUF_FORMAT_INDIRECT);
2024 	rsp->req_lim_delta = cpu_to_be32(ch->rq_size);
2025 	atomic_set(&ch->req_lim, ch->rq_size);
2026 	atomic_set(&ch->req_lim_delta, 0);
2027 
2028 	/* create cm reply */
2029 	rep_param->qp_num = ch->qp->qp_num;
2030 	rep_param->private_data = (void *)rsp;
2031 	rep_param->private_data_len = sizeof(*rsp);
2032 	rep_param->rnr_retry_count = 7;
2033 	rep_param->flow_control = 1;
2034 	rep_param->failover_accepted = 0;
2035 	rep_param->srq = 1;
2036 	rep_param->responder_resources = 4;
2037 	rep_param->initiator_depth = 4;
2038 
2039 	ret = ib_send_cm_rep(cm_id, rep_param);
2040 	if (ret) {
2041 		pr_err("sending SRP_LOGIN_REQ response failed"
2042 		       " (error code = %d)\n", ret);
2043 		goto release_channel;
2044 	}
2045 
2046 	mutex_lock(&sdev->mutex);
2047 	list_add_tail(&ch->list, &sdev->rch_list);
2048 	mutex_unlock(&sdev->mutex);
2049 
2050 	goto out;
2051 
2052 release_channel:
2053 	srpt_disconnect_ch(ch);
2054 	transport_deregister_session_configfs(ch->sess);
2055 	transport_deregister_session(ch->sess);
2056 	ch->sess = NULL;
2057 
2058 destroy_ib:
2059 	srpt_destroy_ch_ib(ch);
2060 
2061 free_ring:
2062 	srpt_free_ioctx_ring((struct srpt_ioctx **)ch->ioctx_ring,
2063 			     ch->sport->sdev, ch->rq_size,
2064 			     ch->rsp_size, DMA_TO_DEVICE);
2065 free_ch:
2066 	kfree(ch);
2067 
2068 reject:
2069 	rej->opcode = SRP_LOGIN_REJ;
2070 	rej->tag = req->tag;
2071 	rej->buf_fmt = cpu_to_be16(SRP_BUF_FORMAT_DIRECT
2072 				   | SRP_BUF_FORMAT_INDIRECT);
2073 
2074 	ib_send_cm_rej(cm_id, IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
2075 			     (void *)rej, sizeof(*rej));
2076 
2077 out:
2078 	kfree(rep_param);
2079 	kfree(rsp);
2080 	kfree(rej);
2081 
2082 	return ret;
2083 }
2084 
2085 static void srpt_cm_rej_recv(struct srpt_rdma_ch *ch,
2086 			     enum ib_cm_rej_reason reason,
2087 			     const u8 *private_data,
2088 			     u8 private_data_len)
2089 {
2090 	char *priv = NULL;
2091 	int i;
2092 
2093 	if (private_data_len && (priv = kmalloc(private_data_len * 3 + 1,
2094 						GFP_KERNEL))) {
2095 		for (i = 0; i < private_data_len; i++)
2096 			sprintf(priv + 3 * i, " %02x", private_data[i]);
2097 	}
2098 	pr_info("Received CM REJ for ch %s-%d; reason %d%s%s.\n",
2099 		ch->sess_name, ch->qp->qp_num, reason, private_data_len ?
2100 		"; private data" : "", priv ? priv : " (?)");
2101 	kfree(priv);
2102 }
2103 
2104 /**
2105  * srpt_cm_rtu_recv() - Process an IB_CM_RTU_RECEIVED or USER_ESTABLISHED event.
2106  *
2107  * An IB_CM_RTU_RECEIVED message indicates that the connection is established
2108  * and that the recipient may begin transmitting (RTU = ready to use).
2109  */
2110 static void srpt_cm_rtu_recv(struct srpt_rdma_ch *ch)
2111 {
2112 	int ret;
2113 
2114 	if (srpt_set_ch_state(ch, CH_LIVE)) {
2115 		ret = srpt_ch_qp_rts(ch, ch->qp);
2116 
2117 		if (ret == 0) {
2118 			/* Trigger wait list processing. */
2119 			ret = srpt_zerolength_write(ch);
2120 			WARN_ONCE(ret < 0, "%d\n", ret);
2121 		} else {
2122 			srpt_close_ch(ch);
2123 		}
2124 	}
2125 }
2126 
2127 /**
2128  * srpt_cm_handler() - IB connection manager callback function.
2129  *
2130  * A non-zero return value will cause the caller destroy the CM ID.
2131  *
2132  * Note: srpt_cm_handler() must only return a non-zero value when transferring
2133  * ownership of the cm_id to a channel by srpt_cm_req_recv() failed. Returning
2134  * a non-zero value in any other case will trigger a race with the
2135  * ib_destroy_cm_id() call in srpt_release_channel().
2136  */
2137 static int srpt_cm_handler(struct ib_cm_id *cm_id, struct ib_cm_event *event)
2138 {
2139 	struct srpt_rdma_ch *ch = cm_id->context;
2140 	int ret;
2141 
2142 	ret = 0;
2143 	switch (event->event) {
2144 	case IB_CM_REQ_RECEIVED:
2145 		ret = srpt_cm_req_recv(cm_id, &event->param.req_rcvd,
2146 				       event->private_data);
2147 		break;
2148 	case IB_CM_REJ_RECEIVED:
2149 		srpt_cm_rej_recv(ch, event->param.rej_rcvd.reason,
2150 				 event->private_data,
2151 				 IB_CM_REJ_PRIVATE_DATA_SIZE);
2152 		break;
2153 	case IB_CM_RTU_RECEIVED:
2154 	case IB_CM_USER_ESTABLISHED:
2155 		srpt_cm_rtu_recv(ch);
2156 		break;
2157 	case IB_CM_DREQ_RECEIVED:
2158 		srpt_disconnect_ch(ch);
2159 		break;
2160 	case IB_CM_DREP_RECEIVED:
2161 		pr_info("Received CM DREP message for ch %s-%d.\n",
2162 			ch->sess_name, ch->qp->qp_num);
2163 		srpt_close_ch(ch);
2164 		break;
2165 	case IB_CM_TIMEWAIT_EXIT:
2166 		pr_info("Received CM TimeWait exit for ch %s-%d.\n",
2167 			ch->sess_name, ch->qp->qp_num);
2168 		srpt_close_ch(ch);
2169 		break;
2170 	case IB_CM_REP_ERROR:
2171 		pr_info("Received CM REP error for ch %s-%d.\n", ch->sess_name,
2172 			ch->qp->qp_num);
2173 		break;
2174 	case IB_CM_DREQ_ERROR:
2175 		pr_info("Received CM DREQ ERROR event.\n");
2176 		break;
2177 	case IB_CM_MRA_RECEIVED:
2178 		pr_info("Received CM MRA event\n");
2179 		break;
2180 	default:
2181 		pr_err("received unrecognized CM event %d\n", event->event);
2182 		break;
2183 	}
2184 
2185 	return ret;
2186 }
2187 
2188 static int srpt_write_pending_status(struct se_cmd *se_cmd)
2189 {
2190 	struct srpt_send_ioctx *ioctx;
2191 
2192 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2193 	return srpt_get_cmd_state(ioctx) == SRPT_STATE_NEED_DATA;
2194 }
2195 
2196 /*
2197  * srpt_write_pending() - Start data transfer from initiator to target (write).
2198  */
2199 static int srpt_write_pending(struct se_cmd *se_cmd)
2200 {
2201 	struct srpt_send_ioctx *ioctx =
2202 		container_of(se_cmd, struct srpt_send_ioctx, cmd);
2203 	struct srpt_rdma_ch *ch = ioctx->ch;
2204 	struct ib_send_wr *first_wr = NULL, *bad_wr;
2205 	struct ib_cqe *cqe = &ioctx->rdma_cqe;
2206 	enum srpt_command_state new_state;
2207 	int ret, i;
2208 
2209 	new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);
2210 	WARN_ON(new_state == SRPT_STATE_DONE);
2211 
2212 	if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) {
2213 		pr_warn("%s: IB send queue full (needed %d)\n",
2214 				__func__, ioctx->n_rdma);
2215 		ret = -ENOMEM;
2216 		goto out_undo;
2217 	}
2218 
2219 	cqe->done = srpt_rdma_read_done;
2220 	for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2221 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2222 
2223 		first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp, ch->sport->port,
2224 				cqe, first_wr);
2225 		cqe = NULL;
2226 	}
2227 
2228 	ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2229 	if (ret) {
2230 		pr_err("%s: ib_post_send() returned %d for %d (avail: %d)\n",
2231 			 __func__, ret, ioctx->n_rdma,
2232 			 atomic_read(&ch->sq_wr_avail));
2233 		goto out_undo;
2234 	}
2235 
2236 	return 0;
2237 out_undo:
2238 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);
2239 	return ret;
2240 }
2241 
2242 static u8 tcm_to_srp_tsk_mgmt_status(const int tcm_mgmt_status)
2243 {
2244 	switch (tcm_mgmt_status) {
2245 	case TMR_FUNCTION_COMPLETE:
2246 		return SRP_TSK_MGMT_SUCCESS;
2247 	case TMR_FUNCTION_REJECTED:
2248 		return SRP_TSK_MGMT_FUNC_NOT_SUPP;
2249 	}
2250 	return SRP_TSK_MGMT_FAILED;
2251 }
2252 
2253 /**
2254  * srpt_queue_response() - Transmits the response to a SCSI command.
2255  *
2256  * Callback function called by the TCM core. Must not block since it can be
2257  * invoked on the context of the IB completion handler.
2258  */
2259 static void srpt_queue_response(struct se_cmd *cmd)
2260 {
2261 	struct srpt_send_ioctx *ioctx =
2262 		container_of(cmd, struct srpt_send_ioctx, cmd);
2263 	struct srpt_rdma_ch *ch = ioctx->ch;
2264 	struct srpt_device *sdev = ch->sport->sdev;
2265 	struct ib_send_wr send_wr, *first_wr = &send_wr, *bad_wr;
2266 	struct ib_sge sge;
2267 	enum srpt_command_state state;
2268 	unsigned long flags;
2269 	int resp_len, ret, i;
2270 	u8 srp_tm_status;
2271 
2272 	BUG_ON(!ch);
2273 
2274 	spin_lock_irqsave(&ioctx->spinlock, flags);
2275 	state = ioctx->state;
2276 	switch (state) {
2277 	case SRPT_STATE_NEW:
2278 	case SRPT_STATE_DATA_IN:
2279 		ioctx->state = SRPT_STATE_CMD_RSP_SENT;
2280 		break;
2281 	case SRPT_STATE_MGMT:
2282 		ioctx->state = SRPT_STATE_MGMT_RSP_SENT;
2283 		break;
2284 	default:
2285 		WARN(true, "ch %p; cmd %d: unexpected command state %d\n",
2286 			ch, ioctx->ioctx.index, ioctx->state);
2287 		break;
2288 	}
2289 	spin_unlock_irqrestore(&ioctx->spinlock, flags);
2290 
2291 	if (unlikely(transport_check_aborted_status(&ioctx->cmd, false)
2292 		     || WARN_ON_ONCE(state == SRPT_STATE_CMD_RSP_SENT))) {
2293 		atomic_inc(&ch->req_lim_delta);
2294 		srpt_abort_cmd(ioctx);
2295 		return;
2296 	}
2297 
2298 	/* For read commands, transfer the data to the initiator. */
2299 	if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&
2300 	    ioctx->cmd.data_length &&
2301 	    !ioctx->queue_status_only) {
2302 		for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {
2303 			struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];
2304 
2305 			first_wr = rdma_rw_ctx_wrs(&ctx->rw, ch->qp,
2306 					ch->sport->port, NULL, first_wr);
2307 		}
2308 	}
2309 
2310 	if (state != SRPT_STATE_MGMT)
2311 		resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,
2312 					      cmd->scsi_status);
2313 	else {
2314 		srp_tm_status
2315 			= tcm_to_srp_tsk_mgmt_status(cmd->se_tmr_req->response);
2316 		resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,
2317 						 ioctx->cmd.tag);
2318 	}
2319 
2320 	atomic_inc(&ch->req_lim);
2321 
2322 	if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,
2323 			&ch->sq_wr_avail) < 0)) {
2324 		pr_warn("%s: IB send queue full (needed %d)\n",
2325 				__func__, ioctx->n_rdma);
2326 		ret = -ENOMEM;
2327 		goto out;
2328 	}
2329 
2330 	ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, resp_len,
2331 				      DMA_TO_DEVICE);
2332 
2333 	sge.addr = ioctx->ioctx.dma;
2334 	sge.length = resp_len;
2335 	sge.lkey = sdev->pd->local_dma_lkey;
2336 
2337 	ioctx->ioctx.cqe.done = srpt_send_done;
2338 	send_wr.next = NULL;
2339 	send_wr.wr_cqe = &ioctx->ioctx.cqe;
2340 	send_wr.sg_list = &sge;
2341 	send_wr.num_sge = 1;
2342 	send_wr.opcode = IB_WR_SEND;
2343 	send_wr.send_flags = IB_SEND_SIGNALED;
2344 
2345 	ret = ib_post_send(ch->qp, first_wr, &bad_wr);
2346 	if (ret < 0) {
2347 		pr_err("%s: sending cmd response failed for tag %llu (%d)\n",
2348 			__func__, ioctx->cmd.tag, ret);
2349 		goto out;
2350 	}
2351 
2352 	return;
2353 
2354 out:
2355 	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);
2356 	atomic_dec(&ch->req_lim);
2357 	srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);
2358 	target_put_sess_cmd(&ioctx->cmd);
2359 }
2360 
2361 static int srpt_queue_data_in(struct se_cmd *cmd)
2362 {
2363 	srpt_queue_response(cmd);
2364 	return 0;
2365 }
2366 
2367 static void srpt_queue_tm_rsp(struct se_cmd *cmd)
2368 {
2369 	srpt_queue_response(cmd);
2370 }
2371 
2372 static void srpt_aborted_task(struct se_cmd *cmd)
2373 {
2374 }
2375 
2376 static int srpt_queue_status(struct se_cmd *cmd)
2377 {
2378 	struct srpt_send_ioctx *ioctx;
2379 
2380 	ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);
2381 	BUG_ON(ioctx->sense_data != cmd->sense_buffer);
2382 	if (cmd->se_cmd_flags &
2383 	    (SCF_TRANSPORT_TASK_SENSE | SCF_EMULATED_TASK_SENSE))
2384 		WARN_ON(cmd->scsi_status != SAM_STAT_CHECK_CONDITION);
2385 	ioctx->queue_status_only = true;
2386 	srpt_queue_response(cmd);
2387 	return 0;
2388 }
2389 
2390 static void srpt_refresh_port_work(struct work_struct *work)
2391 {
2392 	struct srpt_port *sport = container_of(work, struct srpt_port, work);
2393 
2394 	srpt_refresh_port(sport);
2395 }
2396 
2397 /**
2398  * srpt_release_sdev() - Free the channel resources associated with a target.
2399  */
2400 static int srpt_release_sdev(struct srpt_device *sdev)
2401 {
2402 	int i, res;
2403 
2404 	WARN_ON_ONCE(irqs_disabled());
2405 
2406 	BUG_ON(!sdev);
2407 
2408 	mutex_lock(&sdev->mutex);
2409 	for (i = 0; i < ARRAY_SIZE(sdev->port); i++)
2410 		sdev->port[i].enabled = false;
2411 	__srpt_close_all_ch(sdev);
2412 	mutex_unlock(&sdev->mutex);
2413 
2414 	res = wait_event_interruptible(sdev->ch_releaseQ,
2415 				       list_empty_careful(&sdev->rch_list));
2416 	if (res)
2417 		pr_err("%s: interrupted.\n", __func__);
2418 
2419 	return 0;
2420 }
2421 
2422 static struct srpt_port *__srpt_lookup_port(const char *name)
2423 {
2424 	struct ib_device *dev;
2425 	struct srpt_device *sdev;
2426 	struct srpt_port *sport;
2427 	int i;
2428 
2429 	list_for_each_entry(sdev, &srpt_dev_list, list) {
2430 		dev = sdev->device;
2431 		if (!dev)
2432 			continue;
2433 
2434 		for (i = 0; i < dev->phys_port_cnt; i++) {
2435 			sport = &sdev->port[i];
2436 
2437 			if (!strcmp(sport->port_guid, name))
2438 				return sport;
2439 		}
2440 	}
2441 
2442 	return NULL;
2443 }
2444 
2445 static struct srpt_port *srpt_lookup_port(const char *name)
2446 {
2447 	struct srpt_port *sport;
2448 
2449 	spin_lock(&srpt_dev_lock);
2450 	sport = __srpt_lookup_port(name);
2451 	spin_unlock(&srpt_dev_lock);
2452 
2453 	return sport;
2454 }
2455 
2456 /**
2457  * srpt_add_one() - Infiniband device addition callback function.
2458  */
2459 static void srpt_add_one(struct ib_device *device)
2460 {
2461 	struct srpt_device *sdev;
2462 	struct srpt_port *sport;
2463 	struct ib_srq_init_attr srq_attr;
2464 	int i;
2465 
2466 	pr_debug("device = %p, device->dma_ops = %p\n", device,
2467 		 device->dma_ops);
2468 
2469 	sdev = kzalloc(sizeof(*sdev), GFP_KERNEL);
2470 	if (!sdev)
2471 		goto err;
2472 
2473 	sdev->device = device;
2474 	INIT_LIST_HEAD(&sdev->rch_list);
2475 	init_waitqueue_head(&sdev->ch_releaseQ);
2476 	mutex_init(&sdev->mutex);
2477 
2478 	sdev->pd = ib_alloc_pd(device);
2479 	if (IS_ERR(sdev->pd))
2480 		goto free_dev;
2481 
2482 	sdev->srq_size = min(srpt_srq_size, sdev->device->attrs.max_srq_wr);
2483 
2484 	srq_attr.event_handler = srpt_srq_event;
2485 	srq_attr.srq_context = (void *)sdev;
2486 	srq_attr.attr.max_wr = sdev->srq_size;
2487 	srq_attr.attr.max_sge = 1;
2488 	srq_attr.attr.srq_limit = 0;
2489 	srq_attr.srq_type = IB_SRQT_BASIC;
2490 
2491 	sdev->srq = ib_create_srq(sdev->pd, &srq_attr);
2492 	if (IS_ERR(sdev->srq))
2493 		goto err_pd;
2494 
2495 	pr_debug("%s: create SRQ #wr= %d max_allow=%d dev= %s\n",
2496 		 __func__, sdev->srq_size, sdev->device->attrs.max_srq_wr,
2497 		 device->name);
2498 
2499 	if (!srpt_service_guid)
2500 		srpt_service_guid = be64_to_cpu(device->node_guid);
2501 
2502 	sdev->cm_id = ib_create_cm_id(device, srpt_cm_handler, sdev);
2503 	if (IS_ERR(sdev->cm_id))
2504 		goto err_srq;
2505 
2506 	/* print out target login information */
2507 	pr_debug("Target login info: id_ext=%016llx,ioc_guid=%016llx,"
2508 		 "pkey=ffff,service_id=%016llx\n", srpt_service_guid,
2509 		 srpt_service_guid, srpt_service_guid);
2510 
2511 	/*
2512 	 * We do not have a consistent service_id (ie. also id_ext of target_id)
2513 	 * to identify this target. We currently use the guid of the first HCA
2514 	 * in the system as service_id; therefore, the target_id will change
2515 	 * if this HCA is gone bad and replaced by different HCA
2516 	 */
2517 	if (ib_cm_listen(sdev->cm_id, cpu_to_be64(srpt_service_guid), 0))
2518 		goto err_cm;
2519 
2520 	INIT_IB_EVENT_HANDLER(&sdev->event_handler, sdev->device,
2521 			      srpt_event_handler);
2522 	if (ib_register_event_handler(&sdev->event_handler))
2523 		goto err_cm;
2524 
2525 	sdev->ioctx_ring = (struct srpt_recv_ioctx **)
2526 		srpt_alloc_ioctx_ring(sdev, sdev->srq_size,
2527 				      sizeof(*sdev->ioctx_ring[0]),
2528 				      srp_max_req_size, DMA_FROM_DEVICE);
2529 	if (!sdev->ioctx_ring)
2530 		goto err_event;
2531 
2532 	for (i = 0; i < sdev->srq_size; ++i)
2533 		srpt_post_recv(sdev, sdev->ioctx_ring[i]);
2534 
2535 	WARN_ON(sdev->device->phys_port_cnt > ARRAY_SIZE(sdev->port));
2536 
2537 	for (i = 1; i <= sdev->device->phys_port_cnt; i++) {
2538 		sport = &sdev->port[i - 1];
2539 		sport->sdev = sdev;
2540 		sport->port = i;
2541 		sport->port_attrib.srp_max_rdma_size = DEFAULT_MAX_RDMA_SIZE;
2542 		sport->port_attrib.srp_max_rsp_size = DEFAULT_MAX_RSP_SIZE;
2543 		sport->port_attrib.srp_sq_size = DEF_SRPT_SQ_SIZE;
2544 		INIT_WORK(&sport->work, srpt_refresh_port_work);
2545 
2546 		if (srpt_refresh_port(sport)) {
2547 			pr_err("MAD registration failed for %s-%d.\n",
2548 			       sdev->device->name, i);
2549 			goto err_ring;
2550 		}
2551 		snprintf(sport->port_guid, sizeof(sport->port_guid),
2552 			"0x%016llx%016llx",
2553 			be64_to_cpu(sport->gid.global.subnet_prefix),
2554 			be64_to_cpu(sport->gid.global.interface_id));
2555 	}
2556 
2557 	spin_lock(&srpt_dev_lock);
2558 	list_add_tail(&sdev->list, &srpt_dev_list);
2559 	spin_unlock(&srpt_dev_lock);
2560 
2561 out:
2562 	ib_set_client_data(device, &srpt_client, sdev);
2563 	pr_debug("added %s.\n", device->name);
2564 	return;
2565 
2566 err_ring:
2567 	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2568 			     sdev->srq_size, srp_max_req_size,
2569 			     DMA_FROM_DEVICE);
2570 err_event:
2571 	ib_unregister_event_handler(&sdev->event_handler);
2572 err_cm:
2573 	ib_destroy_cm_id(sdev->cm_id);
2574 err_srq:
2575 	ib_destroy_srq(sdev->srq);
2576 err_pd:
2577 	ib_dealloc_pd(sdev->pd);
2578 free_dev:
2579 	kfree(sdev);
2580 err:
2581 	sdev = NULL;
2582 	pr_info("%s(%s) failed.\n", __func__, device->name);
2583 	goto out;
2584 }
2585 
2586 /**
2587  * srpt_remove_one() - InfiniBand device removal callback function.
2588  */
2589 static void srpt_remove_one(struct ib_device *device, void *client_data)
2590 {
2591 	struct srpt_device *sdev = client_data;
2592 	int i;
2593 
2594 	if (!sdev) {
2595 		pr_info("%s(%s): nothing to do.\n", __func__, device->name);
2596 		return;
2597 	}
2598 
2599 	srpt_unregister_mad_agent(sdev);
2600 
2601 	ib_unregister_event_handler(&sdev->event_handler);
2602 
2603 	/* Cancel any work queued by the just unregistered IB event handler. */
2604 	for (i = 0; i < sdev->device->phys_port_cnt; i++)
2605 		cancel_work_sync(&sdev->port[i].work);
2606 
2607 	ib_destroy_cm_id(sdev->cm_id);
2608 
2609 	/*
2610 	 * Unregistering a target must happen after destroying sdev->cm_id
2611 	 * such that no new SRP_LOGIN_REQ information units can arrive while
2612 	 * destroying the target.
2613 	 */
2614 	spin_lock(&srpt_dev_lock);
2615 	list_del(&sdev->list);
2616 	spin_unlock(&srpt_dev_lock);
2617 	srpt_release_sdev(sdev);
2618 
2619 	ib_destroy_srq(sdev->srq);
2620 	ib_dealloc_pd(sdev->pd);
2621 
2622 	srpt_free_ioctx_ring((struct srpt_ioctx **)sdev->ioctx_ring, sdev,
2623 			     sdev->srq_size, srp_max_req_size, DMA_FROM_DEVICE);
2624 	sdev->ioctx_ring = NULL;
2625 	kfree(sdev);
2626 }
2627 
2628 static struct ib_client srpt_client = {
2629 	.name = DRV_NAME,
2630 	.add = srpt_add_one,
2631 	.remove = srpt_remove_one
2632 };
2633 
2634 static int srpt_check_true(struct se_portal_group *se_tpg)
2635 {
2636 	return 1;
2637 }
2638 
2639 static int srpt_check_false(struct se_portal_group *se_tpg)
2640 {
2641 	return 0;
2642 }
2643 
2644 static char *srpt_get_fabric_name(void)
2645 {
2646 	return "srpt";
2647 }
2648 
2649 static char *srpt_get_fabric_wwn(struct se_portal_group *tpg)
2650 {
2651 	struct srpt_port *sport = container_of(tpg, struct srpt_port, port_tpg_1);
2652 
2653 	return sport->port_guid;
2654 }
2655 
2656 static u16 srpt_get_tag(struct se_portal_group *tpg)
2657 {
2658 	return 1;
2659 }
2660 
2661 static u32 srpt_tpg_get_inst_index(struct se_portal_group *se_tpg)
2662 {
2663 	return 1;
2664 }
2665 
2666 static void srpt_release_cmd(struct se_cmd *se_cmd)
2667 {
2668 	struct srpt_send_ioctx *ioctx = container_of(se_cmd,
2669 				struct srpt_send_ioctx, cmd);
2670 	struct srpt_rdma_ch *ch = ioctx->ch;
2671 	unsigned long flags;
2672 
2673 	WARN_ON(ioctx->state != SRPT_STATE_DONE);
2674 
2675 	if (ioctx->n_rw_ctx) {
2676 		srpt_free_rw_ctxs(ch, ioctx);
2677 		ioctx->n_rw_ctx = 0;
2678 	}
2679 
2680 	spin_lock_irqsave(&ch->spinlock, flags);
2681 	list_add(&ioctx->free_list, &ch->free_list);
2682 	spin_unlock_irqrestore(&ch->spinlock, flags);
2683 }
2684 
2685 /**
2686  * srpt_close_session() - Forcibly close a session.
2687  *
2688  * Callback function invoked by the TCM core to clean up sessions associated
2689  * with a node ACL when the user invokes
2690  * rmdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2691  */
2692 static void srpt_close_session(struct se_session *se_sess)
2693 {
2694 	DECLARE_COMPLETION_ONSTACK(release_done);
2695 	struct srpt_rdma_ch *ch = se_sess->fabric_sess_ptr;
2696 	struct srpt_device *sdev = ch->sport->sdev;
2697 	bool wait;
2698 
2699 	pr_debug("ch %s-%d state %d\n", ch->sess_name, ch->qp->qp_num,
2700 		 ch->state);
2701 
2702 	mutex_lock(&sdev->mutex);
2703 	BUG_ON(ch->release_done);
2704 	ch->release_done = &release_done;
2705 	wait = !list_empty(&ch->list);
2706 	srpt_disconnect_ch(ch);
2707 	mutex_unlock(&sdev->mutex);
2708 
2709 	if (!wait)
2710 		return;
2711 
2712 	while (wait_for_completion_timeout(&release_done, 180 * HZ) == 0)
2713 		pr_info("%s(%s-%d state %d): still waiting ...\n", __func__,
2714 			ch->sess_name, ch->qp->qp_num, ch->state);
2715 }
2716 
2717 /**
2718  * srpt_sess_get_index() - Return the value of scsiAttIntrPortIndex (SCSI-MIB).
2719  *
2720  * A quote from RFC 4455 (SCSI-MIB) about this MIB object:
2721  * This object represents an arbitrary integer used to uniquely identify a
2722  * particular attached remote initiator port to a particular SCSI target port
2723  * within a particular SCSI target device within a particular SCSI instance.
2724  */
2725 static u32 srpt_sess_get_index(struct se_session *se_sess)
2726 {
2727 	return 0;
2728 }
2729 
2730 static void srpt_set_default_node_attrs(struct se_node_acl *nacl)
2731 {
2732 }
2733 
2734 /* Note: only used from inside debug printk's by the TCM core. */
2735 static int srpt_get_tcm_cmd_state(struct se_cmd *se_cmd)
2736 {
2737 	struct srpt_send_ioctx *ioctx;
2738 
2739 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);
2740 	return srpt_get_cmd_state(ioctx);
2741 }
2742 
2743 /**
2744  * srpt_parse_i_port_id() - Parse an initiator port ID.
2745  * @name: ASCII representation of a 128-bit initiator port ID.
2746  * @i_port_id: Binary 128-bit port ID.
2747  */
2748 static int srpt_parse_i_port_id(u8 i_port_id[16], const char *name)
2749 {
2750 	const char *p;
2751 	unsigned len, count, leading_zero_bytes;
2752 	int ret, rc;
2753 
2754 	p = name;
2755 	if (strncasecmp(p, "0x", 2) == 0)
2756 		p += 2;
2757 	ret = -EINVAL;
2758 	len = strlen(p);
2759 	if (len % 2)
2760 		goto out;
2761 	count = min(len / 2, 16U);
2762 	leading_zero_bytes = 16 - count;
2763 	memset(i_port_id, 0, leading_zero_bytes);
2764 	rc = hex2bin(i_port_id + leading_zero_bytes, p, count);
2765 	if (rc < 0)
2766 		pr_debug("hex2bin failed for srpt_parse_i_port_id: %d\n", rc);
2767 	ret = 0;
2768 out:
2769 	return ret;
2770 }
2771 
2772 /*
2773  * configfs callback function invoked for
2774  * mkdir /sys/kernel/config/target/$driver/$port/$tpg/acls/$i_port_id
2775  */
2776 static int srpt_init_nodeacl(struct se_node_acl *se_nacl, const char *name)
2777 {
2778 	u8 i_port_id[16];
2779 
2780 	if (srpt_parse_i_port_id(i_port_id, name) < 0) {
2781 		pr_err("invalid initiator port ID %s\n", name);
2782 		return -EINVAL;
2783 	}
2784 	return 0;
2785 }
2786 
2787 static ssize_t srpt_tpg_attrib_srp_max_rdma_size_show(struct config_item *item,
2788 		char *page)
2789 {
2790 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2791 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2792 
2793 	return sprintf(page, "%u\n", sport->port_attrib.srp_max_rdma_size);
2794 }
2795 
2796 static ssize_t srpt_tpg_attrib_srp_max_rdma_size_store(struct config_item *item,
2797 		const char *page, size_t count)
2798 {
2799 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2800 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2801 	unsigned long val;
2802 	int ret;
2803 
2804 	ret = kstrtoul(page, 0, &val);
2805 	if (ret < 0) {
2806 		pr_err("kstrtoul() failed with ret: %d\n", ret);
2807 		return -EINVAL;
2808 	}
2809 	if (val > MAX_SRPT_RDMA_SIZE) {
2810 		pr_err("val: %lu exceeds MAX_SRPT_RDMA_SIZE: %d\n", val,
2811 			MAX_SRPT_RDMA_SIZE);
2812 		return -EINVAL;
2813 	}
2814 	if (val < DEFAULT_MAX_RDMA_SIZE) {
2815 		pr_err("val: %lu smaller than DEFAULT_MAX_RDMA_SIZE: %d\n",
2816 			val, DEFAULT_MAX_RDMA_SIZE);
2817 		return -EINVAL;
2818 	}
2819 	sport->port_attrib.srp_max_rdma_size = val;
2820 
2821 	return count;
2822 }
2823 
2824 static ssize_t srpt_tpg_attrib_srp_max_rsp_size_show(struct config_item *item,
2825 		char *page)
2826 {
2827 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2828 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2829 
2830 	return sprintf(page, "%u\n", sport->port_attrib.srp_max_rsp_size);
2831 }
2832 
2833 static ssize_t srpt_tpg_attrib_srp_max_rsp_size_store(struct config_item *item,
2834 		const char *page, size_t count)
2835 {
2836 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2837 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2838 	unsigned long val;
2839 	int ret;
2840 
2841 	ret = kstrtoul(page, 0, &val);
2842 	if (ret < 0) {
2843 		pr_err("kstrtoul() failed with ret: %d\n", ret);
2844 		return -EINVAL;
2845 	}
2846 	if (val > MAX_SRPT_RSP_SIZE) {
2847 		pr_err("val: %lu exceeds MAX_SRPT_RSP_SIZE: %d\n", val,
2848 			MAX_SRPT_RSP_SIZE);
2849 		return -EINVAL;
2850 	}
2851 	if (val < MIN_MAX_RSP_SIZE) {
2852 		pr_err("val: %lu smaller than MIN_MAX_RSP_SIZE: %d\n", val,
2853 			MIN_MAX_RSP_SIZE);
2854 		return -EINVAL;
2855 	}
2856 	sport->port_attrib.srp_max_rsp_size = val;
2857 
2858 	return count;
2859 }
2860 
2861 static ssize_t srpt_tpg_attrib_srp_sq_size_show(struct config_item *item,
2862 		char *page)
2863 {
2864 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2865 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2866 
2867 	return sprintf(page, "%u\n", sport->port_attrib.srp_sq_size);
2868 }
2869 
2870 static ssize_t srpt_tpg_attrib_srp_sq_size_store(struct config_item *item,
2871 		const char *page, size_t count)
2872 {
2873 	struct se_portal_group *se_tpg = attrib_to_tpg(item);
2874 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2875 	unsigned long val;
2876 	int ret;
2877 
2878 	ret = kstrtoul(page, 0, &val);
2879 	if (ret < 0) {
2880 		pr_err("kstrtoul() failed with ret: %d\n", ret);
2881 		return -EINVAL;
2882 	}
2883 	if (val > MAX_SRPT_SRQ_SIZE) {
2884 		pr_err("val: %lu exceeds MAX_SRPT_SRQ_SIZE: %d\n", val,
2885 			MAX_SRPT_SRQ_SIZE);
2886 		return -EINVAL;
2887 	}
2888 	if (val < MIN_SRPT_SRQ_SIZE) {
2889 		pr_err("val: %lu smaller than MIN_SRPT_SRQ_SIZE: %d\n", val,
2890 			MIN_SRPT_SRQ_SIZE);
2891 		return -EINVAL;
2892 	}
2893 	sport->port_attrib.srp_sq_size = val;
2894 
2895 	return count;
2896 }
2897 
2898 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_max_rdma_size);
2899 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_max_rsp_size);
2900 CONFIGFS_ATTR(srpt_tpg_attrib_,  srp_sq_size);
2901 
2902 static struct configfs_attribute *srpt_tpg_attrib_attrs[] = {
2903 	&srpt_tpg_attrib_attr_srp_max_rdma_size,
2904 	&srpt_tpg_attrib_attr_srp_max_rsp_size,
2905 	&srpt_tpg_attrib_attr_srp_sq_size,
2906 	NULL,
2907 };
2908 
2909 static ssize_t srpt_tpg_enable_show(struct config_item *item, char *page)
2910 {
2911 	struct se_portal_group *se_tpg = to_tpg(item);
2912 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2913 
2914 	return snprintf(page, PAGE_SIZE, "%d\n", (sport->enabled) ? 1: 0);
2915 }
2916 
2917 static ssize_t srpt_tpg_enable_store(struct config_item *item,
2918 		const char *page, size_t count)
2919 {
2920 	struct se_portal_group *se_tpg = to_tpg(item);
2921 	struct srpt_port *sport = container_of(se_tpg, struct srpt_port, port_tpg_1);
2922 	struct srpt_device *sdev = sport->sdev;
2923 	struct srpt_rdma_ch *ch;
2924 	unsigned long tmp;
2925         int ret;
2926 
2927 	ret = kstrtoul(page, 0, &tmp);
2928 	if (ret < 0) {
2929 		pr_err("Unable to extract srpt_tpg_store_enable\n");
2930 		return -EINVAL;
2931 	}
2932 
2933 	if ((tmp != 0) && (tmp != 1)) {
2934 		pr_err("Illegal value for srpt_tpg_store_enable: %lu\n", tmp);
2935 		return -EINVAL;
2936 	}
2937 	if (sport->enabled == tmp)
2938 		goto out;
2939 	sport->enabled = tmp;
2940 	if (sport->enabled)
2941 		goto out;
2942 
2943 	mutex_lock(&sdev->mutex);
2944 	list_for_each_entry(ch, &sdev->rch_list, list) {
2945 		if (ch->sport == sport) {
2946 			pr_debug("%s: ch %p %s-%d\n", __func__, ch,
2947 				 ch->sess_name, ch->qp->qp_num);
2948 			srpt_disconnect_ch(ch);
2949 			srpt_close_ch(ch);
2950 		}
2951 	}
2952 	mutex_unlock(&sdev->mutex);
2953 
2954 out:
2955 	return count;
2956 }
2957 
2958 CONFIGFS_ATTR(srpt_tpg_, enable);
2959 
2960 static struct configfs_attribute *srpt_tpg_attrs[] = {
2961 	&srpt_tpg_attr_enable,
2962 	NULL,
2963 };
2964 
2965 /**
2966  * configfs callback invoked for
2967  * mkdir /sys/kernel/config/target/$driver/$port/$tpg
2968  */
2969 static struct se_portal_group *srpt_make_tpg(struct se_wwn *wwn,
2970 					     struct config_group *group,
2971 					     const char *name)
2972 {
2973 	struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
2974 	int res;
2975 
2976 	/* Initialize sport->port_wwn and sport->port_tpg_1 */
2977 	res = core_tpg_register(&sport->port_wwn, &sport->port_tpg_1, SCSI_PROTOCOL_SRP);
2978 	if (res)
2979 		return ERR_PTR(res);
2980 
2981 	return &sport->port_tpg_1;
2982 }
2983 
2984 /**
2985  * configfs callback invoked for
2986  * rmdir /sys/kernel/config/target/$driver/$port/$tpg
2987  */
2988 static void srpt_drop_tpg(struct se_portal_group *tpg)
2989 {
2990 	struct srpt_port *sport = container_of(tpg,
2991 				struct srpt_port, port_tpg_1);
2992 
2993 	sport->enabled = false;
2994 	core_tpg_deregister(&sport->port_tpg_1);
2995 }
2996 
2997 /**
2998  * configfs callback invoked for
2999  * mkdir /sys/kernel/config/target/$driver/$port
3000  */
3001 static struct se_wwn *srpt_make_tport(struct target_fabric_configfs *tf,
3002 				      struct config_group *group,
3003 				      const char *name)
3004 {
3005 	struct srpt_port *sport;
3006 	int ret;
3007 
3008 	sport = srpt_lookup_port(name);
3009 	pr_debug("make_tport(%s)\n", name);
3010 	ret = -EINVAL;
3011 	if (!sport)
3012 		goto err;
3013 
3014 	return &sport->port_wwn;
3015 
3016 err:
3017 	return ERR_PTR(ret);
3018 }
3019 
3020 /**
3021  * configfs callback invoked for
3022  * rmdir /sys/kernel/config/target/$driver/$port
3023  */
3024 static void srpt_drop_tport(struct se_wwn *wwn)
3025 {
3026 	struct srpt_port *sport = container_of(wwn, struct srpt_port, port_wwn);
3027 
3028 	pr_debug("drop_tport(%s\n", config_item_name(&sport->port_wwn.wwn_group.cg_item));
3029 }
3030 
3031 static ssize_t srpt_wwn_version_show(struct config_item *item, char *buf)
3032 {
3033 	return scnprintf(buf, PAGE_SIZE, "%s\n", DRV_VERSION);
3034 }
3035 
3036 CONFIGFS_ATTR_RO(srpt_wwn_, version);
3037 
3038 static struct configfs_attribute *srpt_wwn_attrs[] = {
3039 	&srpt_wwn_attr_version,
3040 	NULL,
3041 };
3042 
3043 static const struct target_core_fabric_ops srpt_template = {
3044 	.module				= THIS_MODULE,
3045 	.name				= "srpt",
3046 	.get_fabric_name		= srpt_get_fabric_name,
3047 	.tpg_get_wwn			= srpt_get_fabric_wwn,
3048 	.tpg_get_tag			= srpt_get_tag,
3049 	.tpg_check_demo_mode		= srpt_check_false,
3050 	.tpg_check_demo_mode_cache	= srpt_check_true,
3051 	.tpg_check_demo_mode_write_protect = srpt_check_true,
3052 	.tpg_check_prod_mode_write_protect = srpt_check_false,
3053 	.tpg_get_inst_index		= srpt_tpg_get_inst_index,
3054 	.release_cmd			= srpt_release_cmd,
3055 	.check_stop_free		= srpt_check_stop_free,
3056 	.close_session			= srpt_close_session,
3057 	.sess_get_index			= srpt_sess_get_index,
3058 	.sess_get_initiator_sid		= NULL,
3059 	.write_pending			= srpt_write_pending,
3060 	.write_pending_status		= srpt_write_pending_status,
3061 	.set_default_node_attributes	= srpt_set_default_node_attrs,
3062 	.get_cmd_state			= srpt_get_tcm_cmd_state,
3063 	.queue_data_in			= srpt_queue_data_in,
3064 	.queue_status			= srpt_queue_status,
3065 	.queue_tm_rsp			= srpt_queue_tm_rsp,
3066 	.aborted_task			= srpt_aborted_task,
3067 	/*
3068 	 * Setup function pointers for generic logic in
3069 	 * target_core_fabric_configfs.c
3070 	 */
3071 	.fabric_make_wwn		= srpt_make_tport,
3072 	.fabric_drop_wwn		= srpt_drop_tport,
3073 	.fabric_make_tpg		= srpt_make_tpg,
3074 	.fabric_drop_tpg		= srpt_drop_tpg,
3075 	.fabric_init_nodeacl		= srpt_init_nodeacl,
3076 
3077 	.tfc_wwn_attrs			= srpt_wwn_attrs,
3078 	.tfc_tpg_base_attrs		= srpt_tpg_attrs,
3079 	.tfc_tpg_attrib_attrs		= srpt_tpg_attrib_attrs,
3080 };
3081 
3082 /**
3083  * srpt_init_module() - Kernel module initialization.
3084  *
3085  * Note: Since ib_register_client() registers callback functions, and since at
3086  * least one of these callback functions (srpt_add_one()) calls target core
3087  * functions, this driver must be registered with the target core before
3088  * ib_register_client() is called.
3089  */
3090 static int __init srpt_init_module(void)
3091 {
3092 	int ret;
3093 
3094 	ret = -EINVAL;
3095 	if (srp_max_req_size < MIN_MAX_REQ_SIZE) {
3096 		pr_err("invalid value %d for kernel module parameter"
3097 		       " srp_max_req_size -- must be at least %d.\n",
3098 		       srp_max_req_size, MIN_MAX_REQ_SIZE);
3099 		goto out;
3100 	}
3101 
3102 	if (srpt_srq_size < MIN_SRPT_SRQ_SIZE
3103 	    || srpt_srq_size > MAX_SRPT_SRQ_SIZE) {
3104 		pr_err("invalid value %d for kernel module parameter"
3105 		       " srpt_srq_size -- must be in the range [%d..%d].\n",
3106 		       srpt_srq_size, MIN_SRPT_SRQ_SIZE, MAX_SRPT_SRQ_SIZE);
3107 		goto out;
3108 	}
3109 
3110 	ret = target_register_template(&srpt_template);
3111 	if (ret)
3112 		goto out;
3113 
3114 	ret = ib_register_client(&srpt_client);
3115 	if (ret) {
3116 		pr_err("couldn't register IB client\n");
3117 		goto out_unregister_target;
3118 	}
3119 
3120 	return 0;
3121 
3122 out_unregister_target:
3123 	target_unregister_template(&srpt_template);
3124 out:
3125 	return ret;
3126 }
3127 
3128 static void __exit srpt_cleanup_module(void)
3129 {
3130 	ib_unregister_client(&srpt_client);
3131 	target_unregister_template(&srpt_template);
3132 }
3133 
3134 module_init(srpt_init_module);
3135 module_exit(srpt_cleanup_module);
3136