1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*******************************************************************************
3  * IBM Virtual SCSI Target Driver
4  * Copyright (C) 2003-2005 Dave Boutcher (boutcher@us.ibm.com) IBM Corp.
5  *			   Santiago Leon (santil@us.ibm.com) IBM Corp.
6  *			   Linda Xie (lxie@us.ibm.com) IBM Corp.
7  *
8  * Copyright (C) 2005-2011 FUJITA Tomonori <tomof@acm.org>
9  * Copyright (C) 2010 Nicholas A. Bellinger <nab@kernel.org>
10  *
11  * Authors: Bryant G. Ly <bryantly@linux.vnet.ibm.com>
12  * Authors: Michael Cyr <mikecyr@linux.vnet.ibm.com>
13  *
14  ****************************************************************************/
15 
16 #define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
17 
18 #include <linux/module.h>
19 #include <linux/kernel.h>
20 #include <linux/slab.h>
21 #include <linux/types.h>
22 #include <linux/list.h>
23 #include <linux/string.h>
24 #include <linux/delay.h>
25 
26 #include <target/target_core_base.h>
27 #include <target/target_core_fabric.h>
28 
29 #include <asm/hvcall.h>
30 #include <asm/vio.h>
31 
32 #include <scsi/viosrp.h>
33 
34 #include "ibmvscsi_tgt.h"
35 
36 #define IBMVSCSIS_VERSION	"v0.2"
37 
38 #define	INITIAL_SRP_LIMIT	800
39 #define	DEFAULT_MAX_SECTORS	256
40 #define MAX_TXU			1024 * 1024
41 
42 static uint max_vdma_size = MAX_H_COPY_RDMA;
43 
44 static char system_id[SYS_ID_NAME_LEN] = "";
45 static char partition_name[PARTITION_NAMELEN] = "UNKNOWN";
46 static uint partition_number = -1;
47 
48 /* Adapter list and lock to control it */
49 static DEFINE_SPINLOCK(ibmvscsis_dev_lock);
50 static LIST_HEAD(ibmvscsis_dev_list);
51 
52 static long ibmvscsis_parse_command(struct scsi_info *vscsi,
53 				    struct viosrp_crq *crq);
54 
55 static void ibmvscsis_adapter_idle(struct scsi_info *vscsi);
56 
57 static void ibmvscsis_determine_resid(struct se_cmd *se_cmd,
58 				      struct srp_rsp *rsp)
59 {
60 	u32 residual_count = se_cmd->residual_count;
61 
62 	if (!residual_count)
63 		return;
64 
65 	if (se_cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
66 		if (se_cmd->data_direction == DMA_TO_DEVICE) {
67 			/* residual data from an underflow write */
68 			rsp->flags = SRP_RSP_FLAG_DOUNDER;
69 			rsp->data_out_res_cnt = cpu_to_be32(residual_count);
70 		} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
71 			/* residual data from an underflow read */
72 			rsp->flags = SRP_RSP_FLAG_DIUNDER;
73 			rsp->data_in_res_cnt = cpu_to_be32(residual_count);
74 		}
75 	} else if (se_cmd->se_cmd_flags & SCF_OVERFLOW_BIT) {
76 		if (se_cmd->data_direction == DMA_TO_DEVICE) {
77 			/* residual data from an overflow write */
78 			rsp->flags = SRP_RSP_FLAG_DOOVER;
79 			rsp->data_out_res_cnt = cpu_to_be32(residual_count);
80 		} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
81 			/* residual data from an overflow read */
82 			rsp->flags = SRP_RSP_FLAG_DIOVER;
83 			rsp->data_in_res_cnt = cpu_to_be32(residual_count);
84 		}
85 	}
86 }
87 
88 /**
89  * connection_broken() - Determine if the connection to the client is good
90  * @vscsi:	Pointer to our adapter structure
91  *
92  * This function attempts to send a ping MAD to the client. If the call to
93  * queue the request returns H_CLOSED then the connection has been broken
94  * and the function returns TRUE.
95  *
96  * EXECUTION ENVIRONMENT:
97  *	Interrupt or Process environment
98  */
99 static bool connection_broken(struct scsi_info *vscsi)
100 {
101 	struct viosrp_crq *crq;
102 	u64 buffer[2] = { 0, 0 };
103 	long h_return_code;
104 	bool rc = false;
105 
106 	/* create a PING crq */
107 	crq = (struct viosrp_crq *)&buffer;
108 	crq->valid = VALID_CMD_RESP_EL;
109 	crq->format = MESSAGE_IN_CRQ;
110 	crq->status = PING;
111 
112 	h_return_code = h_send_crq(vscsi->dds.unit_id,
113 				   cpu_to_be64(buffer[MSG_HI]),
114 				   cpu_to_be64(buffer[MSG_LOW]));
115 
116 	dev_dbg(&vscsi->dev, "Connection_broken: rc %ld\n", h_return_code);
117 
118 	if (h_return_code == H_CLOSED)
119 		rc = true;
120 
121 	return rc;
122 }
123 
124 /**
125  * ibmvscsis_unregister_command_q() - Helper Function-Unregister Command Queue
126  * @vscsi:	Pointer to our adapter structure
127  *
128  * This function calls h_free_q then frees the interrupt bit etc.
129  * It must release the lock before doing so because of the time it can take
130  * for h_free_crq in PHYP
131  * NOTE: the caller must make sure that state and or flags will prevent
132  *	 interrupt handler from scheduling work.
133  * NOTE: anyone calling this function may need to set the CRQ_CLOSED flag
134  *	 we can't do it here, because we don't have the lock
135  *
136  * EXECUTION ENVIRONMENT:
137  *	Process level
138  */
139 static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi)
140 {
141 	long qrc;
142 	long rc = ADAPT_SUCCESS;
143 	int ticks = 0;
144 
145 	do {
146 		qrc = h_free_crq(vscsi->dds.unit_id);
147 		switch (qrc) {
148 		case H_SUCCESS:
149 			spin_lock_bh(&vscsi->intr_lock);
150 			vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS;
151 			spin_unlock_bh(&vscsi->intr_lock);
152 			break;
153 
154 		case H_HARDWARE:
155 		case H_PARAMETER:
156 			dev_err(&vscsi->dev, "unregister_command_q: error from h_free_crq %ld\n",
157 				qrc);
158 			rc = ERROR;
159 			break;
160 
161 		case H_BUSY:
162 		case H_LONG_BUSY_ORDER_1_MSEC:
163 			/* msleep not good for small values */
164 			usleep_range(1000, 2000);
165 			ticks += 1;
166 			break;
167 		case H_LONG_BUSY_ORDER_10_MSEC:
168 			usleep_range(10000, 20000);
169 			ticks += 10;
170 			break;
171 		case H_LONG_BUSY_ORDER_100_MSEC:
172 			msleep(100);
173 			ticks += 100;
174 			break;
175 		case H_LONG_BUSY_ORDER_1_SEC:
176 			ssleep(1);
177 			ticks += 1000;
178 			break;
179 		case H_LONG_BUSY_ORDER_10_SEC:
180 			ssleep(10);
181 			ticks += 10000;
182 			break;
183 		case H_LONG_BUSY_ORDER_100_SEC:
184 			ssleep(100);
185 			ticks += 100000;
186 			break;
187 		default:
188 			dev_err(&vscsi->dev, "unregister_command_q: unknown error %ld from h_free_crq\n",
189 				qrc);
190 			rc = ERROR;
191 			break;
192 		}
193 
194 		/*
195 		 * dont wait more then 300 seconds
196 		 * ticks are in milliseconds more or less
197 		 */
198 		if (ticks > 300000 && qrc != H_SUCCESS) {
199 			rc = ERROR;
200 			dev_err(&vscsi->dev, "Excessive wait for h_free_crq\n");
201 		}
202 	} while (qrc != H_SUCCESS && rc == ADAPT_SUCCESS);
203 
204 	dev_dbg(&vscsi->dev, "Freeing CRQ: phyp rc %ld, rc %ld\n", qrc, rc);
205 
206 	return rc;
207 }
208 
209 /**
210  * ibmvscsis_delete_client_info() - Helper function to Delete Client Info
211  * @vscsi:	Pointer to our adapter structure
212  * @client_closed:	True if client closed its queue
213  *
214  * Deletes information specific to the client when the client goes away
215  *
216  * EXECUTION ENVIRONMENT:
217  *	Interrupt or Process
218  */
219 static void ibmvscsis_delete_client_info(struct scsi_info *vscsi,
220 					 bool client_closed)
221 {
222 	vscsi->client_cap = 0;
223 
224 	/*
225 	 * Some things we don't want to clear if we're closing the queue,
226 	 * because some clients don't resend the host handshake when they
227 	 * get a transport event.
228 	 */
229 	if (client_closed)
230 		vscsi->client_data.os_type = 0;
231 }
232 
233 /**
234  * ibmvscsis_free_command_q() - Free Command Queue
235  * @vscsi:	Pointer to our adapter structure
236  *
237  * This function calls unregister_command_q, then clears interrupts and
238  * any pending interrupt acknowledgments associated with the command q.
239  * It also clears memory if there is no error.
240  *
241  * PHYP did not meet the PAPR architecture so that we must give up the
242  * lock. This causes a timing hole regarding state change.  To close the
243  * hole this routine does accounting on any change that occurred during
244  * the time the lock is not held.
245  * NOTE: must give up and then acquire the interrupt lock, the caller must
246  *	 make sure that state and or flags will prevent interrupt handler from
247  *	 scheduling work.
248  *
249  * EXECUTION ENVIRONMENT:
250  *	Process level, interrupt lock is held
251  */
252 static long ibmvscsis_free_command_q(struct scsi_info *vscsi)
253 {
254 	int bytes;
255 	u32 flags_under_lock;
256 	u16 state_under_lock;
257 	long rc = ADAPT_SUCCESS;
258 
259 	if (!(vscsi->flags & CRQ_CLOSED)) {
260 		vio_disable_interrupts(vscsi->dma_dev);
261 
262 		state_under_lock = vscsi->new_state;
263 		flags_under_lock = vscsi->flags;
264 		vscsi->phyp_acr_state = 0;
265 		vscsi->phyp_acr_flags = 0;
266 
267 		spin_unlock_bh(&vscsi->intr_lock);
268 		rc = ibmvscsis_unregister_command_q(vscsi);
269 		spin_lock_bh(&vscsi->intr_lock);
270 
271 		if (state_under_lock != vscsi->new_state)
272 			vscsi->phyp_acr_state = vscsi->new_state;
273 
274 		vscsi->phyp_acr_flags = ((~flags_under_lock) & vscsi->flags);
275 
276 		if (rc == ADAPT_SUCCESS) {
277 			bytes = vscsi->cmd_q.size * PAGE_SIZE;
278 			memset(vscsi->cmd_q.base_addr, 0, bytes);
279 			vscsi->cmd_q.index = 0;
280 			vscsi->flags |= CRQ_CLOSED;
281 
282 			ibmvscsis_delete_client_info(vscsi, false);
283 		}
284 
285 		dev_dbg(&vscsi->dev, "free_command_q: flags 0x%x, state 0x%hx, acr_flags 0x%x, acr_state 0x%hx\n",
286 			vscsi->flags, vscsi->state, vscsi->phyp_acr_flags,
287 			vscsi->phyp_acr_state);
288 	}
289 	return rc;
290 }
291 
292 /**
293  * ibmvscsis_cmd_q_dequeue() - Get valid Command element
294  * @mask:	Mask to use in case index wraps
295  * @current_index:	Current index into command queue
296  * @base_addr:	Pointer to start of command queue
297  *
298  * Returns a pointer to a valid command element or NULL, if the command
299  * queue is empty
300  *
301  * EXECUTION ENVIRONMENT:
302  *	Interrupt environment, interrupt lock held
303  */
304 static struct viosrp_crq *ibmvscsis_cmd_q_dequeue(uint mask,
305 						  uint *current_index,
306 						  struct viosrp_crq *base_addr)
307 {
308 	struct viosrp_crq *ptr;
309 
310 	ptr = base_addr + *current_index;
311 
312 	if (ptr->valid) {
313 		*current_index = (*current_index + 1) & mask;
314 		dma_rmb();
315 	} else {
316 		ptr = NULL;
317 	}
318 
319 	return ptr;
320 }
321 
322 /**
323  * ibmvscsis_send_init_message() - send initialize message to the client
324  * @vscsi:	Pointer to our adapter structure
325  * @format:	Which Init Message format to send
326  *
327  * EXECUTION ENVIRONMENT:
328  *	Interrupt environment interrupt lock held
329  */
330 static long ibmvscsis_send_init_message(struct scsi_info *vscsi, u8 format)
331 {
332 	struct viosrp_crq *crq;
333 	u64 buffer[2] = { 0, 0 };
334 	long rc;
335 
336 	crq = (struct viosrp_crq *)&buffer;
337 	crq->valid = VALID_INIT_MSG;
338 	crq->format = format;
339 	rc = h_send_crq(vscsi->dds.unit_id, cpu_to_be64(buffer[MSG_HI]),
340 			cpu_to_be64(buffer[MSG_LOW]));
341 
342 	return rc;
343 }
344 
345 /**
346  * ibmvscsis_check_init_msg() - Check init message valid
347  * @vscsi:	Pointer to our adapter structure
348  * @format:	Pointer to return format of Init Message, if any.
349  *		Set to UNUSED_FORMAT if no Init Message in queue.
350  *
351  * Checks if an initialize message was queued by the initiatior
352  * after the queue was created and before the interrupt was enabled.
353  *
354  * EXECUTION ENVIRONMENT:
355  *	Process level only, interrupt lock held
356  */
357 static long ibmvscsis_check_init_msg(struct scsi_info *vscsi, uint *format)
358 {
359 	struct viosrp_crq *crq;
360 	long rc = ADAPT_SUCCESS;
361 
362 	crq = ibmvscsis_cmd_q_dequeue(vscsi->cmd_q.mask, &vscsi->cmd_q.index,
363 				      vscsi->cmd_q.base_addr);
364 	if (!crq) {
365 		*format = (uint)UNUSED_FORMAT;
366 	} else if (crq->valid == VALID_INIT_MSG && crq->format == INIT_MSG) {
367 		*format = (uint)INIT_MSG;
368 		crq->valid = INVALIDATE_CMD_RESP_EL;
369 		dma_rmb();
370 
371 		/*
372 		 * the caller has ensured no initialize message was
373 		 * sent after the queue was
374 		 * created so there should be no other message on the queue.
375 		 */
376 		crq = ibmvscsis_cmd_q_dequeue(vscsi->cmd_q.mask,
377 					      &vscsi->cmd_q.index,
378 					      vscsi->cmd_q.base_addr);
379 		if (crq) {
380 			*format = (uint)(crq->format);
381 			rc = ERROR;
382 			crq->valid = INVALIDATE_CMD_RESP_EL;
383 			dma_rmb();
384 		}
385 	} else {
386 		*format = (uint)(crq->format);
387 		rc = ERROR;
388 		crq->valid = INVALIDATE_CMD_RESP_EL;
389 		dma_rmb();
390 	}
391 
392 	return rc;
393 }
394 
395 /**
396  * ibmvscsis_disconnect() - Helper function to disconnect
397  * @work:	Pointer to work_struct, gives access to our adapter structure
398  *
399  * An error has occurred or the driver received a Transport event,
400  * and the driver is requesting that the command queue be de-registered
401  * in a safe manner. If there is no outstanding I/O then we can stop the
402  * queue. If we are restarting the queue it will be reflected in the
403  * the state of the adapter.
404  *
405  * EXECUTION ENVIRONMENT:
406  *	Process environment
407  */
408 static void ibmvscsis_disconnect(struct work_struct *work)
409 {
410 	struct scsi_info *vscsi = container_of(work, struct scsi_info,
411 					       proc_work);
412 	u16 new_state;
413 	bool wait_idle = false;
414 
415 	spin_lock_bh(&vscsi->intr_lock);
416 	new_state = vscsi->new_state;
417 	vscsi->new_state = 0;
418 
419 	vscsi->flags |= DISCONNECT_SCHEDULED;
420 	vscsi->flags &= ~SCHEDULE_DISCONNECT;
421 
422 	dev_dbg(&vscsi->dev, "disconnect: flags 0x%x, state 0x%hx\n",
423 		vscsi->flags, vscsi->state);
424 
425 	/*
426 	 * check which state we are in and see if we
427 	 * should transitition to the new state
428 	 */
429 	switch (vscsi->state) {
430 	/* Should never be called while in this state. */
431 	case NO_QUEUE:
432 	/*
433 	 * Can never transition from this state;
434 	 * igonore errors and logout.
435 	 */
436 	case UNCONFIGURING:
437 		break;
438 
439 	/* can transition from this state to UNCONFIGURING */
440 	case ERR_DISCONNECT:
441 		if (new_state == UNCONFIGURING)
442 			vscsi->state = new_state;
443 		break;
444 
445 	/*
446 	 * Can transition from this state to to unconfiguring
447 	 * or err disconnect.
448 	 */
449 	case ERR_DISCONNECT_RECONNECT:
450 		switch (new_state) {
451 		case UNCONFIGURING:
452 		case ERR_DISCONNECT:
453 			vscsi->state = new_state;
454 			break;
455 
456 		case WAIT_IDLE:
457 			break;
458 		default:
459 			break;
460 		}
461 		break;
462 
463 	/* can transition from this state to UNCONFIGURING */
464 	case ERR_DISCONNECTED:
465 		if (new_state == UNCONFIGURING)
466 			vscsi->state = new_state;
467 		break;
468 
469 	case WAIT_ENABLED:
470 		switch (new_state) {
471 		case UNCONFIGURING:
472 			vscsi->state = new_state;
473 			vscsi->flags |= RESPONSE_Q_DOWN;
474 			vscsi->flags &= ~(SCHEDULE_DISCONNECT |
475 					  DISCONNECT_SCHEDULED);
476 			dma_rmb();
477 			if (vscsi->flags & CFG_SLEEPING) {
478 				vscsi->flags &= ~CFG_SLEEPING;
479 				complete(&vscsi->unconfig);
480 			}
481 			break;
482 
483 		/* should never happen */
484 		case ERR_DISCONNECT:
485 		case ERR_DISCONNECT_RECONNECT:
486 		case WAIT_IDLE:
487 			dev_err(&vscsi->dev, "disconnect: invalid state %d for WAIT_IDLE\n",
488 				vscsi->state);
489 			break;
490 		}
491 		break;
492 
493 	case WAIT_IDLE:
494 		switch (new_state) {
495 		case UNCONFIGURING:
496 			vscsi->flags |= RESPONSE_Q_DOWN;
497 			vscsi->state = new_state;
498 			vscsi->flags &= ~(SCHEDULE_DISCONNECT |
499 					  DISCONNECT_SCHEDULED);
500 			ibmvscsis_free_command_q(vscsi);
501 			break;
502 		case ERR_DISCONNECT:
503 		case ERR_DISCONNECT_RECONNECT:
504 			vscsi->state = new_state;
505 			break;
506 		}
507 		break;
508 
509 	/*
510 	 * Initiator has not done a successful srp login
511 	 * or has done a successful srp logout ( adapter was not
512 	 * busy). In the first case there can be responses queued
513 	 * waiting for space on the initiators response queue (MAD)
514 	 * The second case the adapter is idle. Assume the worse case,
515 	 * i.e. the second case.
516 	 */
517 	case WAIT_CONNECTION:
518 	case CONNECTED:
519 	case SRP_PROCESSING:
520 		wait_idle = true;
521 		vscsi->state = new_state;
522 		break;
523 
524 	/* can transition from this state to UNCONFIGURING */
525 	case UNDEFINED:
526 		if (new_state == UNCONFIGURING)
527 			vscsi->state = new_state;
528 		break;
529 	default:
530 		break;
531 	}
532 
533 	if (wait_idle) {
534 		dev_dbg(&vscsi->dev, "disconnect start wait, active %d, sched %d\n",
535 			(int)list_empty(&vscsi->active_q),
536 			(int)list_empty(&vscsi->schedule_q));
537 		if (!list_empty(&vscsi->active_q) ||
538 		    !list_empty(&vscsi->schedule_q)) {
539 			vscsi->flags |= WAIT_FOR_IDLE;
540 			dev_dbg(&vscsi->dev, "disconnect flags 0x%x\n",
541 				vscsi->flags);
542 			/*
543 			 * This routine is can not be called with the interrupt
544 			 * lock held.
545 			 */
546 			spin_unlock_bh(&vscsi->intr_lock);
547 			wait_for_completion(&vscsi->wait_idle);
548 			spin_lock_bh(&vscsi->intr_lock);
549 		}
550 		dev_dbg(&vscsi->dev, "disconnect stop wait\n");
551 
552 		ibmvscsis_adapter_idle(vscsi);
553 	}
554 
555 	spin_unlock_bh(&vscsi->intr_lock);
556 }
557 
558 /**
559  * ibmvscsis_post_disconnect() - Schedule the disconnect
560  * @vscsi:	Pointer to our adapter structure
561  * @new_state:	State to move to after disconnecting
562  * @flag_bits:	Flags to turn on in adapter structure
563  *
564  * If it's already been scheduled, then see if we need to "upgrade"
565  * the new state (if the one passed in is more "severe" than the
566  * previous one).
567  *
568  * PRECONDITION:
569  *	interrupt lock is held
570  */
571 static void ibmvscsis_post_disconnect(struct scsi_info *vscsi, uint new_state,
572 				      uint flag_bits)
573 {
574 	uint state;
575 
576 	/* check the validity of the new state */
577 	switch (new_state) {
578 	case UNCONFIGURING:
579 	case ERR_DISCONNECT:
580 	case ERR_DISCONNECT_RECONNECT:
581 	case WAIT_IDLE:
582 		break;
583 
584 	default:
585 		dev_err(&vscsi->dev, "post_disconnect: Invalid new state %d\n",
586 			new_state);
587 		return;
588 	}
589 
590 	vscsi->flags |= flag_bits;
591 
592 	dev_dbg(&vscsi->dev, "post_disconnect: new_state 0x%x, flag_bits 0x%x, vscsi->flags 0x%x, state %hx\n",
593 		new_state, flag_bits, vscsi->flags, vscsi->state);
594 
595 	if (!(vscsi->flags & (DISCONNECT_SCHEDULED | SCHEDULE_DISCONNECT))) {
596 		vscsi->flags |= SCHEDULE_DISCONNECT;
597 		vscsi->new_state = new_state;
598 
599 		INIT_WORK(&vscsi->proc_work, ibmvscsis_disconnect);
600 		(void)queue_work(vscsi->work_q, &vscsi->proc_work);
601 	} else {
602 		if (vscsi->new_state)
603 			state = vscsi->new_state;
604 		else
605 			state = vscsi->state;
606 
607 		switch (state) {
608 		case NO_QUEUE:
609 		case UNCONFIGURING:
610 			break;
611 
612 		case ERR_DISCONNECTED:
613 		case ERR_DISCONNECT:
614 		case UNDEFINED:
615 			if (new_state == UNCONFIGURING)
616 				vscsi->new_state = new_state;
617 			break;
618 
619 		case ERR_DISCONNECT_RECONNECT:
620 			switch (new_state) {
621 			case UNCONFIGURING:
622 			case ERR_DISCONNECT:
623 				vscsi->new_state = new_state;
624 				break;
625 			default:
626 				break;
627 			}
628 			break;
629 
630 		case WAIT_ENABLED:
631 		case WAIT_IDLE:
632 		case WAIT_CONNECTION:
633 		case CONNECTED:
634 		case SRP_PROCESSING:
635 			vscsi->new_state = new_state;
636 			break;
637 
638 		default:
639 			break;
640 		}
641 	}
642 
643 	dev_dbg(&vscsi->dev, "Leaving post_disconnect: flags 0x%x, new_state 0x%x\n",
644 		vscsi->flags, vscsi->new_state);
645 }
646 
647 /**
648  * ibmvscsis_handle_init_compl_msg() - Respond to an Init Complete Message
649  * @vscsi:	Pointer to our adapter structure
650  *
651  * Must be called with interrupt lock held.
652  */
653 static long ibmvscsis_handle_init_compl_msg(struct scsi_info *vscsi)
654 {
655 	long rc = ADAPT_SUCCESS;
656 
657 	switch (vscsi->state) {
658 	case NO_QUEUE:
659 	case ERR_DISCONNECT:
660 	case ERR_DISCONNECT_RECONNECT:
661 	case ERR_DISCONNECTED:
662 	case UNCONFIGURING:
663 	case UNDEFINED:
664 		rc = ERROR;
665 		break;
666 
667 	case WAIT_CONNECTION:
668 		vscsi->state = CONNECTED;
669 		break;
670 
671 	case WAIT_IDLE:
672 	case SRP_PROCESSING:
673 	case CONNECTED:
674 	case WAIT_ENABLED:
675 	default:
676 		rc = ERROR;
677 		dev_err(&vscsi->dev, "init_msg: invalid state %d to get init compl msg\n",
678 			vscsi->state);
679 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
680 		break;
681 	}
682 
683 	return rc;
684 }
685 
686 /**
687  * ibmvscsis_handle_init_msg() - Respond to an Init Message
688  * @vscsi:	Pointer to our adapter structure
689  *
690  * Must be called with interrupt lock held.
691  */
692 static long ibmvscsis_handle_init_msg(struct scsi_info *vscsi)
693 {
694 	long rc = ADAPT_SUCCESS;
695 
696 	switch (vscsi->state) {
697 	case WAIT_CONNECTION:
698 		rc = ibmvscsis_send_init_message(vscsi, INIT_COMPLETE_MSG);
699 		switch (rc) {
700 		case H_SUCCESS:
701 			vscsi->state = CONNECTED;
702 			break;
703 
704 		case H_PARAMETER:
705 			dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
706 				rc);
707 			ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
708 			break;
709 
710 		case H_DROPPED:
711 			dev_err(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
712 				rc);
713 			rc = ERROR;
714 			ibmvscsis_post_disconnect(vscsi,
715 						  ERR_DISCONNECT_RECONNECT, 0);
716 			break;
717 
718 		case H_CLOSED:
719 			dev_warn(&vscsi->dev, "init_msg: failed to send, rc %ld\n",
720 				 rc);
721 			rc = 0;
722 			break;
723 		}
724 		break;
725 
726 	case UNDEFINED:
727 		rc = ERROR;
728 		break;
729 
730 	case UNCONFIGURING:
731 		break;
732 
733 	case WAIT_ENABLED:
734 	case CONNECTED:
735 	case SRP_PROCESSING:
736 	case WAIT_IDLE:
737 	case NO_QUEUE:
738 	case ERR_DISCONNECT:
739 	case ERR_DISCONNECT_RECONNECT:
740 	case ERR_DISCONNECTED:
741 	default:
742 		rc = ERROR;
743 		dev_err(&vscsi->dev, "init_msg: invalid state %d to get init msg\n",
744 			vscsi->state);
745 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
746 		break;
747 	}
748 
749 	return rc;
750 }
751 
752 /**
753  * ibmvscsis_init_msg() - Respond to an init message
754  * @vscsi:	Pointer to our adapter structure
755  * @crq:	Pointer to CRQ element containing the Init Message
756  *
757  * EXECUTION ENVIRONMENT:
758  *	Interrupt, interrupt lock held
759  */
760 static long ibmvscsis_init_msg(struct scsi_info *vscsi, struct viosrp_crq *crq)
761 {
762 	long rc = ADAPT_SUCCESS;
763 
764 	dev_dbg(&vscsi->dev, "init_msg: state 0x%hx\n", vscsi->state);
765 
766 	rc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
767 		      (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
768 		      0);
769 	if (rc == H_SUCCESS) {
770 		vscsi->client_data.partition_number =
771 			be64_to_cpu(*(u64 *)vscsi->map_buf);
772 		dev_dbg(&vscsi->dev, "init_msg, part num %d\n",
773 			vscsi->client_data.partition_number);
774 	} else {
775 		dev_dbg(&vscsi->dev, "init_msg h_vioctl rc %ld\n", rc);
776 		rc = ADAPT_SUCCESS;
777 	}
778 
779 	if (crq->format == INIT_MSG) {
780 		rc = ibmvscsis_handle_init_msg(vscsi);
781 	} else if (crq->format == INIT_COMPLETE_MSG) {
782 		rc = ibmvscsis_handle_init_compl_msg(vscsi);
783 	} else {
784 		rc = ERROR;
785 		dev_err(&vscsi->dev, "init_msg: invalid format %d\n",
786 			(uint)crq->format);
787 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
788 	}
789 
790 	return rc;
791 }
792 
793 /**
794  * ibmvscsis_establish_new_q() - Establish new CRQ queue
795  * @vscsi:	Pointer to our adapter structure
796  *
797  * Must be called with interrupt lock held.
798  */
799 static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
800 {
801 	long rc = ADAPT_SUCCESS;
802 	uint format;
803 
804 	rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000,
805 		      0, 0, 0, 0);
806 	if (rc == H_SUCCESS)
807 		vscsi->flags |= PREP_FOR_SUSPEND_ENABLED;
808 	else if (rc != H_NOT_FOUND)
809 		dev_err(&vscsi->dev, "Error from Enable Prepare for Suspend: %ld\n",
810 			rc);
811 
812 	vscsi->flags &= PRESERVE_FLAG_FIELDS;
813 	vscsi->rsp_q_timer.timer_pops = 0;
814 	vscsi->debit = 0;
815 	vscsi->credit = 0;
816 
817 	rc = vio_enable_interrupts(vscsi->dma_dev);
818 	if (rc) {
819 		dev_warn(&vscsi->dev, "establish_new_q: failed to enable interrupts, rc %ld\n",
820 			 rc);
821 		return rc;
822 	}
823 
824 	rc = ibmvscsis_check_init_msg(vscsi, &format);
825 	if (rc) {
826 		dev_err(&vscsi->dev, "establish_new_q: check_init_msg failed, rc %ld\n",
827 			rc);
828 		return rc;
829 	}
830 
831 	if (format == UNUSED_FORMAT) {
832 		rc = ibmvscsis_send_init_message(vscsi, INIT_MSG);
833 		switch (rc) {
834 		case H_SUCCESS:
835 		case H_DROPPED:
836 		case H_CLOSED:
837 			rc = ADAPT_SUCCESS;
838 			break;
839 
840 		case H_PARAMETER:
841 		case H_HARDWARE:
842 			break;
843 
844 		default:
845 			vscsi->state = UNDEFINED;
846 			rc = H_HARDWARE;
847 			break;
848 		}
849 	} else if (format == INIT_MSG) {
850 		rc = ibmvscsis_handle_init_msg(vscsi);
851 	}
852 
853 	return rc;
854 }
855 
856 /**
857  * ibmvscsis_reset_queue() - Reset CRQ Queue
858  * @vscsi:	Pointer to our adapter structure
859  *
860  * This function calls h_free_q and then calls h_reg_q and does all
861  * of the bookkeeping to get us back to where we can communicate.
862  *
863  * Actually, we don't always call h_free_crq.  A problem was discovered
864  * where one partition would close and reopen his queue, which would
865  * cause his partner to get a transport event, which would cause him to
866  * close and reopen his queue, which would cause the original partition
867  * to get a transport event, etc., etc.  To prevent this, we don't
868  * actually close our queue if the client initiated the reset, (i.e.
869  * either we got a transport event or we have detected that the client's
870  * queue is gone)
871  *
872  * EXECUTION ENVIRONMENT:
873  *	Process environment, called with interrupt lock held
874  */
875 static void ibmvscsis_reset_queue(struct scsi_info *vscsi)
876 {
877 	int bytes;
878 	long rc = ADAPT_SUCCESS;
879 
880 	dev_dbg(&vscsi->dev, "reset_queue: flags 0x%x\n", vscsi->flags);
881 
882 	/* don't reset, the client did it for us */
883 	if (vscsi->flags & (CLIENT_FAILED | TRANS_EVENT)) {
884 		vscsi->flags &= PRESERVE_FLAG_FIELDS;
885 		vscsi->rsp_q_timer.timer_pops = 0;
886 		vscsi->debit = 0;
887 		vscsi->credit = 0;
888 		vscsi->state = WAIT_CONNECTION;
889 		vio_enable_interrupts(vscsi->dma_dev);
890 	} else {
891 		rc = ibmvscsis_free_command_q(vscsi);
892 		if (rc == ADAPT_SUCCESS) {
893 			vscsi->state = WAIT_CONNECTION;
894 
895 			bytes = vscsi->cmd_q.size * PAGE_SIZE;
896 			rc = h_reg_crq(vscsi->dds.unit_id,
897 				       vscsi->cmd_q.crq_token, bytes);
898 			if (rc == H_CLOSED || rc == H_SUCCESS) {
899 				rc = ibmvscsis_establish_new_q(vscsi);
900 			}
901 
902 			if (rc != ADAPT_SUCCESS) {
903 				dev_dbg(&vscsi->dev, "reset_queue: reg_crq rc %ld\n",
904 					rc);
905 
906 				vscsi->state = ERR_DISCONNECTED;
907 				vscsi->flags |= RESPONSE_Q_DOWN;
908 				ibmvscsis_free_command_q(vscsi);
909 			}
910 		} else {
911 			vscsi->state = ERR_DISCONNECTED;
912 			vscsi->flags |= RESPONSE_Q_DOWN;
913 		}
914 	}
915 }
916 
917 /**
918  * ibmvscsis_free_cmd_resources() - Free command resources
919  * @vscsi:	Pointer to our adapter structure
920  * @cmd:	Command which is not longer in use
921  *
922  * Must be called with interrupt lock held.
923  */
924 static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
925 					 struct ibmvscsis_cmd *cmd)
926 {
927 	struct iu_entry *iue = cmd->iue;
928 
929 	switch (cmd->type) {
930 	case TASK_MANAGEMENT:
931 	case SCSI_CDB:
932 		/*
933 		 * When the queue goes down this value is cleared, so it
934 		 * cannot be cleared in this general purpose function.
935 		 */
936 		if (vscsi->debit)
937 			vscsi->debit -= 1;
938 		break;
939 	case ADAPTER_MAD:
940 		vscsi->flags &= ~PROCESSING_MAD;
941 		break;
942 	case UNSET_TYPE:
943 		break;
944 	default:
945 		dev_err(&vscsi->dev, "free_cmd_resources unknown type %d\n",
946 			cmd->type);
947 		break;
948 	}
949 
950 	cmd->iue = NULL;
951 	list_add_tail(&cmd->list, &vscsi->free_cmd);
952 	srp_iu_put(iue);
953 
954 	if (list_empty(&vscsi->active_q) && list_empty(&vscsi->schedule_q) &&
955 	    list_empty(&vscsi->waiting_rsp) && (vscsi->flags & WAIT_FOR_IDLE)) {
956 		vscsi->flags &= ~WAIT_FOR_IDLE;
957 		complete(&vscsi->wait_idle);
958 	}
959 }
960 
961 /**
962  * ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL
963  * @vscsi:	Pointer to our adapter structure
964  * @idle:	Indicates whether we were called from adapter_idle.  This
965  *		is important to know if we need to do a disconnect, since if
966  *		we're called from adapter_idle, we're still processing the
967  *		current disconnect, so we can't just call post_disconnect.
968  *
969  * This function is called when the adapter is idle when phyp has sent
970  * us a Prepare for Suspend Transport Event.
971  *
972  * EXECUTION ENVIRONMENT:
973  *	Process or interrupt environment called with interrupt lock held
974  */
975 static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle)
976 {
977 	long rc = 0;
978 	struct viosrp_crq *crq;
979 
980 	/* See if there is a Resume event in the queue */
981 	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
982 
983 	dev_dbg(&vscsi->dev, "ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n",
984 		vscsi->flags, vscsi->state, (int)crq->valid);
985 
986 	if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) {
987 		rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0,
988 			      0, 0);
989 		if (rc) {
990 			dev_err(&vscsi->dev, "Ready for Suspend Vioctl failed: %ld\n",
991 				rc);
992 			rc = 0;
993 		}
994 	} else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) &&
995 		    (vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) ||
996 		   ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
997 				     (crq->format != RESUME_FROM_SUSP)))) {
998 		if (idle) {
999 			vscsi->state = ERR_DISCONNECT_RECONNECT;
1000 			ibmvscsis_reset_queue(vscsi);
1001 			rc = -1;
1002 		} else if (vscsi->state == CONNECTED) {
1003 			ibmvscsis_post_disconnect(vscsi,
1004 						  ERR_DISCONNECT_RECONNECT, 0);
1005 		}
1006 
1007 		vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1008 
1009 		if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
1010 				     (crq->format != RESUME_FROM_SUSP)))
1011 			dev_err(&vscsi->dev, "Invalid element in CRQ after Prepare for Suspend");
1012 	}
1013 
1014 	vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED);
1015 
1016 	return rc;
1017 }
1018 
1019 /**
1020  * ibmvscsis_trans_event() - Handle a Transport Event
1021  * @vscsi:	Pointer to our adapter structure
1022  * @crq:	Pointer to CRQ entry containing the Transport Event
1023  *
1024  * Do the logic to close the I_T nexus.  This function may not
1025  * behave to specification.
1026  *
1027  * EXECUTION ENVIRONMENT:
1028  *	Interrupt, interrupt lock held
1029  */
1030 static long ibmvscsis_trans_event(struct scsi_info *vscsi,
1031 				  struct viosrp_crq *crq)
1032 {
1033 	long rc = ADAPT_SUCCESS;
1034 
1035 	dev_dbg(&vscsi->dev, "trans_event: format %d, flags 0x%x, state 0x%hx\n",
1036 		(int)crq->format, vscsi->flags, vscsi->state);
1037 
1038 	switch (crq->format) {
1039 	case MIGRATED:
1040 	case PARTNER_FAILED:
1041 	case PARTNER_DEREGISTER:
1042 		ibmvscsis_delete_client_info(vscsi, true);
1043 		if (crq->format == MIGRATED)
1044 			vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1045 		switch (vscsi->state) {
1046 		case NO_QUEUE:
1047 		case ERR_DISCONNECTED:
1048 		case UNDEFINED:
1049 			break;
1050 
1051 		case UNCONFIGURING:
1052 			vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
1053 			break;
1054 
1055 		case WAIT_ENABLED:
1056 			break;
1057 
1058 		case WAIT_CONNECTION:
1059 			break;
1060 
1061 		case CONNECTED:
1062 			ibmvscsis_post_disconnect(vscsi, WAIT_IDLE,
1063 						  (RESPONSE_Q_DOWN |
1064 						   TRANS_EVENT));
1065 			break;
1066 
1067 		case SRP_PROCESSING:
1068 			if ((vscsi->debit > 0) ||
1069 			    !list_empty(&vscsi->schedule_q) ||
1070 			    !list_empty(&vscsi->waiting_rsp) ||
1071 			    !list_empty(&vscsi->active_q)) {
1072 				dev_dbg(&vscsi->dev, "debit %d, sched %d, wait %d, active %d\n",
1073 					vscsi->debit,
1074 					(int)list_empty(&vscsi->schedule_q),
1075 					(int)list_empty(&vscsi->waiting_rsp),
1076 					(int)list_empty(&vscsi->active_q));
1077 				dev_warn(&vscsi->dev, "connection lost with outstanding work\n");
1078 			} else {
1079 				dev_dbg(&vscsi->dev, "trans_event: SRP Processing, but no outstanding work\n");
1080 			}
1081 
1082 			ibmvscsis_post_disconnect(vscsi, WAIT_IDLE,
1083 						  (RESPONSE_Q_DOWN |
1084 						   TRANS_EVENT));
1085 			break;
1086 
1087 		case ERR_DISCONNECT:
1088 		case ERR_DISCONNECT_RECONNECT:
1089 		case WAIT_IDLE:
1090 			vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
1091 			break;
1092 		}
1093 		break;
1094 
1095 	case PREPARE_FOR_SUSPEND:
1096 		dev_dbg(&vscsi->dev, "Prep for Suspend, crq status = 0x%x\n",
1097 			(int)crq->status);
1098 		switch (vscsi->state) {
1099 		case ERR_DISCONNECTED:
1100 		case WAIT_CONNECTION:
1101 		case CONNECTED:
1102 			ibmvscsis_ready_for_suspend(vscsi, false);
1103 			break;
1104 		case SRP_PROCESSING:
1105 			vscsi->resume_state = vscsi->state;
1106 			vscsi->flags |= PREP_FOR_SUSPEND_PENDING;
1107 			if (crq->status == CRQ_ENTRY_OVERWRITTEN)
1108 				vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE;
1109 			ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
1110 			break;
1111 		case NO_QUEUE:
1112 		case UNDEFINED:
1113 		case UNCONFIGURING:
1114 		case WAIT_ENABLED:
1115 		case ERR_DISCONNECT:
1116 		case ERR_DISCONNECT_RECONNECT:
1117 		case WAIT_IDLE:
1118 			dev_err(&vscsi->dev, "Invalid state for Prepare for Suspend Trans Event: 0x%x\n",
1119 				vscsi->state);
1120 			break;
1121 		}
1122 		break;
1123 
1124 	case RESUME_FROM_SUSP:
1125 		dev_dbg(&vscsi->dev, "Resume from Suspend, crq status = 0x%x\n",
1126 			(int)crq->status);
1127 		if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
1128 			vscsi->flags |= PREP_FOR_SUSPEND_ABORTED;
1129 		} else {
1130 			if ((crq->status == CRQ_ENTRY_OVERWRITTEN) ||
1131 			    (vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) {
1132 				ibmvscsis_post_disconnect(vscsi,
1133 							  ERR_DISCONNECT_RECONNECT,
1134 							  0);
1135 				vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
1136 			}
1137 		}
1138 		break;
1139 
1140 	default:
1141 		rc = ERROR;
1142 		dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
1143 			(uint)crq->format);
1144 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
1145 					  RESPONSE_Q_DOWN);
1146 		break;
1147 	}
1148 
1149 	rc = vscsi->flags & SCHEDULE_DISCONNECT;
1150 
1151 	dev_dbg(&vscsi->dev, "Leaving trans_event: flags 0x%x, state 0x%hx, rc %ld\n",
1152 		vscsi->flags, vscsi->state, rc);
1153 
1154 	return rc;
1155 }
1156 
1157 /**
1158  * ibmvscsis_poll_cmd_q() - Poll Command Queue
1159  * @vscsi:	Pointer to our adapter structure
1160  *
1161  * Called to handle command elements that may have arrived while
1162  * interrupts were disabled.
1163  *
1164  * EXECUTION ENVIRONMENT:
1165  *	intr_lock must be held
1166  */
1167 static void ibmvscsis_poll_cmd_q(struct scsi_info *vscsi)
1168 {
1169 	struct viosrp_crq *crq;
1170 	long rc;
1171 	bool ack = true;
1172 	volatile u8 valid;
1173 
1174 	dev_dbg(&vscsi->dev, "poll_cmd_q: flags 0x%x, state 0x%hx, q index %ud\n",
1175 		vscsi->flags, vscsi->state, vscsi->cmd_q.index);
1176 
1177 	rc = vscsi->flags & SCHEDULE_DISCONNECT;
1178 	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
1179 	valid = crq->valid;
1180 	dma_rmb();
1181 
1182 	while (valid) {
1183 poll_work:
1184 		vscsi->cmd_q.index =
1185 			(vscsi->cmd_q.index + 1) & vscsi->cmd_q.mask;
1186 
1187 		if (!rc) {
1188 			rc = ibmvscsis_parse_command(vscsi, crq);
1189 		} else {
1190 			if ((uint)crq->valid == VALID_TRANS_EVENT) {
1191 				/*
1192 				 * must service the transport layer events even
1193 				 * in an error state, dont break out until all
1194 				 * the consecutive transport events have been
1195 				 * processed
1196 				 */
1197 				rc = ibmvscsis_trans_event(vscsi, crq);
1198 			} else if (vscsi->flags & TRANS_EVENT) {
1199 				/*
1200 				 * if a tranport event has occurred leave
1201 				 * everything but transport events on the queue
1202 				 */
1203 				dev_dbg(&vscsi->dev, "poll_cmd_q, ignoring\n");
1204 
1205 				/*
1206 				 * need to decrement the queue index so we can
1207 				 * look at the elment again
1208 				 */
1209 				if (vscsi->cmd_q.index)
1210 					vscsi->cmd_q.index -= 1;
1211 				else
1212 					/*
1213 					 * index is at 0 it just wrapped.
1214 					 * have it index last element in q
1215 					 */
1216 					vscsi->cmd_q.index = vscsi->cmd_q.mask;
1217 				break;
1218 			}
1219 		}
1220 
1221 		crq->valid = INVALIDATE_CMD_RESP_EL;
1222 
1223 		crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
1224 		valid = crq->valid;
1225 		dma_rmb();
1226 	}
1227 
1228 	if (!rc) {
1229 		if (ack) {
1230 			vio_enable_interrupts(vscsi->dma_dev);
1231 			ack = false;
1232 			dev_dbg(&vscsi->dev, "poll_cmd_q, reenabling interrupts\n");
1233 		}
1234 		valid = crq->valid;
1235 		dma_rmb();
1236 		if (valid)
1237 			goto poll_work;
1238 	}
1239 
1240 	dev_dbg(&vscsi->dev, "Leaving poll_cmd_q: rc %ld\n", rc);
1241 }
1242 
1243 /**
1244  * ibmvscsis_free_cmd_qs() - Free elements in queue
1245  * @vscsi:	Pointer to our adapter structure
1246  *
1247  * Free all of the elements on all queues that are waiting for
1248  * whatever reason.
1249  *
1250  * PRECONDITION:
1251  *	Called with interrupt lock held
1252  */
1253 static void ibmvscsis_free_cmd_qs(struct scsi_info *vscsi)
1254 {
1255 	struct ibmvscsis_cmd *cmd, *nxt;
1256 
1257 	dev_dbg(&vscsi->dev, "free_cmd_qs: waiting_rsp empty %d, timer starter %d\n",
1258 		(int)list_empty(&vscsi->waiting_rsp),
1259 		vscsi->rsp_q_timer.started);
1260 
1261 	list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp, list) {
1262 		list_del(&cmd->list);
1263 		ibmvscsis_free_cmd_resources(vscsi, cmd);
1264 	}
1265 }
1266 
1267 /**
1268  * ibmvscsis_get_free_cmd() - Get free command from list
1269  * @vscsi:	Pointer to our adapter structure
1270  *
1271  * Must be called with interrupt lock held.
1272  */
1273 static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
1274 {
1275 	struct ibmvscsis_cmd *cmd = NULL;
1276 	struct iu_entry *iue;
1277 
1278 	iue = srp_iu_get(&vscsi->target);
1279 	if (iue) {
1280 		cmd = list_first_entry_or_null(&vscsi->free_cmd,
1281 					       struct ibmvscsis_cmd, list);
1282 		if (cmd) {
1283 			if (cmd->abort_cmd)
1284 				cmd->abort_cmd = NULL;
1285 			cmd->flags &= ~(DELAY_SEND);
1286 			list_del(&cmd->list);
1287 			cmd->iue = iue;
1288 			cmd->type = UNSET_TYPE;
1289 			memset(&cmd->se_cmd, 0, sizeof(cmd->se_cmd));
1290 		} else {
1291 			srp_iu_put(iue);
1292 		}
1293 	}
1294 
1295 	return cmd;
1296 }
1297 
1298 /**
1299  * ibmvscsis_adapter_idle() - Helper function to handle idle adapter
1300  * @vscsi:	Pointer to our adapter structure
1301  *
1302  * This function is called when the adapter is idle when the driver
1303  * is attempting to clear an error condition.
1304  * The adapter is considered busy if any of its cmd queues
1305  * are non-empty. This function can be invoked
1306  * from the off level disconnect function.
1307  *
1308  * EXECUTION ENVIRONMENT:
1309  *	Process environment called with interrupt lock held
1310  */
1311 static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
1312 {
1313 	int free_qs = false;
1314 	long rc = 0;
1315 
1316 	dev_dbg(&vscsi->dev, "adapter_idle: flags 0x%x, state 0x%hx\n",
1317 		vscsi->flags, vscsi->state);
1318 
1319 	/* Only need to free qs if we're disconnecting from client */
1320 	if (vscsi->state != WAIT_CONNECTION || vscsi->flags & TRANS_EVENT)
1321 		free_qs = true;
1322 
1323 	switch (vscsi->state) {
1324 	case UNCONFIGURING:
1325 		ibmvscsis_free_command_q(vscsi);
1326 		dma_rmb();
1327 		isync();
1328 		if (vscsi->flags & CFG_SLEEPING) {
1329 			vscsi->flags &= ~CFG_SLEEPING;
1330 			complete(&vscsi->unconfig);
1331 		}
1332 		break;
1333 	case ERR_DISCONNECT_RECONNECT:
1334 		ibmvscsis_reset_queue(vscsi);
1335 		dev_dbg(&vscsi->dev, "adapter_idle, disc_rec: flags 0x%x\n",
1336 			vscsi->flags);
1337 		break;
1338 
1339 	case ERR_DISCONNECT:
1340 		ibmvscsis_free_command_q(vscsi);
1341 		vscsi->flags &= ~(SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED);
1342 		vscsi->flags |= RESPONSE_Q_DOWN;
1343 		if (vscsi->tport.enabled)
1344 			vscsi->state = ERR_DISCONNECTED;
1345 		else
1346 			vscsi->state = WAIT_ENABLED;
1347 		dev_dbg(&vscsi->dev, "adapter_idle, disc: flags 0x%x, state 0x%hx\n",
1348 			vscsi->flags, vscsi->state);
1349 		break;
1350 
1351 	case WAIT_IDLE:
1352 		vscsi->rsp_q_timer.timer_pops = 0;
1353 		vscsi->debit = 0;
1354 		vscsi->credit = 0;
1355 		if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
1356 			vscsi->state = vscsi->resume_state;
1357 			vscsi->resume_state = 0;
1358 			rc = ibmvscsis_ready_for_suspend(vscsi, true);
1359 			vscsi->flags &= ~DISCONNECT_SCHEDULED;
1360 			if (rc)
1361 				break;
1362 		} else if (vscsi->flags & TRANS_EVENT) {
1363 			vscsi->state = WAIT_CONNECTION;
1364 			vscsi->flags &= PRESERVE_FLAG_FIELDS;
1365 		} else {
1366 			vscsi->state = CONNECTED;
1367 			vscsi->flags &= ~DISCONNECT_SCHEDULED;
1368 		}
1369 
1370 		dev_dbg(&vscsi->dev, "adapter_idle, wait: flags 0x%x, state 0x%hx\n",
1371 			vscsi->flags, vscsi->state);
1372 		ibmvscsis_poll_cmd_q(vscsi);
1373 		break;
1374 
1375 	case ERR_DISCONNECTED:
1376 		vscsi->flags &= ~DISCONNECT_SCHEDULED;
1377 		dev_dbg(&vscsi->dev, "adapter_idle, disconnected: flags 0x%x, state 0x%hx\n",
1378 			vscsi->flags, vscsi->state);
1379 		break;
1380 
1381 	default:
1382 		dev_err(&vscsi->dev, "adapter_idle: in invalid state %d\n",
1383 			vscsi->state);
1384 		break;
1385 	}
1386 
1387 	if (free_qs)
1388 		ibmvscsis_free_cmd_qs(vscsi);
1389 
1390 	/*
1391 	 * There is a timing window where we could lose a disconnect request.
1392 	 * The known path to this window occurs during the DISCONNECT_RECONNECT
1393 	 * case above: reset_queue calls free_command_q, which will release the
1394 	 * interrupt lock.  During that time, a new post_disconnect call can be
1395 	 * made with a "more severe" state (DISCONNECT or UNCONFIGURING).
1396 	 * Because the DISCONNECT_SCHEDULED flag is already set, post_disconnect
1397 	 * will only set the new_state.  Now free_command_q reacquires the intr
1398 	 * lock and clears the DISCONNECT_SCHEDULED flag (using PRESERVE_FLAG_
1399 	 * FIELDS), and the disconnect is lost.  This is particularly bad when
1400 	 * the new disconnect was for UNCONFIGURING, since the unconfigure hangs
1401 	 * forever.
1402 	 * Fix is that free command queue sets acr state and acr flags if there
1403 	 * is a change under the lock
1404 	 * note free command queue writes to this state it clears it
1405 	 * before releasing the lock, different drivers call the free command
1406 	 * queue different times so dont initialize above
1407 	 */
1408 	if (vscsi->phyp_acr_state != 0)	{
1409 		/*
1410 		 * set any bits in flags that may have been cleared by
1411 		 * a call to free command queue in switch statement
1412 		 * or reset queue
1413 		 */
1414 		vscsi->flags |= vscsi->phyp_acr_flags;
1415 		ibmvscsis_post_disconnect(vscsi, vscsi->phyp_acr_state, 0);
1416 		vscsi->phyp_acr_state = 0;
1417 		vscsi->phyp_acr_flags = 0;
1418 
1419 		dev_dbg(&vscsi->dev, "adapter_idle: flags 0x%x, state 0x%hx, acr_flags 0x%x, acr_state 0x%hx\n",
1420 			vscsi->flags, vscsi->state, vscsi->phyp_acr_flags,
1421 			vscsi->phyp_acr_state);
1422 	}
1423 
1424 	dev_dbg(&vscsi->dev, "Leaving adapter_idle: flags 0x%x, state 0x%hx, new_state 0x%x\n",
1425 		vscsi->flags, vscsi->state, vscsi->new_state);
1426 }
1427 
1428 /**
1429  * ibmvscsis_copy_crq_packet() - Copy CRQ Packet
1430  * @vscsi:	Pointer to our adapter structure
1431  * @cmd:	Pointer to command element to use to process the request
1432  * @crq:	Pointer to CRQ entry containing the request
1433  *
1434  * Copy the srp information unit from the hosted
1435  * partition using remote dma
1436  *
1437  * EXECUTION ENVIRONMENT:
1438  *	Interrupt, interrupt lock held
1439  */
1440 static long ibmvscsis_copy_crq_packet(struct scsi_info *vscsi,
1441 				      struct ibmvscsis_cmd *cmd,
1442 				      struct viosrp_crq *crq)
1443 {
1444 	struct iu_entry *iue = cmd->iue;
1445 	long rc = 0;
1446 	u16 len;
1447 
1448 	len = be16_to_cpu(crq->IU_length);
1449 	if ((len > SRP_MAX_IU_LEN) || (len == 0)) {
1450 		dev_err(&vscsi->dev, "copy_crq: Invalid len %d passed", len);
1451 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
1452 		return SRP_VIOLATION;
1453 	}
1454 
1455 	rc = h_copy_rdma(len, vscsi->dds.window[REMOTE].liobn,
1456 			 be64_to_cpu(crq->IU_data_ptr),
1457 			 vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma);
1458 
1459 	switch (rc) {
1460 	case H_SUCCESS:
1461 		cmd->init_time = mftb();
1462 		iue->remote_token = crq->IU_data_ptr;
1463 		iue->iu_len = len;
1464 		dev_dbg(&vscsi->dev, "copy_crq: ioba 0x%llx, init_time 0x%llx\n",
1465 			be64_to_cpu(crq->IU_data_ptr), cmd->init_time);
1466 		break;
1467 	case H_PERMISSION:
1468 		if (connection_broken(vscsi))
1469 			ibmvscsis_post_disconnect(vscsi,
1470 						  ERR_DISCONNECT_RECONNECT,
1471 						  (RESPONSE_Q_DOWN |
1472 						   CLIENT_FAILED));
1473 		else
1474 			ibmvscsis_post_disconnect(vscsi,
1475 						  ERR_DISCONNECT_RECONNECT, 0);
1476 
1477 		dev_err(&vscsi->dev, "copy_crq: h_copy_rdma failed, rc %ld\n",
1478 			rc);
1479 		break;
1480 	case H_DEST_PARM:
1481 	case H_SOURCE_PARM:
1482 	default:
1483 		dev_err(&vscsi->dev, "copy_crq: h_copy_rdma failed, rc %ld\n",
1484 			rc);
1485 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
1486 		break;
1487 	}
1488 
1489 	return rc;
1490 }
1491 
1492 /**
1493  * ibmvscsis_adapter_info - Service an Adapter Info MAnagement Data gram
1494  * @vscsi:	Pointer to our adapter structure
1495  * @iue:	Information Unit containing the Adapter Info MAD request
1496  *
1497  * EXECUTION ENVIRONMENT:
1498  *	Interrupt adapter lock is held
1499  */
1500 static long ibmvscsis_adapter_info(struct scsi_info *vscsi,
1501 				   struct iu_entry *iue)
1502 {
1503 	struct viosrp_adapter_info *mad = &vio_iu(iue)->mad.adapter_info;
1504 	struct mad_adapter_info_data *info;
1505 	uint flag_bits = 0;
1506 	dma_addr_t token;
1507 	long rc;
1508 
1509 	mad->common.status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1510 
1511 	if (be16_to_cpu(mad->common.length) > sizeof(*info)) {
1512 		mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1513 		return 0;
1514 	}
1515 
1516 	info = dma_alloc_coherent(&vscsi->dma_dev->dev, sizeof(*info), &token,
1517 				  GFP_ATOMIC);
1518 	if (!info) {
1519 		dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n",
1520 			iue->target);
1521 		mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1522 		return 0;
1523 	}
1524 
1525 	/* Get remote info */
1526 	rc = h_copy_rdma(be16_to_cpu(mad->common.length),
1527 			 vscsi->dds.window[REMOTE].liobn,
1528 			 be64_to_cpu(mad->buffer),
1529 			 vscsi->dds.window[LOCAL].liobn, token);
1530 
1531 	if (rc != H_SUCCESS) {
1532 		if (rc == H_PERMISSION) {
1533 			if (connection_broken(vscsi))
1534 				flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
1535 		}
1536 		dev_warn(&vscsi->dev, "adapter_info: h_copy_rdma from client failed, rc %ld\n",
1537 			 rc);
1538 		dev_dbg(&vscsi->dev, "adapter_info: ioba 0x%llx, flags 0x%x, flag_bits 0x%x\n",
1539 			be64_to_cpu(mad->buffer), vscsi->flags, flag_bits);
1540 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
1541 					  flag_bits);
1542 		goto free_dma;
1543 	}
1544 
1545 	/*
1546 	 * Copy client info, but ignore partition number, which we
1547 	 * already got from phyp - unless we failed to get it from
1548 	 * phyp (e.g. if we're running on a p5 system).
1549 	 */
1550 	if (vscsi->client_data.partition_number == 0)
1551 		vscsi->client_data.partition_number =
1552 			be32_to_cpu(info->partition_number);
1553 	strncpy(vscsi->client_data.srp_version, info->srp_version,
1554 		sizeof(vscsi->client_data.srp_version));
1555 	strncpy(vscsi->client_data.partition_name, info->partition_name,
1556 		sizeof(vscsi->client_data.partition_name));
1557 	vscsi->client_data.mad_version = be32_to_cpu(info->mad_version);
1558 	vscsi->client_data.os_type = be32_to_cpu(info->os_type);
1559 
1560 	/* Copy our info */
1561 	strncpy(info->srp_version, SRP_VERSION,
1562 		sizeof(info->srp_version));
1563 	strncpy(info->partition_name, vscsi->dds.partition_name,
1564 		sizeof(info->partition_name));
1565 	info->partition_number = cpu_to_be32(vscsi->dds.partition_num);
1566 	info->mad_version = cpu_to_be32(MAD_VERSION_1);
1567 	info->os_type = cpu_to_be32(LINUX);
1568 	memset(&info->port_max_txu[0], 0, sizeof(info->port_max_txu));
1569 	info->port_max_txu[0] = cpu_to_be32(MAX_TXU);
1570 
1571 	dma_wmb();
1572 	rc = h_copy_rdma(sizeof(*info), vscsi->dds.window[LOCAL].liobn,
1573 			 token, vscsi->dds.window[REMOTE].liobn,
1574 			 be64_to_cpu(mad->buffer));
1575 	switch (rc) {
1576 	case H_SUCCESS:
1577 		break;
1578 
1579 	case H_SOURCE_PARM:
1580 	case H_DEST_PARM:
1581 	case H_PERMISSION:
1582 		if (connection_broken(vscsi))
1583 			flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
1584 		fallthrough;
1585 	default:
1586 		dev_err(&vscsi->dev, "adapter_info: h_copy_rdma to client failed, rc %ld\n",
1587 			rc);
1588 		ibmvscsis_post_disconnect(vscsi,
1589 					  ERR_DISCONNECT_RECONNECT,
1590 					  flag_bits);
1591 		break;
1592 	}
1593 
1594 free_dma:
1595 	dma_free_coherent(&vscsi->dma_dev->dev, sizeof(*info), info, token);
1596 	dev_dbg(&vscsi->dev, "Leaving adapter_info, rc %ld\n", rc);
1597 
1598 	return rc;
1599 }
1600 
1601 /**
1602  * ibmvscsis_cap_mad() - Service a Capabilities MAnagement Data gram
1603  * @vscsi:	Pointer to our adapter structure
1604  * @iue:	Information Unit containing the Capabilities MAD request
1605  *
1606  * NOTE: if you return an error from this routine you must be
1607  * disconnecting or you will cause a hang
1608  *
1609  * EXECUTION ENVIRONMENT:
1610  *	Interrupt called with adapter lock held
1611  */
1612 static int ibmvscsis_cap_mad(struct scsi_info *vscsi, struct iu_entry *iue)
1613 {
1614 	struct viosrp_capabilities *mad = &vio_iu(iue)->mad.capabilities;
1615 	struct capabilities *cap;
1616 	struct mad_capability_common *common;
1617 	dma_addr_t token;
1618 	u16 olen, len, status, min_len, cap_len;
1619 	u32 flag;
1620 	uint flag_bits = 0;
1621 	long rc = 0;
1622 
1623 	olen = be16_to_cpu(mad->common.length);
1624 	/*
1625 	 * struct capabilities hardcodes a couple capabilities after the
1626 	 * header, but the capabilities can actually be in any order.
1627 	 */
1628 	min_len = offsetof(struct capabilities, migration);
1629 	if ((olen < min_len) || (olen > PAGE_SIZE)) {
1630 		dev_warn(&vscsi->dev, "cap_mad: invalid len %d\n", olen);
1631 		mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1632 		return 0;
1633 	}
1634 
1635 	cap = dma_alloc_coherent(&vscsi->dma_dev->dev, olen, &token,
1636 				 GFP_ATOMIC);
1637 	if (!cap) {
1638 		dev_err(&vscsi->dev, "bad dma_alloc_coherent %p\n",
1639 			iue->target);
1640 		mad->common.status = cpu_to_be16(VIOSRP_MAD_FAILED);
1641 		return 0;
1642 	}
1643 	rc = h_copy_rdma(olen, vscsi->dds.window[REMOTE].liobn,
1644 			 be64_to_cpu(mad->buffer),
1645 			 vscsi->dds.window[LOCAL].liobn, token);
1646 	if (rc == H_SUCCESS) {
1647 		strncpy(cap->name, dev_name(&vscsi->dma_dev->dev),
1648 			SRP_MAX_LOC_LEN);
1649 
1650 		len = olen - min_len;
1651 		status = VIOSRP_MAD_SUCCESS;
1652 		common = (struct mad_capability_common *)&cap->migration;
1653 
1654 		while ((len > 0) && (status == VIOSRP_MAD_SUCCESS) && !rc) {
1655 			dev_dbg(&vscsi->dev, "cap_mad: len left %hd, cap type %d, cap len %hd\n",
1656 				len, be32_to_cpu(common->cap_type),
1657 				be16_to_cpu(common->length));
1658 
1659 			cap_len = be16_to_cpu(common->length);
1660 			if (cap_len > len) {
1661 				dev_err(&vscsi->dev, "cap_mad: cap len mismatch with total len\n");
1662 				status = VIOSRP_MAD_FAILED;
1663 				break;
1664 			}
1665 
1666 			if (cap_len == 0) {
1667 				dev_err(&vscsi->dev, "cap_mad: cap len is 0\n");
1668 				status = VIOSRP_MAD_FAILED;
1669 				break;
1670 			}
1671 
1672 			switch (common->cap_type) {
1673 			default:
1674 				dev_dbg(&vscsi->dev, "cap_mad: unsupported capability\n");
1675 				common->server_support = 0;
1676 				flag = cpu_to_be32((u32)CAP_LIST_SUPPORTED);
1677 				cap->flags &= ~flag;
1678 				break;
1679 			}
1680 
1681 			len = len - cap_len;
1682 			common = (struct mad_capability_common *)
1683 				((char *)common + cap_len);
1684 		}
1685 
1686 		mad->common.status = cpu_to_be16(status);
1687 
1688 		dma_wmb();
1689 		rc = h_copy_rdma(olen, vscsi->dds.window[LOCAL].liobn, token,
1690 				 vscsi->dds.window[REMOTE].liobn,
1691 				 be64_to_cpu(mad->buffer));
1692 
1693 		if (rc != H_SUCCESS) {
1694 			dev_dbg(&vscsi->dev, "cap_mad: failed to copy to client, rc %ld\n",
1695 				rc);
1696 
1697 			if (rc == H_PERMISSION) {
1698 				if (connection_broken(vscsi))
1699 					flag_bits = (RESPONSE_Q_DOWN |
1700 						     CLIENT_FAILED);
1701 			}
1702 
1703 			dev_warn(&vscsi->dev, "cap_mad: error copying data to client, rc %ld\n",
1704 				 rc);
1705 			ibmvscsis_post_disconnect(vscsi,
1706 						  ERR_DISCONNECT_RECONNECT,
1707 						  flag_bits);
1708 		}
1709 	}
1710 
1711 	dma_free_coherent(&vscsi->dma_dev->dev, olen, cap, token);
1712 
1713 	dev_dbg(&vscsi->dev, "Leaving cap_mad, rc %ld, client_cap 0x%x\n",
1714 		rc, vscsi->client_cap);
1715 
1716 	return rc;
1717 }
1718 
1719 /**
1720  * ibmvscsis_process_mad() - Service a MAnagement Data gram
1721  * @vscsi:	Pointer to our adapter structure
1722  * @iue:	Information Unit containing the MAD request
1723  *
1724  * Must be called with interrupt lock held.
1725  */
1726 static long ibmvscsis_process_mad(struct scsi_info *vscsi, struct iu_entry *iue)
1727 {
1728 	struct mad_common *mad = (struct mad_common *)&vio_iu(iue)->mad;
1729 	struct viosrp_empty_iu *empty;
1730 	long rc = ADAPT_SUCCESS;
1731 
1732 	switch (be32_to_cpu(mad->type)) {
1733 	case VIOSRP_EMPTY_IU_TYPE:
1734 		empty = &vio_iu(iue)->mad.empty_iu;
1735 		vscsi->empty_iu_id = be64_to_cpu(empty->buffer);
1736 		vscsi->empty_iu_tag = be64_to_cpu(empty->common.tag);
1737 		mad->status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1738 		break;
1739 	case VIOSRP_ADAPTER_INFO_TYPE:
1740 		rc = ibmvscsis_adapter_info(vscsi, iue);
1741 		break;
1742 	case VIOSRP_CAPABILITIES_TYPE:
1743 		rc = ibmvscsis_cap_mad(vscsi, iue);
1744 		break;
1745 	case VIOSRP_ENABLE_FAST_FAIL:
1746 		if (vscsi->state == CONNECTED) {
1747 			vscsi->fast_fail = true;
1748 			mad->status = cpu_to_be16(VIOSRP_MAD_SUCCESS);
1749 		} else {
1750 			dev_warn(&vscsi->dev, "fast fail mad sent after login\n");
1751 			mad->status = cpu_to_be16(VIOSRP_MAD_FAILED);
1752 		}
1753 		break;
1754 	default:
1755 		mad->status = cpu_to_be16(VIOSRP_MAD_NOT_SUPPORTED);
1756 		break;
1757 	}
1758 
1759 	return rc;
1760 }
1761 
1762 /**
1763  * srp_snd_msg_failed() - Handle an error when sending a response
1764  * @vscsi:	Pointer to our adapter structure
1765  * @rc:		The return code from the h_send_crq command
1766  *
1767  * Must be called with interrupt lock held.
1768  */
1769 static void srp_snd_msg_failed(struct scsi_info *vscsi, long rc)
1770 {
1771 	ktime_t kt;
1772 
1773 	if (rc != H_DROPPED) {
1774 		ibmvscsis_free_cmd_qs(vscsi);
1775 
1776 		if (rc == H_CLOSED)
1777 			vscsi->flags |= CLIENT_FAILED;
1778 
1779 		/* don't flag the same problem multiple times */
1780 		if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1781 			vscsi->flags |= RESPONSE_Q_DOWN;
1782 			if (!(vscsi->state & (ERR_DISCONNECT |
1783 					      ERR_DISCONNECT_RECONNECT |
1784 					      ERR_DISCONNECTED | UNDEFINED))) {
1785 				dev_err(&vscsi->dev, "snd_msg_failed: setting RESPONSE_Q_DOWN, state 0x%hx, flags 0x%x, rc %ld\n",
1786 					vscsi->state, vscsi->flags, rc);
1787 			}
1788 			ibmvscsis_post_disconnect(vscsi,
1789 						  ERR_DISCONNECT_RECONNECT, 0);
1790 		}
1791 		return;
1792 	}
1793 
1794 	/*
1795 	 * The response queue is full.
1796 	 * If the server is processing SRP requests, i.e.
1797 	 * the client has successfully done an
1798 	 * SRP_LOGIN, then it will wait forever for room in
1799 	 * the queue.  However if the system admin
1800 	 * is attempting to unconfigure the server then one
1801 	 * or more children will be in a state where
1802 	 * they are being removed. So if there is even one
1803 	 * child being removed then the driver assumes
1804 	 * the system admin is attempting to break the
1805 	 * connection with the client and MAX_TIMER_POPS
1806 	 * is honored.
1807 	 */
1808 	if ((vscsi->rsp_q_timer.timer_pops < MAX_TIMER_POPS) ||
1809 	    (vscsi->state == SRP_PROCESSING)) {
1810 		dev_dbg(&vscsi->dev, "snd_msg_failed: response queue full, flags 0x%x, timer started %d, pops %d\n",
1811 			vscsi->flags, (int)vscsi->rsp_q_timer.started,
1812 			vscsi->rsp_q_timer.timer_pops);
1813 
1814 		/*
1815 		 * Check if the timer is running; if it
1816 		 * is not then start it up.
1817 		 */
1818 		if (!vscsi->rsp_q_timer.started) {
1819 			if (vscsi->rsp_q_timer.timer_pops <
1820 			    MAX_TIMER_POPS) {
1821 				kt = WAIT_NANO_SECONDS;
1822 			} else {
1823 				/*
1824 				 * slide the timeslice if the maximum
1825 				 * timer pops have already happened
1826 				 */
1827 				kt = ktime_set(WAIT_SECONDS, 0);
1828 			}
1829 
1830 			vscsi->rsp_q_timer.started = true;
1831 			hrtimer_start(&vscsi->rsp_q_timer.timer, kt,
1832 				      HRTIMER_MODE_REL);
1833 		}
1834 	} else {
1835 		/*
1836 		 * TBD: Do we need to worry about this? Need to get
1837 		 *      remove working.
1838 		 */
1839 		/*
1840 		 * waited a long time and it appears the system admin
1841 		 * is bring this driver down
1842 		 */
1843 		vscsi->flags |= RESPONSE_Q_DOWN;
1844 		ibmvscsis_free_cmd_qs(vscsi);
1845 		/*
1846 		 * if the driver is already attempting to disconnect
1847 		 * from the client and has already logged an error
1848 		 * trace this event but don't put it in the error log
1849 		 */
1850 		if (!(vscsi->state & (ERR_DISCONNECT |
1851 				      ERR_DISCONNECT_RECONNECT |
1852 				      ERR_DISCONNECTED | UNDEFINED))) {
1853 			dev_err(&vscsi->dev, "client crq full too long\n");
1854 			ibmvscsis_post_disconnect(vscsi,
1855 						  ERR_DISCONNECT_RECONNECT,
1856 						  0);
1857 		}
1858 	}
1859 }
1860 
1861 /**
1862  * ibmvscsis_send_messages() - Send a Response
1863  * @vscsi:	Pointer to our adapter structure
1864  *
1865  * Send a response, first checking the waiting queue. Responses are
1866  * sent in order they are received. If the response cannot be sent,
1867  * because the client queue is full, it stays on the waiting queue.
1868  *
1869  * PRECONDITION:
1870  *	Called with interrupt lock held
1871  */
1872 static void ibmvscsis_send_messages(struct scsi_info *vscsi)
1873 {
1874 	u64 msg_hi = 0;
1875 	/* note do not attempt to access the IU_data_ptr with this pointer
1876 	 * it is not valid
1877 	 */
1878 	struct viosrp_crq *crq = (struct viosrp_crq *)&msg_hi;
1879 	struct ibmvscsis_cmd *cmd, *nxt;
1880 	long rc = ADAPT_SUCCESS;
1881 	bool retry = false;
1882 
1883 	if (!(vscsi->flags & RESPONSE_Q_DOWN)) {
1884 		do {
1885 			retry = false;
1886 			list_for_each_entry_safe(cmd, nxt, &vscsi->waiting_rsp,
1887 						 list) {
1888 				/*
1889 				 * Check to make sure abort cmd gets processed
1890 				 * prior to the abort tmr cmd
1891 				 */
1892 				if (cmd->flags & DELAY_SEND)
1893 					continue;
1894 
1895 				if (cmd->abort_cmd) {
1896 					retry = true;
1897 					cmd->abort_cmd->flags &= ~(DELAY_SEND);
1898 					cmd->abort_cmd = NULL;
1899 				}
1900 
1901 				/*
1902 				 * If CMD_T_ABORTED w/o CMD_T_TAS scenarios and
1903 				 * the case where LIO issued a
1904 				 * ABORT_TASK: Sending TMR_TASK_DOES_NOT_EXIST
1905 				 * case then we dont send a response, since it
1906 				 * was already done.
1907 				 */
1908 				if (cmd->se_cmd.transport_state & CMD_T_ABORTED &&
1909 				    !(cmd->se_cmd.transport_state & CMD_T_TAS)) {
1910 					list_del(&cmd->list);
1911 					ibmvscsis_free_cmd_resources(vscsi,
1912 								     cmd);
1913 					/*
1914 					 * With a successfully aborted op
1915 					 * through LIO we want to increment the
1916 					 * the vscsi credit so that when we dont
1917 					 * send a rsp to the original scsi abort
1918 					 * op (h_send_crq), but the tm rsp to
1919 					 * the abort is sent, the credit is
1920 					 * correctly sent with the abort tm rsp.
1921 					 * We would need 1 for the abort tm rsp
1922 					 * and 1 credit for the aborted scsi op.
1923 					 * Thus we need to increment here.
1924 					 * Also we want to increment the credit
1925 					 * here because we want to make sure
1926 					 * cmd is actually released first
1927 					 * otherwise the client will think it
1928 					 * it can send a new cmd, and we could
1929 					 * find ourselves short of cmd elements.
1930 					 */
1931 					vscsi->credit += 1;
1932 				} else {
1933 					crq->valid = VALID_CMD_RESP_EL;
1934 					crq->format = cmd->rsp.format;
1935 
1936 					if (cmd->flags & CMD_FAST_FAIL)
1937 						crq->status = VIOSRP_ADAPTER_FAIL;
1938 
1939 					crq->IU_length = cpu_to_be16(cmd->rsp.len);
1940 
1941 					rc = h_send_crq(vscsi->dma_dev->unit_address,
1942 							be64_to_cpu(msg_hi),
1943 							be64_to_cpu(cmd->rsp.tag));
1944 
1945 					dev_dbg(&vscsi->dev, "send_messages: cmd %p, tag 0x%llx, rc %ld\n",
1946 						cmd, be64_to_cpu(cmd->rsp.tag),
1947 						rc);
1948 
1949 					/* if all ok free up the command
1950 					 * element resources
1951 					 */
1952 					if (rc == H_SUCCESS) {
1953 						/* some movement has occurred */
1954 						vscsi->rsp_q_timer.timer_pops = 0;
1955 						list_del(&cmd->list);
1956 
1957 						ibmvscsis_free_cmd_resources(vscsi,
1958 									     cmd);
1959 					} else {
1960 						srp_snd_msg_failed(vscsi, rc);
1961 						break;
1962 					}
1963 				}
1964 			}
1965 		} while (retry);
1966 
1967 		if (!rc) {
1968 			/*
1969 			 * The timer could pop with the queue empty.  If
1970 			 * this happens, rc will always indicate a
1971 			 * success; clear the pop count.
1972 			 */
1973 			vscsi->rsp_q_timer.timer_pops = 0;
1974 		}
1975 	} else {
1976 		ibmvscsis_free_cmd_qs(vscsi);
1977 	}
1978 }
1979 
1980 /* Called with intr lock held */
1981 static void ibmvscsis_send_mad_resp(struct scsi_info *vscsi,
1982 				    struct ibmvscsis_cmd *cmd,
1983 				    struct viosrp_crq *crq)
1984 {
1985 	struct iu_entry *iue = cmd->iue;
1986 	struct mad_common *mad = (struct mad_common *)&vio_iu(iue)->mad;
1987 	uint flag_bits = 0;
1988 	long rc;
1989 
1990 	dma_wmb();
1991 	rc = h_copy_rdma(sizeof(struct mad_common),
1992 			 vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma,
1993 			 vscsi->dds.window[REMOTE].liobn,
1994 			 be64_to_cpu(crq->IU_data_ptr));
1995 	if (!rc) {
1996 		cmd->rsp.format = VIOSRP_MAD_FORMAT;
1997 		cmd->rsp.len = sizeof(struct mad_common);
1998 		cmd->rsp.tag = mad->tag;
1999 		list_add_tail(&cmd->list, &vscsi->waiting_rsp);
2000 		ibmvscsis_send_messages(vscsi);
2001 	} else {
2002 		dev_dbg(&vscsi->dev, "Error sending mad response, rc %ld\n",
2003 			rc);
2004 		if (rc == H_PERMISSION) {
2005 			if (connection_broken(vscsi))
2006 				flag_bits = (RESPONSE_Q_DOWN | CLIENT_FAILED);
2007 		}
2008 		dev_err(&vscsi->dev, "mad: failed to copy to client, rc %ld\n",
2009 			rc);
2010 
2011 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2012 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2013 					  flag_bits);
2014 	}
2015 }
2016 
2017 /**
2018  * ibmvscsis_mad() - Service a MAnagement Data gram.
2019  * @vscsi:	Pointer to our adapter structure
2020  * @crq:	Pointer to the CRQ entry containing the MAD request
2021  *
2022  * EXECUTION ENVIRONMENT:
2023  *	Interrupt, called with adapter lock held
2024  */
2025 static long ibmvscsis_mad(struct scsi_info *vscsi, struct viosrp_crq *crq)
2026 {
2027 	struct iu_entry *iue;
2028 	struct ibmvscsis_cmd *cmd;
2029 	struct mad_common *mad;
2030 	long rc = ADAPT_SUCCESS;
2031 
2032 	switch (vscsi->state) {
2033 		/*
2034 		 * We have not exchanged Init Msgs yet, so this MAD was sent
2035 		 * before the last Transport Event; client will not be
2036 		 * expecting a response.
2037 		 */
2038 	case WAIT_CONNECTION:
2039 		dev_dbg(&vscsi->dev, "mad: in Wait Connection state, ignoring MAD, flags %d\n",
2040 			vscsi->flags);
2041 		return ADAPT_SUCCESS;
2042 
2043 	case SRP_PROCESSING:
2044 	case CONNECTED:
2045 		break;
2046 
2047 		/*
2048 		 * We should never get here while we're in these states.
2049 		 * Just log an error and get out.
2050 		 */
2051 	case UNCONFIGURING:
2052 	case WAIT_IDLE:
2053 	case ERR_DISCONNECT:
2054 	case ERR_DISCONNECT_RECONNECT:
2055 	default:
2056 		dev_err(&vscsi->dev, "mad: invalid adapter state %d for mad\n",
2057 			vscsi->state);
2058 		return ADAPT_SUCCESS;
2059 	}
2060 
2061 	cmd = ibmvscsis_get_free_cmd(vscsi);
2062 	if (!cmd) {
2063 		dev_err(&vscsi->dev, "mad: failed to get cmd, debit %d\n",
2064 			vscsi->debit);
2065 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2066 		return ERROR;
2067 	}
2068 	iue = cmd->iue;
2069 	cmd->type = ADAPTER_MAD;
2070 
2071 	rc = ibmvscsis_copy_crq_packet(vscsi, cmd, crq);
2072 	if (!rc) {
2073 		mad = (struct mad_common *)&vio_iu(iue)->mad;
2074 
2075 		dev_dbg(&vscsi->dev, "mad: type %d\n", be32_to_cpu(mad->type));
2076 
2077 		rc = ibmvscsis_process_mad(vscsi, iue);
2078 
2079 		dev_dbg(&vscsi->dev, "mad: status %hd, rc %ld\n",
2080 			be16_to_cpu(mad->status), rc);
2081 
2082 		if (!rc)
2083 			ibmvscsis_send_mad_resp(vscsi, cmd, crq);
2084 	} else {
2085 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2086 	}
2087 
2088 	dev_dbg(&vscsi->dev, "Leaving mad, rc %ld\n", rc);
2089 	return rc;
2090 }
2091 
2092 /**
2093  * ibmvscsis_login_rsp() - Create/copy a login response notice to the client
2094  * @vscsi:	Pointer to our adapter structure
2095  * @cmd:	Pointer to the command for the SRP Login request
2096  *
2097  * EXECUTION ENVIRONMENT:
2098  *	Interrupt, interrupt lock held
2099  */
2100 static long ibmvscsis_login_rsp(struct scsi_info *vscsi,
2101 				struct ibmvscsis_cmd *cmd)
2102 {
2103 	struct iu_entry *iue = cmd->iue;
2104 	struct srp_login_rsp *rsp = &vio_iu(iue)->srp.login_rsp;
2105 	struct format_code *fmt;
2106 	uint flag_bits = 0;
2107 	long rc = ADAPT_SUCCESS;
2108 
2109 	memset(rsp, 0, sizeof(struct srp_login_rsp));
2110 
2111 	rsp->opcode = SRP_LOGIN_RSP;
2112 	rsp->req_lim_delta = cpu_to_be32(vscsi->request_limit);
2113 	rsp->tag = cmd->rsp.tag;
2114 	rsp->max_it_iu_len = cpu_to_be32(SRP_MAX_IU_LEN);
2115 	rsp->max_ti_iu_len = cpu_to_be32(SRP_MAX_IU_LEN);
2116 	fmt = (struct format_code *)&rsp->buf_fmt;
2117 	fmt->buffers = SUPPORTED_FORMATS;
2118 	vscsi->credit = 0;
2119 
2120 	cmd->rsp.len = sizeof(struct srp_login_rsp);
2121 
2122 	dma_wmb();
2123 	rc = h_copy_rdma(cmd->rsp.len, vscsi->dds.window[LOCAL].liobn,
2124 			 iue->sbuf->dma, vscsi->dds.window[REMOTE].liobn,
2125 			 be64_to_cpu(iue->remote_token));
2126 
2127 	switch (rc) {
2128 	case H_SUCCESS:
2129 		break;
2130 
2131 	case H_PERMISSION:
2132 		if (connection_broken(vscsi))
2133 			flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
2134 		dev_err(&vscsi->dev, "login_rsp: error copying to client, rc %ld\n",
2135 			rc);
2136 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2137 					  flag_bits);
2138 		break;
2139 	case H_SOURCE_PARM:
2140 	case H_DEST_PARM:
2141 	default:
2142 		dev_err(&vscsi->dev, "login_rsp: error copying to client, rc %ld\n",
2143 			rc);
2144 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2145 		break;
2146 	}
2147 
2148 	return rc;
2149 }
2150 
2151 /**
2152  * ibmvscsis_srp_login_rej() - Create/copy a login rejection notice to client
2153  * @vscsi:	Pointer to our adapter structure
2154  * @cmd:	Pointer to the command for the SRP Login request
2155  * @reason:	The reason the SRP Login is being rejected, per SRP protocol
2156  *
2157  * EXECUTION ENVIRONMENT:
2158  *	Interrupt, interrupt lock held
2159  */
2160 static long ibmvscsis_srp_login_rej(struct scsi_info *vscsi,
2161 				    struct ibmvscsis_cmd *cmd, u32 reason)
2162 {
2163 	struct iu_entry *iue = cmd->iue;
2164 	struct srp_login_rej *rej = &vio_iu(iue)->srp.login_rej;
2165 	struct format_code *fmt;
2166 	uint flag_bits = 0;
2167 	long rc = ADAPT_SUCCESS;
2168 
2169 	memset(rej, 0, sizeof(*rej));
2170 
2171 	rej->opcode = SRP_LOGIN_REJ;
2172 	rej->reason = cpu_to_be32(reason);
2173 	rej->tag = cmd->rsp.tag;
2174 	fmt = (struct format_code *)&rej->buf_fmt;
2175 	fmt->buffers = SUPPORTED_FORMATS;
2176 
2177 	cmd->rsp.len = sizeof(*rej);
2178 
2179 	dma_wmb();
2180 	rc = h_copy_rdma(cmd->rsp.len, vscsi->dds.window[LOCAL].liobn,
2181 			 iue->sbuf->dma, vscsi->dds.window[REMOTE].liobn,
2182 			 be64_to_cpu(iue->remote_token));
2183 
2184 	switch (rc) {
2185 	case H_SUCCESS:
2186 		break;
2187 	case H_PERMISSION:
2188 		if (connection_broken(vscsi))
2189 			flag_bits = RESPONSE_Q_DOWN | CLIENT_FAILED;
2190 		dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
2191 			rc);
2192 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT,
2193 					  flag_bits);
2194 		break;
2195 	case H_SOURCE_PARM:
2196 	case H_DEST_PARM:
2197 	default:
2198 		dev_err(&vscsi->dev, "login_rej: error copying to client, rc %ld\n",
2199 			rc);
2200 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2201 		break;
2202 	}
2203 
2204 	return rc;
2205 }
2206 
2207 static int ibmvscsis_make_nexus(struct ibmvscsis_tport *tport)
2208 {
2209 	char *name = tport->tport_name;
2210 	struct ibmvscsis_nexus *nexus;
2211 	struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
2212 	int rc;
2213 
2214 	if (tport->ibmv_nexus) {
2215 		dev_dbg(&vscsi->dev, "tport->ibmv_nexus already exists\n");
2216 		return 0;
2217 	}
2218 
2219 	nexus = kzalloc(sizeof(*nexus), GFP_KERNEL);
2220 	if (!nexus) {
2221 		dev_err(&vscsi->dev, "Unable to allocate struct ibmvscsis_nexus\n");
2222 		return -ENOMEM;
2223 	}
2224 
2225 	nexus->se_sess = target_setup_session(&tport->se_tpg, 0, 0,
2226 					      TARGET_PROT_NORMAL, name, nexus,
2227 					      NULL);
2228 	if (IS_ERR(nexus->se_sess)) {
2229 		rc = PTR_ERR(nexus->se_sess);
2230 		goto transport_init_fail;
2231 	}
2232 
2233 	tport->ibmv_nexus = nexus;
2234 
2235 	return 0;
2236 
2237 transport_init_fail:
2238 	kfree(nexus);
2239 	return rc;
2240 }
2241 
2242 static int ibmvscsis_drop_nexus(struct ibmvscsis_tport *tport)
2243 {
2244 	struct se_session *se_sess;
2245 	struct ibmvscsis_nexus *nexus;
2246 
2247 	nexus = tport->ibmv_nexus;
2248 	if (!nexus)
2249 		return -ENODEV;
2250 
2251 	se_sess = nexus->se_sess;
2252 	if (!se_sess)
2253 		return -ENODEV;
2254 
2255 	/*
2256 	 * Release the SCSI I_T Nexus to the emulated ibmvscsis Target Port
2257 	 */
2258 	target_remove_session(se_sess);
2259 	tport->ibmv_nexus = NULL;
2260 	kfree(nexus);
2261 
2262 	return 0;
2263 }
2264 
2265 /**
2266  * ibmvscsis_srp_login() - Process an SRP Login Request
2267  * @vscsi:	Pointer to our adapter structure
2268  * @cmd:	Command element to use to process the SRP Login request
2269  * @crq:	Pointer to CRQ entry containing the SRP Login request
2270  *
2271  * EXECUTION ENVIRONMENT:
2272  *	Interrupt, called with interrupt lock held
2273  */
2274 static long ibmvscsis_srp_login(struct scsi_info *vscsi,
2275 				struct ibmvscsis_cmd *cmd,
2276 				struct viosrp_crq *crq)
2277 {
2278 	struct iu_entry *iue = cmd->iue;
2279 	struct srp_login_req *req = &vio_iu(iue)->srp.login_req;
2280 	struct port_id {
2281 		__be64 id_extension;
2282 		__be64 io_guid;
2283 	} *iport, *tport;
2284 	struct format_code *fmt;
2285 	u32 reason = 0x0;
2286 	long rc = ADAPT_SUCCESS;
2287 
2288 	iport = (struct port_id *)req->initiator_port_id;
2289 	tport = (struct port_id *)req->target_port_id;
2290 	fmt = (struct format_code *)&req->req_buf_fmt;
2291 	if (be32_to_cpu(req->req_it_iu_len) > SRP_MAX_IU_LEN)
2292 		reason = SRP_LOGIN_REJ_REQ_IT_IU_LENGTH_TOO_LARGE;
2293 	else if (be32_to_cpu(req->req_it_iu_len) < 64)
2294 		reason = SRP_LOGIN_REJ_UNABLE_ESTABLISH_CHANNEL;
2295 	else if ((be64_to_cpu(iport->id_extension) > (MAX_NUM_PORTS - 1)) ||
2296 		 (be64_to_cpu(tport->id_extension) > (MAX_NUM_PORTS - 1)))
2297 		reason = SRP_LOGIN_REJ_UNABLE_ASSOCIATE_CHANNEL;
2298 	else if (req->req_flags & SRP_MULTICHAN_MULTI)
2299 		reason = SRP_LOGIN_REJ_MULTI_CHANNEL_UNSUPPORTED;
2300 	else if (fmt->buffers & (~SUPPORTED_FORMATS))
2301 		reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
2302 	else if ((fmt->buffers & SUPPORTED_FORMATS) == 0)
2303 		reason = SRP_LOGIN_REJ_UNSUPPORTED_DESCRIPTOR_FMT;
2304 
2305 	if (vscsi->state == SRP_PROCESSING)
2306 		reason = SRP_LOGIN_REJ_CHANNEL_LIMIT_REACHED;
2307 
2308 	rc = ibmvscsis_make_nexus(&vscsi->tport);
2309 	if (rc)
2310 		reason = SRP_LOGIN_REJ_UNABLE_ESTABLISH_CHANNEL;
2311 
2312 	cmd->rsp.format = VIOSRP_SRP_FORMAT;
2313 	cmd->rsp.tag = req->tag;
2314 
2315 	dev_dbg(&vscsi->dev, "srp_login: reason 0x%x\n", reason);
2316 
2317 	if (reason)
2318 		rc = ibmvscsis_srp_login_rej(vscsi, cmd, reason);
2319 	else
2320 		rc = ibmvscsis_login_rsp(vscsi, cmd);
2321 
2322 	if (!rc) {
2323 		if (!reason)
2324 			vscsi->state = SRP_PROCESSING;
2325 
2326 		list_add_tail(&cmd->list, &vscsi->waiting_rsp);
2327 		ibmvscsis_send_messages(vscsi);
2328 	} else {
2329 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2330 	}
2331 
2332 	dev_dbg(&vscsi->dev, "Leaving srp_login, rc %ld\n", rc);
2333 	return rc;
2334 }
2335 
2336 /**
2337  * ibmvscsis_srp_i_logout() - Helper Function to close I_T Nexus
2338  * @vscsi:	Pointer to our adapter structure
2339  * @cmd:	Command element to use to process the Implicit Logout request
2340  * @crq:	Pointer to CRQ entry containing the Implicit Logout request
2341  *
2342  * Do the logic to close the I_T nexus.  This function may not
2343  * behave to specification.
2344  *
2345  * EXECUTION ENVIRONMENT:
2346  *	Interrupt, interrupt lock held
2347  */
2348 static long ibmvscsis_srp_i_logout(struct scsi_info *vscsi,
2349 				   struct ibmvscsis_cmd *cmd,
2350 				   struct viosrp_crq *crq)
2351 {
2352 	struct iu_entry *iue = cmd->iue;
2353 	struct srp_i_logout *log_out = &vio_iu(iue)->srp.i_logout;
2354 
2355 	if ((vscsi->debit > 0) || !list_empty(&vscsi->schedule_q) ||
2356 	    !list_empty(&vscsi->waiting_rsp)) {
2357 		dev_err(&vscsi->dev, "i_logout: outstanding work\n");
2358 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
2359 	} else {
2360 		cmd->rsp.format = SRP_FORMAT;
2361 		cmd->rsp.tag = log_out->tag;
2362 		cmd->rsp.len = sizeof(struct mad_common);
2363 		list_add_tail(&cmd->list, &vscsi->waiting_rsp);
2364 		ibmvscsis_send_messages(vscsi);
2365 
2366 		ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
2367 	}
2368 
2369 	return ADAPT_SUCCESS;
2370 }
2371 
2372 /* Called with intr lock held */
2373 static void ibmvscsis_srp_cmd(struct scsi_info *vscsi, struct viosrp_crq *crq)
2374 {
2375 	struct ibmvscsis_cmd *cmd;
2376 	struct iu_entry *iue;
2377 	struct srp_cmd *srp;
2378 	struct srp_tsk_mgmt *tsk;
2379 	long rc;
2380 
2381 	if (vscsi->request_limit - vscsi->debit <= 0) {
2382 		/* Client has exceeded request limit */
2383 		dev_err(&vscsi->dev, "Client exceeded the request limit (%d), debit %d\n",
2384 			vscsi->request_limit, vscsi->debit);
2385 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2386 		return;
2387 	}
2388 
2389 	cmd = ibmvscsis_get_free_cmd(vscsi);
2390 	if (!cmd) {
2391 		dev_err(&vscsi->dev, "srp_cmd failed to get cmd, debit %d\n",
2392 			vscsi->debit);
2393 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2394 		return;
2395 	}
2396 	iue = cmd->iue;
2397 	srp = &vio_iu(iue)->srp.cmd;
2398 
2399 	rc = ibmvscsis_copy_crq_packet(vscsi, cmd, crq);
2400 	if (rc) {
2401 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2402 		return;
2403 	}
2404 
2405 	if (vscsi->state == SRP_PROCESSING) {
2406 		switch (srp->opcode) {
2407 		case SRP_LOGIN_REQ:
2408 			rc = ibmvscsis_srp_login(vscsi, cmd, crq);
2409 			break;
2410 
2411 		case SRP_TSK_MGMT:
2412 			tsk = &vio_iu(iue)->srp.tsk_mgmt;
2413 			dev_dbg(&vscsi->dev, "tsk_mgmt tag: %llu (0x%llx)\n",
2414 				tsk->tag, tsk->tag);
2415 			cmd->rsp.tag = tsk->tag;
2416 			vscsi->debit += 1;
2417 			cmd->type = TASK_MANAGEMENT;
2418 			list_add_tail(&cmd->list, &vscsi->schedule_q);
2419 			queue_work(vscsi->work_q, &cmd->work);
2420 			break;
2421 
2422 		case SRP_CMD:
2423 			dev_dbg(&vscsi->dev, "srp_cmd tag: %llu (0x%llx)\n",
2424 				srp->tag, srp->tag);
2425 			cmd->rsp.tag = srp->tag;
2426 			vscsi->debit += 1;
2427 			cmd->type = SCSI_CDB;
2428 			/*
2429 			 * We want to keep track of work waiting for
2430 			 * the workqueue.
2431 			 */
2432 			list_add_tail(&cmd->list, &vscsi->schedule_q);
2433 			queue_work(vscsi->work_q, &cmd->work);
2434 			break;
2435 
2436 		case SRP_I_LOGOUT:
2437 			rc = ibmvscsis_srp_i_logout(vscsi, cmd, crq);
2438 			break;
2439 
2440 		case SRP_CRED_RSP:
2441 		case SRP_AER_RSP:
2442 		default:
2443 			ibmvscsis_free_cmd_resources(vscsi, cmd);
2444 			dev_err(&vscsi->dev, "invalid srp cmd, opcode %d\n",
2445 				(uint)srp->opcode);
2446 			ibmvscsis_post_disconnect(vscsi,
2447 						  ERR_DISCONNECT_RECONNECT, 0);
2448 			break;
2449 		}
2450 	} else if (srp->opcode == SRP_LOGIN_REQ && vscsi->state == CONNECTED) {
2451 		rc = ibmvscsis_srp_login(vscsi, cmd, crq);
2452 	} else {
2453 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2454 		dev_err(&vscsi->dev, "Invalid state %d to handle srp cmd\n",
2455 			vscsi->state);
2456 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2457 	}
2458 }
2459 
2460 /**
2461  * ibmvscsis_ping_response() - Respond to a ping request
2462  * @vscsi:	Pointer to our adapter structure
2463  *
2464  * Let the client know that the server is alive and waiting on
2465  * its native I/O stack.
2466  * If any type of error occurs from the call to queue a ping
2467  * response then the client is either not accepting or receiving
2468  * interrupts.  Disconnect with an error.
2469  *
2470  * EXECUTION ENVIRONMENT:
2471  *	Interrupt, interrupt lock held
2472  */
2473 static long ibmvscsis_ping_response(struct scsi_info *vscsi)
2474 {
2475 	struct viosrp_crq *crq;
2476 	u64 buffer[2] = { 0, 0 };
2477 	long rc;
2478 
2479 	crq = (struct viosrp_crq *)&buffer;
2480 	crq->valid = VALID_CMD_RESP_EL;
2481 	crq->format = (u8)MESSAGE_IN_CRQ;
2482 	crq->status = PING_RESPONSE;
2483 
2484 	rc = h_send_crq(vscsi->dds.unit_id, cpu_to_be64(buffer[MSG_HI]),
2485 			cpu_to_be64(buffer[MSG_LOW]));
2486 
2487 	switch (rc) {
2488 	case H_SUCCESS:
2489 		break;
2490 	case H_CLOSED:
2491 		vscsi->flags |= CLIENT_FAILED;
2492 		fallthrough;
2493 	case H_DROPPED:
2494 		vscsi->flags |= RESPONSE_Q_DOWN;
2495 		fallthrough;
2496 	case H_REMOTE_PARM:
2497 		dev_err(&vscsi->dev, "ping_response: h_send_crq failed, rc %ld\n",
2498 			rc);
2499 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2500 		break;
2501 	default:
2502 		dev_err(&vscsi->dev, "ping_response: h_send_crq returned unknown rc %ld\n",
2503 			rc);
2504 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
2505 		break;
2506 	}
2507 
2508 	return rc;
2509 }
2510 
2511 /**
2512  * ibmvscsis_parse_command() - Parse an element taken from the cmd rsp queue.
2513  * @vscsi:	Pointer to our adapter structure
2514  * @crq:	Pointer to CRQ element containing the SRP request
2515  *
2516  * This function will return success if the command queue element is valid
2517  * and the srp iu or MAD request it pointed to was also valid.  That does
2518  * not mean that an error was not returned to the client.
2519  *
2520  * EXECUTION ENVIRONMENT:
2521  *	Interrupt, intr lock held
2522  */
2523 static long ibmvscsis_parse_command(struct scsi_info *vscsi,
2524 				    struct viosrp_crq *crq)
2525 {
2526 	long rc = ADAPT_SUCCESS;
2527 
2528 	switch (crq->valid) {
2529 	case VALID_CMD_RESP_EL:
2530 		switch (crq->format) {
2531 		case OS400_FORMAT:
2532 		case AIX_FORMAT:
2533 		case LINUX_FORMAT:
2534 		case MAD_FORMAT:
2535 			if (vscsi->flags & PROCESSING_MAD) {
2536 				rc = ERROR;
2537 				dev_err(&vscsi->dev, "parse_command: already processing mad\n");
2538 				ibmvscsis_post_disconnect(vscsi,
2539 						       ERR_DISCONNECT_RECONNECT,
2540 						       0);
2541 			} else {
2542 				vscsi->flags |= PROCESSING_MAD;
2543 				rc = ibmvscsis_mad(vscsi, crq);
2544 			}
2545 			break;
2546 
2547 		case SRP_FORMAT:
2548 			ibmvscsis_srp_cmd(vscsi, crq);
2549 			break;
2550 
2551 		case MESSAGE_IN_CRQ:
2552 			if (crq->status == PING)
2553 				ibmvscsis_ping_response(vscsi);
2554 			break;
2555 
2556 		default:
2557 			dev_err(&vscsi->dev, "parse_command: invalid format %d\n",
2558 				(uint)crq->format);
2559 			ibmvscsis_post_disconnect(vscsi,
2560 						  ERR_DISCONNECT_RECONNECT, 0);
2561 			break;
2562 		}
2563 		break;
2564 
2565 	case VALID_TRANS_EVENT:
2566 		rc = ibmvscsis_trans_event(vscsi, crq);
2567 		break;
2568 
2569 	case VALID_INIT_MSG:
2570 		rc = ibmvscsis_init_msg(vscsi, crq);
2571 		break;
2572 
2573 	default:
2574 		dev_err(&vscsi->dev, "parse_command: invalid valid field %d\n",
2575 			(uint)crq->valid);
2576 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2577 		break;
2578 	}
2579 
2580 	/*
2581 	 * Return only what the interrupt handler cares
2582 	 * about. Most errors we keep right on trucking.
2583 	 */
2584 	rc = vscsi->flags & SCHEDULE_DISCONNECT;
2585 
2586 	return rc;
2587 }
2588 
2589 static int read_dma_window(struct scsi_info *vscsi)
2590 {
2591 	struct vio_dev *vdev = vscsi->dma_dev;
2592 	const __be32 *dma_window;
2593 	const __be32 *prop;
2594 
2595 	/* TODO Using of_parse_dma_window would be better, but it doesn't give
2596 	 * a way to read multiple windows without already knowing the size of
2597 	 * a window or the number of windows.
2598 	 */
2599 	dma_window = (const __be32 *)vio_get_attribute(vdev,
2600 						       "ibm,my-dma-window",
2601 						       NULL);
2602 	if (!dma_window) {
2603 		dev_err(&vscsi->dev, "Couldn't find ibm,my-dma-window property\n");
2604 		return -1;
2605 	}
2606 
2607 	vscsi->dds.window[LOCAL].liobn = be32_to_cpu(*dma_window);
2608 	dma_window++;
2609 
2610 	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells",
2611 						 NULL);
2612 	if (!prop) {
2613 		dev_warn(&vscsi->dev, "Couldn't find ibm,#dma-address-cells property\n");
2614 		dma_window++;
2615 	} else {
2616 		dma_window += be32_to_cpu(*prop);
2617 	}
2618 
2619 	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells",
2620 						 NULL);
2621 	if (!prop) {
2622 		dev_warn(&vscsi->dev, "Couldn't find ibm,#dma-size-cells property\n");
2623 		dma_window++;
2624 	} else {
2625 		dma_window += be32_to_cpu(*prop);
2626 	}
2627 
2628 	/* dma_window should point to the second window now */
2629 	vscsi->dds.window[REMOTE].liobn = be32_to_cpu(*dma_window);
2630 
2631 	return 0;
2632 }
2633 
2634 static struct ibmvscsis_tport *ibmvscsis_lookup_port(const char *name)
2635 {
2636 	struct ibmvscsis_tport *tport = NULL;
2637 	struct vio_dev *vdev;
2638 	struct scsi_info *vscsi;
2639 
2640 	spin_lock_bh(&ibmvscsis_dev_lock);
2641 	list_for_each_entry(vscsi, &ibmvscsis_dev_list, list) {
2642 		vdev = vscsi->dma_dev;
2643 		if (!strcmp(dev_name(&vdev->dev), name)) {
2644 			tport = &vscsi->tport;
2645 			break;
2646 		}
2647 	}
2648 	spin_unlock_bh(&ibmvscsis_dev_lock);
2649 
2650 	return tport;
2651 }
2652 
2653 /**
2654  * ibmvscsis_parse_cmd() - Parse SRP Command
2655  * @vscsi:	Pointer to our adapter structure
2656  * @cmd:	Pointer to command element with SRP command
2657  *
2658  * Parse the srp command; if it is valid then submit it to tcm.
2659  * Note: The return code does not reflect the status of the SCSI CDB.
2660  *
2661  * EXECUTION ENVIRONMENT:
2662  *	Process level
2663  */
2664 static void ibmvscsis_parse_cmd(struct scsi_info *vscsi,
2665 				struct ibmvscsis_cmd *cmd)
2666 {
2667 	struct iu_entry *iue = cmd->iue;
2668 	struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
2669 	struct ibmvscsis_nexus *nexus;
2670 	u64 data_len = 0;
2671 	enum dma_data_direction dir;
2672 	int attr = 0;
2673 	int rc = 0;
2674 
2675 	nexus = vscsi->tport.ibmv_nexus;
2676 	/*
2677 	 * additional length in bytes.  Note that the SRP spec says that
2678 	 * additional length is in 4-byte words, but technically the
2679 	 * additional length field is only the upper 6 bits of the byte.
2680 	 * The lower 2 bits are reserved.  If the lower 2 bits are 0 (as
2681 	 * all reserved fields should be), then interpreting the byte as
2682 	 * an int will yield the length in bytes.
2683 	 */
2684 	if (srp->add_cdb_len & 0x03) {
2685 		dev_err(&vscsi->dev, "parse_cmd: reserved bits set in IU\n");
2686 		spin_lock_bh(&vscsi->intr_lock);
2687 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2688 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2689 		spin_unlock_bh(&vscsi->intr_lock);
2690 		return;
2691 	}
2692 
2693 	if (srp_get_desc_table(srp, &dir, &data_len)) {
2694 		dev_err(&vscsi->dev, "0x%llx: parsing SRP descriptor table failed.\n",
2695 			srp->tag);
2696 		goto fail;
2697 	}
2698 
2699 	cmd->rsp.sol_not = srp->sol_not;
2700 
2701 	switch (srp->task_attr) {
2702 	case SRP_SIMPLE_TASK:
2703 		attr = TCM_SIMPLE_TAG;
2704 		break;
2705 	case SRP_ORDERED_TASK:
2706 		attr = TCM_ORDERED_TAG;
2707 		break;
2708 	case SRP_HEAD_TASK:
2709 		attr = TCM_HEAD_TAG;
2710 		break;
2711 	case SRP_ACA_TASK:
2712 		attr = TCM_ACA_TAG;
2713 		break;
2714 	default:
2715 		dev_err(&vscsi->dev, "Invalid task attribute %d\n",
2716 			srp->task_attr);
2717 		goto fail;
2718 	}
2719 
2720 	cmd->se_cmd.tag = be64_to_cpu(srp->tag);
2721 
2722 	spin_lock_bh(&vscsi->intr_lock);
2723 	list_add_tail(&cmd->list, &vscsi->active_q);
2724 	spin_unlock_bh(&vscsi->intr_lock);
2725 
2726 	srp->lun.scsi_lun[0] &= 0x3f;
2727 
2728 	rc = target_submit_cmd(&cmd->se_cmd, nexus->se_sess, srp->cdb,
2729 			       cmd->sense_buf, scsilun_to_int(&srp->lun),
2730 			       data_len, attr, dir, 0);
2731 	if (rc) {
2732 		dev_err(&vscsi->dev, "target_submit_cmd failed, rc %d\n", rc);
2733 		spin_lock_bh(&vscsi->intr_lock);
2734 		list_del(&cmd->list);
2735 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2736 		spin_unlock_bh(&vscsi->intr_lock);
2737 		goto fail;
2738 	}
2739 	return;
2740 
2741 fail:
2742 	spin_lock_bh(&vscsi->intr_lock);
2743 	ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT_RECONNECT, 0);
2744 	spin_unlock_bh(&vscsi->intr_lock);
2745 }
2746 
2747 /**
2748  * ibmvscsis_parse_task() - Parse SRP Task Management Request
2749  * @vscsi:	Pointer to our adapter structure
2750  * @cmd:	Pointer to command element with SRP task management request
2751  *
2752  * Parse the srp task management request; if it is valid then submit it to tcm.
2753  * Note: The return code does not reflect the status of the task management
2754  * request.
2755  *
2756  * EXECUTION ENVIRONMENT:
2757  *	Processor level
2758  */
2759 static void ibmvscsis_parse_task(struct scsi_info *vscsi,
2760 				 struct ibmvscsis_cmd *cmd)
2761 {
2762 	struct iu_entry *iue = cmd->iue;
2763 	struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
2764 	int tcm_type;
2765 	u64 tag_to_abort = 0;
2766 	int rc = 0;
2767 	struct ibmvscsis_nexus *nexus;
2768 
2769 	nexus = vscsi->tport.ibmv_nexus;
2770 
2771 	cmd->rsp.sol_not = srp_tsk->sol_not;
2772 
2773 	switch (srp_tsk->tsk_mgmt_func) {
2774 	case SRP_TSK_ABORT_TASK:
2775 		tcm_type = TMR_ABORT_TASK;
2776 		tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
2777 		break;
2778 	case SRP_TSK_ABORT_TASK_SET:
2779 		tcm_type = TMR_ABORT_TASK_SET;
2780 		break;
2781 	case SRP_TSK_CLEAR_TASK_SET:
2782 		tcm_type = TMR_CLEAR_TASK_SET;
2783 		break;
2784 	case SRP_TSK_LUN_RESET:
2785 		tcm_type = TMR_LUN_RESET;
2786 		break;
2787 	case SRP_TSK_CLEAR_ACA:
2788 		tcm_type = TMR_CLEAR_ACA;
2789 		break;
2790 	default:
2791 		dev_err(&vscsi->dev, "unknown task mgmt func %d\n",
2792 			srp_tsk->tsk_mgmt_func);
2793 		cmd->se_cmd.se_tmr_req->response =
2794 			TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
2795 		rc = -1;
2796 		break;
2797 	}
2798 
2799 	if (!rc) {
2800 		cmd->se_cmd.tag = be64_to_cpu(srp_tsk->tag);
2801 
2802 		spin_lock_bh(&vscsi->intr_lock);
2803 		list_add_tail(&cmd->list, &vscsi->active_q);
2804 		spin_unlock_bh(&vscsi->intr_lock);
2805 
2806 		srp_tsk->lun.scsi_lun[0] &= 0x3f;
2807 
2808 		dev_dbg(&vscsi->dev, "calling submit_tmr, func %d\n",
2809 			srp_tsk->tsk_mgmt_func);
2810 		rc = target_submit_tmr(&cmd->se_cmd, nexus->se_sess, NULL,
2811 				       scsilun_to_int(&srp_tsk->lun), srp_tsk,
2812 				       tcm_type, GFP_KERNEL, tag_to_abort, 0);
2813 		if (rc) {
2814 			dev_err(&vscsi->dev, "target_submit_tmr failed, rc %d\n",
2815 				rc);
2816 			spin_lock_bh(&vscsi->intr_lock);
2817 			list_del(&cmd->list);
2818 			spin_unlock_bh(&vscsi->intr_lock);
2819 			cmd->se_cmd.se_tmr_req->response =
2820 				TMR_FUNCTION_REJECTED;
2821 		}
2822 	}
2823 
2824 	if (rc)
2825 		transport_send_check_condition_and_sense(&cmd->se_cmd, 0, 0);
2826 }
2827 
2828 static void ibmvscsis_scheduler(struct work_struct *work)
2829 {
2830 	struct ibmvscsis_cmd *cmd = container_of(work, struct ibmvscsis_cmd,
2831 						 work);
2832 	struct scsi_info *vscsi = cmd->adapter;
2833 
2834 	spin_lock_bh(&vscsi->intr_lock);
2835 
2836 	/* Remove from schedule_q */
2837 	list_del(&cmd->list);
2838 
2839 	/* Don't submit cmd if we're disconnecting */
2840 	if (vscsi->flags & (SCHEDULE_DISCONNECT | DISCONNECT_SCHEDULED)) {
2841 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2842 
2843 		/* ibmvscsis_disconnect might be waiting for us */
2844 		if (list_empty(&vscsi->active_q) &&
2845 		    list_empty(&vscsi->schedule_q) &&
2846 		    (vscsi->flags & WAIT_FOR_IDLE)) {
2847 			vscsi->flags &= ~WAIT_FOR_IDLE;
2848 			complete(&vscsi->wait_idle);
2849 		}
2850 
2851 		spin_unlock_bh(&vscsi->intr_lock);
2852 		return;
2853 	}
2854 
2855 	spin_unlock_bh(&vscsi->intr_lock);
2856 
2857 	switch (cmd->type) {
2858 	case SCSI_CDB:
2859 		ibmvscsis_parse_cmd(vscsi, cmd);
2860 		break;
2861 	case TASK_MANAGEMENT:
2862 		ibmvscsis_parse_task(vscsi, cmd);
2863 		break;
2864 	default:
2865 		dev_err(&vscsi->dev, "scheduler, invalid cmd type %d\n",
2866 			cmd->type);
2867 		spin_lock_bh(&vscsi->intr_lock);
2868 		ibmvscsis_free_cmd_resources(vscsi, cmd);
2869 		spin_unlock_bh(&vscsi->intr_lock);
2870 		break;
2871 	}
2872 }
2873 
2874 static int ibmvscsis_alloc_cmds(struct scsi_info *vscsi, int num)
2875 {
2876 	struct ibmvscsis_cmd *cmd;
2877 	int i;
2878 
2879 	INIT_LIST_HEAD(&vscsi->free_cmd);
2880 	vscsi->cmd_pool = kcalloc(num, sizeof(struct ibmvscsis_cmd),
2881 				  GFP_KERNEL);
2882 	if (!vscsi->cmd_pool)
2883 		return -ENOMEM;
2884 
2885 	for (i = 0, cmd = (struct ibmvscsis_cmd *)vscsi->cmd_pool; i < num;
2886 	     i++, cmd++) {
2887 		cmd->abort_cmd = NULL;
2888 		cmd->adapter = vscsi;
2889 		INIT_WORK(&cmd->work, ibmvscsis_scheduler);
2890 		list_add_tail(&cmd->list, &vscsi->free_cmd);
2891 	}
2892 
2893 	return 0;
2894 }
2895 
2896 static void ibmvscsis_free_cmds(struct scsi_info *vscsi)
2897 {
2898 	kfree(vscsi->cmd_pool);
2899 	vscsi->cmd_pool = NULL;
2900 	INIT_LIST_HEAD(&vscsi->free_cmd);
2901 }
2902 
2903 /**
2904  * ibmvscsis_service_wait_q() - Service Waiting Queue
2905  * @timer:	Pointer to timer which has expired
2906  *
2907  * This routine is called when the timer pops to service the waiting
2908  * queue. Elements on the queue have completed, their responses have been
2909  * copied to the client, but the client's response queue was full so
2910  * the queue message could not be sent. The routine grabs the proper locks
2911  * and calls send messages.
2912  *
2913  * EXECUTION ENVIRONMENT:
2914  *	called at interrupt level
2915  */
2916 static enum hrtimer_restart ibmvscsis_service_wait_q(struct hrtimer *timer)
2917 {
2918 	struct timer_cb *p_timer = container_of(timer, struct timer_cb, timer);
2919 	struct scsi_info *vscsi = container_of(p_timer, struct scsi_info,
2920 					       rsp_q_timer);
2921 
2922 	spin_lock_bh(&vscsi->intr_lock);
2923 	p_timer->timer_pops += 1;
2924 	p_timer->started = false;
2925 	ibmvscsis_send_messages(vscsi);
2926 	spin_unlock_bh(&vscsi->intr_lock);
2927 
2928 	return HRTIMER_NORESTART;
2929 }
2930 
2931 static long ibmvscsis_alloctimer(struct scsi_info *vscsi)
2932 {
2933 	struct timer_cb *p_timer;
2934 
2935 	p_timer = &vscsi->rsp_q_timer;
2936 	hrtimer_init(&p_timer->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
2937 
2938 	p_timer->timer.function = ibmvscsis_service_wait_q;
2939 	p_timer->started = false;
2940 	p_timer->timer_pops = 0;
2941 
2942 	return ADAPT_SUCCESS;
2943 }
2944 
2945 static void ibmvscsis_freetimer(struct scsi_info *vscsi)
2946 {
2947 	struct timer_cb *p_timer;
2948 
2949 	p_timer = &vscsi->rsp_q_timer;
2950 
2951 	(void)hrtimer_cancel(&p_timer->timer);
2952 
2953 	p_timer->started = false;
2954 	p_timer->timer_pops = 0;
2955 }
2956 
2957 static irqreturn_t ibmvscsis_interrupt(int dummy, void *data)
2958 {
2959 	struct scsi_info *vscsi = data;
2960 
2961 	vio_disable_interrupts(vscsi->dma_dev);
2962 	tasklet_schedule(&vscsi->work_task);
2963 
2964 	return IRQ_HANDLED;
2965 }
2966 
2967 /**
2968  * ibmvscsis_enable_change_state() - Set new state based on enabled status
2969  * @vscsi:	Pointer to our adapter structure
2970  *
2971  * This function determines our new state now that we are enabled.  This
2972  * may involve sending an Init Complete message to the client.
2973  *
2974  * Must be called with interrupt lock held.
2975  */
2976 static long ibmvscsis_enable_change_state(struct scsi_info *vscsi)
2977 {
2978 	int bytes;
2979 	long rc = ADAPT_SUCCESS;
2980 
2981 	bytes = vscsi->cmd_q.size * PAGE_SIZE;
2982 	rc = h_reg_crq(vscsi->dds.unit_id, vscsi->cmd_q.crq_token, bytes);
2983 	if (rc == H_CLOSED || rc == H_SUCCESS) {
2984 		vscsi->state = WAIT_CONNECTION;
2985 		rc = ibmvscsis_establish_new_q(vscsi);
2986 	}
2987 
2988 	if (rc != ADAPT_SUCCESS) {
2989 		vscsi->state = ERR_DISCONNECTED;
2990 		vscsi->flags |= RESPONSE_Q_DOWN;
2991 	}
2992 
2993 	return rc;
2994 }
2995 
2996 /**
2997  * ibmvscsis_create_command_q() - Create Command Queue
2998  * @vscsi:	Pointer to our adapter structure
2999  * @num_cmds:	Currently unused.  In the future, may be used to determine
3000  *		the size of the CRQ.
3001  *
3002  * Allocates memory for command queue maps remote memory into an ioba
3003  * initializes the command response queue
3004  *
3005  * EXECUTION ENVIRONMENT:
3006  *	Process level only
3007  */
3008 static long ibmvscsis_create_command_q(struct scsi_info *vscsi, int num_cmds)
3009 {
3010 	int pages;
3011 	struct vio_dev *vdev = vscsi->dma_dev;
3012 
3013 	/* We might support multiple pages in the future, but just 1 for now */
3014 	pages = 1;
3015 
3016 	vscsi->cmd_q.size = pages;
3017 
3018 	vscsi->cmd_q.base_addr =
3019 		(struct viosrp_crq *)get_zeroed_page(GFP_KERNEL);
3020 	if (!vscsi->cmd_q.base_addr)
3021 		return -ENOMEM;
3022 
3023 	vscsi->cmd_q.mask = ((uint)pages * CRQ_PER_PAGE) - 1;
3024 
3025 	vscsi->cmd_q.crq_token = dma_map_single(&vdev->dev,
3026 						vscsi->cmd_q.base_addr,
3027 						PAGE_SIZE, DMA_BIDIRECTIONAL);
3028 	if (dma_mapping_error(&vdev->dev, vscsi->cmd_q.crq_token)) {
3029 		free_page((unsigned long)vscsi->cmd_q.base_addr);
3030 		return -ENOMEM;
3031 	}
3032 
3033 	return 0;
3034 }
3035 
3036 /**
3037  * ibmvscsis_destroy_command_q - Destroy Command Queue
3038  * @vscsi:	Pointer to our adapter structure
3039  *
3040  * Releases memory for command queue and unmaps mapped remote memory.
3041  *
3042  * EXECUTION ENVIRONMENT:
3043  *	Process level only
3044  */
3045 static void ibmvscsis_destroy_command_q(struct scsi_info *vscsi)
3046 {
3047 	dma_unmap_single(&vscsi->dma_dev->dev, vscsi->cmd_q.crq_token,
3048 			 PAGE_SIZE, DMA_BIDIRECTIONAL);
3049 	free_page((unsigned long)vscsi->cmd_q.base_addr);
3050 	vscsi->cmd_q.base_addr = NULL;
3051 	vscsi->state = NO_QUEUE;
3052 }
3053 
3054 static u8 ibmvscsis_fast_fail(struct scsi_info *vscsi,
3055 			      struct ibmvscsis_cmd *cmd)
3056 {
3057 	struct iu_entry *iue = cmd->iue;
3058 	struct se_cmd *se_cmd = &cmd->se_cmd;
3059 	struct srp_cmd *srp = (struct srp_cmd *)iue->sbuf->buf;
3060 	struct scsi_sense_hdr sshdr;
3061 	u8 rc = se_cmd->scsi_status;
3062 
3063 	if (vscsi->fast_fail && (READ_CMD(srp->cdb) || WRITE_CMD(srp->cdb)))
3064 		if (scsi_normalize_sense(se_cmd->sense_buffer,
3065 					 se_cmd->scsi_sense_length, &sshdr))
3066 			if (sshdr.sense_key == HARDWARE_ERROR &&
3067 			    (se_cmd->residual_count == 0 ||
3068 			     se_cmd->residual_count == se_cmd->data_length)) {
3069 				rc = NO_SENSE;
3070 				cmd->flags |= CMD_FAST_FAIL;
3071 			}
3072 
3073 	return rc;
3074 }
3075 
3076 /**
3077  * srp_build_response() - Build an SRP response buffer
3078  * @vscsi:	Pointer to our adapter structure
3079  * @cmd:	Pointer to command for which to send the response
3080  * @len_p:	Where to return the length of the IU response sent.  This
3081  *		is needed to construct the CRQ response.
3082  *
3083  * Build the SRP response buffer and copy it to the client's memory space.
3084  */
3085 static long srp_build_response(struct scsi_info *vscsi,
3086 			       struct ibmvscsis_cmd *cmd, uint *len_p)
3087 {
3088 	struct iu_entry *iue = cmd->iue;
3089 	struct se_cmd *se_cmd = &cmd->se_cmd;
3090 	struct srp_rsp *rsp;
3091 	uint len;
3092 	u32 rsp_code;
3093 	char *data;
3094 	u32 *tsk_status;
3095 	long rc = ADAPT_SUCCESS;
3096 
3097 	spin_lock_bh(&vscsi->intr_lock);
3098 
3099 	rsp = &vio_iu(iue)->srp.rsp;
3100 	len = sizeof(*rsp);
3101 	memset(rsp, 0, len);
3102 	data = rsp->data;
3103 
3104 	rsp->opcode = SRP_RSP;
3105 
3106 	rsp->req_lim_delta = cpu_to_be32(1 + vscsi->credit);
3107 	rsp->tag = cmd->rsp.tag;
3108 	rsp->flags = 0;
3109 
3110 	if (cmd->type == SCSI_CDB) {
3111 		rsp->status = ibmvscsis_fast_fail(vscsi, cmd);
3112 		if (rsp->status) {
3113 			dev_dbg(&vscsi->dev, "build_resp: cmd %p, scsi status %d\n",
3114 				cmd, (int)rsp->status);
3115 			ibmvscsis_determine_resid(se_cmd, rsp);
3116 			if (se_cmd->scsi_sense_length && se_cmd->sense_buffer) {
3117 				rsp->sense_data_len =
3118 					cpu_to_be32(se_cmd->scsi_sense_length);
3119 				rsp->flags |= SRP_RSP_FLAG_SNSVALID;
3120 				len += se_cmd->scsi_sense_length;
3121 				memcpy(data, se_cmd->sense_buffer,
3122 				       se_cmd->scsi_sense_length);
3123 			}
3124 			rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3125 				UCSOLNT_RESP_SHIFT;
3126 		} else if (cmd->flags & CMD_FAST_FAIL) {
3127 			dev_dbg(&vscsi->dev, "build_resp: cmd %p, fast fail\n",
3128 				cmd);
3129 			rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3130 				UCSOLNT_RESP_SHIFT;
3131 		} else {
3132 			rsp->sol_not = (cmd->rsp.sol_not & SCSOLNT) >>
3133 				SCSOLNT_RESP_SHIFT;
3134 		}
3135 	} else {
3136 		/* this is task management */
3137 		rsp->status = 0;
3138 		rsp->resp_data_len = cpu_to_be32(4);
3139 		rsp->flags |= SRP_RSP_FLAG_RSPVALID;
3140 
3141 		switch (se_cmd->se_tmr_req->response) {
3142 		case TMR_FUNCTION_COMPLETE:
3143 		case TMR_TASK_DOES_NOT_EXIST:
3144 			rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_COMPLETE;
3145 			rsp->sol_not = (cmd->rsp.sol_not & SCSOLNT) >>
3146 				SCSOLNT_RESP_SHIFT;
3147 			break;
3148 		case TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED:
3149 		case TMR_LUN_DOES_NOT_EXIST:
3150 			rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_NOT_SUPPORTED;
3151 			rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3152 				UCSOLNT_RESP_SHIFT;
3153 			break;
3154 		case TMR_FUNCTION_FAILED:
3155 		case TMR_FUNCTION_REJECTED:
3156 		default:
3157 			rsp_code = SRP_TASK_MANAGEMENT_FUNCTION_FAILED;
3158 			rsp->sol_not = (cmd->rsp.sol_not & UCSOLNT) >>
3159 				UCSOLNT_RESP_SHIFT;
3160 			break;
3161 		}
3162 
3163 		tsk_status = (u32 *)data;
3164 		*tsk_status = cpu_to_be32(rsp_code);
3165 		data = (char *)(tsk_status + 1);
3166 		len += 4;
3167 	}
3168 
3169 	dma_wmb();
3170 	rc = h_copy_rdma(len, vscsi->dds.window[LOCAL].liobn, iue->sbuf->dma,
3171 			 vscsi->dds.window[REMOTE].liobn,
3172 			 be64_to_cpu(iue->remote_token));
3173 
3174 	switch (rc) {
3175 	case H_SUCCESS:
3176 		vscsi->credit = 0;
3177 		*len_p = len;
3178 		break;
3179 	case H_PERMISSION:
3180 		if (connection_broken(vscsi))
3181 			vscsi->flags |= RESPONSE_Q_DOWN | CLIENT_FAILED;
3182 
3183 		dev_err(&vscsi->dev, "build_response: error copying to client, rc %ld, flags 0x%x, state 0x%hx\n",
3184 			rc, vscsi->flags, vscsi->state);
3185 		break;
3186 	case H_SOURCE_PARM:
3187 	case H_DEST_PARM:
3188 	default:
3189 		dev_err(&vscsi->dev, "build_response: error copying to client, rc %ld\n",
3190 			rc);
3191 		break;
3192 	}
3193 
3194 	spin_unlock_bh(&vscsi->intr_lock);
3195 
3196 	return rc;
3197 }
3198 
3199 static int ibmvscsis_rdma(struct ibmvscsis_cmd *cmd, struct scatterlist *sg,
3200 			  int nsg, struct srp_direct_buf *md, int nmd,
3201 			  enum dma_data_direction dir, unsigned int bytes)
3202 {
3203 	struct iu_entry *iue = cmd->iue;
3204 	struct srp_target *target = iue->target;
3205 	struct scsi_info *vscsi = target->ldata;
3206 	struct scatterlist *sgp;
3207 	dma_addr_t client_ioba, server_ioba;
3208 	ulong buf_len;
3209 	ulong client_len, server_len;
3210 	int md_idx;
3211 	long tx_len;
3212 	long rc = 0;
3213 
3214 	if (bytes == 0)
3215 		return 0;
3216 
3217 	sgp = sg;
3218 	client_len = 0;
3219 	server_len = 0;
3220 	md_idx = 0;
3221 	tx_len = bytes;
3222 
3223 	do {
3224 		if (client_len == 0) {
3225 			if (md_idx >= nmd) {
3226 				dev_err(&vscsi->dev, "rdma: ran out of client memory descriptors\n");
3227 				rc = -EIO;
3228 				break;
3229 			}
3230 			client_ioba = be64_to_cpu(md[md_idx].va);
3231 			client_len = be32_to_cpu(md[md_idx].len);
3232 		}
3233 		if (server_len == 0) {
3234 			if (!sgp) {
3235 				dev_err(&vscsi->dev, "rdma: ran out of scatter/gather list\n");
3236 				rc = -EIO;
3237 				break;
3238 			}
3239 			server_ioba = sg_dma_address(sgp);
3240 			server_len = sg_dma_len(sgp);
3241 		}
3242 
3243 		buf_len = tx_len;
3244 
3245 		if (buf_len > client_len)
3246 			buf_len = client_len;
3247 
3248 		if (buf_len > server_len)
3249 			buf_len = server_len;
3250 
3251 		if (buf_len > max_vdma_size)
3252 			buf_len = max_vdma_size;
3253 
3254 		if (dir == DMA_TO_DEVICE) {
3255 			/* read from client */
3256 			rc = h_copy_rdma(buf_len,
3257 					 vscsi->dds.window[REMOTE].liobn,
3258 					 client_ioba,
3259 					 vscsi->dds.window[LOCAL].liobn,
3260 					 server_ioba);
3261 		} else {
3262 			/* The h_copy_rdma will cause phyp, running in another
3263 			 * partition, to read memory, so we need to make sure
3264 			 * the data has been written out, hence these syncs.
3265 			 */
3266 			/* ensure that everything is in memory */
3267 			isync();
3268 			/* ensure that memory has been made visible */
3269 			dma_wmb();
3270 			rc = h_copy_rdma(buf_len,
3271 					 vscsi->dds.window[LOCAL].liobn,
3272 					 server_ioba,
3273 					 vscsi->dds.window[REMOTE].liobn,
3274 					 client_ioba);
3275 		}
3276 		switch (rc) {
3277 		case H_SUCCESS:
3278 			break;
3279 		case H_PERMISSION:
3280 		case H_SOURCE_PARM:
3281 		case H_DEST_PARM:
3282 			if (connection_broken(vscsi)) {
3283 				spin_lock_bh(&vscsi->intr_lock);
3284 				vscsi->flags |=
3285 					(RESPONSE_Q_DOWN | CLIENT_FAILED);
3286 				spin_unlock_bh(&vscsi->intr_lock);
3287 			}
3288 			dev_err(&vscsi->dev, "rdma: h_copy_rdma failed, rc %ld\n",
3289 				rc);
3290 			break;
3291 
3292 		default:
3293 			dev_err(&vscsi->dev, "rdma: unknown error %ld from h_copy_rdma\n",
3294 				rc);
3295 			break;
3296 		}
3297 
3298 		if (!rc) {
3299 			tx_len -= buf_len;
3300 			if (tx_len) {
3301 				client_len -= buf_len;
3302 				if (client_len == 0)
3303 					md_idx++;
3304 				else
3305 					client_ioba += buf_len;
3306 
3307 				server_len -= buf_len;
3308 				if (server_len == 0)
3309 					sgp = sg_next(sgp);
3310 				else
3311 					server_ioba += buf_len;
3312 			} else {
3313 				break;
3314 			}
3315 		}
3316 	} while (!rc);
3317 
3318 	return rc;
3319 }
3320 
3321 /**
3322  * ibmvscsis_handle_crq() - Handle CRQ
3323  * @data:	Pointer to our adapter structure
3324  *
3325  * Read the command elements from the command queue and copy the payloads
3326  * associated with the command elements to local memory and execute the
3327  * SRP requests.
3328  *
3329  * Note: this is an edge triggered interrupt. It can not be shared.
3330  */
3331 static void ibmvscsis_handle_crq(unsigned long data)
3332 {
3333 	struct scsi_info *vscsi = (struct scsi_info *)data;
3334 	struct viosrp_crq *crq;
3335 	long rc;
3336 	bool ack = true;
3337 	volatile u8 valid;
3338 
3339 	spin_lock_bh(&vscsi->intr_lock);
3340 
3341 	dev_dbg(&vscsi->dev, "got interrupt\n");
3342 
3343 	/*
3344 	 * if we are in a path where we are waiting for all pending commands
3345 	 * to complete because we received a transport event and anything in
3346 	 * the command queue is for a new connection, do nothing
3347 	 */
3348 	if (TARGET_STOP(vscsi)) {
3349 		vio_enable_interrupts(vscsi->dma_dev);
3350 
3351 		dev_dbg(&vscsi->dev, "handle_crq, don't process: flags 0x%x, state 0x%hx\n",
3352 			vscsi->flags, vscsi->state);
3353 		spin_unlock_bh(&vscsi->intr_lock);
3354 		return;
3355 	}
3356 
3357 	rc = vscsi->flags & SCHEDULE_DISCONNECT;
3358 	crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
3359 	valid = crq->valid;
3360 	dma_rmb();
3361 
3362 	while (valid) {
3363 		/*
3364 		 * These are edege triggered interrupts. After dropping out of
3365 		 * the while loop, the code must check for work since an
3366 		 * interrupt could be lost, and an elment be left on the queue,
3367 		 * hence the label.
3368 		 */
3369 cmd_work:
3370 		vscsi->cmd_q.index =
3371 			(vscsi->cmd_q.index + 1) & vscsi->cmd_q.mask;
3372 
3373 		if (!rc) {
3374 			rc = ibmvscsis_parse_command(vscsi, crq);
3375 		} else {
3376 			if ((uint)crq->valid == VALID_TRANS_EVENT) {
3377 				/*
3378 				 * must service the transport layer events even
3379 				 * in an error state, dont break out until all
3380 				 * the consecutive transport events have been
3381 				 * processed
3382 				 */
3383 				rc = ibmvscsis_trans_event(vscsi, crq);
3384 			} else if (vscsi->flags & TRANS_EVENT) {
3385 				/*
3386 				 * if a transport event has occurred leave
3387 				 * everything but transport events on the queue
3388 				 *
3389 				 * need to decrement the queue index so we can
3390 				 * look at the element again
3391 				 */
3392 				if (vscsi->cmd_q.index)
3393 					vscsi->cmd_q.index -= 1;
3394 				else
3395 					/*
3396 					 * index is at 0 it just wrapped.
3397 					 * have it index last element in q
3398 					 */
3399 					vscsi->cmd_q.index = vscsi->cmd_q.mask;
3400 				break;
3401 			}
3402 		}
3403 
3404 		crq->valid = INVALIDATE_CMD_RESP_EL;
3405 
3406 		crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
3407 		valid = crq->valid;
3408 		dma_rmb();
3409 	}
3410 
3411 	if (!rc) {
3412 		if (ack) {
3413 			vio_enable_interrupts(vscsi->dma_dev);
3414 			ack = false;
3415 			dev_dbg(&vscsi->dev, "handle_crq, reenabling interrupts\n");
3416 		}
3417 		valid = crq->valid;
3418 		dma_rmb();
3419 		if (valid)
3420 			goto cmd_work;
3421 	} else {
3422 		dev_dbg(&vscsi->dev, "handle_crq, error: flags 0x%x, state 0x%hx, crq index 0x%x\n",
3423 			vscsi->flags, vscsi->state, vscsi->cmd_q.index);
3424 	}
3425 
3426 	dev_dbg(&vscsi->dev, "Leaving handle_crq: schedule_q empty %d, flags 0x%x, state 0x%hx\n",
3427 		(int)list_empty(&vscsi->schedule_q), vscsi->flags,
3428 		vscsi->state);
3429 
3430 	spin_unlock_bh(&vscsi->intr_lock);
3431 }
3432 
3433 static int ibmvscsis_probe(struct vio_dev *vdev,
3434 			   const struct vio_device_id *id)
3435 {
3436 	struct scsi_info *vscsi;
3437 	int rc = 0;
3438 	long hrc = 0;
3439 	char wq_name[24];
3440 
3441 	vscsi = kzalloc(sizeof(*vscsi), GFP_KERNEL);
3442 	if (!vscsi) {
3443 		rc = -ENOMEM;
3444 		dev_err(&vdev->dev, "probe: allocation of adapter failed\n");
3445 		return rc;
3446 	}
3447 
3448 	vscsi->dma_dev = vdev;
3449 	vscsi->dev = vdev->dev;
3450 	INIT_LIST_HEAD(&vscsi->schedule_q);
3451 	INIT_LIST_HEAD(&vscsi->waiting_rsp);
3452 	INIT_LIST_HEAD(&vscsi->active_q);
3453 
3454 	snprintf(vscsi->tport.tport_name, IBMVSCSIS_NAMELEN, "%s",
3455 		 dev_name(&vdev->dev));
3456 
3457 	dev_dbg(&vscsi->dev, "probe tport_name: %s\n", vscsi->tport.tport_name);
3458 
3459 	rc = read_dma_window(vscsi);
3460 	if (rc)
3461 		goto free_adapter;
3462 	dev_dbg(&vscsi->dev, "Probe: liobn 0x%x, riobn 0x%x\n",
3463 		vscsi->dds.window[LOCAL].liobn,
3464 		vscsi->dds.window[REMOTE].liobn);
3465 
3466 	snprintf(vscsi->eye, sizeof(vscsi->eye), "VSCSI %s", vdev->name);
3467 
3468 	vscsi->dds.unit_id = vdev->unit_address;
3469 	strscpy(vscsi->dds.partition_name, partition_name,
3470 		sizeof(vscsi->dds.partition_name));
3471 	vscsi->dds.partition_num = partition_number;
3472 
3473 	spin_lock_bh(&ibmvscsis_dev_lock);
3474 	list_add_tail(&vscsi->list, &ibmvscsis_dev_list);
3475 	spin_unlock_bh(&ibmvscsis_dev_lock);
3476 
3477 	/*
3478 	 * TBD: How do we determine # of cmds to request?  Do we know how
3479 	 * many "children" we have?
3480 	 */
3481 	vscsi->request_limit = INITIAL_SRP_LIMIT;
3482 	rc = srp_target_alloc(&vscsi->target, &vdev->dev, vscsi->request_limit,
3483 			      SRP_MAX_IU_LEN);
3484 	if (rc)
3485 		goto rem_list;
3486 
3487 	vscsi->target.ldata = vscsi;
3488 
3489 	rc = ibmvscsis_alloc_cmds(vscsi, vscsi->request_limit);
3490 	if (rc) {
3491 		dev_err(&vscsi->dev, "alloc_cmds failed, rc %d, num %d\n",
3492 			rc, vscsi->request_limit);
3493 		goto free_target;
3494 	}
3495 
3496 	/*
3497 	 * Note: the lock is used in freeing timers, so must initialize
3498 	 * first so that ordering in case of error is correct.
3499 	 */
3500 	spin_lock_init(&vscsi->intr_lock);
3501 
3502 	rc = ibmvscsis_alloctimer(vscsi);
3503 	if (rc) {
3504 		dev_err(&vscsi->dev, "probe: alloctimer failed, rc %d\n", rc);
3505 		goto free_cmds;
3506 	}
3507 
3508 	rc = ibmvscsis_create_command_q(vscsi, 256);
3509 	if (rc) {
3510 		dev_err(&vscsi->dev, "probe: create_command_q failed, rc %d\n",
3511 			rc);
3512 		goto free_timer;
3513 	}
3514 
3515 	vscsi->map_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
3516 	if (!vscsi->map_buf) {
3517 		rc = -ENOMEM;
3518 		dev_err(&vscsi->dev, "probe: allocating cmd buffer failed\n");
3519 		goto destroy_queue;
3520 	}
3521 
3522 	vscsi->map_ioba = dma_map_single(&vdev->dev, vscsi->map_buf, PAGE_SIZE,
3523 					 DMA_BIDIRECTIONAL);
3524 	if (dma_mapping_error(&vdev->dev, vscsi->map_ioba)) {
3525 		rc = -ENOMEM;
3526 		dev_err(&vscsi->dev, "probe: error mapping command buffer\n");
3527 		goto free_buf;
3528 	}
3529 
3530 	hrc = h_vioctl(vscsi->dds.unit_id, H_GET_PARTNER_INFO,
3531 		       (u64)vscsi->map_ioba | ((u64)PAGE_SIZE << 32), 0, 0, 0,
3532 		       0);
3533 	if (hrc == H_SUCCESS)
3534 		vscsi->client_data.partition_number =
3535 			be64_to_cpu(*(u64 *)vscsi->map_buf);
3536 	/*
3537 	 * We expect the VIOCTL to fail if we're configured as "any
3538 	 * client can connect" and the client isn't activated yet.
3539 	 * We'll make the call again when he sends an init msg.
3540 	 */
3541 	dev_dbg(&vscsi->dev, "probe hrc %ld, client partition num %d\n",
3542 		hrc, vscsi->client_data.partition_number);
3543 
3544 	tasklet_init(&vscsi->work_task, ibmvscsis_handle_crq,
3545 		     (unsigned long)vscsi);
3546 
3547 	init_completion(&vscsi->wait_idle);
3548 	init_completion(&vscsi->unconfig);
3549 
3550 	snprintf(wq_name, 24, "ibmvscsis%s", dev_name(&vdev->dev));
3551 	vscsi->work_q = create_workqueue(wq_name);
3552 	if (!vscsi->work_q) {
3553 		rc = -ENOMEM;
3554 		dev_err(&vscsi->dev, "create_workqueue failed\n");
3555 		goto unmap_buf;
3556 	}
3557 
3558 	rc = request_irq(vdev->irq, ibmvscsis_interrupt, 0, "ibmvscsis", vscsi);
3559 	if (rc) {
3560 		rc = -EPERM;
3561 		dev_err(&vscsi->dev, "probe: request_irq failed, rc %d\n", rc);
3562 		goto destroy_WQ;
3563 	}
3564 
3565 	vscsi->state = WAIT_ENABLED;
3566 
3567 	dev_set_drvdata(&vdev->dev, vscsi);
3568 
3569 	return 0;
3570 
3571 destroy_WQ:
3572 	destroy_workqueue(vscsi->work_q);
3573 unmap_buf:
3574 	dma_unmap_single(&vdev->dev, vscsi->map_ioba, PAGE_SIZE,
3575 			 DMA_BIDIRECTIONAL);
3576 free_buf:
3577 	kfree(vscsi->map_buf);
3578 destroy_queue:
3579 	tasklet_kill(&vscsi->work_task);
3580 	ibmvscsis_unregister_command_q(vscsi);
3581 	ibmvscsis_destroy_command_q(vscsi);
3582 free_timer:
3583 	ibmvscsis_freetimer(vscsi);
3584 free_cmds:
3585 	ibmvscsis_free_cmds(vscsi);
3586 free_target:
3587 	srp_target_free(&vscsi->target);
3588 rem_list:
3589 	spin_lock_bh(&ibmvscsis_dev_lock);
3590 	list_del(&vscsi->list);
3591 	spin_unlock_bh(&ibmvscsis_dev_lock);
3592 free_adapter:
3593 	kfree(vscsi);
3594 
3595 	return rc;
3596 }
3597 
3598 static int ibmvscsis_remove(struct vio_dev *vdev)
3599 {
3600 	struct scsi_info *vscsi = dev_get_drvdata(&vdev->dev);
3601 
3602 	dev_dbg(&vscsi->dev, "remove (%s)\n", dev_name(&vscsi->dma_dev->dev));
3603 
3604 	spin_lock_bh(&vscsi->intr_lock);
3605 	ibmvscsis_post_disconnect(vscsi, UNCONFIGURING, 0);
3606 	vscsi->flags |= CFG_SLEEPING;
3607 	spin_unlock_bh(&vscsi->intr_lock);
3608 	wait_for_completion(&vscsi->unconfig);
3609 
3610 	vio_disable_interrupts(vdev);
3611 	free_irq(vdev->irq, vscsi);
3612 	destroy_workqueue(vscsi->work_q);
3613 	dma_unmap_single(&vdev->dev, vscsi->map_ioba, PAGE_SIZE,
3614 			 DMA_BIDIRECTIONAL);
3615 	kfree(vscsi->map_buf);
3616 	tasklet_kill(&vscsi->work_task);
3617 	ibmvscsis_destroy_command_q(vscsi);
3618 	ibmvscsis_freetimer(vscsi);
3619 	ibmvscsis_free_cmds(vscsi);
3620 	srp_target_free(&vscsi->target);
3621 	spin_lock_bh(&ibmvscsis_dev_lock);
3622 	list_del(&vscsi->list);
3623 	spin_unlock_bh(&ibmvscsis_dev_lock);
3624 	kfree(vscsi);
3625 
3626 	return 0;
3627 }
3628 
3629 static ssize_t system_id_show(struct device *dev,
3630 			      struct device_attribute *attr, char *buf)
3631 {
3632 	return snprintf(buf, PAGE_SIZE, "%s\n", system_id);
3633 }
3634 
3635 static ssize_t partition_number_show(struct device *dev,
3636 				     struct device_attribute *attr, char *buf)
3637 {
3638 	return snprintf(buf, PAGE_SIZE, "%x\n", partition_number);
3639 }
3640 
3641 static ssize_t unit_address_show(struct device *dev,
3642 				 struct device_attribute *attr, char *buf)
3643 {
3644 	struct scsi_info *vscsi = container_of(dev, struct scsi_info, dev);
3645 
3646 	return snprintf(buf, PAGE_SIZE, "%x\n", vscsi->dma_dev->unit_address);
3647 }
3648 
3649 static int ibmvscsis_get_system_info(void)
3650 {
3651 	struct device_node *rootdn, *vdevdn;
3652 	const char *id, *model, *name;
3653 	const uint *num;
3654 
3655 	rootdn = of_find_node_by_path("/");
3656 	if (!rootdn)
3657 		return -ENOENT;
3658 
3659 	model = of_get_property(rootdn, "model", NULL);
3660 	id = of_get_property(rootdn, "system-id", NULL);
3661 	if (model && id)
3662 		snprintf(system_id, sizeof(system_id), "%s-%s", model, id);
3663 
3664 	name = of_get_property(rootdn, "ibm,partition-name", NULL);
3665 	if (name)
3666 		strncpy(partition_name, name, sizeof(partition_name));
3667 
3668 	num = of_get_property(rootdn, "ibm,partition-no", NULL);
3669 	if (num)
3670 		partition_number = of_read_number(num, 1);
3671 
3672 	of_node_put(rootdn);
3673 
3674 	vdevdn = of_find_node_by_path("/vdevice");
3675 	if (vdevdn) {
3676 		const uint *mvds;
3677 
3678 		mvds = of_get_property(vdevdn, "ibm,max-virtual-dma-size",
3679 				       NULL);
3680 		if (mvds)
3681 			max_vdma_size = *mvds;
3682 		of_node_put(vdevdn);
3683 	}
3684 
3685 	return 0;
3686 }
3687 
3688 static char *ibmvscsis_get_fabric_wwn(struct se_portal_group *se_tpg)
3689 {
3690 	struct ibmvscsis_tport *tport =
3691 		container_of(se_tpg, struct ibmvscsis_tport, se_tpg);
3692 
3693 	return tport->tport_name;
3694 }
3695 
3696 static u16 ibmvscsis_get_tag(struct se_portal_group *se_tpg)
3697 {
3698 	struct ibmvscsis_tport *tport =
3699 		container_of(se_tpg, struct ibmvscsis_tport, se_tpg);
3700 
3701 	return tport->tport_tpgt;
3702 }
3703 
3704 static u32 ibmvscsis_get_default_depth(struct se_portal_group *se_tpg)
3705 {
3706 	return 1;
3707 }
3708 
3709 static int ibmvscsis_check_true(struct se_portal_group *se_tpg)
3710 {
3711 	return 1;
3712 }
3713 
3714 static int ibmvscsis_check_false(struct se_portal_group *se_tpg)
3715 {
3716 	return 0;
3717 }
3718 
3719 static u32 ibmvscsis_tpg_get_inst_index(struct se_portal_group *se_tpg)
3720 {
3721 	return 1;
3722 }
3723 
3724 static int ibmvscsis_check_stop_free(struct se_cmd *se_cmd)
3725 {
3726 	return target_put_sess_cmd(se_cmd);
3727 }
3728 
3729 static void ibmvscsis_release_cmd(struct se_cmd *se_cmd)
3730 {
3731 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3732 						 se_cmd);
3733 	struct scsi_info *vscsi = cmd->adapter;
3734 
3735 	spin_lock_bh(&vscsi->intr_lock);
3736 	/* Remove from active_q */
3737 	list_move_tail(&cmd->list, &vscsi->waiting_rsp);
3738 	ibmvscsis_send_messages(vscsi);
3739 	spin_unlock_bh(&vscsi->intr_lock);
3740 }
3741 
3742 static u32 ibmvscsis_sess_get_index(struct se_session *se_sess)
3743 {
3744 	return 0;
3745 }
3746 
3747 static int ibmvscsis_write_pending(struct se_cmd *se_cmd)
3748 {
3749 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3750 						 se_cmd);
3751 	struct scsi_info *vscsi = cmd->adapter;
3752 	struct iu_entry *iue = cmd->iue;
3753 	int rc;
3754 
3755 	/*
3756 	 * If CLIENT_FAILED OR RESPONSE_Q_DOWN, then just return success
3757 	 * since LIO can't do anything about it, and we dont want to
3758 	 * attempt an srp_transfer_data.
3759 	 */
3760 	if ((vscsi->flags & (CLIENT_FAILED | RESPONSE_Q_DOWN))) {
3761 		dev_err(&vscsi->dev, "write_pending failed since: %d\n",
3762 			vscsi->flags);
3763 		return -EIO;
3764 
3765 	}
3766 
3767 	rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma,
3768 			       1, 1);
3769 	if (rc) {
3770 		dev_err(&vscsi->dev, "srp_transfer_data() failed: %d\n", rc);
3771 		return -EIO;
3772 	}
3773 	/*
3774 	 * We now tell TCM to add this WRITE CDB directly into the TCM storage
3775 	 * object execution queue.
3776 	 */
3777 	target_execute_cmd(se_cmd);
3778 	return 0;
3779 }
3780 
3781 static void ibmvscsis_set_default_node_attrs(struct se_node_acl *nacl)
3782 {
3783 }
3784 
3785 static int ibmvscsis_get_cmd_state(struct se_cmd *se_cmd)
3786 {
3787 	return 0;
3788 }
3789 
3790 static int ibmvscsis_queue_data_in(struct se_cmd *se_cmd)
3791 {
3792 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3793 						 se_cmd);
3794 	struct iu_entry *iue = cmd->iue;
3795 	struct scsi_info *vscsi = cmd->adapter;
3796 	uint len = 0;
3797 	int rc;
3798 
3799 	rc = srp_transfer_data(cmd, &vio_iu(iue)->srp.cmd, ibmvscsis_rdma, 1,
3800 			       1);
3801 	if (rc) {
3802 		dev_err(&vscsi->dev, "srp_transfer_data failed: %d\n", rc);
3803 		se_cmd->scsi_sense_length = 18;
3804 		memset(se_cmd->sense_buffer, 0, se_cmd->scsi_sense_length);
3805 		/* Logical Unit Communication Time-out asc/ascq = 0x0801 */
3806 		scsi_build_sense_buffer(0, se_cmd->sense_buffer, MEDIUM_ERROR,
3807 					0x08, 0x01);
3808 	}
3809 
3810 	srp_build_response(vscsi, cmd, &len);
3811 	cmd->rsp.format = SRP_FORMAT;
3812 	cmd->rsp.len = len;
3813 
3814 	return 0;
3815 }
3816 
3817 static int ibmvscsis_queue_status(struct se_cmd *se_cmd)
3818 {
3819 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3820 						 se_cmd);
3821 	struct scsi_info *vscsi = cmd->adapter;
3822 	uint len;
3823 
3824 	dev_dbg(&vscsi->dev, "queue_status %p\n", se_cmd);
3825 
3826 	srp_build_response(vscsi, cmd, &len);
3827 	cmd->rsp.format = SRP_FORMAT;
3828 	cmd->rsp.len = len;
3829 
3830 	return 0;
3831 }
3832 
3833 static void ibmvscsis_queue_tm_rsp(struct se_cmd *se_cmd)
3834 {
3835 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3836 						 se_cmd);
3837 	struct scsi_info *vscsi = cmd->adapter;
3838 	struct ibmvscsis_cmd *cmd_itr;
3839 	struct iu_entry *iue = iue = cmd->iue;
3840 	struct srp_tsk_mgmt *srp_tsk = &vio_iu(iue)->srp.tsk_mgmt;
3841 	u64 tag_to_abort = be64_to_cpu(srp_tsk->task_tag);
3842 	uint len;
3843 
3844 	dev_dbg(&vscsi->dev, "queue_tm_rsp %p, status %d\n",
3845 		se_cmd, (int)se_cmd->se_tmr_req->response);
3846 
3847 	if (srp_tsk->tsk_mgmt_func == SRP_TSK_ABORT_TASK &&
3848 	    cmd->se_cmd.se_tmr_req->response == TMR_TASK_DOES_NOT_EXIST) {
3849 		spin_lock_bh(&vscsi->intr_lock);
3850 		list_for_each_entry(cmd_itr, &vscsi->active_q, list) {
3851 			if (tag_to_abort == cmd_itr->se_cmd.tag) {
3852 				cmd_itr->abort_cmd = cmd;
3853 				cmd->flags |= DELAY_SEND;
3854 				break;
3855 			}
3856 		}
3857 		spin_unlock_bh(&vscsi->intr_lock);
3858 	}
3859 
3860 	srp_build_response(vscsi, cmd, &len);
3861 	cmd->rsp.format = SRP_FORMAT;
3862 	cmd->rsp.len = len;
3863 }
3864 
3865 static void ibmvscsis_aborted_task(struct se_cmd *se_cmd)
3866 {
3867 	struct ibmvscsis_cmd *cmd = container_of(se_cmd, struct ibmvscsis_cmd,
3868 						 se_cmd);
3869 	struct scsi_info *vscsi = cmd->adapter;
3870 
3871 	dev_dbg(&vscsi->dev, "ibmvscsis_aborted_task %p task_tag: %llu\n",
3872 		se_cmd, se_cmd->tag);
3873 }
3874 
3875 static struct se_wwn *ibmvscsis_make_tport(struct target_fabric_configfs *tf,
3876 					   struct config_group *group,
3877 					   const char *name)
3878 {
3879 	struct ibmvscsis_tport *tport;
3880 	struct scsi_info *vscsi;
3881 
3882 	tport = ibmvscsis_lookup_port(name);
3883 	if (tport) {
3884 		vscsi = container_of(tport, struct scsi_info, tport);
3885 		tport->tport_proto_id = SCSI_PROTOCOL_SRP;
3886 		dev_dbg(&vscsi->dev, "make_tport(%s), pointer:%p, tport_id:%x\n",
3887 			name, tport, tport->tport_proto_id);
3888 		return &tport->tport_wwn;
3889 	}
3890 
3891 	return ERR_PTR(-EINVAL);
3892 }
3893 
3894 static void ibmvscsis_drop_tport(struct se_wwn *wwn)
3895 {
3896 	struct ibmvscsis_tport *tport = container_of(wwn,
3897 						     struct ibmvscsis_tport,
3898 						     tport_wwn);
3899 	struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
3900 
3901 	dev_dbg(&vscsi->dev, "drop_tport(%s)\n",
3902 		config_item_name(&tport->tport_wwn.wwn_group.cg_item));
3903 }
3904 
3905 static struct se_portal_group *ibmvscsis_make_tpg(struct se_wwn *wwn,
3906 						  const char *name)
3907 {
3908 	struct ibmvscsis_tport *tport =
3909 		container_of(wwn, struct ibmvscsis_tport, tport_wwn);
3910 	u16 tpgt;
3911 	int rc;
3912 
3913 	if (strstr(name, "tpgt_") != name)
3914 		return ERR_PTR(-EINVAL);
3915 	rc = kstrtou16(name + 5, 0, &tpgt);
3916 	if (rc)
3917 		return ERR_PTR(rc);
3918 	tport->tport_tpgt = tpgt;
3919 
3920 	tport->releasing = false;
3921 
3922 	rc = core_tpg_register(&tport->tport_wwn, &tport->se_tpg,
3923 			       tport->tport_proto_id);
3924 	if (rc)
3925 		return ERR_PTR(rc);
3926 
3927 	return &tport->se_tpg;
3928 }
3929 
3930 static void ibmvscsis_drop_tpg(struct se_portal_group *se_tpg)
3931 {
3932 	struct ibmvscsis_tport *tport = container_of(se_tpg,
3933 						     struct ibmvscsis_tport,
3934 						     se_tpg);
3935 
3936 	tport->releasing = true;
3937 	tport->enabled = false;
3938 
3939 	/*
3940 	 * Release the virtual I_T Nexus for this ibmvscsis TPG
3941 	 */
3942 	ibmvscsis_drop_nexus(tport);
3943 	/*
3944 	 * Deregister the se_tpg from TCM..
3945 	 */
3946 	core_tpg_deregister(se_tpg);
3947 }
3948 
3949 static ssize_t ibmvscsis_wwn_version_show(struct config_item *item,
3950 					  char *page)
3951 {
3952 	return scnprintf(page, PAGE_SIZE, "%s\n", IBMVSCSIS_VERSION);
3953 }
3954 CONFIGFS_ATTR_RO(ibmvscsis_wwn_, version);
3955 
3956 static struct configfs_attribute *ibmvscsis_wwn_attrs[] = {
3957 	&ibmvscsis_wwn_attr_version,
3958 	NULL,
3959 };
3960 
3961 static ssize_t ibmvscsis_tpg_enable_show(struct config_item *item,
3962 					 char *page)
3963 {
3964 	struct se_portal_group *se_tpg = to_tpg(item);
3965 	struct ibmvscsis_tport *tport = container_of(se_tpg,
3966 						     struct ibmvscsis_tport,
3967 						     se_tpg);
3968 
3969 	return snprintf(page, PAGE_SIZE, "%d\n", (tport->enabled) ? 1 : 0);
3970 }
3971 
3972 static ssize_t ibmvscsis_tpg_enable_store(struct config_item *item,
3973 					  const char *page, size_t count)
3974 {
3975 	struct se_portal_group *se_tpg = to_tpg(item);
3976 	struct ibmvscsis_tport *tport = container_of(se_tpg,
3977 						     struct ibmvscsis_tport,
3978 						     se_tpg);
3979 	struct scsi_info *vscsi = container_of(tport, struct scsi_info, tport);
3980 	unsigned long tmp;
3981 	int rc;
3982 	long lrc;
3983 
3984 	rc = kstrtoul(page, 0, &tmp);
3985 	if (rc < 0) {
3986 		dev_err(&vscsi->dev, "Unable to extract srpt_tpg_store_enable\n");
3987 		return -EINVAL;
3988 	}
3989 
3990 	if ((tmp != 0) && (tmp != 1)) {
3991 		dev_err(&vscsi->dev, "Illegal value for srpt_tpg_store_enable\n");
3992 		return -EINVAL;
3993 	}
3994 
3995 	if (tmp) {
3996 		spin_lock_bh(&vscsi->intr_lock);
3997 		tport->enabled = true;
3998 		lrc = ibmvscsis_enable_change_state(vscsi);
3999 		if (lrc)
4000 			dev_err(&vscsi->dev, "enable_change_state failed, rc %ld state %d\n",
4001 				lrc, vscsi->state);
4002 		spin_unlock_bh(&vscsi->intr_lock);
4003 	} else {
4004 		spin_lock_bh(&vscsi->intr_lock);
4005 		tport->enabled = false;
4006 		/* This simulates the server going down */
4007 		ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT, 0);
4008 		spin_unlock_bh(&vscsi->intr_lock);
4009 	}
4010 
4011 	dev_dbg(&vscsi->dev, "tpg_enable_store, tmp %ld, state %d\n", tmp,
4012 		vscsi->state);
4013 
4014 	return count;
4015 }
4016 CONFIGFS_ATTR(ibmvscsis_tpg_, enable);
4017 
4018 static struct configfs_attribute *ibmvscsis_tpg_attrs[] = {
4019 	&ibmvscsis_tpg_attr_enable,
4020 	NULL,
4021 };
4022 
4023 static const struct target_core_fabric_ops ibmvscsis_ops = {
4024 	.module				= THIS_MODULE,
4025 	.fabric_name			= "ibmvscsis",
4026 	.max_data_sg_nents		= MAX_TXU / PAGE_SIZE,
4027 	.tpg_get_wwn			= ibmvscsis_get_fabric_wwn,
4028 	.tpg_get_tag			= ibmvscsis_get_tag,
4029 	.tpg_get_default_depth		= ibmvscsis_get_default_depth,
4030 	.tpg_check_demo_mode		= ibmvscsis_check_true,
4031 	.tpg_check_demo_mode_cache	= ibmvscsis_check_true,
4032 	.tpg_check_demo_mode_write_protect = ibmvscsis_check_false,
4033 	.tpg_check_prod_mode_write_protect = ibmvscsis_check_false,
4034 	.tpg_get_inst_index		= ibmvscsis_tpg_get_inst_index,
4035 	.check_stop_free		= ibmvscsis_check_stop_free,
4036 	.release_cmd			= ibmvscsis_release_cmd,
4037 	.sess_get_index			= ibmvscsis_sess_get_index,
4038 	.write_pending			= ibmvscsis_write_pending,
4039 	.set_default_node_attributes	= ibmvscsis_set_default_node_attrs,
4040 	.get_cmd_state			= ibmvscsis_get_cmd_state,
4041 	.queue_data_in			= ibmvscsis_queue_data_in,
4042 	.queue_status			= ibmvscsis_queue_status,
4043 	.queue_tm_rsp			= ibmvscsis_queue_tm_rsp,
4044 	.aborted_task			= ibmvscsis_aborted_task,
4045 	/*
4046 	 * Setup function pointers for logic in target_core_fabric_configfs.c
4047 	 */
4048 	.fabric_make_wwn		= ibmvscsis_make_tport,
4049 	.fabric_drop_wwn		= ibmvscsis_drop_tport,
4050 	.fabric_make_tpg		= ibmvscsis_make_tpg,
4051 	.fabric_drop_tpg		= ibmvscsis_drop_tpg,
4052 
4053 	.tfc_wwn_attrs			= ibmvscsis_wwn_attrs,
4054 	.tfc_tpg_base_attrs		= ibmvscsis_tpg_attrs,
4055 };
4056 
4057 static void ibmvscsis_dev_release(struct device *dev) {};
4058 
4059 static struct device_attribute dev_attr_system_id =
4060 	__ATTR(system_id, S_IRUGO, system_id_show, NULL);
4061 
4062 static struct device_attribute dev_attr_partition_number =
4063 	__ATTR(partition_number, S_IRUGO, partition_number_show, NULL);
4064 
4065 static struct device_attribute dev_attr_unit_address =
4066 	__ATTR(unit_address, S_IRUGO, unit_address_show, NULL);
4067 
4068 static struct attribute *ibmvscsis_dev_attrs[] = {
4069 	&dev_attr_system_id.attr,
4070 	&dev_attr_partition_number.attr,
4071 	&dev_attr_unit_address.attr,
4072 };
4073 ATTRIBUTE_GROUPS(ibmvscsis_dev);
4074 
4075 static struct class ibmvscsis_class = {
4076 	.name		= "ibmvscsis",
4077 	.dev_release	= ibmvscsis_dev_release,
4078 	.dev_groups	= ibmvscsis_dev_groups,
4079 };
4080 
4081 static const struct vio_device_id ibmvscsis_device_table[] = {
4082 	{ "v-scsi-host", "IBM,v-scsi-host" },
4083 	{ "", "" }
4084 };
4085 MODULE_DEVICE_TABLE(vio, ibmvscsis_device_table);
4086 
4087 static struct vio_driver ibmvscsis_driver = {
4088 	.name = "ibmvscsis",
4089 	.id_table = ibmvscsis_device_table,
4090 	.probe = ibmvscsis_probe,
4091 	.remove = ibmvscsis_remove,
4092 };
4093 
4094 /*
4095  * ibmvscsis_init() - Kernel Module initialization
4096  *
4097  * Note: vio_register_driver() registers callback functions, and at least one
4098  * of those callback functions calls TCM - Linux IO Target Subsystem, thus
4099  * the SCSI Target template must be registered before vio_register_driver()
4100  * is called.
4101  */
4102 static int __init ibmvscsis_init(void)
4103 {
4104 	int rc = 0;
4105 
4106 	rc = ibmvscsis_get_system_info();
4107 	if (rc) {
4108 		pr_err("rc %d from get_system_info\n", rc);
4109 		goto out;
4110 	}
4111 
4112 	rc = class_register(&ibmvscsis_class);
4113 	if (rc) {
4114 		pr_err("failed class register\n");
4115 		goto out;
4116 	}
4117 
4118 	rc = target_register_template(&ibmvscsis_ops);
4119 	if (rc) {
4120 		pr_err("rc %d from target_register_template\n", rc);
4121 		goto unregister_class;
4122 	}
4123 
4124 	rc = vio_register_driver(&ibmvscsis_driver);
4125 	if (rc) {
4126 		pr_err("rc %d from vio_register_driver\n", rc);
4127 		goto unregister_target;
4128 	}
4129 
4130 	return 0;
4131 
4132 unregister_target:
4133 	target_unregister_template(&ibmvscsis_ops);
4134 unregister_class:
4135 	class_unregister(&ibmvscsis_class);
4136 out:
4137 	return rc;
4138 }
4139 
4140 static void __exit ibmvscsis_exit(void)
4141 {
4142 	pr_info("Unregister IBM virtual SCSI host driver\n");
4143 	vio_unregister_driver(&ibmvscsis_driver);
4144 	target_unregister_template(&ibmvscsis_ops);
4145 	class_unregister(&ibmvscsis_class);
4146 }
4147 
4148 MODULE_DESCRIPTION("IBMVSCSIS fabric driver");
4149 MODULE_AUTHOR("Bryant G. Ly and Michael Cyr");
4150 MODULE_LICENSE("GPL");
4151 MODULE_VERSION(IBMVSCSIS_VERSION);
4152 module_init(ibmvscsis_init);
4153 module_exit(ibmvscsis_exit);
4154