xref: /openbmc/linux/drivers/scsi/mpi3mr/mpi3mr_os.c (revision 0c6dfa75)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Driver for Broadcom MPI3 Storage Controllers
4  *
5  * Copyright (C) 2017-2023 Broadcom Inc.
6  *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
7  *
8  */
9 
10 #include "mpi3mr.h"
11 
12 /* global driver scop variables */
13 LIST_HEAD(mrioc_list);
14 DEFINE_SPINLOCK(mrioc_list_lock);
15 static int mrioc_ids;
16 static int warn_non_secure_ctlr;
17 atomic64_t event_counter;
18 
19 MODULE_AUTHOR(MPI3MR_DRIVER_AUTHOR);
20 MODULE_DESCRIPTION(MPI3MR_DRIVER_DESC);
21 MODULE_LICENSE(MPI3MR_DRIVER_LICENSE);
22 MODULE_VERSION(MPI3MR_DRIVER_VERSION);
23 
24 /* Module parameters*/
25 int prot_mask = -1;
26 module_param(prot_mask, int, 0);
27 MODULE_PARM_DESC(prot_mask, "Host protection capabilities mask, def=0x07");
28 
29 static int prot_guard_mask = 3;
30 module_param(prot_guard_mask, int, 0);
31 MODULE_PARM_DESC(prot_guard_mask, " Host protection guard mask, def=3");
32 static int logging_level;
33 module_param(logging_level, int, 0);
34 MODULE_PARM_DESC(logging_level,
35 	" bits for enabling additional logging info (default=0)");
36 static int max_sgl_entries = MPI3MR_DEFAULT_SGL_ENTRIES;
37 module_param(max_sgl_entries, int, 0444);
38 MODULE_PARM_DESC(max_sgl_entries,
39 	"Preferred max number of SG entries to be used for a single I/O\n"
40 	"The actual value will be determined by the driver\n"
41 	"(Minimum=256, Maximum=2048, default=256)");
42 
43 /* Forward declarations*/
44 static void mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
45 	struct mpi3mr_drv_cmd *cmdparam, u32 event_ctx);
46 
47 #define MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION	(0xFFFF)
48 
49 #define MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH	(0xFFFE)
50 
51 /**
52  * mpi3mr_host_tag_for_scmd - Get host tag for a scmd
53  * @mrioc: Adapter instance reference
54  * @scmd: SCSI command reference
55  *
56  * Calculate the host tag based on block tag for a given scmd.
57  *
58  * Return: Valid host tag or MPI3MR_HOSTTAG_INVALID.
59  */
60 static u16 mpi3mr_host_tag_for_scmd(struct mpi3mr_ioc *mrioc,
61 	struct scsi_cmnd *scmd)
62 {
63 	struct scmd_priv *priv = NULL;
64 	u32 unique_tag;
65 	u16 host_tag, hw_queue;
66 
67 	unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
68 
69 	hw_queue = blk_mq_unique_tag_to_hwq(unique_tag);
70 	if (hw_queue >= mrioc->num_op_reply_q)
71 		return MPI3MR_HOSTTAG_INVALID;
72 	host_tag = blk_mq_unique_tag_to_tag(unique_tag);
73 
74 	if (WARN_ON(host_tag >= mrioc->max_host_ios))
75 		return MPI3MR_HOSTTAG_INVALID;
76 
77 	priv = scsi_cmd_priv(scmd);
78 	/*host_tag 0 is invalid hence incrementing by 1*/
79 	priv->host_tag = host_tag + 1;
80 	priv->scmd = scmd;
81 	priv->in_lld_scope = 1;
82 	priv->req_q_idx = hw_queue;
83 	priv->meta_chain_idx = -1;
84 	priv->chain_idx = -1;
85 	priv->meta_sg_valid = 0;
86 	return priv->host_tag;
87 }
88 
89 /**
90  * mpi3mr_scmd_from_host_tag - Get SCSI command from host tag
91  * @mrioc: Adapter instance reference
92  * @host_tag: Host tag
93  * @qidx: Operational queue index
94  *
95  * Identify the block tag from the host tag and queue index and
96  * retrieve associated scsi command using scsi_host_find_tag().
97  *
98  * Return: SCSI command reference or NULL.
99  */
100 static struct scsi_cmnd *mpi3mr_scmd_from_host_tag(
101 	struct mpi3mr_ioc *mrioc, u16 host_tag, u16 qidx)
102 {
103 	struct scsi_cmnd *scmd = NULL;
104 	struct scmd_priv *priv = NULL;
105 	u32 unique_tag = host_tag - 1;
106 
107 	if (WARN_ON(host_tag > mrioc->max_host_ios))
108 		goto out;
109 
110 	unique_tag |= (qidx << BLK_MQ_UNIQUE_TAG_BITS);
111 
112 	scmd = scsi_host_find_tag(mrioc->shost, unique_tag);
113 	if (scmd) {
114 		priv = scsi_cmd_priv(scmd);
115 		if (!priv->in_lld_scope)
116 			scmd = NULL;
117 	}
118 out:
119 	return scmd;
120 }
121 
122 /**
123  * mpi3mr_clear_scmd_priv - Cleanup SCSI command private date
124  * @mrioc: Adapter instance reference
125  * @scmd: SCSI command reference
126  *
127  * Invalidate the SCSI command private data to mark the command
128  * is not in LLD scope anymore.
129  *
130  * Return: Nothing.
131  */
132 static void mpi3mr_clear_scmd_priv(struct mpi3mr_ioc *mrioc,
133 	struct scsi_cmnd *scmd)
134 {
135 	struct scmd_priv *priv = NULL;
136 
137 	priv = scsi_cmd_priv(scmd);
138 
139 	if (WARN_ON(priv->in_lld_scope == 0))
140 		return;
141 	priv->host_tag = MPI3MR_HOSTTAG_INVALID;
142 	priv->req_q_idx = 0xFFFF;
143 	priv->scmd = NULL;
144 	priv->in_lld_scope = 0;
145 	priv->meta_sg_valid = 0;
146 	if (priv->chain_idx >= 0) {
147 		clear_bit(priv->chain_idx, mrioc->chain_bitmap);
148 		priv->chain_idx = -1;
149 	}
150 	if (priv->meta_chain_idx >= 0) {
151 		clear_bit(priv->meta_chain_idx, mrioc->chain_bitmap);
152 		priv->meta_chain_idx = -1;
153 	}
154 }
155 
156 static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_ioc *mrioc, u16 handle,
157 	struct mpi3mr_drv_cmd *cmdparam, u8 iou_rc);
158 static void mpi3mr_fwevt_worker(struct work_struct *work);
159 
160 /**
161  * mpi3mr_fwevt_free - firmware event memory dealloctor
162  * @r: k reference pointer of the firmware event
163  *
164  * Free firmware event memory when no reference.
165  */
166 static void mpi3mr_fwevt_free(struct kref *r)
167 {
168 	kfree(container_of(r, struct mpi3mr_fwevt, ref_count));
169 }
170 
171 /**
172  * mpi3mr_fwevt_get - k reference incrementor
173  * @fwevt: Firmware event reference
174  *
175  * Increment firmware event reference count.
176  */
177 static void mpi3mr_fwevt_get(struct mpi3mr_fwevt *fwevt)
178 {
179 	kref_get(&fwevt->ref_count);
180 }
181 
182 /**
183  * mpi3mr_fwevt_put - k reference decrementor
184  * @fwevt: Firmware event reference
185  *
186  * decrement firmware event reference count.
187  */
188 static void mpi3mr_fwevt_put(struct mpi3mr_fwevt *fwevt)
189 {
190 	kref_put(&fwevt->ref_count, mpi3mr_fwevt_free);
191 }
192 
193 /**
194  * mpi3mr_alloc_fwevt - Allocate firmware event
195  * @len: length of firmware event data to allocate
196  *
197  * Allocate firmware event with required length and initialize
198  * the reference counter.
199  *
200  * Return: firmware event reference.
201  */
202 static struct mpi3mr_fwevt *mpi3mr_alloc_fwevt(int len)
203 {
204 	struct mpi3mr_fwevt *fwevt;
205 
206 	fwevt = kzalloc(sizeof(*fwevt) + len, GFP_ATOMIC);
207 	if (!fwevt)
208 		return NULL;
209 
210 	kref_init(&fwevt->ref_count);
211 	return fwevt;
212 }
213 
214 /**
215  * mpi3mr_fwevt_add_to_list - Add firmware event to the list
216  * @mrioc: Adapter instance reference
217  * @fwevt: Firmware event reference
218  *
219  * Add the given firmware event to the firmware event list.
220  *
221  * Return: Nothing.
222  */
223 static void mpi3mr_fwevt_add_to_list(struct mpi3mr_ioc *mrioc,
224 	struct mpi3mr_fwevt *fwevt)
225 {
226 	unsigned long flags;
227 
228 	if (!mrioc->fwevt_worker_thread)
229 		return;
230 
231 	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
232 	/* get fwevt reference count while adding it to fwevt_list */
233 	mpi3mr_fwevt_get(fwevt);
234 	INIT_LIST_HEAD(&fwevt->list);
235 	list_add_tail(&fwevt->list, &mrioc->fwevt_list);
236 	INIT_WORK(&fwevt->work, mpi3mr_fwevt_worker);
237 	/* get fwevt reference count while enqueueing it to worker queue */
238 	mpi3mr_fwevt_get(fwevt);
239 	queue_work(mrioc->fwevt_worker_thread, &fwevt->work);
240 	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
241 }
242 
243 /**
244  * mpi3mr_fwevt_del_from_list - Delete firmware event from list
245  * @mrioc: Adapter instance reference
246  * @fwevt: Firmware event reference
247  *
248  * Delete the given firmware event from the firmware event list.
249  *
250  * Return: Nothing.
251  */
252 static void mpi3mr_fwevt_del_from_list(struct mpi3mr_ioc *mrioc,
253 	struct mpi3mr_fwevt *fwevt)
254 {
255 	unsigned long flags;
256 
257 	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
258 	if (!list_empty(&fwevt->list)) {
259 		list_del_init(&fwevt->list);
260 		/*
261 		 * Put fwevt reference count after
262 		 * removing it from fwevt_list
263 		 */
264 		mpi3mr_fwevt_put(fwevt);
265 	}
266 	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
267 }
268 
269 /**
270  * mpi3mr_dequeue_fwevt - Dequeue firmware event from the list
271  * @mrioc: Adapter instance reference
272  *
273  * Dequeue a firmware event from the firmware event list.
274  *
275  * Return: firmware event.
276  */
277 static struct mpi3mr_fwevt *mpi3mr_dequeue_fwevt(
278 	struct mpi3mr_ioc *mrioc)
279 {
280 	unsigned long flags;
281 	struct mpi3mr_fwevt *fwevt = NULL;
282 
283 	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
284 	if (!list_empty(&mrioc->fwevt_list)) {
285 		fwevt = list_first_entry(&mrioc->fwevt_list,
286 		    struct mpi3mr_fwevt, list);
287 		list_del_init(&fwevt->list);
288 		/*
289 		 * Put fwevt reference count after
290 		 * removing it from fwevt_list
291 		 */
292 		mpi3mr_fwevt_put(fwevt);
293 	}
294 	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
295 
296 	return fwevt;
297 }
298 
299 /**
300  * mpi3mr_cancel_work - cancel firmware event
301  * @fwevt: fwevt object which needs to be canceled
302  *
303  * Return: Nothing.
304  */
305 static void mpi3mr_cancel_work(struct mpi3mr_fwevt *fwevt)
306 {
307 	/*
308 	 * Wait on the fwevt to complete. If this returns 1, then
309 	 * the event was never executed.
310 	 *
311 	 * If it did execute, we wait for it to finish, and the put will
312 	 * happen from mpi3mr_process_fwevt()
313 	 */
314 	if (cancel_work_sync(&fwevt->work)) {
315 		/*
316 		 * Put fwevt reference count after
317 		 * dequeuing it from worker queue
318 		 */
319 		mpi3mr_fwevt_put(fwevt);
320 		/*
321 		 * Put fwevt reference count to neutralize
322 		 * kref_init increment
323 		 */
324 		mpi3mr_fwevt_put(fwevt);
325 	}
326 }
327 
328 /**
329  * mpi3mr_cleanup_fwevt_list - Cleanup firmware event list
330  * @mrioc: Adapter instance reference
331  *
332  * Flush all pending firmware events from the firmware event
333  * list.
334  *
335  * Return: Nothing.
336  */
337 void mpi3mr_cleanup_fwevt_list(struct mpi3mr_ioc *mrioc)
338 {
339 	struct mpi3mr_fwevt *fwevt = NULL;
340 
341 	if ((list_empty(&mrioc->fwevt_list) && !mrioc->current_event) ||
342 	    !mrioc->fwevt_worker_thread)
343 		return;
344 
345 	while ((fwevt = mpi3mr_dequeue_fwevt(mrioc)))
346 		mpi3mr_cancel_work(fwevt);
347 
348 	if (mrioc->current_event) {
349 		fwevt = mrioc->current_event;
350 		/*
351 		 * Don't call cancel_work_sync() API for the
352 		 * fwevt work if the controller reset is
353 		 * get called as part of processing the
354 		 * same fwevt work (or) when worker thread is
355 		 * waiting for device add/remove APIs to complete.
356 		 * Otherwise we will see deadlock.
357 		 */
358 		if (current_work() == &fwevt->work || fwevt->pending_at_sml) {
359 			fwevt->discard = 1;
360 			return;
361 		}
362 
363 		mpi3mr_cancel_work(fwevt);
364 	}
365 }
366 
367 /**
368  * mpi3mr_queue_qd_reduction_event - Queue TG QD reduction event
369  * @mrioc: Adapter instance reference
370  * @tg: Throttle group information pointer
371  *
372  * Accessor to queue on synthetically generated driver event to
373  * the event worker thread, the driver event will be used to
374  * reduce the QD of all VDs in the TG from the worker thread.
375  *
376  * Return: None.
377  */
378 static void mpi3mr_queue_qd_reduction_event(struct mpi3mr_ioc *mrioc,
379 	struct mpi3mr_throttle_group_info *tg)
380 {
381 	struct mpi3mr_fwevt *fwevt;
382 	u16 sz = sizeof(struct mpi3mr_throttle_group_info *);
383 
384 	/*
385 	 * If the QD reduction event is already queued due to throttle and if
386 	 * the QD is not restored through device info change event
387 	 * then dont queue further reduction events
388 	 */
389 	if (tg->fw_qd != tg->modified_qd)
390 		return;
391 
392 	fwevt = mpi3mr_alloc_fwevt(sz);
393 	if (!fwevt) {
394 		ioc_warn(mrioc, "failed to queue TG QD reduction event\n");
395 		return;
396 	}
397 	*(struct mpi3mr_throttle_group_info **)fwevt->event_data = tg;
398 	fwevt->mrioc = mrioc;
399 	fwevt->event_id = MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION;
400 	fwevt->send_ack = 0;
401 	fwevt->process_evt = 1;
402 	fwevt->evt_ctx = 0;
403 	fwevt->event_data_size = sz;
404 	tg->modified_qd = max_t(u16, (tg->fw_qd * tg->qd_reduction) / 10, 8);
405 
406 	dprint_event_bh(mrioc, "qd reduction event queued for tg_id(%d)\n",
407 	    tg->id);
408 	mpi3mr_fwevt_add_to_list(mrioc, fwevt);
409 }
410 
411 /**
412  * mpi3mr_invalidate_devhandles -Invalidate device handles
413  * @mrioc: Adapter instance reference
414  *
415  * Invalidate the device handles in the target device structures
416  * . Called post reset prior to reinitializing the controller.
417  *
418  * Return: Nothing.
419  */
420 void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc)
421 {
422 	struct mpi3mr_tgt_dev *tgtdev;
423 	struct mpi3mr_stgt_priv_data *tgt_priv;
424 
425 	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
426 		tgtdev->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
427 		if (tgtdev->starget && tgtdev->starget->hostdata) {
428 			tgt_priv = tgtdev->starget->hostdata;
429 			tgt_priv->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
430 			tgt_priv->io_throttle_enabled = 0;
431 			tgt_priv->io_divert = 0;
432 			tgt_priv->throttle_group = NULL;
433 			tgt_priv->wslen = 0;
434 			if (tgtdev->host_exposed)
435 				atomic_set(&tgt_priv->block_io, 1);
436 		}
437 	}
438 }
439 
440 /**
441  * mpi3mr_print_scmd - print individual SCSI command
442  * @rq: Block request
443  * @data: Adapter instance reference
444  *
445  * Print the SCSI command details if it is in LLD scope.
446  *
447  * Return: true always.
448  */
449 static bool mpi3mr_print_scmd(struct request *rq, void *data)
450 {
451 	struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data;
452 	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
453 	struct scmd_priv *priv = NULL;
454 
455 	if (scmd) {
456 		priv = scsi_cmd_priv(scmd);
457 		if (!priv->in_lld_scope)
458 			goto out;
459 
460 		ioc_info(mrioc, "%s :Host Tag = %d, qid = %d\n",
461 		    __func__, priv->host_tag, priv->req_q_idx + 1);
462 		scsi_print_command(scmd);
463 	}
464 
465 out:
466 	return(true);
467 }
468 
469 /**
470  * mpi3mr_flush_scmd - Flush individual SCSI command
471  * @rq: Block request
472  * @data: Adapter instance reference
473  *
474  * Return the SCSI command to the upper layers if it is in LLD
475  * scope.
476  *
477  * Return: true always.
478  */
479 
480 static bool mpi3mr_flush_scmd(struct request *rq, void *data)
481 {
482 	struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data;
483 	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
484 	struct scmd_priv *priv = NULL;
485 
486 	if (scmd) {
487 		priv = scsi_cmd_priv(scmd);
488 		if (!priv->in_lld_scope)
489 			goto out;
490 
491 		if (priv->meta_sg_valid)
492 			dma_unmap_sg(&mrioc->pdev->dev, scsi_prot_sglist(scmd),
493 			    scsi_prot_sg_count(scmd), scmd->sc_data_direction);
494 		mpi3mr_clear_scmd_priv(mrioc, scmd);
495 		scsi_dma_unmap(scmd);
496 		scmd->result = DID_RESET << 16;
497 		scsi_print_command(scmd);
498 		scsi_done(scmd);
499 		mrioc->flush_io_count++;
500 	}
501 
502 out:
503 	return(true);
504 }
505 
506 /**
507  * mpi3mr_count_dev_pending - Count commands pending for a lun
508  * @rq: Block request
509  * @data: SCSI device reference
510  *
511  * This is an iterator function called for each SCSI command in
512  * a host and if the command is pending in the LLD for the
513  * specific device(lun) then device specific pending I/O counter
514  * is updated in the device structure.
515  *
516  * Return: true always.
517  */
518 
519 static bool mpi3mr_count_dev_pending(struct request *rq, void *data)
520 {
521 	struct scsi_device *sdev = (struct scsi_device *)data;
522 	struct mpi3mr_sdev_priv_data *sdev_priv_data = sdev->hostdata;
523 	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
524 	struct scmd_priv *priv;
525 
526 	if (scmd) {
527 		priv = scsi_cmd_priv(scmd);
528 		if (!priv->in_lld_scope)
529 			goto out;
530 		if (scmd->device == sdev)
531 			sdev_priv_data->pend_count++;
532 	}
533 
534 out:
535 	return true;
536 }
537 
538 /**
539  * mpi3mr_count_tgt_pending - Count commands pending for target
540  * @rq: Block request
541  * @data: SCSI target reference
542  *
543  * This is an iterator function called for each SCSI command in
544  * a host and if the command is pending in the LLD for the
545  * specific target then target specific pending I/O counter is
546  * updated in the target structure.
547  *
548  * Return: true always.
549  */
550 
551 static bool mpi3mr_count_tgt_pending(struct request *rq, void *data)
552 {
553 	struct scsi_target *starget = (struct scsi_target *)data;
554 	struct mpi3mr_stgt_priv_data *stgt_priv_data = starget->hostdata;
555 	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
556 	struct scmd_priv *priv;
557 
558 	if (scmd) {
559 		priv = scsi_cmd_priv(scmd);
560 		if (!priv->in_lld_scope)
561 			goto out;
562 		if (scmd->device && (scsi_target(scmd->device) == starget))
563 			stgt_priv_data->pend_count++;
564 	}
565 
566 out:
567 	return true;
568 }
569 
570 /**
571  * mpi3mr_flush_host_io -  Flush host I/Os
572  * @mrioc: Adapter instance reference
573  *
574  * Flush all of the pending I/Os by calling
575  * blk_mq_tagset_busy_iter() for each possible tag. This is
576  * executed post controller reset
577  *
578  * Return: Nothing.
579  */
580 void mpi3mr_flush_host_io(struct mpi3mr_ioc *mrioc)
581 {
582 	struct Scsi_Host *shost = mrioc->shost;
583 
584 	mrioc->flush_io_count = 0;
585 	ioc_info(mrioc, "%s :Flushing Host I/O cmds post reset\n", __func__);
586 	blk_mq_tagset_busy_iter(&shost->tag_set,
587 	    mpi3mr_flush_scmd, (void *)mrioc);
588 	ioc_info(mrioc, "%s :Flushed %d Host I/O cmds\n", __func__,
589 	    mrioc->flush_io_count);
590 }
591 
592 /**
593  * mpi3mr_flush_cmds_for_unrecovered_controller - Flush all pending cmds
594  * @mrioc: Adapter instance reference
595  *
596  * This function waits for currently running IO poll threads to
597  * exit and then flushes all host I/Os and any internal pending
598  * cmds. This is executed after controller is marked as
599  * unrecoverable.
600  *
601  * Return: Nothing.
602  */
603 void mpi3mr_flush_cmds_for_unrecovered_controller(struct mpi3mr_ioc *mrioc)
604 {
605 	struct Scsi_Host *shost = mrioc->shost;
606 	int i;
607 
608 	if (!mrioc->unrecoverable)
609 		return;
610 
611 	if (mrioc->op_reply_qinfo) {
612 		for (i = 0; i < mrioc->num_queues; i++) {
613 			while (atomic_read(&mrioc->op_reply_qinfo[i].in_use))
614 				udelay(500);
615 			atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
616 		}
617 	}
618 	mrioc->flush_io_count = 0;
619 	blk_mq_tagset_busy_iter(&shost->tag_set,
620 	    mpi3mr_flush_scmd, (void *)mrioc);
621 	mpi3mr_flush_delayed_cmd_lists(mrioc);
622 	mpi3mr_flush_drv_cmds(mrioc);
623 }
624 
625 /**
626  * mpi3mr_alloc_tgtdev - target device allocator
627  *
628  * Allocate target device instance and initialize the reference
629  * count
630  *
631  * Return: target device instance.
632  */
633 static struct mpi3mr_tgt_dev *mpi3mr_alloc_tgtdev(void)
634 {
635 	struct mpi3mr_tgt_dev *tgtdev;
636 
637 	tgtdev = kzalloc(sizeof(*tgtdev), GFP_ATOMIC);
638 	if (!tgtdev)
639 		return NULL;
640 	kref_init(&tgtdev->ref_count);
641 	return tgtdev;
642 }
643 
644 /**
645  * mpi3mr_tgtdev_add_to_list -Add tgtdevice to the list
646  * @mrioc: Adapter instance reference
647  * @tgtdev: Target device
648  *
649  * Add the target device to the target device list
650  *
651  * Return: Nothing.
652  */
653 static void mpi3mr_tgtdev_add_to_list(struct mpi3mr_ioc *mrioc,
654 	struct mpi3mr_tgt_dev *tgtdev)
655 {
656 	unsigned long flags;
657 
658 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
659 	mpi3mr_tgtdev_get(tgtdev);
660 	INIT_LIST_HEAD(&tgtdev->list);
661 	list_add_tail(&tgtdev->list, &mrioc->tgtdev_list);
662 	tgtdev->state = MPI3MR_DEV_CREATED;
663 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
664 }
665 
666 /**
667  * mpi3mr_tgtdev_del_from_list -Delete tgtdevice from the list
668  * @mrioc: Adapter instance reference
669  * @tgtdev: Target device
670  * @must_delete: Must delete the target device from the list irrespective
671  * of the device state.
672  *
673  * Remove the target device from the target device list
674  *
675  * Return: Nothing.
676  */
677 static void mpi3mr_tgtdev_del_from_list(struct mpi3mr_ioc *mrioc,
678 	struct mpi3mr_tgt_dev *tgtdev, bool must_delete)
679 {
680 	unsigned long flags;
681 
682 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
683 	if ((tgtdev->state == MPI3MR_DEV_REMOVE_HS_STARTED) || (must_delete == true)) {
684 		if (!list_empty(&tgtdev->list)) {
685 			list_del_init(&tgtdev->list);
686 			tgtdev->state = MPI3MR_DEV_DELETED;
687 			mpi3mr_tgtdev_put(tgtdev);
688 		}
689 	}
690 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
691 }
692 
693 /**
694  * __mpi3mr_get_tgtdev_by_handle -Get tgtdev from device handle
695  * @mrioc: Adapter instance reference
696  * @handle: Device handle
697  *
698  * Accessor to retrieve target device from the device handle.
699  * Non Lock version
700  *
701  * Return: Target device reference.
702  */
703 static struct mpi3mr_tgt_dev  *__mpi3mr_get_tgtdev_by_handle(
704 	struct mpi3mr_ioc *mrioc, u16 handle)
705 {
706 	struct mpi3mr_tgt_dev *tgtdev;
707 
708 	assert_spin_locked(&mrioc->tgtdev_lock);
709 	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list)
710 		if (tgtdev->dev_handle == handle)
711 			goto found_tgtdev;
712 	return NULL;
713 
714 found_tgtdev:
715 	mpi3mr_tgtdev_get(tgtdev);
716 	return tgtdev;
717 }
718 
719 /**
720  * mpi3mr_get_tgtdev_by_handle -Get tgtdev from device handle
721  * @mrioc: Adapter instance reference
722  * @handle: Device handle
723  *
724  * Accessor to retrieve target device from the device handle.
725  * Lock version
726  *
727  * Return: Target device reference.
728  */
729 struct mpi3mr_tgt_dev *mpi3mr_get_tgtdev_by_handle(
730 	struct mpi3mr_ioc *mrioc, u16 handle)
731 {
732 	struct mpi3mr_tgt_dev *tgtdev;
733 	unsigned long flags;
734 
735 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
736 	tgtdev = __mpi3mr_get_tgtdev_by_handle(mrioc, handle);
737 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
738 	return tgtdev;
739 }
740 
741 /**
742  * __mpi3mr_get_tgtdev_by_perst_id -Get tgtdev from persist ID
743  * @mrioc: Adapter instance reference
744  * @persist_id: Persistent ID
745  *
746  * Accessor to retrieve target device from the Persistent ID.
747  * Non Lock version
748  *
749  * Return: Target device reference.
750  */
751 static struct mpi3mr_tgt_dev  *__mpi3mr_get_tgtdev_by_perst_id(
752 	struct mpi3mr_ioc *mrioc, u16 persist_id)
753 {
754 	struct mpi3mr_tgt_dev *tgtdev;
755 
756 	assert_spin_locked(&mrioc->tgtdev_lock);
757 	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list)
758 		if (tgtdev->perst_id == persist_id)
759 			goto found_tgtdev;
760 	return NULL;
761 
762 found_tgtdev:
763 	mpi3mr_tgtdev_get(tgtdev);
764 	return tgtdev;
765 }
766 
767 /**
768  * mpi3mr_get_tgtdev_by_perst_id -Get tgtdev from persistent ID
769  * @mrioc: Adapter instance reference
770  * @persist_id: Persistent ID
771  *
772  * Accessor to retrieve target device from the Persistent ID.
773  * Lock version
774  *
775  * Return: Target device reference.
776  */
777 static struct mpi3mr_tgt_dev *mpi3mr_get_tgtdev_by_perst_id(
778 	struct mpi3mr_ioc *mrioc, u16 persist_id)
779 {
780 	struct mpi3mr_tgt_dev *tgtdev;
781 	unsigned long flags;
782 
783 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
784 	tgtdev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, persist_id);
785 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
786 	return tgtdev;
787 }
788 
789 /**
790  * __mpi3mr_get_tgtdev_from_tgtpriv -Get tgtdev from tgt private
791  * @mrioc: Adapter instance reference
792  * @tgt_priv: Target private data
793  *
794  * Accessor to return target device from the target private
795  * data. Non Lock version
796  *
797  * Return: Target device reference.
798  */
799 static struct mpi3mr_tgt_dev  *__mpi3mr_get_tgtdev_from_tgtpriv(
800 	struct mpi3mr_ioc *mrioc, struct mpi3mr_stgt_priv_data *tgt_priv)
801 {
802 	struct mpi3mr_tgt_dev *tgtdev;
803 
804 	assert_spin_locked(&mrioc->tgtdev_lock);
805 	tgtdev = tgt_priv->tgt_dev;
806 	if (tgtdev)
807 		mpi3mr_tgtdev_get(tgtdev);
808 	return tgtdev;
809 }
810 
811 /**
812  * mpi3mr_set_io_divert_for_all_vd_in_tg -set divert for TG VDs
813  * @mrioc: Adapter instance reference
814  * @tg: Throttle group information pointer
815  * @divert_value: 1 or 0
816  *
817  * Accessor to set io_divert flag for each device associated
818  * with the given throttle group with the given value.
819  *
820  * Return: None.
821  */
822 static void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_ioc *mrioc,
823 	struct mpi3mr_throttle_group_info *tg, u8 divert_value)
824 {
825 	unsigned long flags;
826 	struct mpi3mr_tgt_dev *tgtdev;
827 	struct mpi3mr_stgt_priv_data *tgt_priv;
828 
829 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
830 	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
831 		if (tgtdev->starget && tgtdev->starget->hostdata) {
832 			tgt_priv = tgtdev->starget->hostdata;
833 			if (tgt_priv->throttle_group == tg)
834 				tgt_priv->io_divert = divert_value;
835 		}
836 	}
837 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
838 }
839 
840 /**
841  * mpi3mr_print_device_event_notice - print notice related to post processing of
842  *					device event after controller reset.
843  *
844  * @mrioc: Adapter instance reference
845  * @device_add: true for device add event and false for device removal event
846  *
847  * Return: None.
848  */
849 void mpi3mr_print_device_event_notice(struct mpi3mr_ioc *mrioc,
850 	bool device_add)
851 {
852 	ioc_notice(mrioc, "Device %s was in progress before the reset and\n",
853 	    (device_add ? "addition" : "removal"));
854 	ioc_notice(mrioc, "completed after reset, verify whether the exposed devices\n");
855 	ioc_notice(mrioc, "are matched with attached devices for correctness\n");
856 }
857 
858 /**
859  * mpi3mr_remove_tgtdev_from_host - Remove dev from upper layers
860  * @mrioc: Adapter instance reference
861  * @tgtdev: Target device structure
862  *
863  * Checks whether the device is exposed to upper layers and if it
864  * is then remove the device from upper layers by calling
865  * scsi_remove_target().
866  *
867  * Return: 0 on success, non zero on failure.
868  */
869 void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc,
870 	struct mpi3mr_tgt_dev *tgtdev)
871 {
872 	struct mpi3mr_stgt_priv_data *tgt_priv;
873 
874 	ioc_info(mrioc, "%s :Removing handle(0x%04x), wwid(0x%016llx)\n",
875 	    __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid);
876 	if (tgtdev->starget && tgtdev->starget->hostdata) {
877 		tgt_priv = tgtdev->starget->hostdata;
878 		atomic_set(&tgt_priv->block_io, 0);
879 		tgt_priv->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
880 	}
881 
882 	if (!mrioc->sas_transport_enabled || (tgtdev->dev_type !=
883 	    MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl) {
884 		if (tgtdev->starget) {
885 			if (mrioc->current_event)
886 				mrioc->current_event->pending_at_sml = 1;
887 			scsi_remove_target(&tgtdev->starget->dev);
888 			tgtdev->host_exposed = 0;
889 			if (mrioc->current_event) {
890 				mrioc->current_event->pending_at_sml = 0;
891 				if (mrioc->current_event->discard) {
892 					mpi3mr_print_device_event_notice(mrioc,
893 					    false);
894 					return;
895 				}
896 			}
897 		}
898 	} else
899 		mpi3mr_remove_tgtdev_from_sas_transport(mrioc, tgtdev);
900 
901 	ioc_info(mrioc, "%s :Removed handle(0x%04x), wwid(0x%016llx)\n",
902 	    __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid);
903 }
904 
905 /**
906  * mpi3mr_report_tgtdev_to_host - Expose device to upper layers
907  * @mrioc: Adapter instance reference
908  * @perst_id: Persistent ID of the device
909  *
910  * Checks whether the device can be exposed to upper layers and
911  * if it is not then expose the device to upper layers by
912  * calling scsi_scan_target().
913  *
914  * Return: 0 on success, non zero on failure.
915  */
916 static int mpi3mr_report_tgtdev_to_host(struct mpi3mr_ioc *mrioc,
917 	u16 perst_id)
918 {
919 	int retval = 0;
920 	struct mpi3mr_tgt_dev *tgtdev;
921 
922 	if (mrioc->reset_in_progress)
923 		return -1;
924 
925 	tgtdev = mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id);
926 	if (!tgtdev) {
927 		retval = -1;
928 		goto out;
929 	}
930 	if (tgtdev->is_hidden || tgtdev->host_exposed) {
931 		retval = -1;
932 		goto out;
933 	}
934 	if (!mrioc->sas_transport_enabled || (tgtdev->dev_type !=
935 	    MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl){
936 		tgtdev->host_exposed = 1;
937 		if (mrioc->current_event)
938 			mrioc->current_event->pending_at_sml = 1;
939 		scsi_scan_target(&mrioc->shost->shost_gendev,
940 		    mrioc->scsi_device_channel, tgtdev->perst_id,
941 		    SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
942 		if (!tgtdev->starget)
943 			tgtdev->host_exposed = 0;
944 		if (mrioc->current_event) {
945 			mrioc->current_event->pending_at_sml = 0;
946 			if (mrioc->current_event->discard) {
947 				mpi3mr_print_device_event_notice(mrioc, true);
948 				goto out;
949 			}
950 		}
951 	} else
952 		mpi3mr_report_tgtdev_to_sas_transport(mrioc, tgtdev);
953 out:
954 	if (tgtdev)
955 		mpi3mr_tgtdev_put(tgtdev);
956 
957 	return retval;
958 }
959 
960 /**
961  * mpi3mr_change_queue_depth- Change QD callback handler
962  * @sdev: SCSI device reference
963  * @q_depth: Queue depth
964  *
965  * Validate and limit QD and call scsi_change_queue_depth.
966  *
967  * Return: return value of scsi_change_queue_depth
968  */
969 static int mpi3mr_change_queue_depth(struct scsi_device *sdev,
970 	int q_depth)
971 {
972 	struct scsi_target *starget = scsi_target(sdev);
973 	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
974 	int retval = 0;
975 
976 	if (!sdev->tagged_supported)
977 		q_depth = 1;
978 	if (q_depth > shost->can_queue)
979 		q_depth = shost->can_queue;
980 	else if (!q_depth)
981 		q_depth = MPI3MR_DEFAULT_SDEV_QD;
982 	retval = scsi_change_queue_depth(sdev, q_depth);
983 	sdev->max_queue_depth = sdev->queue_depth;
984 
985 	return retval;
986 }
987 
988 /**
989  * mpi3mr_update_sdev - Update SCSI device information
990  * @sdev: SCSI device reference
991  * @data: target device reference
992  *
993  * This is an iterator function called for each SCSI device in a
994  * target to update the target specific information into each
995  * SCSI device.
996  *
997  * Return: Nothing.
998  */
999 static void
1000 mpi3mr_update_sdev(struct scsi_device *sdev, void *data)
1001 {
1002 	struct mpi3mr_tgt_dev *tgtdev;
1003 
1004 	tgtdev = (struct mpi3mr_tgt_dev *)data;
1005 	if (!tgtdev)
1006 		return;
1007 
1008 	mpi3mr_change_queue_depth(sdev, tgtdev->q_depth);
1009 	switch (tgtdev->dev_type) {
1010 	case MPI3_DEVICE_DEVFORM_PCIE:
1011 		/*The block layer hw sector size = 512*/
1012 		if ((tgtdev->dev_spec.pcie_inf.dev_info &
1013 		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) ==
1014 		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) {
1015 			blk_queue_max_hw_sectors(sdev->request_queue,
1016 			    tgtdev->dev_spec.pcie_inf.mdts / 512);
1017 			if (tgtdev->dev_spec.pcie_inf.pgsz == 0)
1018 				blk_queue_virt_boundary(sdev->request_queue,
1019 				    ((1 << MPI3MR_DEFAULT_PGSZEXP) - 1));
1020 			else
1021 				blk_queue_virt_boundary(sdev->request_queue,
1022 				    ((1 << tgtdev->dev_spec.pcie_inf.pgsz) - 1));
1023 		}
1024 		break;
1025 	default:
1026 		break;
1027 	}
1028 }
1029 
1030 /**
1031  * mpi3mr_rfresh_tgtdevs - Refresh target device exposure
1032  * @mrioc: Adapter instance reference
1033  *
1034  * This is executed post controller reset to identify any
1035  * missing devices during reset and remove from the upper layers
1036  * or expose any newly detected device to the upper layers.
1037  *
1038  * Return: Nothing.
1039  */
1040 
1041 void mpi3mr_rfresh_tgtdevs(struct mpi3mr_ioc *mrioc)
1042 {
1043 	struct mpi3mr_tgt_dev *tgtdev, *tgtdev_next;
1044 	struct mpi3mr_stgt_priv_data *tgt_priv;
1045 
1046 	dprint_reset(mrioc, "refresh target devices: check for removals\n");
1047 	list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list,
1048 	    list) {
1049 		if ((tgtdev->dev_handle == MPI3MR_INVALID_DEV_HANDLE) &&
1050 		    tgtdev->host_exposed && tgtdev->starget &&
1051 		    tgtdev->starget->hostdata) {
1052 			tgt_priv = tgtdev->starget->hostdata;
1053 			tgt_priv->dev_removed = 1;
1054 			atomic_set(&tgt_priv->block_io, 0);
1055 		}
1056 	}
1057 
1058 	list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list,
1059 	    list) {
1060 		if (tgtdev->dev_handle == MPI3MR_INVALID_DEV_HANDLE) {
1061 			dprint_reset(mrioc, "removing target device with perst_id(%d)\n",
1062 			    tgtdev->perst_id);
1063 			if (tgtdev->host_exposed)
1064 				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1065 			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, true);
1066 			mpi3mr_tgtdev_put(tgtdev);
1067 		}
1068 	}
1069 
1070 	tgtdev = NULL;
1071 	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
1072 		if ((tgtdev->dev_handle != MPI3MR_INVALID_DEV_HANDLE) &&
1073 		    !tgtdev->is_hidden && !tgtdev->host_exposed)
1074 			mpi3mr_report_tgtdev_to_host(mrioc, tgtdev->perst_id);
1075 	}
1076 }
1077 
1078 /**
1079  * mpi3mr_update_tgtdev - DevStatusChange evt bottomhalf
1080  * @mrioc: Adapter instance reference
1081  * @tgtdev: Target device internal structure
1082  * @dev_pg0: New device page0
1083  * @is_added: Flag to indicate the device is just added
1084  *
1085  * Update the information from the device page0 into the driver
1086  * cached target device structure.
1087  *
1088  * Return: Nothing.
1089  */
1090 static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc,
1091 	struct mpi3mr_tgt_dev *tgtdev, struct mpi3_device_page0 *dev_pg0,
1092 	bool is_added)
1093 {
1094 	u16 flags = 0;
1095 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
1096 	struct mpi3mr_enclosure_node *enclosure_dev = NULL;
1097 	u8 prot_mask = 0;
1098 
1099 	tgtdev->perst_id = le16_to_cpu(dev_pg0->persistent_id);
1100 	tgtdev->dev_handle = le16_to_cpu(dev_pg0->dev_handle);
1101 	tgtdev->dev_type = dev_pg0->device_form;
1102 	tgtdev->io_unit_port = dev_pg0->io_unit_port;
1103 	tgtdev->encl_handle = le16_to_cpu(dev_pg0->enclosure_handle);
1104 	tgtdev->parent_handle = le16_to_cpu(dev_pg0->parent_dev_handle);
1105 	tgtdev->slot = le16_to_cpu(dev_pg0->slot);
1106 	tgtdev->q_depth = le16_to_cpu(dev_pg0->queue_depth);
1107 	tgtdev->wwid = le64_to_cpu(dev_pg0->wwid);
1108 	tgtdev->devpg0_flag = le16_to_cpu(dev_pg0->flags);
1109 
1110 	if (tgtdev->encl_handle)
1111 		enclosure_dev = mpi3mr_enclosure_find_by_handle(mrioc,
1112 		    tgtdev->encl_handle);
1113 	if (enclosure_dev)
1114 		tgtdev->enclosure_logical_id = le64_to_cpu(
1115 		    enclosure_dev->pg0.enclosure_logical_id);
1116 
1117 	flags = tgtdev->devpg0_flag;
1118 
1119 	tgtdev->is_hidden = (flags & MPI3_DEVICE0_FLAGS_HIDDEN);
1120 
1121 	if (is_added == true)
1122 		tgtdev->io_throttle_enabled =
1123 		    (flags & MPI3_DEVICE0_FLAGS_IO_THROTTLING_REQUIRED) ? 1 : 0;
1124 
1125 	switch (flags & MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_MASK) {
1126 	case MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_256_LB:
1127 		tgtdev->wslen = MPI3MR_WRITE_SAME_MAX_LEN_256_BLKS;
1128 		break;
1129 	case MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_2048_LB:
1130 		tgtdev->wslen = MPI3MR_WRITE_SAME_MAX_LEN_2048_BLKS;
1131 		break;
1132 	case MPI3_DEVICE0_FLAGS_MAX_WRITE_SAME_NO_LIMIT:
1133 	default:
1134 		tgtdev->wslen = 0;
1135 		break;
1136 	}
1137 
1138 	if (tgtdev->starget && tgtdev->starget->hostdata) {
1139 		scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
1140 		    tgtdev->starget->hostdata;
1141 		scsi_tgt_priv_data->perst_id = tgtdev->perst_id;
1142 		scsi_tgt_priv_data->dev_handle = tgtdev->dev_handle;
1143 		scsi_tgt_priv_data->dev_type = tgtdev->dev_type;
1144 		scsi_tgt_priv_data->io_throttle_enabled =
1145 		    tgtdev->io_throttle_enabled;
1146 		if (is_added == true)
1147 			atomic_set(&scsi_tgt_priv_data->block_io, 0);
1148 		scsi_tgt_priv_data->wslen = tgtdev->wslen;
1149 	}
1150 
1151 	switch (dev_pg0->access_status) {
1152 	case MPI3_DEVICE0_ASTATUS_NO_ERRORS:
1153 	case MPI3_DEVICE0_ASTATUS_PREPARE:
1154 	case MPI3_DEVICE0_ASTATUS_NEEDS_INITIALIZATION:
1155 	case MPI3_DEVICE0_ASTATUS_DEVICE_MISSING_DELAY:
1156 		break;
1157 	default:
1158 		tgtdev->is_hidden = 1;
1159 		break;
1160 	}
1161 
1162 	switch (tgtdev->dev_type) {
1163 	case MPI3_DEVICE_DEVFORM_SAS_SATA:
1164 	{
1165 		struct mpi3_device0_sas_sata_format *sasinf =
1166 		    &dev_pg0->device_specific.sas_sata_format;
1167 		u16 dev_info = le16_to_cpu(sasinf->device_info);
1168 
1169 		tgtdev->dev_spec.sas_sata_inf.dev_info = dev_info;
1170 		tgtdev->dev_spec.sas_sata_inf.sas_address =
1171 		    le64_to_cpu(sasinf->sas_address);
1172 		tgtdev->dev_spec.sas_sata_inf.phy_id = sasinf->phy_num;
1173 		tgtdev->dev_spec.sas_sata_inf.attached_phy_id =
1174 		    sasinf->attached_phy_identifier;
1175 		if ((dev_info & MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_MASK) !=
1176 		    MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_END_DEVICE)
1177 			tgtdev->is_hidden = 1;
1178 		else if (!(dev_info & (MPI3_SAS_DEVICE_INFO_STP_SATA_TARGET |
1179 		    MPI3_SAS_DEVICE_INFO_SSP_TARGET)))
1180 			tgtdev->is_hidden = 1;
1181 
1182 		if (((tgtdev->devpg0_flag &
1183 		    MPI3_DEVICE0_FLAGS_ATT_METHOD_DIR_ATTACHED)
1184 		    && (tgtdev->devpg0_flag &
1185 		    MPI3_DEVICE0_FLAGS_ATT_METHOD_VIRTUAL)) ||
1186 		    (tgtdev->parent_handle == 0xFFFF))
1187 			tgtdev->non_stl = 1;
1188 		if (tgtdev->dev_spec.sas_sata_inf.hba_port)
1189 			tgtdev->dev_spec.sas_sata_inf.hba_port->port_id =
1190 			    dev_pg0->io_unit_port;
1191 		break;
1192 	}
1193 	case MPI3_DEVICE_DEVFORM_PCIE:
1194 	{
1195 		struct mpi3_device0_pcie_format *pcieinf =
1196 		    &dev_pg0->device_specific.pcie_format;
1197 		u16 dev_info = le16_to_cpu(pcieinf->device_info);
1198 
1199 		tgtdev->dev_spec.pcie_inf.dev_info = dev_info;
1200 		tgtdev->dev_spec.pcie_inf.capb =
1201 		    le32_to_cpu(pcieinf->capabilities);
1202 		tgtdev->dev_spec.pcie_inf.mdts = MPI3MR_DEFAULT_MDTS;
1203 		/* 2^12 = 4096 */
1204 		tgtdev->dev_spec.pcie_inf.pgsz = 12;
1205 		if (dev_pg0->access_status == MPI3_DEVICE0_ASTATUS_NO_ERRORS) {
1206 			tgtdev->dev_spec.pcie_inf.mdts =
1207 			    le32_to_cpu(pcieinf->maximum_data_transfer_size);
1208 			tgtdev->dev_spec.pcie_inf.pgsz = pcieinf->page_size;
1209 			tgtdev->dev_spec.pcie_inf.reset_to =
1210 			    max_t(u8, pcieinf->controller_reset_to,
1211 			     MPI3MR_INTADMCMD_TIMEOUT);
1212 			tgtdev->dev_spec.pcie_inf.abort_to =
1213 			    max_t(u8, pcieinf->nvme_abort_to,
1214 			    MPI3MR_INTADMCMD_TIMEOUT);
1215 		}
1216 		if (tgtdev->dev_spec.pcie_inf.mdts > (1024 * 1024))
1217 			tgtdev->dev_spec.pcie_inf.mdts = (1024 * 1024);
1218 		if (((dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) !=
1219 		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) &&
1220 		    ((dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) !=
1221 		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_SCSI_DEVICE))
1222 			tgtdev->is_hidden = 1;
1223 		tgtdev->non_stl = 1;
1224 		if (!mrioc->shost)
1225 			break;
1226 		prot_mask = scsi_host_get_prot(mrioc->shost);
1227 		if (prot_mask & SHOST_DIX_TYPE0_PROTECTION) {
1228 			scsi_host_set_prot(mrioc->shost, prot_mask & 0x77);
1229 			ioc_info(mrioc,
1230 			    "%s : Disabling DIX0 prot capability\n", __func__);
1231 			ioc_info(mrioc,
1232 			    "because HBA does not support DIX0 operation on NVME drives\n");
1233 		}
1234 		break;
1235 	}
1236 	case MPI3_DEVICE_DEVFORM_VD:
1237 	{
1238 		struct mpi3_device0_vd_format *vdinf =
1239 		    &dev_pg0->device_specific.vd_format;
1240 		struct mpi3mr_throttle_group_info *tg = NULL;
1241 		u16 vdinf_io_throttle_group =
1242 		    le16_to_cpu(vdinf->io_throttle_group);
1243 
1244 		tgtdev->dev_spec.vd_inf.state = vdinf->vd_state;
1245 		if (vdinf->vd_state == MPI3_DEVICE0_VD_STATE_OFFLINE)
1246 			tgtdev->is_hidden = 1;
1247 		tgtdev->non_stl = 1;
1248 		tgtdev->dev_spec.vd_inf.tg_id = vdinf_io_throttle_group;
1249 		tgtdev->dev_spec.vd_inf.tg_high =
1250 		    le16_to_cpu(vdinf->io_throttle_group_high) * 2048;
1251 		tgtdev->dev_spec.vd_inf.tg_low =
1252 		    le16_to_cpu(vdinf->io_throttle_group_low) * 2048;
1253 		if (vdinf_io_throttle_group < mrioc->num_io_throttle_group) {
1254 			tg = mrioc->throttle_groups + vdinf_io_throttle_group;
1255 			tg->id = vdinf_io_throttle_group;
1256 			tg->high = tgtdev->dev_spec.vd_inf.tg_high;
1257 			tg->low = tgtdev->dev_spec.vd_inf.tg_low;
1258 			tg->qd_reduction =
1259 			    tgtdev->dev_spec.vd_inf.tg_qd_reduction;
1260 			if (is_added == true)
1261 				tg->fw_qd = tgtdev->q_depth;
1262 			tg->modified_qd = tgtdev->q_depth;
1263 		}
1264 		tgtdev->dev_spec.vd_inf.tg = tg;
1265 		if (scsi_tgt_priv_data)
1266 			scsi_tgt_priv_data->throttle_group = tg;
1267 		break;
1268 	}
1269 	default:
1270 		break;
1271 	}
1272 }
1273 
1274 /**
1275  * mpi3mr_devstatuschg_evt_bh - DevStatusChange evt bottomhalf
1276  * @mrioc: Adapter instance reference
1277  * @fwevt: Firmware event information.
1278  *
1279  * Process Device status Change event and based on device's new
1280  * information, either expose the device to the upper layers, or
1281  * remove the device from upper layers.
1282  *
1283  * Return: Nothing.
1284  */
1285 static void mpi3mr_devstatuschg_evt_bh(struct mpi3mr_ioc *mrioc,
1286 	struct mpi3mr_fwevt *fwevt)
1287 {
1288 	u16 dev_handle = 0;
1289 	u8 uhide = 0, delete = 0, cleanup = 0;
1290 	struct mpi3mr_tgt_dev *tgtdev = NULL;
1291 	struct mpi3_event_data_device_status_change *evtdata =
1292 	    (struct mpi3_event_data_device_status_change *)fwevt->event_data;
1293 
1294 	dev_handle = le16_to_cpu(evtdata->dev_handle);
1295 	ioc_info(mrioc,
1296 	    "%s :device status change: handle(0x%04x): reason code(0x%x)\n",
1297 	    __func__, dev_handle, evtdata->reason_code);
1298 	switch (evtdata->reason_code) {
1299 	case MPI3_EVENT_DEV_STAT_RC_HIDDEN:
1300 		delete = 1;
1301 		break;
1302 	case MPI3_EVENT_DEV_STAT_RC_NOT_HIDDEN:
1303 		uhide = 1;
1304 		break;
1305 	case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING:
1306 		delete = 1;
1307 		cleanup = 1;
1308 		break;
1309 	default:
1310 		ioc_info(mrioc, "%s :Unhandled reason code(0x%x)\n", __func__,
1311 		    evtdata->reason_code);
1312 		break;
1313 	}
1314 
1315 	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
1316 	if (!tgtdev)
1317 		goto out;
1318 	if (uhide) {
1319 		tgtdev->is_hidden = 0;
1320 		if (!tgtdev->host_exposed)
1321 			mpi3mr_report_tgtdev_to_host(mrioc, tgtdev->perst_id);
1322 	}
1323 
1324 	if (delete)
1325 		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1326 
1327 	if (cleanup) {
1328 		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, false);
1329 		mpi3mr_tgtdev_put(tgtdev);
1330 	}
1331 
1332 out:
1333 	if (tgtdev)
1334 		mpi3mr_tgtdev_put(tgtdev);
1335 }
1336 
1337 /**
1338  * mpi3mr_devinfochg_evt_bh - DeviceInfoChange evt bottomhalf
1339  * @mrioc: Adapter instance reference
1340  * @dev_pg0: New device page0
1341  *
1342  * Process Device Info Change event and based on device's new
1343  * information, either expose the device to the upper layers, or
1344  * remove the device from upper layers or update the details of
1345  * the device.
1346  *
1347  * Return: Nothing.
1348  */
1349 static void mpi3mr_devinfochg_evt_bh(struct mpi3mr_ioc *mrioc,
1350 	struct mpi3_device_page0 *dev_pg0)
1351 {
1352 	struct mpi3mr_tgt_dev *tgtdev = NULL;
1353 	u16 dev_handle = 0, perst_id = 0;
1354 
1355 	perst_id = le16_to_cpu(dev_pg0->persistent_id);
1356 	dev_handle = le16_to_cpu(dev_pg0->dev_handle);
1357 	ioc_info(mrioc,
1358 	    "%s :Device info change: handle(0x%04x): persist_id(0x%x)\n",
1359 	    __func__, dev_handle, perst_id);
1360 	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
1361 	if (!tgtdev)
1362 		goto out;
1363 	mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, false);
1364 	if (!tgtdev->is_hidden && !tgtdev->host_exposed)
1365 		mpi3mr_report_tgtdev_to_host(mrioc, perst_id);
1366 	if (tgtdev->is_hidden && tgtdev->host_exposed)
1367 		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1368 	if (!tgtdev->is_hidden && tgtdev->host_exposed && tgtdev->starget)
1369 		starget_for_each_device(tgtdev->starget, (void *)tgtdev,
1370 		    mpi3mr_update_sdev);
1371 out:
1372 	if (tgtdev)
1373 		mpi3mr_tgtdev_put(tgtdev);
1374 }
1375 
1376 /**
1377  * mpi3mr_free_enclosure_list - release enclosures
1378  * @mrioc: Adapter instance reference
1379  *
1380  * Free memory allocated during encloure add.
1381  *
1382  * Return nothing.
1383  */
1384 void mpi3mr_free_enclosure_list(struct mpi3mr_ioc *mrioc)
1385 {
1386 	struct mpi3mr_enclosure_node *enclosure_dev, *enclosure_dev_next;
1387 
1388 	list_for_each_entry_safe(enclosure_dev,
1389 	    enclosure_dev_next, &mrioc->enclosure_list, list) {
1390 		list_del(&enclosure_dev->list);
1391 		kfree(enclosure_dev);
1392 	}
1393 }
1394 
1395 /**
1396  * mpi3mr_enclosure_find_by_handle - enclosure search by handle
1397  * @mrioc: Adapter instance reference
1398  * @handle: Firmware device handle of the enclosure
1399  *
1400  * This searches for enclosure device based on handle, then returns the
1401  * enclosure object.
1402  *
1403  * Return: Enclosure object reference or NULL
1404  */
1405 struct mpi3mr_enclosure_node *mpi3mr_enclosure_find_by_handle(
1406 	struct mpi3mr_ioc *mrioc, u16 handle)
1407 {
1408 	struct mpi3mr_enclosure_node *enclosure_dev, *r = NULL;
1409 
1410 	list_for_each_entry(enclosure_dev, &mrioc->enclosure_list, list) {
1411 		if (le16_to_cpu(enclosure_dev->pg0.enclosure_handle) != handle)
1412 			continue;
1413 		r = enclosure_dev;
1414 		goto out;
1415 	}
1416 out:
1417 	return r;
1418 }
1419 
1420 /**
1421  * mpi3mr_encldev_add_chg_evt_debug - debug for enclosure event
1422  * @mrioc: Adapter instance reference
1423  * @encl_pg0: Enclosure page 0.
1424  * @is_added: Added event or not
1425  *
1426  * Return nothing.
1427  */
1428 static void mpi3mr_encldev_add_chg_evt_debug(struct mpi3mr_ioc *mrioc,
1429 	struct mpi3_enclosure_page0 *encl_pg0, u8 is_added)
1430 {
1431 	char *reason_str = NULL;
1432 
1433 	if (!(mrioc->logging_level & MPI3_DEBUG_EVENT_WORK_TASK))
1434 		return;
1435 
1436 	if (is_added)
1437 		reason_str = "enclosure added";
1438 	else
1439 		reason_str = "enclosure dev status changed";
1440 
1441 	ioc_info(mrioc,
1442 	    "%s: handle(0x%04x), enclosure logical id(0x%016llx)\n",
1443 	    reason_str, le16_to_cpu(encl_pg0->enclosure_handle),
1444 	    (unsigned long long)le64_to_cpu(encl_pg0->enclosure_logical_id));
1445 	ioc_info(mrioc,
1446 	    "number of slots(%d), port(%d), flags(0x%04x), present(%d)\n",
1447 	    le16_to_cpu(encl_pg0->num_slots), encl_pg0->io_unit_port,
1448 	    le16_to_cpu(encl_pg0->flags),
1449 	    ((le16_to_cpu(encl_pg0->flags) &
1450 	      MPI3_ENCLS0_FLAGS_ENCL_DEV_PRESENT_MASK) >> 4));
1451 }
1452 
1453 /**
1454  * mpi3mr_encldev_add_chg_evt_bh - Enclosure evt bottomhalf
1455  * @mrioc: Adapter instance reference
1456  * @fwevt: Firmware event reference
1457  *
1458  * Prints information about the Enclosure device status or
1459  * Enclosure add events if logging is enabled and add or remove
1460  * the enclosure from the controller's internal list of
1461  * enclosures.
1462  *
1463  * Return: Nothing.
1464  */
1465 static void mpi3mr_encldev_add_chg_evt_bh(struct mpi3mr_ioc *mrioc,
1466 	struct mpi3mr_fwevt *fwevt)
1467 {
1468 	struct mpi3mr_enclosure_node *enclosure_dev = NULL;
1469 	struct mpi3_enclosure_page0 *encl_pg0;
1470 	u16 encl_handle;
1471 	u8 added, present;
1472 
1473 	encl_pg0 = (struct mpi3_enclosure_page0 *) fwevt->event_data;
1474 	added = (fwevt->event_id == MPI3_EVENT_ENCL_DEVICE_ADDED) ? 1 : 0;
1475 	mpi3mr_encldev_add_chg_evt_debug(mrioc, encl_pg0, added);
1476 
1477 
1478 	encl_handle = le16_to_cpu(encl_pg0->enclosure_handle);
1479 	present = ((le16_to_cpu(encl_pg0->flags) &
1480 	      MPI3_ENCLS0_FLAGS_ENCL_DEV_PRESENT_MASK) >> 4);
1481 
1482 	if (encl_handle)
1483 		enclosure_dev = mpi3mr_enclosure_find_by_handle(mrioc,
1484 		    encl_handle);
1485 	if (!enclosure_dev && present) {
1486 		enclosure_dev =
1487 			kzalloc(sizeof(struct mpi3mr_enclosure_node),
1488 			    GFP_KERNEL);
1489 		if (!enclosure_dev)
1490 			return;
1491 		list_add_tail(&enclosure_dev->list,
1492 		    &mrioc->enclosure_list);
1493 	}
1494 	if (enclosure_dev) {
1495 		if (!present) {
1496 			list_del(&enclosure_dev->list);
1497 			kfree(enclosure_dev);
1498 		} else
1499 			memcpy(&enclosure_dev->pg0, encl_pg0,
1500 			    sizeof(enclosure_dev->pg0));
1501 
1502 	}
1503 }
1504 
1505 /**
1506  * mpi3mr_sastopochg_evt_debug - SASTopoChange details
1507  * @mrioc: Adapter instance reference
1508  * @event_data: SAS topology change list event data
1509  *
1510  * Prints information about the SAS topology change event.
1511  *
1512  * Return: Nothing.
1513  */
1514 static void
1515 mpi3mr_sastopochg_evt_debug(struct mpi3mr_ioc *mrioc,
1516 	struct mpi3_event_data_sas_topology_change_list *event_data)
1517 {
1518 	int i;
1519 	u16 handle;
1520 	u8 reason_code, phy_number;
1521 	char *status_str = NULL;
1522 	u8 link_rate, prev_link_rate;
1523 
1524 	switch (event_data->exp_status) {
1525 	case MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING:
1526 		status_str = "remove";
1527 		break;
1528 	case MPI3_EVENT_SAS_TOPO_ES_RESPONDING:
1529 		status_str =  "responding";
1530 		break;
1531 	case MPI3_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING:
1532 		status_str = "remove delay";
1533 		break;
1534 	case MPI3_EVENT_SAS_TOPO_ES_NO_EXPANDER:
1535 		status_str = "direct attached";
1536 		break;
1537 	default:
1538 		status_str = "unknown status";
1539 		break;
1540 	}
1541 	ioc_info(mrioc, "%s :sas topology change: (%s)\n",
1542 	    __func__, status_str);
1543 	ioc_info(mrioc,
1544 	    "%s :\texpander_handle(0x%04x), port(%d), enclosure_handle(0x%04x) start_phy(%02d), num_entries(%d)\n",
1545 	    __func__, le16_to_cpu(event_data->expander_dev_handle),
1546 	    event_data->io_unit_port,
1547 	    le16_to_cpu(event_data->enclosure_handle),
1548 	    event_data->start_phy_num, event_data->num_entries);
1549 	for (i = 0; i < event_data->num_entries; i++) {
1550 		handle = le16_to_cpu(event_data->phy_entry[i].attached_dev_handle);
1551 		if (!handle)
1552 			continue;
1553 		phy_number = event_data->start_phy_num + i;
1554 		reason_code = event_data->phy_entry[i].status &
1555 		    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
1556 		switch (reason_code) {
1557 		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
1558 			status_str = "target remove";
1559 			break;
1560 		case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING:
1561 			status_str = "delay target remove";
1562 			break;
1563 		case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
1564 			status_str = "link status change";
1565 			break;
1566 		case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE:
1567 			status_str = "link status no change";
1568 			break;
1569 		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
1570 			status_str = "target responding";
1571 			break;
1572 		default:
1573 			status_str = "unknown";
1574 			break;
1575 		}
1576 		link_rate = event_data->phy_entry[i].link_rate >> 4;
1577 		prev_link_rate = event_data->phy_entry[i].link_rate & 0xF;
1578 		ioc_info(mrioc,
1579 		    "%s :\tphy(%02d), attached_handle(0x%04x): %s: link rate: new(0x%02x), old(0x%02x)\n",
1580 		    __func__, phy_number, handle, status_str, link_rate,
1581 		    prev_link_rate);
1582 	}
1583 }
1584 
1585 /**
1586  * mpi3mr_sastopochg_evt_bh - SASTopologyChange evt bottomhalf
1587  * @mrioc: Adapter instance reference
1588  * @fwevt: Firmware event reference
1589  *
1590  * Prints information about the SAS topology change event and
1591  * for "not responding" event code, removes the device from the
1592  * upper layers.
1593  *
1594  * Return: Nothing.
1595  */
1596 static void mpi3mr_sastopochg_evt_bh(struct mpi3mr_ioc *mrioc,
1597 	struct mpi3mr_fwevt *fwevt)
1598 {
1599 	struct mpi3_event_data_sas_topology_change_list *event_data =
1600 	    (struct mpi3_event_data_sas_topology_change_list *)fwevt->event_data;
1601 	int i;
1602 	u16 handle;
1603 	u8 reason_code;
1604 	u64 exp_sas_address = 0, parent_sas_address = 0;
1605 	struct mpi3mr_hba_port *hba_port = NULL;
1606 	struct mpi3mr_tgt_dev *tgtdev = NULL;
1607 	struct mpi3mr_sas_node *sas_expander = NULL;
1608 	unsigned long flags;
1609 	u8 link_rate, prev_link_rate, parent_phy_number;
1610 
1611 	mpi3mr_sastopochg_evt_debug(mrioc, event_data);
1612 	if (mrioc->sas_transport_enabled) {
1613 		hba_port = mpi3mr_get_hba_port_by_id(mrioc,
1614 		    event_data->io_unit_port);
1615 		if (le16_to_cpu(event_data->expander_dev_handle)) {
1616 			spin_lock_irqsave(&mrioc->sas_node_lock, flags);
1617 			sas_expander = __mpi3mr_expander_find_by_handle(mrioc,
1618 			    le16_to_cpu(event_data->expander_dev_handle));
1619 			if (sas_expander) {
1620 				exp_sas_address = sas_expander->sas_address;
1621 				hba_port = sas_expander->hba_port;
1622 			}
1623 			spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
1624 			parent_sas_address = exp_sas_address;
1625 		} else
1626 			parent_sas_address = mrioc->sas_hba.sas_address;
1627 	}
1628 
1629 	for (i = 0; i < event_data->num_entries; i++) {
1630 		if (fwevt->discard)
1631 			return;
1632 		handle = le16_to_cpu(event_data->phy_entry[i].attached_dev_handle);
1633 		if (!handle)
1634 			continue;
1635 		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
1636 		if (!tgtdev)
1637 			continue;
1638 
1639 		reason_code = event_data->phy_entry[i].status &
1640 		    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
1641 
1642 		switch (reason_code) {
1643 		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
1644 			if (tgtdev->host_exposed)
1645 				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1646 			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, false);
1647 			mpi3mr_tgtdev_put(tgtdev);
1648 			break;
1649 		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
1650 		case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
1651 		case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE:
1652 		{
1653 			if (!mrioc->sas_transport_enabled || tgtdev->non_stl
1654 			    || tgtdev->is_hidden)
1655 				break;
1656 			link_rate = event_data->phy_entry[i].link_rate >> 4;
1657 			prev_link_rate = event_data->phy_entry[i].link_rate & 0xF;
1658 			if (link_rate == prev_link_rate)
1659 				break;
1660 			if (!parent_sas_address)
1661 				break;
1662 			parent_phy_number = event_data->start_phy_num + i;
1663 			mpi3mr_update_links(mrioc, parent_sas_address, handle,
1664 			    parent_phy_number, link_rate, hba_port);
1665 			break;
1666 		}
1667 		default:
1668 			break;
1669 		}
1670 		if (tgtdev)
1671 			mpi3mr_tgtdev_put(tgtdev);
1672 	}
1673 
1674 	if (mrioc->sas_transport_enabled && (event_data->exp_status ==
1675 	    MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING)) {
1676 		if (sas_expander)
1677 			mpi3mr_expander_remove(mrioc, exp_sas_address,
1678 			    hba_port);
1679 	}
1680 }
1681 
1682 /**
1683  * mpi3mr_pcietopochg_evt_debug - PCIeTopoChange details
1684  * @mrioc: Adapter instance reference
1685  * @event_data: PCIe topology change list event data
1686  *
1687  * Prints information about the PCIe topology change event.
1688  *
1689  * Return: Nothing.
1690  */
1691 static void
1692 mpi3mr_pcietopochg_evt_debug(struct mpi3mr_ioc *mrioc,
1693 	struct mpi3_event_data_pcie_topology_change_list *event_data)
1694 {
1695 	int i;
1696 	u16 handle;
1697 	u16 reason_code;
1698 	u8 port_number;
1699 	char *status_str = NULL;
1700 	u8 link_rate, prev_link_rate;
1701 
1702 	switch (event_data->switch_status) {
1703 	case MPI3_EVENT_PCIE_TOPO_SS_NOT_RESPONDING:
1704 		status_str = "remove";
1705 		break;
1706 	case MPI3_EVENT_PCIE_TOPO_SS_RESPONDING:
1707 		status_str =  "responding";
1708 		break;
1709 	case MPI3_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING:
1710 		status_str = "remove delay";
1711 		break;
1712 	case MPI3_EVENT_PCIE_TOPO_SS_NO_PCIE_SWITCH:
1713 		status_str = "direct attached";
1714 		break;
1715 	default:
1716 		status_str = "unknown status";
1717 		break;
1718 	}
1719 	ioc_info(mrioc, "%s :pcie topology change: (%s)\n",
1720 	    __func__, status_str);
1721 	ioc_info(mrioc,
1722 	    "%s :\tswitch_handle(0x%04x), enclosure_handle(0x%04x) start_port(%02d), num_entries(%d)\n",
1723 	    __func__, le16_to_cpu(event_data->switch_dev_handle),
1724 	    le16_to_cpu(event_data->enclosure_handle),
1725 	    event_data->start_port_num, event_data->num_entries);
1726 	for (i = 0; i < event_data->num_entries; i++) {
1727 		handle =
1728 		    le16_to_cpu(event_data->port_entry[i].attached_dev_handle);
1729 		if (!handle)
1730 			continue;
1731 		port_number = event_data->start_port_num + i;
1732 		reason_code = event_data->port_entry[i].port_status;
1733 		switch (reason_code) {
1734 		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
1735 			status_str = "target remove";
1736 			break;
1737 		case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
1738 			status_str = "delay target remove";
1739 			break;
1740 		case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
1741 			status_str = "link status change";
1742 			break;
1743 		case MPI3_EVENT_PCIE_TOPO_PS_NO_CHANGE:
1744 			status_str = "link status no change";
1745 			break;
1746 		case MPI3_EVENT_PCIE_TOPO_PS_RESPONDING:
1747 			status_str = "target responding";
1748 			break;
1749 		default:
1750 			status_str = "unknown";
1751 			break;
1752 		}
1753 		link_rate = event_data->port_entry[i].current_port_info &
1754 		    MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
1755 		prev_link_rate = event_data->port_entry[i].previous_port_info &
1756 		    MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
1757 		ioc_info(mrioc,
1758 		    "%s :\tport(%02d), attached_handle(0x%04x): %s: link rate: new(0x%02x), old(0x%02x)\n",
1759 		    __func__, port_number, handle, status_str, link_rate,
1760 		    prev_link_rate);
1761 	}
1762 }
1763 
1764 /**
1765  * mpi3mr_pcietopochg_evt_bh - PCIeTopologyChange evt bottomhalf
1766  * @mrioc: Adapter instance reference
1767  * @fwevt: Firmware event reference
1768  *
1769  * Prints information about the PCIe topology change event and
1770  * for "not responding" event code, removes the device from the
1771  * upper layers.
1772  *
1773  * Return: Nothing.
1774  */
1775 static void mpi3mr_pcietopochg_evt_bh(struct mpi3mr_ioc *mrioc,
1776 	struct mpi3mr_fwevt *fwevt)
1777 {
1778 	struct mpi3_event_data_pcie_topology_change_list *event_data =
1779 	    (struct mpi3_event_data_pcie_topology_change_list *)fwevt->event_data;
1780 	int i;
1781 	u16 handle;
1782 	u8 reason_code;
1783 	struct mpi3mr_tgt_dev *tgtdev = NULL;
1784 
1785 	mpi3mr_pcietopochg_evt_debug(mrioc, event_data);
1786 
1787 	for (i = 0; i < event_data->num_entries; i++) {
1788 		if (fwevt->discard)
1789 			return;
1790 		handle =
1791 		    le16_to_cpu(event_data->port_entry[i].attached_dev_handle);
1792 		if (!handle)
1793 			continue;
1794 		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
1795 		if (!tgtdev)
1796 			continue;
1797 
1798 		reason_code = event_data->port_entry[i].port_status;
1799 
1800 		switch (reason_code) {
1801 		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
1802 			if (tgtdev->host_exposed)
1803 				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1804 			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, false);
1805 			mpi3mr_tgtdev_put(tgtdev);
1806 			break;
1807 		default:
1808 			break;
1809 		}
1810 		if (tgtdev)
1811 			mpi3mr_tgtdev_put(tgtdev);
1812 	}
1813 }
1814 
1815 /**
1816  * mpi3mr_logdata_evt_bh -  Log data event bottomhalf
1817  * @mrioc: Adapter instance reference
1818  * @fwevt: Firmware event reference
1819  *
1820  * Extracts the event data and calls application interfacing
1821  * function to process the event further.
1822  *
1823  * Return: Nothing.
1824  */
1825 static void mpi3mr_logdata_evt_bh(struct mpi3mr_ioc *mrioc,
1826 	struct mpi3mr_fwevt *fwevt)
1827 {
1828 	mpi3mr_app_save_logdata(mrioc, fwevt->event_data,
1829 	    fwevt->event_data_size);
1830 }
1831 
1832 /**
1833  * mpi3mr_update_sdev_qd - Update SCSI device queue depath
1834  * @sdev: SCSI device reference
1835  * @data: Queue depth reference
1836  *
1837  * This is an iterator function called for each SCSI device in a
1838  * target to update the QD of each SCSI device.
1839  *
1840  * Return: Nothing.
1841  */
1842 static void mpi3mr_update_sdev_qd(struct scsi_device *sdev, void *data)
1843 {
1844 	u16 *q_depth = (u16 *)data;
1845 
1846 	scsi_change_queue_depth(sdev, (int)*q_depth);
1847 	sdev->max_queue_depth = sdev->queue_depth;
1848 }
1849 
1850 /**
1851  * mpi3mr_set_qd_for_all_vd_in_tg -set QD for TG VDs
1852  * @mrioc: Adapter instance reference
1853  * @tg: Throttle group information pointer
1854  *
1855  * Accessor to reduce QD for each device associated with the
1856  * given throttle group.
1857  *
1858  * Return: None.
1859  */
1860 static void mpi3mr_set_qd_for_all_vd_in_tg(struct mpi3mr_ioc *mrioc,
1861 	struct mpi3mr_throttle_group_info *tg)
1862 {
1863 	unsigned long flags;
1864 	struct mpi3mr_tgt_dev *tgtdev;
1865 	struct mpi3mr_stgt_priv_data *tgt_priv;
1866 
1867 
1868 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
1869 	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
1870 		if (tgtdev->starget && tgtdev->starget->hostdata) {
1871 			tgt_priv = tgtdev->starget->hostdata;
1872 			if (tgt_priv->throttle_group == tg) {
1873 				dprint_event_bh(mrioc,
1874 				    "updating qd due to throttling for persist_id(%d) original_qd(%d), reduced_qd (%d)\n",
1875 				    tgt_priv->perst_id, tgtdev->q_depth,
1876 				    tg->modified_qd);
1877 				starget_for_each_device(tgtdev->starget,
1878 				    (void *)&tg->modified_qd,
1879 				    mpi3mr_update_sdev_qd);
1880 			}
1881 		}
1882 	}
1883 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
1884 }
1885 
1886 /**
1887  * mpi3mr_fwevt_bh - Firmware event bottomhalf handler
1888  * @mrioc: Adapter instance reference
1889  * @fwevt: Firmware event reference
1890  *
1891  * Identifies the firmware event and calls corresponding bottomg
1892  * half handler and sends event acknowledgment if required.
1893  *
1894  * Return: Nothing.
1895  */
1896 static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc,
1897 	struct mpi3mr_fwevt *fwevt)
1898 {
1899 	struct mpi3_device_page0 *dev_pg0 = NULL;
1900 	u16 perst_id, handle, dev_info;
1901 	struct mpi3_device0_sas_sata_format *sasinf = NULL;
1902 
1903 	mpi3mr_fwevt_del_from_list(mrioc, fwevt);
1904 	mrioc->current_event = fwevt;
1905 
1906 	if (mrioc->stop_drv_processing)
1907 		goto out;
1908 
1909 	if (mrioc->unrecoverable) {
1910 		dprint_event_bh(mrioc,
1911 		    "ignoring event(0x%02x) in bottom half handler due to unrecoverable controller\n",
1912 		    fwevt->event_id);
1913 		goto out;
1914 	}
1915 
1916 	if (!fwevt->process_evt)
1917 		goto evt_ack;
1918 
1919 	switch (fwevt->event_id) {
1920 	case MPI3_EVENT_DEVICE_ADDED:
1921 	{
1922 		dev_pg0 = (struct mpi3_device_page0 *)fwevt->event_data;
1923 		perst_id = le16_to_cpu(dev_pg0->persistent_id);
1924 		handle = le16_to_cpu(dev_pg0->dev_handle);
1925 		if (perst_id != MPI3_DEVICE0_PERSISTENTID_INVALID)
1926 			mpi3mr_report_tgtdev_to_host(mrioc, perst_id);
1927 		else if (mrioc->sas_transport_enabled &&
1928 		    (dev_pg0->device_form == MPI3_DEVICE_DEVFORM_SAS_SATA)) {
1929 			sasinf = &dev_pg0->device_specific.sas_sata_format;
1930 			dev_info = le16_to_cpu(sasinf->device_info);
1931 			if (!mrioc->sas_hba.num_phys)
1932 				mpi3mr_sas_host_add(mrioc);
1933 			else
1934 				mpi3mr_sas_host_refresh(mrioc);
1935 
1936 			if (mpi3mr_is_expander_device(dev_info))
1937 				mpi3mr_expander_add(mrioc, handle);
1938 		}
1939 		break;
1940 	}
1941 	case MPI3_EVENT_DEVICE_INFO_CHANGED:
1942 	{
1943 		dev_pg0 = (struct mpi3_device_page0 *)fwevt->event_data;
1944 		perst_id = le16_to_cpu(dev_pg0->persistent_id);
1945 		if (perst_id != MPI3_DEVICE0_PERSISTENTID_INVALID)
1946 			mpi3mr_devinfochg_evt_bh(mrioc, dev_pg0);
1947 		break;
1948 	}
1949 	case MPI3_EVENT_DEVICE_STATUS_CHANGE:
1950 	{
1951 		mpi3mr_devstatuschg_evt_bh(mrioc, fwevt);
1952 		break;
1953 	}
1954 	case MPI3_EVENT_ENCL_DEVICE_ADDED:
1955 	case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
1956 	{
1957 		mpi3mr_encldev_add_chg_evt_bh(mrioc, fwevt);
1958 		break;
1959 	}
1960 
1961 	case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
1962 	{
1963 		mpi3mr_sastopochg_evt_bh(mrioc, fwevt);
1964 		break;
1965 	}
1966 	case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
1967 	{
1968 		mpi3mr_pcietopochg_evt_bh(mrioc, fwevt);
1969 		break;
1970 	}
1971 	case MPI3_EVENT_LOG_DATA:
1972 	{
1973 		mpi3mr_logdata_evt_bh(mrioc, fwevt);
1974 		break;
1975 	}
1976 	case MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION:
1977 	{
1978 		struct mpi3mr_throttle_group_info *tg;
1979 
1980 		tg = *(struct mpi3mr_throttle_group_info **)fwevt->event_data;
1981 		dprint_event_bh(mrioc,
1982 		    "qd reduction event processed for tg_id(%d) reduction_needed(%d)\n",
1983 		    tg->id, tg->need_qd_reduction);
1984 		if (tg->need_qd_reduction) {
1985 			mpi3mr_set_qd_for_all_vd_in_tg(mrioc, tg);
1986 			tg->need_qd_reduction = 0;
1987 		}
1988 		break;
1989 	}
1990 	case MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH:
1991 	{
1992 		while (mrioc->device_refresh_on)
1993 			msleep(500);
1994 
1995 		dprint_event_bh(mrioc,
1996 		    "scan for non responding and newly added devices after soft reset started\n");
1997 		if (mrioc->sas_transport_enabled) {
1998 			mpi3mr_refresh_sas_ports(mrioc);
1999 			mpi3mr_refresh_expanders(mrioc);
2000 		}
2001 		mpi3mr_rfresh_tgtdevs(mrioc);
2002 		ioc_info(mrioc,
2003 		    "scan for non responding and newly added devices after soft reset completed\n");
2004 		break;
2005 	}
2006 	default:
2007 		break;
2008 	}
2009 
2010 evt_ack:
2011 	if (fwevt->send_ack)
2012 		mpi3mr_process_event_ack(mrioc, fwevt->event_id,
2013 		    fwevt->evt_ctx);
2014 out:
2015 	/* Put fwevt reference count to neutralize kref_init increment */
2016 	mpi3mr_fwevt_put(fwevt);
2017 	mrioc->current_event = NULL;
2018 }
2019 
2020 /**
2021  * mpi3mr_fwevt_worker - Firmware event worker
2022  * @work: Work struct containing firmware event
2023  *
2024  * Extracts the firmware event and calls mpi3mr_fwevt_bh.
2025  *
2026  * Return: Nothing.
2027  */
2028 static void mpi3mr_fwevt_worker(struct work_struct *work)
2029 {
2030 	struct mpi3mr_fwevt *fwevt = container_of(work, struct mpi3mr_fwevt,
2031 	    work);
2032 	mpi3mr_fwevt_bh(fwevt->mrioc, fwevt);
2033 	/*
2034 	 * Put fwevt reference count after
2035 	 * dequeuing it from worker queue
2036 	 */
2037 	mpi3mr_fwevt_put(fwevt);
2038 }
2039 
2040 /**
2041  * mpi3mr_create_tgtdev - Create and add a target device
2042  * @mrioc: Adapter instance reference
2043  * @dev_pg0: Device Page 0 data
2044  *
2045  * If the device specified by the device page 0 data is not
2046  * present in the driver's internal list, allocate the memory
2047  * for the device, populate the data and add to the list, else
2048  * update the device data.  The key is persistent ID.
2049  *
2050  * Return: 0 on success, -ENOMEM on memory allocation failure
2051  */
2052 static int mpi3mr_create_tgtdev(struct mpi3mr_ioc *mrioc,
2053 	struct mpi3_device_page0 *dev_pg0)
2054 {
2055 	int retval = 0;
2056 	struct mpi3mr_tgt_dev *tgtdev = NULL;
2057 	u16 perst_id = 0;
2058 	unsigned long flags;
2059 
2060 	perst_id = le16_to_cpu(dev_pg0->persistent_id);
2061 	if (perst_id == MPI3_DEVICE0_PERSISTENTID_INVALID)
2062 		return retval;
2063 
2064 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
2065 	tgtdev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id);
2066 	if (tgtdev)
2067 		tgtdev->state = MPI3MR_DEV_CREATED;
2068 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
2069 
2070 	if (tgtdev) {
2071 		mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, true);
2072 		mpi3mr_tgtdev_put(tgtdev);
2073 	} else {
2074 		tgtdev = mpi3mr_alloc_tgtdev();
2075 		if (!tgtdev)
2076 			return -ENOMEM;
2077 		mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, true);
2078 		mpi3mr_tgtdev_add_to_list(mrioc, tgtdev);
2079 	}
2080 
2081 	return retval;
2082 }
2083 
2084 /**
2085  * mpi3mr_flush_delayed_cmd_lists - Flush pending commands
2086  * @mrioc: Adapter instance reference
2087  *
2088  * Flush pending commands in the delayed lists due to a
2089  * controller reset or driver removal as a cleanup.
2090  *
2091  * Return: Nothing
2092  */
2093 void mpi3mr_flush_delayed_cmd_lists(struct mpi3mr_ioc *mrioc)
2094 {
2095 	struct delayed_dev_rmhs_node *_rmhs_node;
2096 	struct delayed_evt_ack_node *_evtack_node;
2097 
2098 	dprint_reset(mrioc, "flushing delayed dev_remove_hs commands\n");
2099 	while (!list_empty(&mrioc->delayed_rmhs_list)) {
2100 		_rmhs_node = list_entry(mrioc->delayed_rmhs_list.next,
2101 		    struct delayed_dev_rmhs_node, list);
2102 		list_del(&_rmhs_node->list);
2103 		kfree(_rmhs_node);
2104 	}
2105 	dprint_reset(mrioc, "flushing delayed event ack commands\n");
2106 	while (!list_empty(&mrioc->delayed_evtack_cmds_list)) {
2107 		_evtack_node = list_entry(mrioc->delayed_evtack_cmds_list.next,
2108 		    struct delayed_evt_ack_node, list);
2109 		list_del(&_evtack_node->list);
2110 		kfree(_evtack_node);
2111 	}
2112 }
2113 
2114 /**
2115  * mpi3mr_dev_rmhs_complete_iou - Device removal IOUC completion
2116  * @mrioc: Adapter instance reference
2117  * @drv_cmd: Internal command tracker
2118  *
2119  * Issues a target reset TM to the firmware from the device
2120  * removal TM pend list or retry the removal handshake sequence
2121  * based on the IOU control request IOC status.
2122  *
2123  * Return: Nothing
2124  */
2125 static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_ioc *mrioc,
2126 	struct mpi3mr_drv_cmd *drv_cmd)
2127 {
2128 	u16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
2129 	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
2130 
2131 	if (drv_cmd->state & MPI3MR_CMD_RESET)
2132 		goto clear_drv_cmd;
2133 
2134 	ioc_info(mrioc,
2135 	    "%s :dev_rmhs_iouctrl_complete:handle(0x%04x), ioc_status(0x%04x), loginfo(0x%08x)\n",
2136 	    __func__, drv_cmd->dev_handle, drv_cmd->ioc_status,
2137 	    drv_cmd->ioc_loginfo);
2138 	if (drv_cmd->ioc_status != MPI3_IOCSTATUS_SUCCESS) {
2139 		if (drv_cmd->retry_count < MPI3MR_DEV_RMHS_RETRY_COUNT) {
2140 			drv_cmd->retry_count++;
2141 			ioc_info(mrioc,
2142 			    "%s :dev_rmhs_iouctrl_complete: handle(0x%04x)retrying handshake retry=%d\n",
2143 			    __func__, drv_cmd->dev_handle,
2144 			    drv_cmd->retry_count);
2145 			mpi3mr_dev_rmhs_send_tm(mrioc, drv_cmd->dev_handle,
2146 			    drv_cmd, drv_cmd->iou_rc);
2147 			return;
2148 		}
2149 		ioc_err(mrioc,
2150 		    "%s :dev removal handshake failed after all retries: handle(0x%04x)\n",
2151 		    __func__, drv_cmd->dev_handle);
2152 	} else {
2153 		ioc_info(mrioc,
2154 		    "%s :dev removal handshake completed successfully: handle(0x%04x)\n",
2155 		    __func__, drv_cmd->dev_handle);
2156 		clear_bit(drv_cmd->dev_handle, mrioc->removepend_bitmap);
2157 	}
2158 
2159 	if (!list_empty(&mrioc->delayed_rmhs_list)) {
2160 		delayed_dev_rmhs = list_entry(mrioc->delayed_rmhs_list.next,
2161 		    struct delayed_dev_rmhs_node, list);
2162 		drv_cmd->dev_handle = delayed_dev_rmhs->handle;
2163 		drv_cmd->retry_count = 0;
2164 		drv_cmd->iou_rc = delayed_dev_rmhs->iou_rc;
2165 		ioc_info(mrioc,
2166 		    "%s :dev_rmhs_iouctrl_complete: processing delayed TM: handle(0x%04x)\n",
2167 		    __func__, drv_cmd->dev_handle);
2168 		mpi3mr_dev_rmhs_send_tm(mrioc, drv_cmd->dev_handle, drv_cmd,
2169 		    drv_cmd->iou_rc);
2170 		list_del(&delayed_dev_rmhs->list);
2171 		kfree(delayed_dev_rmhs);
2172 		return;
2173 	}
2174 
2175 clear_drv_cmd:
2176 	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2177 	drv_cmd->callback = NULL;
2178 	drv_cmd->retry_count = 0;
2179 	drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
2180 	clear_bit(cmd_idx, mrioc->devrem_bitmap);
2181 }
2182 
2183 /**
2184  * mpi3mr_dev_rmhs_complete_tm - Device removal TM completion
2185  * @mrioc: Adapter instance reference
2186  * @drv_cmd: Internal command tracker
2187  *
2188  * Issues a target reset TM to the firmware from the device
2189  * removal TM pend list or issue IO unit control request as
2190  * part of device removal or hidden acknowledgment handshake.
2191  *
2192  * Return: Nothing
2193  */
2194 static void mpi3mr_dev_rmhs_complete_tm(struct mpi3mr_ioc *mrioc,
2195 	struct mpi3mr_drv_cmd *drv_cmd)
2196 {
2197 	struct mpi3_iounit_control_request iou_ctrl;
2198 	u16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
2199 	struct mpi3_scsi_task_mgmt_reply *tm_reply = NULL;
2200 	int retval;
2201 
2202 	if (drv_cmd->state & MPI3MR_CMD_RESET)
2203 		goto clear_drv_cmd;
2204 
2205 	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
2206 		tm_reply = (struct mpi3_scsi_task_mgmt_reply *)drv_cmd->reply;
2207 
2208 	if (tm_reply)
2209 		pr_info(IOCNAME
2210 		    "dev_rmhs_tr_complete:handle(0x%04x), ioc_status(0x%04x), loginfo(0x%08x), term_count(%d)\n",
2211 		    mrioc->name, drv_cmd->dev_handle, drv_cmd->ioc_status,
2212 		    drv_cmd->ioc_loginfo,
2213 		    le32_to_cpu(tm_reply->termination_count));
2214 
2215 	pr_info(IOCNAME "Issuing IOU CTL: handle(0x%04x) dev_rmhs idx(%d)\n",
2216 	    mrioc->name, drv_cmd->dev_handle, cmd_idx);
2217 
2218 	memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2219 
2220 	drv_cmd->state = MPI3MR_CMD_PENDING;
2221 	drv_cmd->is_waiting = 0;
2222 	drv_cmd->callback = mpi3mr_dev_rmhs_complete_iou;
2223 	iou_ctrl.operation = drv_cmd->iou_rc;
2224 	iou_ctrl.param16[0] = cpu_to_le16(drv_cmd->dev_handle);
2225 	iou_ctrl.host_tag = cpu_to_le16(drv_cmd->host_tag);
2226 	iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2227 
2228 	retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl, sizeof(iou_ctrl),
2229 	    1);
2230 	if (retval) {
2231 		pr_err(IOCNAME "Issue DevRmHsTMIOUCTL: Admin post failed\n",
2232 		    mrioc->name);
2233 		goto clear_drv_cmd;
2234 	}
2235 
2236 	return;
2237 clear_drv_cmd:
2238 	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2239 	drv_cmd->callback = NULL;
2240 	drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
2241 	drv_cmd->retry_count = 0;
2242 	clear_bit(cmd_idx, mrioc->devrem_bitmap);
2243 }
2244 
2245 /**
2246  * mpi3mr_dev_rmhs_send_tm - Issue TM for device removal
2247  * @mrioc: Adapter instance reference
2248  * @handle: Device handle
2249  * @cmdparam: Internal command tracker
2250  * @iou_rc: IO unit reason code
2251  *
2252  * Issues a target reset TM to the firmware or add it to a pend
2253  * list as part of device removal or hidden acknowledgment
2254  * handshake.
2255  *
2256  * Return: Nothing
2257  */
2258 static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_ioc *mrioc, u16 handle,
2259 	struct mpi3mr_drv_cmd *cmdparam, u8 iou_rc)
2260 {
2261 	struct mpi3_scsi_task_mgmt_request tm_req;
2262 	int retval = 0;
2263 	u16 cmd_idx = MPI3MR_NUM_DEVRMCMD;
2264 	u8 retrycount = 5;
2265 	struct mpi3mr_drv_cmd *drv_cmd = cmdparam;
2266 	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
2267 	struct mpi3mr_tgt_dev *tgtdev = NULL;
2268 	unsigned long flags;
2269 
2270 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
2271 	tgtdev = __mpi3mr_get_tgtdev_by_handle(mrioc, handle);
2272 	if (tgtdev && (iou_rc == MPI3_CTRL_OP_REMOVE_DEVICE))
2273 		tgtdev->state = MPI3MR_DEV_REMOVE_HS_STARTED;
2274 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
2275 
2276 	if (drv_cmd)
2277 		goto issue_cmd;
2278 	do {
2279 		cmd_idx = find_first_zero_bit(mrioc->devrem_bitmap,
2280 		    MPI3MR_NUM_DEVRMCMD);
2281 		if (cmd_idx < MPI3MR_NUM_DEVRMCMD) {
2282 			if (!test_and_set_bit(cmd_idx, mrioc->devrem_bitmap))
2283 				break;
2284 			cmd_idx = MPI3MR_NUM_DEVRMCMD;
2285 		}
2286 	} while (retrycount--);
2287 
2288 	if (cmd_idx >= MPI3MR_NUM_DEVRMCMD) {
2289 		delayed_dev_rmhs = kzalloc(sizeof(*delayed_dev_rmhs),
2290 		    GFP_ATOMIC);
2291 		if (!delayed_dev_rmhs)
2292 			return;
2293 		INIT_LIST_HEAD(&delayed_dev_rmhs->list);
2294 		delayed_dev_rmhs->handle = handle;
2295 		delayed_dev_rmhs->iou_rc = iou_rc;
2296 		list_add_tail(&delayed_dev_rmhs->list,
2297 		    &mrioc->delayed_rmhs_list);
2298 		ioc_info(mrioc, "%s :DevRmHs: tr:handle(0x%04x) is postponed\n",
2299 		    __func__, handle);
2300 		return;
2301 	}
2302 	drv_cmd = &mrioc->dev_rmhs_cmds[cmd_idx];
2303 
2304 issue_cmd:
2305 	cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
2306 	ioc_info(mrioc,
2307 	    "%s :Issuing TR TM: for devhandle 0x%04x with dev_rmhs %d\n",
2308 	    __func__, handle, cmd_idx);
2309 
2310 	memset(&tm_req, 0, sizeof(tm_req));
2311 	if (drv_cmd->state & MPI3MR_CMD_PENDING) {
2312 		ioc_err(mrioc, "%s :Issue TM: Command is in use\n", __func__);
2313 		goto out;
2314 	}
2315 	drv_cmd->state = MPI3MR_CMD_PENDING;
2316 	drv_cmd->is_waiting = 0;
2317 	drv_cmd->callback = mpi3mr_dev_rmhs_complete_tm;
2318 	drv_cmd->dev_handle = handle;
2319 	drv_cmd->iou_rc = iou_rc;
2320 	tm_req.dev_handle = cpu_to_le16(handle);
2321 	tm_req.task_type = MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
2322 	tm_req.host_tag = cpu_to_le16(drv_cmd->host_tag);
2323 	tm_req.task_host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INVALID);
2324 	tm_req.function = MPI3_FUNCTION_SCSI_TASK_MGMT;
2325 
2326 	set_bit(handle, mrioc->removepend_bitmap);
2327 	retval = mpi3mr_admin_request_post(mrioc, &tm_req, sizeof(tm_req), 1);
2328 	if (retval) {
2329 		ioc_err(mrioc, "%s :Issue DevRmHsTM: Admin Post failed\n",
2330 		    __func__);
2331 		goto out_failed;
2332 	}
2333 out:
2334 	return;
2335 out_failed:
2336 	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2337 	drv_cmd->callback = NULL;
2338 	drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
2339 	drv_cmd->retry_count = 0;
2340 	clear_bit(cmd_idx, mrioc->devrem_bitmap);
2341 }
2342 
2343 /**
2344  * mpi3mr_complete_evt_ack - event ack request completion
2345  * @mrioc: Adapter instance reference
2346  * @drv_cmd: Internal command tracker
2347  *
2348  * This is the completion handler for non blocking event
2349  * acknowledgment sent to the firmware and this will issue any
2350  * pending event acknowledgment request.
2351  *
2352  * Return: Nothing
2353  */
2354 static void mpi3mr_complete_evt_ack(struct mpi3mr_ioc *mrioc,
2355 	struct mpi3mr_drv_cmd *drv_cmd)
2356 {
2357 	u16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
2358 	struct delayed_evt_ack_node *delayed_evtack = NULL;
2359 
2360 	if (drv_cmd->state & MPI3MR_CMD_RESET)
2361 		goto clear_drv_cmd;
2362 
2363 	if (drv_cmd->ioc_status != MPI3_IOCSTATUS_SUCCESS) {
2364 		dprint_event_th(mrioc,
2365 		    "immediate event ack failed with ioc_status(0x%04x) log_info(0x%08x)\n",
2366 		    (drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2367 		    drv_cmd->ioc_loginfo);
2368 	}
2369 
2370 	if (!list_empty(&mrioc->delayed_evtack_cmds_list)) {
2371 		delayed_evtack =
2372 			list_entry(mrioc->delayed_evtack_cmds_list.next,
2373 			    struct delayed_evt_ack_node, list);
2374 		mpi3mr_send_event_ack(mrioc, delayed_evtack->event, drv_cmd,
2375 		    delayed_evtack->event_ctx);
2376 		list_del(&delayed_evtack->list);
2377 		kfree(delayed_evtack);
2378 		return;
2379 	}
2380 clear_drv_cmd:
2381 	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2382 	drv_cmd->callback = NULL;
2383 	clear_bit(cmd_idx, mrioc->evtack_cmds_bitmap);
2384 }
2385 
2386 /**
2387  * mpi3mr_send_event_ack - Issue event acknwoledgment request
2388  * @mrioc: Adapter instance reference
2389  * @event: MPI3 event id
2390  * @cmdparam: Internal command tracker
2391  * @event_ctx: event context
2392  *
2393  * Issues event acknowledgment request to the firmware if there
2394  * is a free command to send the event ack else it to a pend
2395  * list so that it will be processed on a completion of a prior
2396  * event acknowledgment .
2397  *
2398  * Return: Nothing
2399  */
2400 static void mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
2401 	struct mpi3mr_drv_cmd *cmdparam, u32 event_ctx)
2402 {
2403 	struct mpi3_event_ack_request evtack_req;
2404 	int retval = 0;
2405 	u8 retrycount = 5;
2406 	u16 cmd_idx = MPI3MR_NUM_EVTACKCMD;
2407 	struct mpi3mr_drv_cmd *drv_cmd = cmdparam;
2408 	struct delayed_evt_ack_node *delayed_evtack = NULL;
2409 
2410 	if (drv_cmd) {
2411 		dprint_event_th(mrioc,
2412 		    "sending delayed event ack in the top half for event(0x%02x), event_ctx(0x%08x)\n",
2413 		    event, event_ctx);
2414 		goto issue_cmd;
2415 	}
2416 	dprint_event_th(mrioc,
2417 	    "sending event ack in the top half for event(0x%02x), event_ctx(0x%08x)\n",
2418 	    event, event_ctx);
2419 	do {
2420 		cmd_idx = find_first_zero_bit(mrioc->evtack_cmds_bitmap,
2421 		    MPI3MR_NUM_EVTACKCMD);
2422 		if (cmd_idx < MPI3MR_NUM_EVTACKCMD) {
2423 			if (!test_and_set_bit(cmd_idx,
2424 			    mrioc->evtack_cmds_bitmap))
2425 				break;
2426 			cmd_idx = MPI3MR_NUM_EVTACKCMD;
2427 		}
2428 	} while (retrycount--);
2429 
2430 	if (cmd_idx >= MPI3MR_NUM_EVTACKCMD) {
2431 		delayed_evtack = kzalloc(sizeof(*delayed_evtack),
2432 		    GFP_ATOMIC);
2433 		if (!delayed_evtack)
2434 			return;
2435 		INIT_LIST_HEAD(&delayed_evtack->list);
2436 		delayed_evtack->event = event;
2437 		delayed_evtack->event_ctx = event_ctx;
2438 		list_add_tail(&delayed_evtack->list,
2439 		    &mrioc->delayed_evtack_cmds_list);
2440 		dprint_event_th(mrioc,
2441 		    "event ack in the top half for event(0x%02x), event_ctx(0x%08x) is postponed\n",
2442 		    event, event_ctx);
2443 		return;
2444 	}
2445 	drv_cmd = &mrioc->evtack_cmds[cmd_idx];
2446 
2447 issue_cmd:
2448 	cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
2449 
2450 	memset(&evtack_req, 0, sizeof(evtack_req));
2451 	if (drv_cmd->state & MPI3MR_CMD_PENDING) {
2452 		dprint_event_th(mrioc,
2453 		    "sending event ack failed due to command in use\n");
2454 		goto out;
2455 	}
2456 	drv_cmd->state = MPI3MR_CMD_PENDING;
2457 	drv_cmd->is_waiting = 0;
2458 	drv_cmd->callback = mpi3mr_complete_evt_ack;
2459 	evtack_req.host_tag = cpu_to_le16(drv_cmd->host_tag);
2460 	evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
2461 	evtack_req.event = event;
2462 	evtack_req.event_context = cpu_to_le32(event_ctx);
2463 	retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
2464 	    sizeof(evtack_req), 1);
2465 	if (retval) {
2466 		dprint_event_th(mrioc,
2467 		    "posting event ack request is failed\n");
2468 		goto out_failed;
2469 	}
2470 
2471 	dprint_event_th(mrioc,
2472 	    "event ack in the top half for event(0x%02x), event_ctx(0x%08x) is posted\n",
2473 	    event, event_ctx);
2474 out:
2475 	return;
2476 out_failed:
2477 	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2478 	drv_cmd->callback = NULL;
2479 	clear_bit(cmd_idx, mrioc->evtack_cmds_bitmap);
2480 }
2481 
2482 /**
2483  * mpi3mr_pcietopochg_evt_th - PCIETopologyChange evt tophalf
2484  * @mrioc: Adapter instance reference
2485  * @event_reply: event data
2486  *
2487  * Checks for the reason code and based on that either block I/O
2488  * to device, or unblock I/O to the device, or start the device
2489  * removal handshake with reason as remove with the firmware for
2490  * PCIe devices.
2491  *
2492  * Return: Nothing
2493  */
2494 static void mpi3mr_pcietopochg_evt_th(struct mpi3mr_ioc *mrioc,
2495 	struct mpi3_event_notification_reply *event_reply)
2496 {
2497 	struct mpi3_event_data_pcie_topology_change_list *topo_evt =
2498 	    (struct mpi3_event_data_pcie_topology_change_list *)event_reply->event_data;
2499 	int i;
2500 	u16 handle;
2501 	u8 reason_code;
2502 	struct mpi3mr_tgt_dev *tgtdev = NULL;
2503 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
2504 
2505 	for (i = 0; i < topo_evt->num_entries; i++) {
2506 		handle = le16_to_cpu(topo_evt->port_entry[i].attached_dev_handle);
2507 		if (!handle)
2508 			continue;
2509 		reason_code = topo_evt->port_entry[i].port_status;
2510 		scsi_tgt_priv_data =  NULL;
2511 		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
2512 		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata)
2513 			scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
2514 			    tgtdev->starget->hostdata;
2515 		switch (reason_code) {
2516 		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
2517 			if (scsi_tgt_priv_data) {
2518 				scsi_tgt_priv_data->dev_removed = 1;
2519 				scsi_tgt_priv_data->dev_removedelay = 0;
2520 				atomic_set(&scsi_tgt_priv_data->block_io, 0);
2521 			}
2522 			mpi3mr_dev_rmhs_send_tm(mrioc, handle, NULL,
2523 			    MPI3_CTRL_OP_REMOVE_DEVICE);
2524 			break;
2525 		case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
2526 			if (scsi_tgt_priv_data) {
2527 				scsi_tgt_priv_data->dev_removedelay = 1;
2528 				atomic_inc(&scsi_tgt_priv_data->block_io);
2529 			}
2530 			break;
2531 		case MPI3_EVENT_PCIE_TOPO_PS_RESPONDING:
2532 			if (scsi_tgt_priv_data &&
2533 			    scsi_tgt_priv_data->dev_removedelay) {
2534 				scsi_tgt_priv_data->dev_removedelay = 0;
2535 				atomic_dec_if_positive
2536 				    (&scsi_tgt_priv_data->block_io);
2537 			}
2538 			break;
2539 		case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
2540 		default:
2541 			break;
2542 		}
2543 		if (tgtdev)
2544 			mpi3mr_tgtdev_put(tgtdev);
2545 	}
2546 }
2547 
2548 /**
2549  * mpi3mr_sastopochg_evt_th - SASTopologyChange evt tophalf
2550  * @mrioc: Adapter instance reference
2551  * @event_reply: event data
2552  *
2553  * Checks for the reason code and based on that either block I/O
2554  * to device, or unblock I/O to the device, or start the device
2555  * removal handshake with reason as remove with the firmware for
2556  * SAS/SATA devices.
2557  *
2558  * Return: Nothing
2559  */
2560 static void mpi3mr_sastopochg_evt_th(struct mpi3mr_ioc *mrioc,
2561 	struct mpi3_event_notification_reply *event_reply)
2562 {
2563 	struct mpi3_event_data_sas_topology_change_list *topo_evt =
2564 	    (struct mpi3_event_data_sas_topology_change_list *)event_reply->event_data;
2565 	int i;
2566 	u16 handle;
2567 	u8 reason_code;
2568 	struct mpi3mr_tgt_dev *tgtdev = NULL;
2569 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
2570 
2571 	for (i = 0; i < topo_evt->num_entries; i++) {
2572 		handle = le16_to_cpu(topo_evt->phy_entry[i].attached_dev_handle);
2573 		if (!handle)
2574 			continue;
2575 		reason_code = topo_evt->phy_entry[i].status &
2576 		    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
2577 		scsi_tgt_priv_data =  NULL;
2578 		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
2579 		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata)
2580 			scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
2581 			    tgtdev->starget->hostdata;
2582 		switch (reason_code) {
2583 		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
2584 			if (scsi_tgt_priv_data) {
2585 				scsi_tgt_priv_data->dev_removed = 1;
2586 				scsi_tgt_priv_data->dev_removedelay = 0;
2587 				atomic_set(&scsi_tgt_priv_data->block_io, 0);
2588 			}
2589 			mpi3mr_dev_rmhs_send_tm(mrioc, handle, NULL,
2590 			    MPI3_CTRL_OP_REMOVE_DEVICE);
2591 			break;
2592 		case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING:
2593 			if (scsi_tgt_priv_data) {
2594 				scsi_tgt_priv_data->dev_removedelay = 1;
2595 				atomic_inc(&scsi_tgt_priv_data->block_io);
2596 			}
2597 			break;
2598 		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
2599 			if (scsi_tgt_priv_data &&
2600 			    scsi_tgt_priv_data->dev_removedelay) {
2601 				scsi_tgt_priv_data->dev_removedelay = 0;
2602 				atomic_dec_if_positive
2603 				    (&scsi_tgt_priv_data->block_io);
2604 			}
2605 			break;
2606 		case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
2607 		default:
2608 			break;
2609 		}
2610 		if (tgtdev)
2611 			mpi3mr_tgtdev_put(tgtdev);
2612 	}
2613 }
2614 
2615 /**
2616  * mpi3mr_devstatuschg_evt_th - DeviceStatusChange evt tophalf
2617  * @mrioc: Adapter instance reference
2618  * @event_reply: event data
2619  *
2620  * Checks for the reason code and based on that either block I/O
2621  * to device, or unblock I/O to the device, or start the device
2622  * removal handshake with reason as remove/hide acknowledgment
2623  * with the firmware.
2624  *
2625  * Return: Nothing
2626  */
2627 static void mpi3mr_devstatuschg_evt_th(struct mpi3mr_ioc *mrioc,
2628 	struct mpi3_event_notification_reply *event_reply)
2629 {
2630 	u16 dev_handle = 0;
2631 	u8 ublock = 0, block = 0, hide = 0, delete = 0, remove = 0;
2632 	struct mpi3mr_tgt_dev *tgtdev = NULL;
2633 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
2634 	struct mpi3_event_data_device_status_change *evtdata =
2635 	    (struct mpi3_event_data_device_status_change *)event_reply->event_data;
2636 
2637 	if (mrioc->stop_drv_processing)
2638 		goto out;
2639 
2640 	dev_handle = le16_to_cpu(evtdata->dev_handle);
2641 
2642 	switch (evtdata->reason_code) {
2643 	case MPI3_EVENT_DEV_STAT_RC_INT_DEVICE_RESET_STRT:
2644 	case MPI3_EVENT_DEV_STAT_RC_INT_IT_NEXUS_RESET_STRT:
2645 		block = 1;
2646 		break;
2647 	case MPI3_EVENT_DEV_STAT_RC_HIDDEN:
2648 		delete = 1;
2649 		hide = 1;
2650 		break;
2651 	case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING:
2652 		delete = 1;
2653 		remove = 1;
2654 		break;
2655 	case MPI3_EVENT_DEV_STAT_RC_INT_DEVICE_RESET_CMP:
2656 	case MPI3_EVENT_DEV_STAT_RC_INT_IT_NEXUS_RESET_CMP:
2657 		ublock = 1;
2658 		break;
2659 	default:
2660 		break;
2661 	}
2662 
2663 	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
2664 	if (!tgtdev)
2665 		goto out;
2666 	if (hide)
2667 		tgtdev->is_hidden = hide;
2668 	if (tgtdev->starget && tgtdev->starget->hostdata) {
2669 		scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
2670 		    tgtdev->starget->hostdata;
2671 		if (block)
2672 			atomic_inc(&scsi_tgt_priv_data->block_io);
2673 		if (delete)
2674 			scsi_tgt_priv_data->dev_removed = 1;
2675 		if (ublock)
2676 			atomic_dec_if_positive(&scsi_tgt_priv_data->block_io);
2677 	}
2678 	if (remove)
2679 		mpi3mr_dev_rmhs_send_tm(mrioc, dev_handle, NULL,
2680 		    MPI3_CTRL_OP_REMOVE_DEVICE);
2681 	if (hide)
2682 		mpi3mr_dev_rmhs_send_tm(mrioc, dev_handle, NULL,
2683 		    MPI3_CTRL_OP_HIDDEN_ACK);
2684 
2685 out:
2686 	if (tgtdev)
2687 		mpi3mr_tgtdev_put(tgtdev);
2688 }
2689 
2690 /**
2691  * mpi3mr_preparereset_evt_th - Prepare for reset event tophalf
2692  * @mrioc: Adapter instance reference
2693  * @event_reply: event data
2694  *
2695  * Blocks and unblocks host level I/O based on the reason code
2696  *
2697  * Return: Nothing
2698  */
2699 static void mpi3mr_preparereset_evt_th(struct mpi3mr_ioc *mrioc,
2700 	struct mpi3_event_notification_reply *event_reply)
2701 {
2702 	struct mpi3_event_data_prepare_for_reset *evtdata =
2703 	    (struct mpi3_event_data_prepare_for_reset *)event_reply->event_data;
2704 
2705 	if (evtdata->reason_code == MPI3_EVENT_PREPARE_RESET_RC_START) {
2706 		dprint_event_th(mrioc,
2707 		    "prepare for reset event top half with rc=start\n");
2708 		if (mrioc->prepare_for_reset)
2709 			return;
2710 		mrioc->prepare_for_reset = 1;
2711 		mrioc->prepare_for_reset_timeout_counter = 0;
2712 	} else if (evtdata->reason_code == MPI3_EVENT_PREPARE_RESET_RC_ABORT) {
2713 		dprint_event_th(mrioc,
2714 		    "prepare for reset top half with rc=abort\n");
2715 		mrioc->prepare_for_reset = 0;
2716 		mrioc->prepare_for_reset_timeout_counter = 0;
2717 	}
2718 	if ((event_reply->msg_flags & MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_MASK)
2719 	    == MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_REQUIRED)
2720 		mpi3mr_send_event_ack(mrioc, event_reply->event, NULL,
2721 		    le32_to_cpu(event_reply->event_context));
2722 }
2723 
2724 /**
2725  * mpi3mr_energypackchg_evt_th - Energy pack change evt tophalf
2726  * @mrioc: Adapter instance reference
2727  * @event_reply: event data
2728  *
2729  * Identifies the new shutdown timeout value and update.
2730  *
2731  * Return: Nothing
2732  */
2733 static void mpi3mr_energypackchg_evt_th(struct mpi3mr_ioc *mrioc,
2734 	struct mpi3_event_notification_reply *event_reply)
2735 {
2736 	struct mpi3_event_data_energy_pack_change *evtdata =
2737 	    (struct mpi3_event_data_energy_pack_change *)event_reply->event_data;
2738 	u16 shutdown_timeout = le16_to_cpu(evtdata->shutdown_timeout);
2739 
2740 	if (shutdown_timeout <= 0) {
2741 		ioc_warn(mrioc,
2742 		    "%s :Invalid Shutdown Timeout received = %d\n",
2743 		    __func__, shutdown_timeout);
2744 		return;
2745 	}
2746 
2747 	ioc_info(mrioc,
2748 	    "%s :Previous Shutdown Timeout Value = %d New Shutdown Timeout Value = %d\n",
2749 	    __func__, mrioc->facts.shutdown_timeout, shutdown_timeout);
2750 	mrioc->facts.shutdown_timeout = shutdown_timeout;
2751 }
2752 
2753 /**
2754  * mpi3mr_cablemgmt_evt_th - Cable management event tophalf
2755  * @mrioc: Adapter instance reference
2756  * @event_reply: event data
2757  *
2758  * Displays Cable manegemt event details.
2759  *
2760  * Return: Nothing
2761  */
2762 static void mpi3mr_cablemgmt_evt_th(struct mpi3mr_ioc *mrioc,
2763 	struct mpi3_event_notification_reply *event_reply)
2764 {
2765 	struct mpi3_event_data_cable_management *evtdata =
2766 	    (struct mpi3_event_data_cable_management *)event_reply->event_data;
2767 
2768 	switch (evtdata->status) {
2769 	case MPI3_EVENT_CABLE_MGMT_STATUS_INSUFFICIENT_POWER:
2770 	{
2771 		ioc_info(mrioc, "An active cable with receptacle_id %d cannot be powered.\n"
2772 		    "Devices connected to this cable are not detected.\n"
2773 		    "This cable requires %d mW of power.\n",
2774 		    evtdata->receptacle_id,
2775 		    le32_to_cpu(evtdata->active_cable_power_requirement));
2776 		break;
2777 	}
2778 	case MPI3_EVENT_CABLE_MGMT_STATUS_DEGRADED:
2779 	{
2780 		ioc_info(mrioc, "A cable with receptacle_id %d is not running at optimal speed\n",
2781 		    evtdata->receptacle_id);
2782 		break;
2783 	}
2784 	default:
2785 		break;
2786 	}
2787 }
2788 
2789 /**
2790  * mpi3mr_add_event_wait_for_device_refresh - Add Wait for Device Refresh Event
2791  * @mrioc: Adapter instance reference
2792  *
2793  * Add driver specific event to make sure that the driver won't process the
2794  * events until all the devices are refreshed during soft reset.
2795  *
2796  * Return: Nothing
2797  */
2798 void mpi3mr_add_event_wait_for_device_refresh(struct mpi3mr_ioc *mrioc)
2799 {
2800 	struct mpi3mr_fwevt *fwevt = NULL;
2801 
2802 	fwevt = mpi3mr_alloc_fwevt(0);
2803 	if (!fwevt) {
2804 		dprint_event_th(mrioc,
2805 		    "failed to schedule bottom half handler for event(0x%02x)\n",
2806 		    MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH);
2807 		return;
2808 	}
2809 	fwevt->mrioc = mrioc;
2810 	fwevt->event_id = MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH;
2811 	fwevt->send_ack = 0;
2812 	fwevt->process_evt = 1;
2813 	fwevt->evt_ctx = 0;
2814 	fwevt->event_data_size = 0;
2815 	mpi3mr_fwevt_add_to_list(mrioc, fwevt);
2816 }
2817 
2818 /**
2819  * mpi3mr_os_handle_events - Firmware event handler
2820  * @mrioc: Adapter instance reference
2821  * @event_reply: event data
2822  *
2823  * Identify whteher the event has to handled and acknowledged
2824  * and either process the event in the tophalf and/or schedule a
2825  * bottom half through mpi3mr_fwevt_worker.
2826  *
2827  * Return: Nothing
2828  */
2829 void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc,
2830 	struct mpi3_event_notification_reply *event_reply)
2831 {
2832 	u16 evt_type, sz;
2833 	struct mpi3mr_fwevt *fwevt = NULL;
2834 	bool ack_req = 0, process_evt_bh = 0;
2835 
2836 	if (mrioc->stop_drv_processing)
2837 		return;
2838 
2839 	if ((event_reply->msg_flags & MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_MASK)
2840 	    == MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_REQUIRED)
2841 		ack_req = 1;
2842 
2843 	evt_type = event_reply->event;
2844 
2845 	switch (evt_type) {
2846 	case MPI3_EVENT_DEVICE_ADDED:
2847 	{
2848 		struct mpi3_device_page0 *dev_pg0 =
2849 		    (struct mpi3_device_page0 *)event_reply->event_data;
2850 		if (mpi3mr_create_tgtdev(mrioc, dev_pg0))
2851 			ioc_err(mrioc,
2852 			    "%s :Failed to add device in the device add event\n",
2853 			    __func__);
2854 		else
2855 			process_evt_bh = 1;
2856 		break;
2857 	}
2858 	case MPI3_EVENT_DEVICE_STATUS_CHANGE:
2859 	{
2860 		process_evt_bh = 1;
2861 		mpi3mr_devstatuschg_evt_th(mrioc, event_reply);
2862 		break;
2863 	}
2864 	case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
2865 	{
2866 		process_evt_bh = 1;
2867 		mpi3mr_sastopochg_evt_th(mrioc, event_reply);
2868 		break;
2869 	}
2870 	case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
2871 	{
2872 		process_evt_bh = 1;
2873 		mpi3mr_pcietopochg_evt_th(mrioc, event_reply);
2874 		break;
2875 	}
2876 	case MPI3_EVENT_PREPARE_FOR_RESET:
2877 	{
2878 		mpi3mr_preparereset_evt_th(mrioc, event_reply);
2879 		ack_req = 0;
2880 		break;
2881 	}
2882 	case MPI3_EVENT_DEVICE_INFO_CHANGED:
2883 	case MPI3_EVENT_LOG_DATA:
2884 	case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
2885 	case MPI3_EVENT_ENCL_DEVICE_ADDED:
2886 	{
2887 		process_evt_bh = 1;
2888 		break;
2889 	}
2890 	case MPI3_EVENT_ENERGY_PACK_CHANGE:
2891 	{
2892 		mpi3mr_energypackchg_evt_th(mrioc, event_reply);
2893 		break;
2894 	}
2895 	case MPI3_EVENT_CABLE_MGMT:
2896 	{
2897 		mpi3mr_cablemgmt_evt_th(mrioc, event_reply);
2898 		break;
2899 	}
2900 	case MPI3_EVENT_SAS_DISCOVERY:
2901 	case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
2902 	case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
2903 	case MPI3_EVENT_PCIE_ENUMERATION:
2904 		break;
2905 	default:
2906 		ioc_info(mrioc, "%s :event 0x%02x is not handled\n",
2907 		    __func__, evt_type);
2908 		break;
2909 	}
2910 	if (process_evt_bh || ack_req) {
2911 		sz = event_reply->event_data_length * 4;
2912 		fwevt = mpi3mr_alloc_fwevt(sz);
2913 		if (!fwevt) {
2914 			ioc_info(mrioc, "%s :failure at %s:%d/%s()!\n",
2915 			    __func__, __FILE__, __LINE__, __func__);
2916 			return;
2917 		}
2918 
2919 		memcpy(fwevt->event_data, event_reply->event_data, sz);
2920 		fwevt->mrioc = mrioc;
2921 		fwevt->event_id = evt_type;
2922 		fwevt->send_ack = ack_req;
2923 		fwevt->process_evt = process_evt_bh;
2924 		fwevt->evt_ctx = le32_to_cpu(event_reply->event_context);
2925 		mpi3mr_fwevt_add_to_list(mrioc, fwevt);
2926 	}
2927 }
2928 
2929 /**
2930  * mpi3mr_setup_eedp - Setup EEDP information in MPI3 SCSI IO
2931  * @mrioc: Adapter instance reference
2932  * @scmd: SCSI command reference
2933  * @scsiio_req: MPI3 SCSI IO request
2934  *
2935  * Identifies the protection information flags from the SCSI
2936  * command and set appropriate flags in the MPI3 SCSI IO
2937  * request.
2938  *
2939  * Return: Nothing
2940  */
2941 static void mpi3mr_setup_eedp(struct mpi3mr_ioc *mrioc,
2942 	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req)
2943 {
2944 	u16 eedp_flags = 0;
2945 	unsigned char prot_op = scsi_get_prot_op(scmd);
2946 
2947 	switch (prot_op) {
2948 	case SCSI_PROT_NORMAL:
2949 		return;
2950 	case SCSI_PROT_READ_STRIP:
2951 		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK_REMOVE;
2952 		break;
2953 	case SCSI_PROT_WRITE_INSERT:
2954 		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_INSERT;
2955 		break;
2956 	case SCSI_PROT_READ_INSERT:
2957 		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_INSERT;
2958 		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2959 		break;
2960 	case SCSI_PROT_WRITE_STRIP:
2961 		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK_REMOVE;
2962 		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2963 		break;
2964 	case SCSI_PROT_READ_PASS:
2965 		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK;
2966 		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2967 		break;
2968 	case SCSI_PROT_WRITE_PASS:
2969 		if (scmd->prot_flags & SCSI_PROT_IP_CHECKSUM) {
2970 			eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK_REGEN;
2971 			scsiio_req->sgl[0].eedp.application_tag_translation_mask =
2972 			    0xffff;
2973 		} else
2974 			eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK;
2975 
2976 		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2977 		break;
2978 	default:
2979 		return;
2980 	}
2981 
2982 	if (scmd->prot_flags & SCSI_PROT_GUARD_CHECK)
2983 		eedp_flags |= MPI3_EEDPFLAGS_CHK_GUARD;
2984 
2985 	if (scmd->prot_flags & SCSI_PROT_IP_CHECKSUM)
2986 		eedp_flags |= MPI3_EEDPFLAGS_HOST_GUARD_IP_CHKSUM;
2987 
2988 	if (scmd->prot_flags & SCSI_PROT_REF_CHECK) {
2989 		eedp_flags |= MPI3_EEDPFLAGS_CHK_REF_TAG |
2990 			MPI3_EEDPFLAGS_INCR_PRI_REF_TAG;
2991 		scsiio_req->cdb.eedp32.primary_reference_tag =
2992 			cpu_to_be32(scsi_prot_ref_tag(scmd));
2993 	}
2994 
2995 	if (scmd->prot_flags & SCSI_PROT_REF_INCREMENT)
2996 		eedp_flags |= MPI3_EEDPFLAGS_INCR_PRI_REF_TAG;
2997 
2998 	eedp_flags |= MPI3_EEDPFLAGS_ESC_MODE_APPTAG_DISABLE;
2999 
3000 	switch (scsi_prot_interval(scmd)) {
3001 	case 512:
3002 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_512;
3003 		break;
3004 	case 520:
3005 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_520;
3006 		break;
3007 	case 4080:
3008 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4080;
3009 		break;
3010 	case 4088:
3011 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4088;
3012 		break;
3013 	case 4096:
3014 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4096;
3015 		break;
3016 	case 4104:
3017 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4104;
3018 		break;
3019 	case 4160:
3020 		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4160;
3021 		break;
3022 	default:
3023 		break;
3024 	}
3025 
3026 	scsiio_req->sgl[0].eedp.eedp_flags = cpu_to_le16(eedp_flags);
3027 	scsiio_req->sgl[0].eedp.flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_EXTENDED;
3028 }
3029 
3030 /**
3031  * mpi3mr_build_sense_buffer - Map sense information
3032  * @desc: Sense type
3033  * @buf: Sense buffer to populate
3034  * @key: Sense key
3035  * @asc: Additional sense code
3036  * @ascq: Additional sense code qualifier
3037  *
3038  * Maps the given sense information into either descriptor or
3039  * fixed format sense data.
3040  *
3041  * Return: Nothing
3042  */
3043 static inline void mpi3mr_build_sense_buffer(int desc, u8 *buf, u8 key,
3044 	u8 asc, u8 ascq)
3045 {
3046 	if (desc) {
3047 		buf[0] = 0x72;	/* descriptor, current */
3048 		buf[1] = key;
3049 		buf[2] = asc;
3050 		buf[3] = ascq;
3051 		buf[7] = 0;
3052 	} else {
3053 		buf[0] = 0x70;	/* fixed, current */
3054 		buf[2] = key;
3055 		buf[7] = 0xa;
3056 		buf[12] = asc;
3057 		buf[13] = ascq;
3058 	}
3059 }
3060 
3061 /**
3062  * mpi3mr_map_eedp_error - Map EEDP errors from IOC status
3063  * @scmd: SCSI command reference
3064  * @ioc_status: status of MPI3 request
3065  *
3066  * Maps the EEDP error status of the SCSI IO request to sense
3067  * data.
3068  *
3069  * Return: Nothing
3070  */
3071 static void mpi3mr_map_eedp_error(struct scsi_cmnd *scmd,
3072 	u16 ioc_status)
3073 {
3074 	u8 ascq = 0;
3075 
3076 	switch (ioc_status) {
3077 	case MPI3_IOCSTATUS_EEDP_GUARD_ERROR:
3078 		ascq = 0x01;
3079 		break;
3080 	case MPI3_IOCSTATUS_EEDP_APP_TAG_ERROR:
3081 		ascq = 0x02;
3082 		break;
3083 	case MPI3_IOCSTATUS_EEDP_REF_TAG_ERROR:
3084 		ascq = 0x03;
3085 		break;
3086 	default:
3087 		ascq = 0x00;
3088 		break;
3089 	}
3090 
3091 	mpi3mr_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
3092 	    0x10, ascq);
3093 	scmd->result = (DID_ABORT << 16) | SAM_STAT_CHECK_CONDITION;
3094 }
3095 
3096 /**
3097  * mpi3mr_process_op_reply_desc - reply descriptor handler
3098  * @mrioc: Adapter instance reference
3099  * @reply_desc: Operational reply descriptor
3100  * @reply_dma: place holder for reply DMA address
3101  * @qidx: Operational queue index
3102  *
3103  * Process the operational reply descriptor and identifies the
3104  * descriptor type. Based on the descriptor map the MPI3 request
3105  * status to a SCSI command status and calls scsi_done call
3106  * back.
3107  *
3108  * Return: Nothing
3109  */
3110 void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
3111 	struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma, u16 qidx)
3112 {
3113 	u16 reply_desc_type, host_tag = 0;
3114 	u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
3115 	u32 ioc_loginfo = 0;
3116 	struct mpi3_status_reply_descriptor *status_desc = NULL;
3117 	struct mpi3_address_reply_descriptor *addr_desc = NULL;
3118 	struct mpi3_success_reply_descriptor *success_desc = NULL;
3119 	struct mpi3_scsi_io_reply *scsi_reply = NULL;
3120 	struct scsi_cmnd *scmd = NULL;
3121 	struct scmd_priv *priv = NULL;
3122 	u8 *sense_buf = NULL;
3123 	u8 scsi_state = 0, scsi_status = 0, sense_state = 0;
3124 	u32 xfer_count = 0, sense_count = 0, resp_data = 0;
3125 	u16 dev_handle = 0xFFFF;
3126 	struct scsi_sense_hdr sshdr;
3127 	struct mpi3mr_stgt_priv_data *stgt_priv_data = NULL;
3128 	struct mpi3mr_sdev_priv_data *sdev_priv_data = NULL;
3129 	u32 ioc_pend_data_len = 0, tg_pend_data_len = 0, data_len_blks = 0;
3130 	struct mpi3mr_throttle_group_info *tg = NULL;
3131 	u8 throttle_enabled_dev = 0;
3132 
3133 	*reply_dma = 0;
3134 	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
3135 	    MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
3136 	switch (reply_desc_type) {
3137 	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
3138 		status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
3139 		host_tag = le16_to_cpu(status_desc->host_tag);
3140 		ioc_status = le16_to_cpu(status_desc->ioc_status);
3141 		if (ioc_status &
3142 		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
3143 			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
3144 		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
3145 		break;
3146 	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
3147 		addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
3148 		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
3149 		scsi_reply = mpi3mr_get_reply_virt_addr(mrioc,
3150 		    *reply_dma);
3151 		if (!scsi_reply) {
3152 			panic("%s: scsi_reply is NULL, this shouldn't happen\n",
3153 			    mrioc->name);
3154 			goto out;
3155 		}
3156 		host_tag = le16_to_cpu(scsi_reply->host_tag);
3157 		ioc_status = le16_to_cpu(scsi_reply->ioc_status);
3158 		scsi_status = scsi_reply->scsi_status;
3159 		scsi_state = scsi_reply->scsi_state;
3160 		dev_handle = le16_to_cpu(scsi_reply->dev_handle);
3161 		sense_state = (scsi_state & MPI3_SCSI_STATE_SENSE_MASK);
3162 		xfer_count = le32_to_cpu(scsi_reply->transfer_count);
3163 		sense_count = le32_to_cpu(scsi_reply->sense_count);
3164 		resp_data = le32_to_cpu(scsi_reply->response_data);
3165 		sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
3166 		    le64_to_cpu(scsi_reply->sense_data_buffer_address));
3167 		if (ioc_status &
3168 		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
3169 			ioc_loginfo = le32_to_cpu(scsi_reply->ioc_log_info);
3170 		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
3171 		if (sense_state == MPI3_SCSI_STATE_SENSE_BUFF_Q_EMPTY)
3172 			panic("%s: Ran out of sense buffers\n", mrioc->name);
3173 		break;
3174 	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
3175 		success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
3176 		host_tag = le16_to_cpu(success_desc->host_tag);
3177 		break;
3178 	default:
3179 		break;
3180 	}
3181 	scmd = mpi3mr_scmd_from_host_tag(mrioc, host_tag, qidx);
3182 	if (!scmd) {
3183 		panic("%s: Cannot Identify scmd for host_tag 0x%x\n",
3184 		    mrioc->name, host_tag);
3185 		goto out;
3186 	}
3187 	priv = scsi_cmd_priv(scmd);
3188 
3189 	data_len_blks = scsi_bufflen(scmd) >> 9;
3190 	sdev_priv_data = scmd->device->hostdata;
3191 	if (sdev_priv_data) {
3192 		stgt_priv_data = sdev_priv_data->tgt_priv_data;
3193 		if (stgt_priv_data) {
3194 			tg = stgt_priv_data->throttle_group;
3195 			throttle_enabled_dev =
3196 			    stgt_priv_data->io_throttle_enabled;
3197 		}
3198 	}
3199 	if (unlikely((data_len_blks >= mrioc->io_throttle_data_length) &&
3200 	    throttle_enabled_dev)) {
3201 		ioc_pend_data_len = atomic_sub_return(data_len_blks,
3202 		    &mrioc->pend_large_data_sz);
3203 		if (tg) {
3204 			tg_pend_data_len = atomic_sub_return(data_len_blks,
3205 			    &tg->pend_large_data_sz);
3206 			if (tg->io_divert  && ((ioc_pend_data_len <=
3207 			    mrioc->io_throttle_low) &&
3208 			    (tg_pend_data_len <= tg->low))) {
3209 				tg->io_divert = 0;
3210 				mpi3mr_set_io_divert_for_all_vd_in_tg(
3211 				    mrioc, tg, 0);
3212 			}
3213 		} else {
3214 			if (ioc_pend_data_len <= mrioc->io_throttle_low)
3215 				stgt_priv_data->io_divert = 0;
3216 		}
3217 	} else if (unlikely((stgt_priv_data && stgt_priv_data->io_divert))) {
3218 		ioc_pend_data_len = atomic_read(&mrioc->pend_large_data_sz);
3219 		if (!tg) {
3220 			if (ioc_pend_data_len <= mrioc->io_throttle_low)
3221 				stgt_priv_data->io_divert = 0;
3222 
3223 		} else if (ioc_pend_data_len <= mrioc->io_throttle_low) {
3224 			tg_pend_data_len = atomic_read(&tg->pend_large_data_sz);
3225 			if (tg->io_divert  && (tg_pend_data_len <= tg->low)) {
3226 				tg->io_divert = 0;
3227 				mpi3mr_set_io_divert_for_all_vd_in_tg(
3228 				    mrioc, tg, 0);
3229 			}
3230 		}
3231 	}
3232 
3233 	if (success_desc) {
3234 		scmd->result = DID_OK << 16;
3235 		goto out_success;
3236 	}
3237 
3238 	scsi_set_resid(scmd, scsi_bufflen(scmd) - xfer_count);
3239 	if (ioc_status == MPI3_IOCSTATUS_SCSI_DATA_UNDERRUN &&
3240 	    xfer_count == 0 && (scsi_status == MPI3_SCSI_STATUS_BUSY ||
3241 	    scsi_status == MPI3_SCSI_STATUS_RESERVATION_CONFLICT ||
3242 	    scsi_status == MPI3_SCSI_STATUS_TASK_SET_FULL))
3243 		ioc_status = MPI3_IOCSTATUS_SUCCESS;
3244 
3245 	if ((sense_state == MPI3_SCSI_STATE_SENSE_VALID) && sense_count &&
3246 	    sense_buf) {
3247 		u32 sz = min_t(u32, SCSI_SENSE_BUFFERSIZE, sense_count);
3248 
3249 		memcpy(scmd->sense_buffer, sense_buf, sz);
3250 	}
3251 
3252 	switch (ioc_status) {
3253 	case MPI3_IOCSTATUS_BUSY:
3254 	case MPI3_IOCSTATUS_INSUFFICIENT_RESOURCES:
3255 		scmd->result = SAM_STAT_BUSY;
3256 		break;
3257 	case MPI3_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
3258 		scmd->result = DID_NO_CONNECT << 16;
3259 		break;
3260 	case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED:
3261 		scmd->result = DID_SOFT_ERROR << 16;
3262 		break;
3263 	case MPI3_IOCSTATUS_SCSI_TASK_TERMINATED:
3264 	case MPI3_IOCSTATUS_SCSI_EXT_TERMINATED:
3265 		scmd->result = DID_RESET << 16;
3266 		break;
3267 	case MPI3_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
3268 		if ((xfer_count == 0) || (scmd->underflow > xfer_count))
3269 			scmd->result = DID_SOFT_ERROR << 16;
3270 		else
3271 			scmd->result = (DID_OK << 16) | scsi_status;
3272 		break;
3273 	case MPI3_IOCSTATUS_SCSI_DATA_UNDERRUN:
3274 		scmd->result = (DID_OK << 16) | scsi_status;
3275 		if (sense_state == MPI3_SCSI_STATE_SENSE_VALID)
3276 			break;
3277 		if (xfer_count < scmd->underflow) {
3278 			if (scsi_status == SAM_STAT_BUSY)
3279 				scmd->result = SAM_STAT_BUSY;
3280 			else
3281 				scmd->result = DID_SOFT_ERROR << 16;
3282 		} else if ((scsi_state & (MPI3_SCSI_STATE_NO_SCSI_STATUS)) ||
3283 		    (sense_state != MPI3_SCSI_STATE_SENSE_NOT_AVAILABLE))
3284 			scmd->result = DID_SOFT_ERROR << 16;
3285 		else if (scsi_state & MPI3_SCSI_STATE_TERMINATED)
3286 			scmd->result = DID_RESET << 16;
3287 		break;
3288 	case MPI3_IOCSTATUS_SCSI_DATA_OVERRUN:
3289 		scsi_set_resid(scmd, 0);
3290 		fallthrough;
3291 	case MPI3_IOCSTATUS_SCSI_RECOVERED_ERROR:
3292 	case MPI3_IOCSTATUS_SUCCESS:
3293 		scmd->result = (DID_OK << 16) | scsi_status;
3294 		if ((scsi_state & (MPI3_SCSI_STATE_NO_SCSI_STATUS)) ||
3295 		    (sense_state == MPI3_SCSI_STATE_SENSE_FAILED) ||
3296 			(sense_state == MPI3_SCSI_STATE_SENSE_BUFF_Q_EMPTY))
3297 			scmd->result = DID_SOFT_ERROR << 16;
3298 		else if (scsi_state & MPI3_SCSI_STATE_TERMINATED)
3299 			scmd->result = DID_RESET << 16;
3300 		break;
3301 	case MPI3_IOCSTATUS_EEDP_GUARD_ERROR:
3302 	case MPI3_IOCSTATUS_EEDP_REF_TAG_ERROR:
3303 	case MPI3_IOCSTATUS_EEDP_APP_TAG_ERROR:
3304 		mpi3mr_map_eedp_error(scmd, ioc_status);
3305 		break;
3306 	case MPI3_IOCSTATUS_SCSI_PROTOCOL_ERROR:
3307 	case MPI3_IOCSTATUS_INVALID_FUNCTION:
3308 	case MPI3_IOCSTATUS_INVALID_SGL:
3309 	case MPI3_IOCSTATUS_INTERNAL_ERROR:
3310 	case MPI3_IOCSTATUS_INVALID_FIELD:
3311 	case MPI3_IOCSTATUS_INVALID_STATE:
3312 	case MPI3_IOCSTATUS_SCSI_IO_DATA_ERROR:
3313 	case MPI3_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
3314 	case MPI3_IOCSTATUS_INSUFFICIENT_POWER:
3315 	default:
3316 		scmd->result = DID_SOFT_ERROR << 16;
3317 		break;
3318 	}
3319 
3320 	if (scmd->result != (DID_OK << 16) && (scmd->cmnd[0] != ATA_12) &&
3321 	    (scmd->cmnd[0] != ATA_16) &&
3322 	    mrioc->logging_level & MPI3_DEBUG_SCSI_ERROR) {
3323 		ioc_info(mrioc, "%s :scmd->result 0x%x\n", __func__,
3324 		    scmd->result);
3325 		scsi_print_command(scmd);
3326 		ioc_info(mrioc,
3327 		    "%s :Command issued to handle 0x%02x returned with error 0x%04x loginfo 0x%08x, qid %d\n",
3328 		    __func__, dev_handle, ioc_status, ioc_loginfo,
3329 		    priv->req_q_idx + 1);
3330 		ioc_info(mrioc,
3331 		    " host_tag %d scsi_state 0x%02x scsi_status 0x%02x, xfer_cnt %d resp_data 0x%x\n",
3332 		    host_tag, scsi_state, scsi_status, xfer_count, resp_data);
3333 		if (sense_buf) {
3334 			scsi_normalize_sense(sense_buf, sense_count, &sshdr);
3335 			ioc_info(mrioc,
3336 			    "%s :sense_count 0x%x, sense_key 0x%x ASC 0x%x, ASCQ 0x%x\n",
3337 			    __func__, sense_count, sshdr.sense_key,
3338 			    sshdr.asc, sshdr.ascq);
3339 		}
3340 	}
3341 out_success:
3342 	if (priv->meta_sg_valid) {
3343 		dma_unmap_sg(&mrioc->pdev->dev, scsi_prot_sglist(scmd),
3344 		    scsi_prot_sg_count(scmd), scmd->sc_data_direction);
3345 	}
3346 	mpi3mr_clear_scmd_priv(mrioc, scmd);
3347 	scsi_dma_unmap(scmd);
3348 	scsi_done(scmd);
3349 out:
3350 	if (sense_buf)
3351 		mpi3mr_repost_sense_buf(mrioc,
3352 		    le64_to_cpu(scsi_reply->sense_data_buffer_address));
3353 }
3354 
3355 /**
3356  * mpi3mr_get_chain_idx - get free chain buffer index
3357  * @mrioc: Adapter instance reference
3358  *
3359  * Try to get a free chain buffer index from the free pool.
3360  *
3361  * Return: -1 on failure or the free chain buffer index
3362  */
3363 static int mpi3mr_get_chain_idx(struct mpi3mr_ioc *mrioc)
3364 {
3365 	u8 retry_count = 5;
3366 	int cmd_idx = -1;
3367 	unsigned long flags;
3368 
3369 	spin_lock_irqsave(&mrioc->chain_buf_lock, flags);
3370 	do {
3371 		cmd_idx = find_first_zero_bit(mrioc->chain_bitmap,
3372 		    mrioc->chain_buf_count);
3373 		if (cmd_idx < mrioc->chain_buf_count) {
3374 			set_bit(cmd_idx, mrioc->chain_bitmap);
3375 			break;
3376 		}
3377 		cmd_idx = -1;
3378 	} while (retry_count--);
3379 	spin_unlock_irqrestore(&mrioc->chain_buf_lock, flags);
3380 	return cmd_idx;
3381 }
3382 
3383 /**
3384  * mpi3mr_prepare_sg_scmd - build scatter gather list
3385  * @mrioc: Adapter instance reference
3386  * @scmd: SCSI command reference
3387  * @scsiio_req: MPI3 SCSI IO request
3388  *
3389  * This function maps SCSI command's data and protection SGEs to
3390  * MPI request SGEs. If required additional 4K chain buffer is
3391  * used to send the SGEs.
3392  *
3393  * Return: 0 on success, -ENOMEM on dma_map_sg failure
3394  */
3395 static int mpi3mr_prepare_sg_scmd(struct mpi3mr_ioc *mrioc,
3396 	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req)
3397 {
3398 	dma_addr_t chain_dma;
3399 	struct scatterlist *sg_scmd;
3400 	void *sg_local, *chain;
3401 	u32 chain_length;
3402 	int sges_left, chain_idx;
3403 	u32 sges_in_segment;
3404 	u8 simple_sgl_flags;
3405 	u8 simple_sgl_flags_last;
3406 	u8 last_chain_sgl_flags;
3407 	struct chain_element *chain_req;
3408 	struct scmd_priv *priv = NULL;
3409 	u32 meta_sg = le32_to_cpu(scsiio_req->flags) &
3410 	    MPI3_SCSIIO_FLAGS_DMAOPERATION_HOST_PI;
3411 
3412 	priv = scsi_cmd_priv(scmd);
3413 
3414 	simple_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE |
3415 	    MPI3_SGE_FLAGS_DLAS_SYSTEM;
3416 	simple_sgl_flags_last = simple_sgl_flags |
3417 	    MPI3_SGE_FLAGS_END_OF_LIST;
3418 	last_chain_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_LAST_CHAIN |
3419 	    MPI3_SGE_FLAGS_DLAS_SYSTEM;
3420 
3421 	if (meta_sg)
3422 		sg_local = &scsiio_req->sgl[MPI3_SCSIIO_METASGL_INDEX];
3423 	else
3424 		sg_local = &scsiio_req->sgl;
3425 
3426 	if (!scsiio_req->data_length && !meta_sg) {
3427 		mpi3mr_build_zero_len_sge(sg_local);
3428 		return 0;
3429 	}
3430 
3431 	if (meta_sg) {
3432 		sg_scmd = scsi_prot_sglist(scmd);
3433 		sges_left = dma_map_sg(&mrioc->pdev->dev,
3434 		    scsi_prot_sglist(scmd),
3435 		    scsi_prot_sg_count(scmd),
3436 		    scmd->sc_data_direction);
3437 		priv->meta_sg_valid = 1; /* To unmap meta sg DMA */
3438 	} else {
3439 		sg_scmd = scsi_sglist(scmd);
3440 		sges_left = scsi_dma_map(scmd);
3441 	}
3442 
3443 	if (sges_left < 0) {
3444 		sdev_printk(KERN_ERR, scmd->device,
3445 		    "scsi_dma_map failed: request for %d bytes!\n",
3446 		    scsi_bufflen(scmd));
3447 		return -ENOMEM;
3448 	}
3449 	if (sges_left > mrioc->max_sgl_entries) {
3450 		sdev_printk(KERN_ERR, scmd->device,
3451 		    "scsi_dma_map returned unsupported sge count %d!\n",
3452 		    sges_left);
3453 		return -ENOMEM;
3454 	}
3455 
3456 	sges_in_segment = (mrioc->facts.op_req_sz -
3457 	    offsetof(struct mpi3_scsi_io_request, sgl)) / sizeof(struct mpi3_sge_common);
3458 
3459 	if (scsiio_req->sgl[0].eedp.flags ==
3460 	    MPI3_SGE_FLAGS_ELEMENT_TYPE_EXTENDED && !meta_sg) {
3461 		sg_local += sizeof(struct mpi3_sge_common);
3462 		sges_in_segment--;
3463 		/* Reserve 1st segment (scsiio_req->sgl[0]) for eedp */
3464 	}
3465 
3466 	if (scsiio_req->msg_flags ==
3467 	    MPI3_SCSIIO_MSGFLAGS_METASGL_VALID && !meta_sg) {
3468 		sges_in_segment--;
3469 		/* Reserve last segment (scsiio_req->sgl[3]) for meta sg */
3470 	}
3471 
3472 	if (meta_sg)
3473 		sges_in_segment = 1;
3474 
3475 	if (sges_left <= sges_in_segment)
3476 		goto fill_in_last_segment;
3477 
3478 	/* fill in main message segment when there is a chain following */
3479 	while (sges_in_segment > 1) {
3480 		mpi3mr_add_sg_single(sg_local, simple_sgl_flags,
3481 		    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
3482 		sg_scmd = sg_next(sg_scmd);
3483 		sg_local += sizeof(struct mpi3_sge_common);
3484 		sges_left--;
3485 		sges_in_segment--;
3486 	}
3487 
3488 	chain_idx = mpi3mr_get_chain_idx(mrioc);
3489 	if (chain_idx < 0)
3490 		return -1;
3491 	chain_req = &mrioc->chain_sgl_list[chain_idx];
3492 	if (meta_sg)
3493 		priv->meta_chain_idx = chain_idx;
3494 	else
3495 		priv->chain_idx = chain_idx;
3496 
3497 	chain = chain_req->addr;
3498 	chain_dma = chain_req->dma_addr;
3499 	sges_in_segment = sges_left;
3500 	chain_length = sges_in_segment * sizeof(struct mpi3_sge_common);
3501 
3502 	mpi3mr_add_sg_single(sg_local, last_chain_sgl_flags,
3503 	    chain_length, chain_dma);
3504 
3505 	sg_local = chain;
3506 
3507 fill_in_last_segment:
3508 	while (sges_left > 0) {
3509 		if (sges_left == 1)
3510 			mpi3mr_add_sg_single(sg_local,
3511 			    simple_sgl_flags_last, sg_dma_len(sg_scmd),
3512 			    sg_dma_address(sg_scmd));
3513 		else
3514 			mpi3mr_add_sg_single(sg_local, simple_sgl_flags,
3515 			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
3516 		sg_scmd = sg_next(sg_scmd);
3517 		sg_local += sizeof(struct mpi3_sge_common);
3518 		sges_left--;
3519 	}
3520 
3521 	return 0;
3522 }
3523 
3524 /**
3525  * mpi3mr_build_sg_scmd - build scatter gather list for SCSI IO
3526  * @mrioc: Adapter instance reference
3527  * @scmd: SCSI command reference
3528  * @scsiio_req: MPI3 SCSI IO request
3529  *
3530  * This function calls mpi3mr_prepare_sg_scmd for constructing
3531  * both data SGEs and protection information SGEs in the MPI
3532  * format from the SCSI Command as appropriate .
3533  *
3534  * Return: return value of mpi3mr_prepare_sg_scmd.
3535  */
3536 static int mpi3mr_build_sg_scmd(struct mpi3mr_ioc *mrioc,
3537 	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req)
3538 {
3539 	int ret;
3540 
3541 	ret = mpi3mr_prepare_sg_scmd(mrioc, scmd, scsiio_req);
3542 	if (ret)
3543 		return ret;
3544 
3545 	if (scsiio_req->msg_flags == MPI3_SCSIIO_MSGFLAGS_METASGL_VALID) {
3546 		/* There is a valid meta sg */
3547 		scsiio_req->flags |=
3548 		    cpu_to_le32(MPI3_SCSIIO_FLAGS_DMAOPERATION_HOST_PI);
3549 		ret = mpi3mr_prepare_sg_scmd(mrioc, scmd, scsiio_req);
3550 	}
3551 
3552 	return ret;
3553 }
3554 
3555 /**
3556  * mpi3mr_tm_response_name -  get TM response as a string
3557  * @resp_code: TM response code
3558  *
3559  * Convert known task management response code as a readable
3560  * string.
3561  *
3562  * Return: response code string.
3563  */
3564 static const char *mpi3mr_tm_response_name(u8 resp_code)
3565 {
3566 	char *desc;
3567 
3568 	switch (resp_code) {
3569 	case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE:
3570 		desc = "task management request completed";
3571 		break;
3572 	case MPI3_SCSITASKMGMT_RSPCODE_INVALID_FRAME:
3573 		desc = "invalid frame";
3574 		break;
3575 	case MPI3_SCSITASKMGMT_RSPCODE_TM_FUNCTION_NOT_SUPPORTED:
3576 		desc = "task management request not supported";
3577 		break;
3578 	case MPI3_SCSITASKMGMT_RSPCODE_TM_FAILED:
3579 		desc = "task management request failed";
3580 		break;
3581 	case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED:
3582 		desc = "task management request succeeded";
3583 		break;
3584 	case MPI3_SCSITASKMGMT_RSPCODE_TM_INVALID_LUN:
3585 		desc = "invalid LUN";
3586 		break;
3587 	case MPI3_SCSITASKMGMT_RSPCODE_TM_OVERLAPPED_TAG:
3588 		desc = "overlapped tag attempted";
3589 		break;
3590 	case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC:
3591 		desc = "task queued, however not sent to target";
3592 		break;
3593 	case MPI3_SCSITASKMGMT_RSPCODE_TM_NVME_DENIED:
3594 		desc = "task management request denied by NVMe device";
3595 		break;
3596 	default:
3597 		desc = "unknown";
3598 		break;
3599 	}
3600 
3601 	return desc;
3602 }
3603 
3604 inline void mpi3mr_poll_pend_io_completions(struct mpi3mr_ioc *mrioc)
3605 {
3606 	int i;
3607 	int num_of_reply_queues =
3608 	    mrioc->num_op_reply_q + mrioc->op_reply_q_offset;
3609 
3610 	for (i = mrioc->op_reply_q_offset; i < num_of_reply_queues; i++)
3611 		mpi3mr_process_op_reply_q(mrioc,
3612 		    mrioc->intr_info[i].op_reply_q);
3613 }
3614 
3615 /**
3616  * mpi3mr_issue_tm - Issue Task Management request
3617  * @mrioc: Adapter instance reference
3618  * @tm_type: Task Management type
3619  * @handle: Device handle
3620  * @lun: lun ID
3621  * @htag: Host tag of the TM request
3622  * @timeout: TM timeout value
3623  * @drv_cmd: Internal command tracker
3624  * @resp_code: Response code place holder
3625  * @scmd: SCSI command
3626  *
3627  * Issues a Task Management Request to the controller for a
3628  * specified target, lun and command and wait for its completion
3629  * and check TM response. Recover the TM if it timed out by
3630  * issuing controller reset.
3631  *
3632  * Return: 0 on success, non-zero on errors
3633  */
3634 int mpi3mr_issue_tm(struct mpi3mr_ioc *mrioc, u8 tm_type,
3635 	u16 handle, uint lun, u16 htag, ulong timeout,
3636 	struct mpi3mr_drv_cmd *drv_cmd,
3637 	u8 *resp_code, struct scsi_cmnd *scmd)
3638 {
3639 	struct mpi3_scsi_task_mgmt_request tm_req;
3640 	struct mpi3_scsi_task_mgmt_reply *tm_reply = NULL;
3641 	int retval = 0;
3642 	struct mpi3mr_tgt_dev *tgtdev = NULL;
3643 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
3644 	struct scmd_priv *cmd_priv = NULL;
3645 	struct scsi_device *sdev = NULL;
3646 	struct mpi3mr_sdev_priv_data *sdev_priv_data = NULL;
3647 
3648 	ioc_info(mrioc, "%s :Issue TM: TM type (0x%x) for devhandle 0x%04x\n",
3649 	     __func__, tm_type, handle);
3650 	if (mrioc->unrecoverable) {
3651 		retval = -1;
3652 		ioc_err(mrioc, "%s :Issue TM: Unrecoverable controller\n",
3653 		    __func__);
3654 		goto out;
3655 	}
3656 
3657 	memset(&tm_req, 0, sizeof(tm_req));
3658 	mutex_lock(&drv_cmd->mutex);
3659 	if (drv_cmd->state & MPI3MR_CMD_PENDING) {
3660 		retval = -1;
3661 		ioc_err(mrioc, "%s :Issue TM: Command is in use\n", __func__);
3662 		mutex_unlock(&drv_cmd->mutex);
3663 		goto out;
3664 	}
3665 	if (mrioc->reset_in_progress) {
3666 		retval = -1;
3667 		ioc_err(mrioc, "%s :Issue TM: Reset in progress\n", __func__);
3668 		mutex_unlock(&drv_cmd->mutex);
3669 		goto out;
3670 	}
3671 
3672 	drv_cmd->state = MPI3MR_CMD_PENDING;
3673 	drv_cmd->is_waiting = 1;
3674 	drv_cmd->callback = NULL;
3675 	tm_req.dev_handle = cpu_to_le16(handle);
3676 	tm_req.task_type = tm_type;
3677 	tm_req.host_tag = cpu_to_le16(htag);
3678 
3679 	int_to_scsilun(lun, (struct scsi_lun *)tm_req.lun);
3680 	tm_req.function = MPI3_FUNCTION_SCSI_TASK_MGMT;
3681 
3682 	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
3683 
3684 	if (scmd) {
3685 		sdev = scmd->device;
3686 		sdev_priv_data = sdev->hostdata;
3687 		scsi_tgt_priv_data = ((sdev_priv_data) ?
3688 		    sdev_priv_data->tgt_priv_data : NULL);
3689 	} else {
3690 		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata)
3691 			scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
3692 			    tgtdev->starget->hostdata;
3693 	}
3694 
3695 	if (scsi_tgt_priv_data)
3696 		atomic_inc(&scsi_tgt_priv_data->block_io);
3697 
3698 	if (tgtdev && (tgtdev->dev_type == MPI3_DEVICE_DEVFORM_PCIE)) {
3699 		if (cmd_priv && tgtdev->dev_spec.pcie_inf.abort_to)
3700 			timeout = tgtdev->dev_spec.pcie_inf.abort_to;
3701 		else if (!cmd_priv && tgtdev->dev_spec.pcie_inf.reset_to)
3702 			timeout = tgtdev->dev_spec.pcie_inf.reset_to;
3703 	}
3704 
3705 	init_completion(&drv_cmd->done);
3706 	retval = mpi3mr_admin_request_post(mrioc, &tm_req, sizeof(tm_req), 1);
3707 	if (retval) {
3708 		ioc_err(mrioc, "%s :Issue TM: Admin Post failed\n", __func__);
3709 		goto out_unlock;
3710 	}
3711 	wait_for_completion_timeout(&drv_cmd->done, (timeout * HZ));
3712 
3713 	if (!(drv_cmd->state & MPI3MR_CMD_COMPLETE)) {
3714 		drv_cmd->is_waiting = 0;
3715 		retval = -1;
3716 		if (!(drv_cmd->state & MPI3MR_CMD_RESET)) {
3717 			dprint_tm(mrioc,
3718 			    "task management request timed out after %ld seconds\n",
3719 			    timeout);
3720 			if (mrioc->logging_level & MPI3_DEBUG_TM)
3721 				dprint_dump_req(&tm_req, sizeof(tm_req)/4);
3722 			mpi3mr_soft_reset_handler(mrioc,
3723 			    MPI3MR_RESET_FROM_TM_TIMEOUT, 1);
3724 		}
3725 		goto out_unlock;
3726 	}
3727 
3728 	if (!(drv_cmd->state & MPI3MR_CMD_REPLY_VALID)) {
3729 		dprint_tm(mrioc, "invalid task management reply message\n");
3730 		retval = -1;
3731 		goto out_unlock;
3732 	}
3733 
3734 	tm_reply = (struct mpi3_scsi_task_mgmt_reply *)drv_cmd->reply;
3735 
3736 	switch (drv_cmd->ioc_status) {
3737 	case MPI3_IOCSTATUS_SUCCESS:
3738 		*resp_code = le32_to_cpu(tm_reply->response_data) &
3739 			MPI3MR_RI_MASK_RESPCODE;
3740 		break;
3741 	case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED:
3742 		*resp_code = MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE;
3743 		break;
3744 	default:
3745 		dprint_tm(mrioc,
3746 		    "task management request to handle(0x%04x) is failed with ioc_status(0x%04x) log_info(0x%08x)\n",
3747 		    handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo);
3748 		retval = -1;
3749 		goto out_unlock;
3750 	}
3751 
3752 	switch (*resp_code) {
3753 	case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED:
3754 	case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE:
3755 		break;
3756 	case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC:
3757 		if (tm_type != MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK)
3758 			retval = -1;
3759 		break;
3760 	default:
3761 		retval = -1;
3762 		break;
3763 	}
3764 
3765 	dprint_tm(mrioc,
3766 	    "task management request type(%d) completed for handle(0x%04x) with ioc_status(0x%04x), log_info(0x%08x), termination_count(%d), response:%s(0x%x)\n",
3767 	    tm_type, handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo,
3768 	    le32_to_cpu(tm_reply->termination_count),
3769 	    mpi3mr_tm_response_name(*resp_code), *resp_code);
3770 
3771 	if (!retval) {
3772 		mpi3mr_ioc_disable_intr(mrioc);
3773 		mpi3mr_poll_pend_io_completions(mrioc);
3774 		mpi3mr_ioc_enable_intr(mrioc);
3775 		mpi3mr_poll_pend_io_completions(mrioc);
3776 		mpi3mr_process_admin_reply_q(mrioc);
3777 	}
3778 	switch (tm_type) {
3779 	case MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
3780 		if (!scsi_tgt_priv_data)
3781 			break;
3782 		scsi_tgt_priv_data->pend_count = 0;
3783 		blk_mq_tagset_busy_iter(&mrioc->shost->tag_set,
3784 		    mpi3mr_count_tgt_pending,
3785 		    (void *)scsi_tgt_priv_data->starget);
3786 		break;
3787 	case MPI3_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
3788 		if (!sdev_priv_data)
3789 			break;
3790 		sdev_priv_data->pend_count = 0;
3791 		blk_mq_tagset_busy_iter(&mrioc->shost->tag_set,
3792 		    mpi3mr_count_dev_pending, (void *)sdev);
3793 		break;
3794 	default:
3795 		break;
3796 	}
3797 
3798 out_unlock:
3799 	drv_cmd->state = MPI3MR_CMD_NOTUSED;
3800 	mutex_unlock(&drv_cmd->mutex);
3801 	if (scsi_tgt_priv_data)
3802 		atomic_dec_if_positive(&scsi_tgt_priv_data->block_io);
3803 	if (tgtdev)
3804 		mpi3mr_tgtdev_put(tgtdev);
3805 out:
3806 	return retval;
3807 }
3808 
3809 /**
3810  * mpi3mr_bios_param - BIOS param callback
3811  * @sdev: SCSI device reference
3812  * @bdev: Block device reference
3813  * @capacity: Capacity in logical sectors
3814  * @params: Parameter array
3815  *
3816  * Just the parameters with heads/secots/cylinders.
3817  *
3818  * Return: 0 always
3819  */
3820 static int mpi3mr_bios_param(struct scsi_device *sdev,
3821 	struct block_device *bdev, sector_t capacity, int params[])
3822 {
3823 	int heads;
3824 	int sectors;
3825 	sector_t cylinders;
3826 	ulong dummy;
3827 
3828 	heads = 64;
3829 	sectors = 32;
3830 
3831 	dummy = heads * sectors;
3832 	cylinders = capacity;
3833 	sector_div(cylinders, dummy);
3834 
3835 	if ((ulong)capacity >= 0x200000) {
3836 		heads = 255;
3837 		sectors = 63;
3838 		dummy = heads * sectors;
3839 		cylinders = capacity;
3840 		sector_div(cylinders, dummy);
3841 	}
3842 
3843 	params[0] = heads;
3844 	params[1] = sectors;
3845 	params[2] = cylinders;
3846 	return 0;
3847 }
3848 
3849 /**
3850  * mpi3mr_map_queues - Map queues callback handler
3851  * @shost: SCSI host reference
3852  *
3853  * Maps default and poll queues.
3854  *
3855  * Return: return zero.
3856  */
3857 static void mpi3mr_map_queues(struct Scsi_Host *shost)
3858 {
3859 	struct mpi3mr_ioc *mrioc = shost_priv(shost);
3860 	int i, qoff, offset;
3861 	struct blk_mq_queue_map *map = NULL;
3862 
3863 	offset = mrioc->op_reply_q_offset;
3864 
3865 	for (i = 0, qoff = 0; i < HCTX_MAX_TYPES; i++) {
3866 		map = &shost->tag_set.map[i];
3867 
3868 		map->nr_queues  = 0;
3869 
3870 		if (i == HCTX_TYPE_DEFAULT)
3871 			map->nr_queues = mrioc->default_qcount;
3872 		else if (i == HCTX_TYPE_POLL)
3873 			map->nr_queues = mrioc->active_poll_qcount;
3874 
3875 		if (!map->nr_queues) {
3876 			BUG_ON(i == HCTX_TYPE_DEFAULT);
3877 			continue;
3878 		}
3879 
3880 		/*
3881 		 * The poll queue(s) doesn't have an IRQ (and hence IRQ
3882 		 * affinity), so use the regular blk-mq cpu mapping
3883 		 */
3884 		map->queue_offset = qoff;
3885 		if (i != HCTX_TYPE_POLL)
3886 			blk_mq_pci_map_queues(map, mrioc->pdev, offset);
3887 		else
3888 			blk_mq_map_queues(map);
3889 
3890 		qoff += map->nr_queues;
3891 		offset += map->nr_queues;
3892 	}
3893 }
3894 
3895 /**
3896  * mpi3mr_get_fw_pending_ios - Calculate pending I/O count
3897  * @mrioc: Adapter instance reference
3898  *
3899  * Calculate the pending I/Os for the controller and return.
3900  *
3901  * Return: Number of pending I/Os
3902  */
3903 static inline int mpi3mr_get_fw_pending_ios(struct mpi3mr_ioc *mrioc)
3904 {
3905 	u16 i;
3906 	uint pend_ios = 0;
3907 
3908 	for (i = 0; i < mrioc->num_op_reply_q; i++)
3909 		pend_ios += atomic_read(&mrioc->op_reply_qinfo[i].pend_ios);
3910 	return pend_ios;
3911 }
3912 
3913 /**
3914  * mpi3mr_print_pending_host_io - print pending I/Os
3915  * @mrioc: Adapter instance reference
3916  *
3917  * Print number of pending I/Os and each I/O details prior to
3918  * reset for debug purpose.
3919  *
3920  * Return: Nothing
3921  */
3922 static void mpi3mr_print_pending_host_io(struct mpi3mr_ioc *mrioc)
3923 {
3924 	struct Scsi_Host *shost = mrioc->shost;
3925 
3926 	ioc_info(mrioc, "%s :Pending commands prior to reset: %d\n",
3927 	    __func__, mpi3mr_get_fw_pending_ios(mrioc));
3928 	blk_mq_tagset_busy_iter(&shost->tag_set,
3929 	    mpi3mr_print_scmd, (void *)mrioc);
3930 }
3931 
3932 /**
3933  * mpi3mr_wait_for_host_io - block for I/Os to complete
3934  * @mrioc: Adapter instance reference
3935  * @timeout: time out in seconds
3936  * Waits for pending I/Os for the given adapter to complete or
3937  * to hit the timeout.
3938  *
3939  * Return: Nothing
3940  */
3941 void mpi3mr_wait_for_host_io(struct mpi3mr_ioc *mrioc, u32 timeout)
3942 {
3943 	enum mpi3mr_iocstate iocstate;
3944 	int i = 0;
3945 
3946 	iocstate = mpi3mr_get_iocstate(mrioc);
3947 	if (iocstate != MRIOC_STATE_READY)
3948 		return;
3949 
3950 	if (!mpi3mr_get_fw_pending_ios(mrioc))
3951 		return;
3952 	ioc_info(mrioc,
3953 	    "%s :Waiting for %d seconds prior to reset for %d I/O\n",
3954 	    __func__, timeout, mpi3mr_get_fw_pending_ios(mrioc));
3955 
3956 	for (i = 0; i < timeout; i++) {
3957 		if (!mpi3mr_get_fw_pending_ios(mrioc))
3958 			break;
3959 		iocstate = mpi3mr_get_iocstate(mrioc);
3960 		if (iocstate != MRIOC_STATE_READY)
3961 			break;
3962 		msleep(1000);
3963 	}
3964 
3965 	ioc_info(mrioc, "%s :Pending I/Os after wait is: %d\n", __func__,
3966 	    mpi3mr_get_fw_pending_ios(mrioc));
3967 }
3968 
3969 /**
3970  * mpi3mr_setup_divert_ws - Setup Divert IO flag for write same
3971  * @mrioc: Adapter instance reference
3972  * @scmd: SCSI command reference
3973  * @scsiio_req: MPI3 SCSI IO request
3974  * @scsiio_flags: Pointer to MPI3 SCSI IO Flags
3975  * @wslen: write same max length
3976  *
3977  * Gets values of unmap, ndob and number of blocks from write
3978  * same scsi io and based on these values it sets divert IO flag
3979  * and reason for diverting IO to firmware.
3980  *
3981  * Return: Nothing
3982  */
3983 static inline void mpi3mr_setup_divert_ws(struct mpi3mr_ioc *mrioc,
3984 	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req,
3985 	u32 *scsiio_flags, u16 wslen)
3986 {
3987 	u8 unmap = 0, ndob = 0;
3988 	u8 opcode = scmd->cmnd[0];
3989 	u32 num_blocks = 0;
3990 	u16 sa = (scmd->cmnd[8] << 8) | (scmd->cmnd[9]);
3991 
3992 	if (opcode == WRITE_SAME_16) {
3993 		unmap = scmd->cmnd[1] & 0x08;
3994 		ndob = scmd->cmnd[1] & 0x01;
3995 		num_blocks = get_unaligned_be32(scmd->cmnd + 10);
3996 	} else if ((opcode == VARIABLE_LENGTH_CMD) && (sa == WRITE_SAME_32)) {
3997 		unmap = scmd->cmnd[10] & 0x08;
3998 		ndob = scmd->cmnd[10] & 0x01;
3999 		num_blocks = get_unaligned_be32(scmd->cmnd + 28);
4000 	} else
4001 		return;
4002 
4003 	if ((unmap) && (ndob) && (num_blocks > wslen)) {
4004 		scsiio_req->msg_flags |=
4005 		    MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE;
4006 		*scsiio_flags |=
4007 			MPI3_SCSIIO_FLAGS_DIVERT_REASON_WRITE_SAME_TOO_LARGE;
4008 	}
4009 }
4010 
4011 /**
4012  * mpi3mr_eh_host_reset - Host reset error handling callback
4013  * @scmd: SCSI command reference
4014  *
4015  * Issue controller reset if the scmd is for a Physical Device,
4016  * if the scmd is for RAID volume, then wait for
4017  * MPI3MR_RAID_ERRREC_RESET_TIMEOUT and checke whether any
4018  * pending I/Os prior to issuing reset to the controller.
4019  *
4020  * Return: SUCCESS of successful reset else FAILED
4021  */
4022 static int mpi3mr_eh_host_reset(struct scsi_cmnd *scmd)
4023 {
4024 	struct mpi3mr_ioc *mrioc = shost_priv(scmd->device->host);
4025 	struct mpi3mr_stgt_priv_data *stgt_priv_data;
4026 	struct mpi3mr_sdev_priv_data *sdev_priv_data;
4027 	u8 dev_type = MPI3_DEVICE_DEVFORM_VD;
4028 	int retval = FAILED, ret;
4029 
4030 	sdev_priv_data = scmd->device->hostdata;
4031 	if (sdev_priv_data && sdev_priv_data->tgt_priv_data) {
4032 		stgt_priv_data = sdev_priv_data->tgt_priv_data;
4033 		dev_type = stgt_priv_data->dev_type;
4034 	}
4035 
4036 	if (dev_type == MPI3_DEVICE_DEVFORM_VD) {
4037 		mpi3mr_wait_for_host_io(mrioc,
4038 		    MPI3MR_RAID_ERRREC_RESET_TIMEOUT);
4039 		if (!mpi3mr_get_fw_pending_ios(mrioc)) {
4040 			retval = SUCCESS;
4041 			goto out;
4042 		}
4043 	}
4044 
4045 	mpi3mr_print_pending_host_io(mrioc);
4046 	ret = mpi3mr_soft_reset_handler(mrioc,
4047 	    MPI3MR_RESET_FROM_EH_HOS, 1);
4048 	if (ret)
4049 		goto out;
4050 
4051 	retval = SUCCESS;
4052 out:
4053 	sdev_printk(KERN_INFO, scmd->device,
4054 	    "Host reset is %s for scmd(%p)\n",
4055 	    ((retval == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4056 
4057 	return retval;
4058 }
4059 
4060 /**
4061  * mpi3mr_eh_target_reset - Target reset error handling callback
4062  * @scmd: SCSI command reference
4063  *
4064  * Issue Target reset Task Management and verify the scmd is
4065  * terminated successfully and return status accordingly.
4066  *
4067  * Return: SUCCESS of successful termination of the scmd else
4068  *         FAILED
4069  */
4070 static int mpi3mr_eh_target_reset(struct scsi_cmnd *scmd)
4071 {
4072 	struct mpi3mr_ioc *mrioc = shost_priv(scmd->device->host);
4073 	struct mpi3mr_stgt_priv_data *stgt_priv_data;
4074 	struct mpi3mr_sdev_priv_data *sdev_priv_data;
4075 	u16 dev_handle;
4076 	u8 resp_code = 0;
4077 	int retval = FAILED, ret = 0;
4078 
4079 	sdev_printk(KERN_INFO, scmd->device,
4080 	    "Attempting Target Reset! scmd(%p)\n", scmd);
4081 	scsi_print_command(scmd);
4082 
4083 	sdev_priv_data = scmd->device->hostdata;
4084 	if (!sdev_priv_data || !sdev_priv_data->tgt_priv_data) {
4085 		sdev_printk(KERN_INFO, scmd->device,
4086 		    "SCSI device is not available\n");
4087 		retval = SUCCESS;
4088 		goto out;
4089 	}
4090 
4091 	stgt_priv_data = sdev_priv_data->tgt_priv_data;
4092 	dev_handle = stgt_priv_data->dev_handle;
4093 	if (stgt_priv_data->dev_removed) {
4094 		struct scmd_priv *cmd_priv = scsi_cmd_priv(scmd);
4095 		sdev_printk(KERN_INFO, scmd->device,
4096 		    "%s:target(handle = 0x%04x) is removed, target reset is not issued\n",
4097 		    mrioc->name, dev_handle);
4098 		if (!cmd_priv->in_lld_scope || cmd_priv->host_tag == MPI3MR_HOSTTAG_INVALID)
4099 			retval = SUCCESS;
4100 		else
4101 			retval = FAILED;
4102 		goto out;
4103 	}
4104 	sdev_printk(KERN_INFO, scmd->device,
4105 	    "Target Reset is issued to handle(0x%04x)\n",
4106 	    dev_handle);
4107 
4108 	ret = mpi3mr_issue_tm(mrioc,
4109 	    MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET, dev_handle,
4110 	    sdev_priv_data->lun_id, MPI3MR_HOSTTAG_BLK_TMS,
4111 	    MPI3MR_RESETTM_TIMEOUT, &mrioc->host_tm_cmds, &resp_code, scmd);
4112 
4113 	if (ret)
4114 		goto out;
4115 
4116 	if (stgt_priv_data->pend_count) {
4117 		sdev_printk(KERN_INFO, scmd->device,
4118 		    "%s: target has %d pending commands, target reset is failed\n",
4119 		    mrioc->name, stgt_priv_data->pend_count);
4120 		goto out;
4121 	}
4122 
4123 	retval = SUCCESS;
4124 out:
4125 	sdev_printk(KERN_INFO, scmd->device,
4126 	    "%s: target reset is %s for scmd(%p)\n", mrioc->name,
4127 	    ((retval == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4128 
4129 	return retval;
4130 }
4131 
4132 /**
4133  * mpi3mr_eh_dev_reset- Device reset error handling callback
4134  * @scmd: SCSI command reference
4135  *
4136  * Issue lun reset Task Management and verify the scmd is
4137  * terminated successfully and return status accordingly.
4138  *
4139  * Return: SUCCESS of successful termination of the scmd else
4140  *         FAILED
4141  */
4142 static int mpi3mr_eh_dev_reset(struct scsi_cmnd *scmd)
4143 {
4144 	struct mpi3mr_ioc *mrioc = shost_priv(scmd->device->host);
4145 	struct mpi3mr_stgt_priv_data *stgt_priv_data;
4146 	struct mpi3mr_sdev_priv_data *sdev_priv_data;
4147 	u16 dev_handle;
4148 	u8 resp_code = 0;
4149 	int retval = FAILED, ret = 0;
4150 
4151 	sdev_printk(KERN_INFO, scmd->device,
4152 	    "Attempting Device(lun) Reset! scmd(%p)\n", scmd);
4153 	scsi_print_command(scmd);
4154 
4155 	sdev_priv_data = scmd->device->hostdata;
4156 	if (!sdev_priv_data || !sdev_priv_data->tgt_priv_data) {
4157 		sdev_printk(KERN_INFO, scmd->device,
4158 		    "SCSI device is not available\n");
4159 		retval = SUCCESS;
4160 		goto out;
4161 	}
4162 
4163 	stgt_priv_data = sdev_priv_data->tgt_priv_data;
4164 	dev_handle = stgt_priv_data->dev_handle;
4165 	if (stgt_priv_data->dev_removed) {
4166 		struct scmd_priv *cmd_priv = scsi_cmd_priv(scmd);
4167 		sdev_printk(KERN_INFO, scmd->device,
4168 		    "%s: device(handle = 0x%04x) is removed, device(LUN) reset is not issued\n",
4169 		    mrioc->name, dev_handle);
4170 		if (!cmd_priv->in_lld_scope || cmd_priv->host_tag == MPI3MR_HOSTTAG_INVALID)
4171 			retval = SUCCESS;
4172 		else
4173 			retval = FAILED;
4174 		goto out;
4175 	}
4176 	sdev_printk(KERN_INFO, scmd->device,
4177 	    "Device(lun) Reset is issued to handle(0x%04x)\n", dev_handle);
4178 
4179 	ret = mpi3mr_issue_tm(mrioc,
4180 	    MPI3_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, dev_handle,
4181 	    sdev_priv_data->lun_id, MPI3MR_HOSTTAG_BLK_TMS,
4182 	    MPI3MR_RESETTM_TIMEOUT, &mrioc->host_tm_cmds, &resp_code, scmd);
4183 
4184 	if (ret)
4185 		goto out;
4186 
4187 	if (sdev_priv_data->pend_count) {
4188 		sdev_printk(KERN_INFO, scmd->device,
4189 		    "%s: device has %d pending commands, device(LUN) reset is failed\n",
4190 		    mrioc->name, sdev_priv_data->pend_count);
4191 		goto out;
4192 	}
4193 	retval = SUCCESS;
4194 out:
4195 	sdev_printk(KERN_INFO, scmd->device,
4196 	    "%s: device(LUN) reset is %s for scmd(%p)\n", mrioc->name,
4197 	    ((retval == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4198 
4199 	return retval;
4200 }
4201 
4202 /**
4203  * mpi3mr_scan_start - Scan start callback handler
4204  * @shost: SCSI host reference
4205  *
4206  * Issue port enable request asynchronously.
4207  *
4208  * Return: Nothing
4209  */
4210 static void mpi3mr_scan_start(struct Scsi_Host *shost)
4211 {
4212 	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4213 
4214 	mrioc->scan_started = 1;
4215 	ioc_info(mrioc, "%s :Issuing Port Enable\n", __func__);
4216 	if (mpi3mr_issue_port_enable(mrioc, 1)) {
4217 		ioc_err(mrioc, "%s :Issuing port enable failed\n", __func__);
4218 		mrioc->scan_started = 0;
4219 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
4220 	}
4221 }
4222 
4223 /**
4224  * mpi3mr_scan_finished - Scan finished callback handler
4225  * @shost: SCSI host reference
4226  * @time: Jiffies from the scan start
4227  *
4228  * Checks whether the port enable is completed or timedout or
4229  * failed and set the scan status accordingly after taking any
4230  * recovery if required.
4231  *
4232  * Return: 1 on scan finished or timed out, 0 for in progress
4233  */
4234 static int mpi3mr_scan_finished(struct Scsi_Host *shost,
4235 	unsigned long time)
4236 {
4237 	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4238 	u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
4239 	u32 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4240 
4241 	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
4242 	    (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
4243 		ioc_err(mrioc, "port enable failed due to fault or reset\n");
4244 		mpi3mr_print_fault_info(mrioc);
4245 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
4246 		mrioc->scan_started = 0;
4247 		mrioc->init_cmds.is_waiting = 0;
4248 		mrioc->init_cmds.callback = NULL;
4249 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4250 	}
4251 
4252 	if (time >= (pe_timeout * HZ)) {
4253 		ioc_err(mrioc, "port enable failed due to time out\n");
4254 		mpi3mr_check_rh_fault_ioc(mrioc,
4255 		    MPI3MR_RESET_FROM_PE_TIMEOUT);
4256 		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
4257 		mrioc->scan_started = 0;
4258 		mrioc->init_cmds.is_waiting = 0;
4259 		mrioc->init_cmds.callback = NULL;
4260 		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4261 	}
4262 
4263 	if (mrioc->scan_started)
4264 		return 0;
4265 
4266 	if (mrioc->scan_failed) {
4267 		ioc_err(mrioc,
4268 		    "port enable failed with status=0x%04x\n",
4269 		    mrioc->scan_failed);
4270 	} else
4271 		ioc_info(mrioc, "port enable is successfully completed\n");
4272 
4273 	mpi3mr_start_watchdog(mrioc);
4274 	mrioc->is_driver_loading = 0;
4275 	mrioc->stop_bsgs = 0;
4276 	return 1;
4277 }
4278 
4279 /**
4280  * mpi3mr_slave_destroy - Slave destroy callback handler
4281  * @sdev: SCSI device reference
4282  *
4283  * Cleanup and free per device(lun) private data.
4284  *
4285  * Return: Nothing.
4286  */
4287 static void mpi3mr_slave_destroy(struct scsi_device *sdev)
4288 {
4289 	struct Scsi_Host *shost;
4290 	struct mpi3mr_ioc *mrioc;
4291 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4292 	struct mpi3mr_tgt_dev *tgt_dev = NULL;
4293 	unsigned long flags;
4294 	struct scsi_target *starget;
4295 	struct sas_rphy *rphy = NULL;
4296 
4297 	if (!sdev->hostdata)
4298 		return;
4299 
4300 	starget = scsi_target(sdev);
4301 	shost = dev_to_shost(&starget->dev);
4302 	mrioc = shost_priv(shost);
4303 	scsi_tgt_priv_data = starget->hostdata;
4304 
4305 	scsi_tgt_priv_data->num_luns--;
4306 
4307 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4308 	if (starget->channel == mrioc->scsi_device_channel)
4309 		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4310 	else if (mrioc->sas_transport_enabled && !starget->channel) {
4311 		rphy = dev_to_rphy(starget->dev.parent);
4312 		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4313 		    rphy->identify.sas_address, rphy);
4314 	}
4315 
4316 	if (tgt_dev && (!scsi_tgt_priv_data->num_luns))
4317 		tgt_dev->starget = NULL;
4318 	if (tgt_dev)
4319 		mpi3mr_tgtdev_put(tgt_dev);
4320 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4321 
4322 	kfree(sdev->hostdata);
4323 	sdev->hostdata = NULL;
4324 }
4325 
4326 /**
4327  * mpi3mr_target_destroy - Target destroy callback handler
4328  * @starget: SCSI target reference
4329  *
4330  * Cleanup and free per target private data.
4331  *
4332  * Return: Nothing.
4333  */
4334 static void mpi3mr_target_destroy(struct scsi_target *starget)
4335 {
4336 	struct Scsi_Host *shost;
4337 	struct mpi3mr_ioc *mrioc;
4338 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4339 	struct mpi3mr_tgt_dev *tgt_dev;
4340 	unsigned long flags;
4341 
4342 	if (!starget->hostdata)
4343 		return;
4344 
4345 	shost = dev_to_shost(&starget->dev);
4346 	mrioc = shost_priv(shost);
4347 	scsi_tgt_priv_data = starget->hostdata;
4348 
4349 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4350 	tgt_dev = __mpi3mr_get_tgtdev_from_tgtpriv(mrioc, scsi_tgt_priv_data);
4351 	if (tgt_dev && (tgt_dev->starget == starget) &&
4352 	    (tgt_dev->perst_id == starget->id))
4353 		tgt_dev->starget = NULL;
4354 	if (tgt_dev) {
4355 		scsi_tgt_priv_data->tgt_dev = NULL;
4356 		scsi_tgt_priv_data->perst_id = 0;
4357 		mpi3mr_tgtdev_put(tgt_dev);
4358 		mpi3mr_tgtdev_put(tgt_dev);
4359 	}
4360 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4361 
4362 	kfree(starget->hostdata);
4363 	starget->hostdata = NULL;
4364 }
4365 
4366 /**
4367  * mpi3mr_slave_configure - Slave configure callback handler
4368  * @sdev: SCSI device reference
4369  *
4370  * Configure queue depth, max hardware sectors and virt boundary
4371  * as required
4372  *
4373  * Return: 0 always.
4374  */
4375 static int mpi3mr_slave_configure(struct scsi_device *sdev)
4376 {
4377 	struct scsi_target *starget;
4378 	struct Scsi_Host *shost;
4379 	struct mpi3mr_ioc *mrioc;
4380 	struct mpi3mr_tgt_dev *tgt_dev = NULL;
4381 	unsigned long flags;
4382 	int retval = 0;
4383 	struct sas_rphy *rphy = NULL;
4384 
4385 	starget = scsi_target(sdev);
4386 	shost = dev_to_shost(&starget->dev);
4387 	mrioc = shost_priv(shost);
4388 
4389 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4390 	if (starget->channel == mrioc->scsi_device_channel)
4391 		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4392 	else if (mrioc->sas_transport_enabled && !starget->channel) {
4393 		rphy = dev_to_rphy(starget->dev.parent);
4394 		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4395 		    rphy->identify.sas_address, rphy);
4396 	}
4397 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4398 	if (!tgt_dev)
4399 		return -ENXIO;
4400 
4401 	mpi3mr_change_queue_depth(sdev, tgt_dev->q_depth);
4402 
4403 	sdev->eh_timeout = MPI3MR_EH_SCMD_TIMEOUT;
4404 	blk_queue_rq_timeout(sdev->request_queue, MPI3MR_SCMD_TIMEOUT);
4405 
4406 	switch (tgt_dev->dev_type) {
4407 	case MPI3_DEVICE_DEVFORM_PCIE:
4408 		/*The block layer hw sector size = 512*/
4409 		if ((tgt_dev->dev_spec.pcie_inf.dev_info &
4410 		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) ==
4411 		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) {
4412 			blk_queue_max_hw_sectors(sdev->request_queue,
4413 			    tgt_dev->dev_spec.pcie_inf.mdts / 512);
4414 			if (tgt_dev->dev_spec.pcie_inf.pgsz == 0)
4415 				blk_queue_virt_boundary(sdev->request_queue,
4416 				    ((1 << MPI3MR_DEFAULT_PGSZEXP) - 1));
4417 			else
4418 				blk_queue_virt_boundary(sdev->request_queue,
4419 				    ((1 << tgt_dev->dev_spec.pcie_inf.pgsz) - 1));
4420 		}
4421 		break;
4422 	default:
4423 		break;
4424 	}
4425 
4426 	mpi3mr_tgtdev_put(tgt_dev);
4427 
4428 	return retval;
4429 }
4430 
4431 /**
4432  * mpi3mr_slave_alloc -Slave alloc callback handler
4433  * @sdev: SCSI device reference
4434  *
4435  * Allocate per device(lun) private data and initialize it.
4436  *
4437  * Return: 0 on success -ENOMEM on memory allocation failure.
4438  */
4439 static int mpi3mr_slave_alloc(struct scsi_device *sdev)
4440 {
4441 	struct Scsi_Host *shost;
4442 	struct mpi3mr_ioc *mrioc;
4443 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4444 	struct mpi3mr_tgt_dev *tgt_dev = NULL;
4445 	struct mpi3mr_sdev_priv_data *scsi_dev_priv_data;
4446 	unsigned long flags;
4447 	struct scsi_target *starget;
4448 	int retval = 0;
4449 	struct sas_rphy *rphy = NULL;
4450 
4451 	starget = scsi_target(sdev);
4452 	shost = dev_to_shost(&starget->dev);
4453 	mrioc = shost_priv(shost);
4454 	scsi_tgt_priv_data = starget->hostdata;
4455 
4456 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4457 
4458 	if (starget->channel == mrioc->scsi_device_channel)
4459 		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4460 	else if (mrioc->sas_transport_enabled && !starget->channel) {
4461 		rphy = dev_to_rphy(starget->dev.parent);
4462 		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4463 		    rphy->identify.sas_address, rphy);
4464 	}
4465 
4466 	if (tgt_dev) {
4467 		if (tgt_dev->starget == NULL)
4468 			tgt_dev->starget = starget;
4469 		mpi3mr_tgtdev_put(tgt_dev);
4470 		retval = 0;
4471 	} else {
4472 		spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4473 		return -ENXIO;
4474 	}
4475 
4476 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4477 
4478 	scsi_dev_priv_data = kzalloc(sizeof(*scsi_dev_priv_data), GFP_KERNEL);
4479 	if (!scsi_dev_priv_data)
4480 		return -ENOMEM;
4481 
4482 	scsi_dev_priv_data->lun_id = sdev->lun;
4483 	scsi_dev_priv_data->tgt_priv_data = scsi_tgt_priv_data;
4484 	sdev->hostdata = scsi_dev_priv_data;
4485 
4486 	scsi_tgt_priv_data->num_luns++;
4487 
4488 	return retval;
4489 }
4490 
4491 /**
4492  * mpi3mr_target_alloc - Target alloc callback handler
4493  * @starget: SCSI target reference
4494  *
4495  * Allocate per target private data and initialize it.
4496  *
4497  * Return: 0 on success -ENOMEM on memory allocation failure.
4498  */
4499 static int mpi3mr_target_alloc(struct scsi_target *starget)
4500 {
4501 	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
4502 	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4503 	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4504 	struct mpi3mr_tgt_dev *tgt_dev;
4505 	unsigned long flags;
4506 	int retval = 0;
4507 	struct sas_rphy *rphy = NULL;
4508 
4509 	scsi_tgt_priv_data = kzalloc(sizeof(*scsi_tgt_priv_data), GFP_KERNEL);
4510 	if (!scsi_tgt_priv_data)
4511 		return -ENOMEM;
4512 
4513 	starget->hostdata = scsi_tgt_priv_data;
4514 
4515 	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4516 	if (starget->channel == mrioc->scsi_device_channel) {
4517 		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4518 		if (tgt_dev && !tgt_dev->is_hidden) {
4519 			scsi_tgt_priv_data->starget = starget;
4520 			scsi_tgt_priv_data->dev_handle = tgt_dev->dev_handle;
4521 			scsi_tgt_priv_data->perst_id = tgt_dev->perst_id;
4522 			scsi_tgt_priv_data->dev_type = tgt_dev->dev_type;
4523 			scsi_tgt_priv_data->tgt_dev = tgt_dev;
4524 			tgt_dev->starget = starget;
4525 			atomic_set(&scsi_tgt_priv_data->block_io, 0);
4526 			retval = 0;
4527 			if ((tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_PCIE) &&
4528 			    ((tgt_dev->dev_spec.pcie_inf.dev_info &
4529 			    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) ==
4530 			    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) &&
4531 			    ((tgt_dev->dev_spec.pcie_inf.dev_info &
4532 			    MPI3_DEVICE0_PCIE_DEVICE_INFO_PITYPE_MASK) !=
4533 			    MPI3_DEVICE0_PCIE_DEVICE_INFO_PITYPE_0))
4534 				scsi_tgt_priv_data->dev_nvme_dif = 1;
4535 			scsi_tgt_priv_data->io_throttle_enabled = tgt_dev->io_throttle_enabled;
4536 			scsi_tgt_priv_data->wslen = tgt_dev->wslen;
4537 			if (tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_VD)
4538 				scsi_tgt_priv_data->throttle_group = tgt_dev->dev_spec.vd_inf.tg;
4539 		} else
4540 			retval = -ENXIO;
4541 	} else if (mrioc->sas_transport_enabled && !starget->channel) {
4542 		rphy = dev_to_rphy(starget->dev.parent);
4543 		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4544 		    rphy->identify.sas_address, rphy);
4545 		if (tgt_dev && !tgt_dev->is_hidden && !tgt_dev->non_stl &&
4546 		    (tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_SAS_SATA)) {
4547 			scsi_tgt_priv_data->starget = starget;
4548 			scsi_tgt_priv_data->dev_handle = tgt_dev->dev_handle;
4549 			scsi_tgt_priv_data->perst_id = tgt_dev->perst_id;
4550 			scsi_tgt_priv_data->dev_type = tgt_dev->dev_type;
4551 			scsi_tgt_priv_data->tgt_dev = tgt_dev;
4552 			scsi_tgt_priv_data->io_throttle_enabled = tgt_dev->io_throttle_enabled;
4553 			scsi_tgt_priv_data->wslen = tgt_dev->wslen;
4554 			tgt_dev->starget = starget;
4555 			atomic_set(&scsi_tgt_priv_data->block_io, 0);
4556 			retval = 0;
4557 		} else
4558 			retval = -ENXIO;
4559 	}
4560 	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4561 
4562 	return retval;
4563 }
4564 
4565 /**
4566  * mpi3mr_check_return_unmap - Whether an unmap is allowed
4567  * @mrioc: Adapter instance reference
4568  * @scmd: SCSI Command reference
4569  *
4570  * The controller hardware cannot handle certain unmap commands
4571  * for NVMe drives, this routine checks those and return true
4572  * and completes the SCSI command with proper status and sense
4573  * data.
4574  *
4575  * Return: TRUE for not  allowed unmap, FALSE otherwise.
4576  */
4577 static bool mpi3mr_check_return_unmap(struct mpi3mr_ioc *mrioc,
4578 	struct scsi_cmnd *scmd)
4579 {
4580 	unsigned char *buf;
4581 	u16 param_len, desc_len, trunc_param_len;
4582 
4583 	trunc_param_len = param_len = get_unaligned_be16(scmd->cmnd + 7);
4584 
4585 	if (mrioc->pdev->revision) {
4586 		if ((param_len > 24) && ((param_len - 8) & 0xF)) {
4587 			trunc_param_len -= (param_len - 8) & 0xF;
4588 			dprint_scsi_command(mrioc, scmd, MPI3_DEBUG_SCSI_ERROR);
4589 			dprint_scsi_err(mrioc,
4590 			    "truncating param_len from (%d) to (%d)\n",
4591 			    param_len, trunc_param_len);
4592 			put_unaligned_be16(trunc_param_len, scmd->cmnd + 7);
4593 			dprint_scsi_command(mrioc, scmd, MPI3_DEBUG_SCSI_ERROR);
4594 		}
4595 		return false;
4596 	}
4597 
4598 	if (!param_len) {
4599 		ioc_warn(mrioc,
4600 		    "%s: cdb received with zero parameter length\n",
4601 		    __func__);
4602 		scsi_print_command(scmd);
4603 		scmd->result = DID_OK << 16;
4604 		scsi_done(scmd);
4605 		return true;
4606 	}
4607 
4608 	if (param_len < 24) {
4609 		ioc_warn(mrioc,
4610 		    "%s: cdb received with invalid param_len: %d\n",
4611 		    __func__, param_len);
4612 		scsi_print_command(scmd);
4613 		scmd->result = SAM_STAT_CHECK_CONDITION;
4614 		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4615 		    0x1A, 0);
4616 		scsi_done(scmd);
4617 		return true;
4618 	}
4619 	if (param_len != scsi_bufflen(scmd)) {
4620 		ioc_warn(mrioc,
4621 		    "%s: cdb received with param_len: %d bufflen: %d\n",
4622 		    __func__, param_len, scsi_bufflen(scmd));
4623 		scsi_print_command(scmd);
4624 		scmd->result = SAM_STAT_CHECK_CONDITION;
4625 		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4626 		    0x1A, 0);
4627 		scsi_done(scmd);
4628 		return true;
4629 	}
4630 	buf = kzalloc(scsi_bufflen(scmd), GFP_ATOMIC);
4631 	if (!buf) {
4632 		scsi_print_command(scmd);
4633 		scmd->result = SAM_STAT_CHECK_CONDITION;
4634 		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4635 		    0x55, 0x03);
4636 		scsi_done(scmd);
4637 		return true;
4638 	}
4639 	scsi_sg_copy_to_buffer(scmd, buf, scsi_bufflen(scmd));
4640 	desc_len = get_unaligned_be16(&buf[2]);
4641 
4642 	if (desc_len < 16) {
4643 		ioc_warn(mrioc,
4644 		    "%s: Invalid descriptor length in param list: %d\n",
4645 		    __func__, desc_len);
4646 		scsi_print_command(scmd);
4647 		scmd->result = SAM_STAT_CHECK_CONDITION;
4648 		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4649 		    0x26, 0);
4650 		scsi_done(scmd);
4651 		kfree(buf);
4652 		return true;
4653 	}
4654 
4655 	if (param_len > (desc_len + 8)) {
4656 		trunc_param_len = desc_len + 8;
4657 		scsi_print_command(scmd);
4658 		dprint_scsi_err(mrioc,
4659 		    "truncating param_len(%d) to desc_len+8(%d)\n",
4660 		    param_len, trunc_param_len);
4661 		put_unaligned_be16(trunc_param_len, scmd->cmnd + 7);
4662 		scsi_print_command(scmd);
4663 	}
4664 
4665 	kfree(buf);
4666 	return false;
4667 }
4668 
4669 /**
4670  * mpi3mr_allow_scmd_to_fw - Command is allowed during shutdown
4671  * @scmd: SCSI Command reference
4672  *
4673  * Checks whether a cdb is allowed during shutdown or not.
4674  *
4675  * Return: TRUE for allowed commands, FALSE otherwise.
4676  */
4677 
4678 inline bool mpi3mr_allow_scmd_to_fw(struct scsi_cmnd *scmd)
4679 {
4680 	switch (scmd->cmnd[0]) {
4681 	case SYNCHRONIZE_CACHE:
4682 	case START_STOP:
4683 		return true;
4684 	default:
4685 		return false;
4686 	}
4687 }
4688 
4689 /**
4690  * mpi3mr_qcmd - I/O request despatcher
4691  * @shost: SCSI Host reference
4692  * @scmd: SCSI Command reference
4693  *
4694  * Issues the SCSI Command as an MPI3 request.
4695  *
4696  * Return: 0 on successful queueing of the request or if the
4697  *         request is completed with failure.
4698  *         SCSI_MLQUEUE_DEVICE_BUSY when the device is busy.
4699  *         SCSI_MLQUEUE_HOST_BUSY when the host queue is full.
4700  */
4701 static int mpi3mr_qcmd(struct Scsi_Host *shost,
4702 	struct scsi_cmnd *scmd)
4703 {
4704 	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4705 	struct mpi3mr_stgt_priv_data *stgt_priv_data;
4706 	struct mpi3mr_sdev_priv_data *sdev_priv_data;
4707 	struct scmd_priv *scmd_priv_data = NULL;
4708 	struct mpi3_scsi_io_request *scsiio_req = NULL;
4709 	struct op_req_qinfo *op_req_q = NULL;
4710 	int retval = 0;
4711 	u16 dev_handle;
4712 	u16 host_tag;
4713 	u32 scsiio_flags = 0, data_len_blks = 0;
4714 	struct request *rq = scsi_cmd_to_rq(scmd);
4715 	int iprio_class;
4716 	u8 is_pcie_dev = 0;
4717 	u32 tracked_io_sz = 0;
4718 	u32 ioc_pend_data_len = 0, tg_pend_data_len = 0;
4719 	struct mpi3mr_throttle_group_info *tg = NULL;
4720 
4721 	if (mrioc->unrecoverable) {
4722 		scmd->result = DID_ERROR << 16;
4723 		scsi_done(scmd);
4724 		goto out;
4725 	}
4726 
4727 	sdev_priv_data = scmd->device->hostdata;
4728 	if (!sdev_priv_data || !sdev_priv_data->tgt_priv_data) {
4729 		scmd->result = DID_NO_CONNECT << 16;
4730 		scsi_done(scmd);
4731 		goto out;
4732 	}
4733 
4734 	if (mrioc->stop_drv_processing &&
4735 	    !(mpi3mr_allow_scmd_to_fw(scmd))) {
4736 		scmd->result = DID_NO_CONNECT << 16;
4737 		scsi_done(scmd);
4738 		goto out;
4739 	}
4740 
4741 	stgt_priv_data = sdev_priv_data->tgt_priv_data;
4742 	dev_handle = stgt_priv_data->dev_handle;
4743 
4744 	/* Avoid error handling escalation when device is removed or blocked */
4745 
4746 	if (scmd->device->host->shost_state == SHOST_RECOVERY &&
4747 		scmd->cmnd[0] == TEST_UNIT_READY &&
4748 		(stgt_priv_data->dev_removed || (dev_handle == MPI3MR_INVALID_DEV_HANDLE))) {
4749 		scsi_build_sense(scmd, 0, UNIT_ATTENTION, 0x29, 0x07);
4750 		scsi_done(scmd);
4751 		goto out;
4752 	}
4753 
4754 	if (mrioc->reset_in_progress) {
4755 		retval = SCSI_MLQUEUE_HOST_BUSY;
4756 		goto out;
4757 	}
4758 
4759 	if (atomic_read(&stgt_priv_data->block_io)) {
4760 		if (mrioc->stop_drv_processing) {
4761 			scmd->result = DID_NO_CONNECT << 16;
4762 			scsi_done(scmd);
4763 			goto out;
4764 		}
4765 		retval = SCSI_MLQUEUE_DEVICE_BUSY;
4766 		goto out;
4767 	}
4768 
4769 	if (dev_handle == MPI3MR_INVALID_DEV_HANDLE) {
4770 		scmd->result = DID_NO_CONNECT << 16;
4771 		scsi_done(scmd);
4772 		goto out;
4773 	}
4774 	if (stgt_priv_data->dev_removed) {
4775 		scmd->result = DID_NO_CONNECT << 16;
4776 		scsi_done(scmd);
4777 		goto out;
4778 	}
4779 
4780 	if (stgt_priv_data->dev_type == MPI3_DEVICE_DEVFORM_PCIE)
4781 		is_pcie_dev = 1;
4782 	if ((scmd->cmnd[0] == UNMAP) && is_pcie_dev &&
4783 	    (mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&
4784 	    mpi3mr_check_return_unmap(mrioc, scmd))
4785 		goto out;
4786 
4787 	host_tag = mpi3mr_host_tag_for_scmd(mrioc, scmd);
4788 	if (host_tag == MPI3MR_HOSTTAG_INVALID) {
4789 		scmd->result = DID_ERROR << 16;
4790 		scsi_done(scmd);
4791 		goto out;
4792 	}
4793 
4794 	if (scmd->sc_data_direction == DMA_FROM_DEVICE)
4795 		scsiio_flags = MPI3_SCSIIO_FLAGS_DATADIRECTION_READ;
4796 	else if (scmd->sc_data_direction == DMA_TO_DEVICE)
4797 		scsiio_flags = MPI3_SCSIIO_FLAGS_DATADIRECTION_WRITE;
4798 	else
4799 		scsiio_flags = MPI3_SCSIIO_FLAGS_DATADIRECTION_NO_DATA_TRANSFER;
4800 
4801 	scsiio_flags |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_SIMPLEQ;
4802 
4803 	if (sdev_priv_data->ncq_prio_enable) {
4804 		iprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
4805 		if (iprio_class == IOPRIO_CLASS_RT)
4806 			scsiio_flags |= 1 << MPI3_SCSIIO_FLAGS_CMDPRI_SHIFT;
4807 	}
4808 
4809 	if (scmd->cmd_len > 16)
4810 		scsiio_flags |= MPI3_SCSIIO_FLAGS_CDB_GREATER_THAN_16;
4811 
4812 	scmd_priv_data = scsi_cmd_priv(scmd);
4813 	memset(scmd_priv_data->mpi3mr_scsiio_req, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
4814 	scsiio_req = (struct mpi3_scsi_io_request *)scmd_priv_data->mpi3mr_scsiio_req;
4815 	scsiio_req->function = MPI3_FUNCTION_SCSI_IO;
4816 	scsiio_req->host_tag = cpu_to_le16(host_tag);
4817 
4818 	mpi3mr_setup_eedp(mrioc, scmd, scsiio_req);
4819 
4820 	if (stgt_priv_data->wslen)
4821 		mpi3mr_setup_divert_ws(mrioc, scmd, scsiio_req, &scsiio_flags,
4822 		    stgt_priv_data->wslen);
4823 
4824 	memcpy(scsiio_req->cdb.cdb32, scmd->cmnd, scmd->cmd_len);
4825 	scsiio_req->data_length = cpu_to_le32(scsi_bufflen(scmd));
4826 	scsiio_req->dev_handle = cpu_to_le16(dev_handle);
4827 	scsiio_req->flags = cpu_to_le32(scsiio_flags);
4828 	int_to_scsilun(sdev_priv_data->lun_id,
4829 	    (struct scsi_lun *)scsiio_req->lun);
4830 
4831 	if (mpi3mr_build_sg_scmd(mrioc, scmd, scsiio_req)) {
4832 		mpi3mr_clear_scmd_priv(mrioc, scmd);
4833 		retval = SCSI_MLQUEUE_HOST_BUSY;
4834 		goto out;
4835 	}
4836 	op_req_q = &mrioc->req_qinfo[scmd_priv_data->req_q_idx];
4837 	data_len_blks = scsi_bufflen(scmd) >> 9;
4838 	if ((data_len_blks >= mrioc->io_throttle_data_length) &&
4839 	    stgt_priv_data->io_throttle_enabled) {
4840 		tracked_io_sz = data_len_blks;
4841 		tg = stgt_priv_data->throttle_group;
4842 		if (tg) {
4843 			ioc_pend_data_len = atomic_add_return(data_len_blks,
4844 			    &mrioc->pend_large_data_sz);
4845 			tg_pend_data_len = atomic_add_return(data_len_blks,
4846 			    &tg->pend_large_data_sz);
4847 			if (!tg->io_divert  && ((ioc_pend_data_len >=
4848 			    mrioc->io_throttle_high) ||
4849 			    (tg_pend_data_len >= tg->high))) {
4850 				tg->io_divert = 1;
4851 				tg->need_qd_reduction = 1;
4852 				mpi3mr_set_io_divert_for_all_vd_in_tg(mrioc,
4853 				    tg, 1);
4854 				mpi3mr_queue_qd_reduction_event(mrioc, tg);
4855 			}
4856 		} else {
4857 			ioc_pend_data_len = atomic_add_return(data_len_blks,
4858 			    &mrioc->pend_large_data_sz);
4859 			if (ioc_pend_data_len >= mrioc->io_throttle_high)
4860 				stgt_priv_data->io_divert = 1;
4861 		}
4862 	}
4863 
4864 	if (stgt_priv_data->io_divert) {
4865 		scsiio_req->msg_flags |=
4866 		    MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE;
4867 		scsiio_flags |= MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING;
4868 	}
4869 	scsiio_req->flags = cpu_to_le32(scsiio_flags);
4870 
4871 	if (mpi3mr_op_request_post(mrioc, op_req_q,
4872 	    scmd_priv_data->mpi3mr_scsiio_req)) {
4873 		mpi3mr_clear_scmd_priv(mrioc, scmd);
4874 		retval = SCSI_MLQUEUE_HOST_BUSY;
4875 		if (tracked_io_sz) {
4876 			atomic_sub(tracked_io_sz, &mrioc->pend_large_data_sz);
4877 			if (tg)
4878 				atomic_sub(tracked_io_sz,
4879 				    &tg->pend_large_data_sz);
4880 		}
4881 		goto out;
4882 	}
4883 
4884 out:
4885 	return retval;
4886 }
4887 
4888 static const struct scsi_host_template mpi3mr_driver_template = {
4889 	.module				= THIS_MODULE,
4890 	.name				= "MPI3 Storage Controller",
4891 	.proc_name			= MPI3MR_DRIVER_NAME,
4892 	.queuecommand			= mpi3mr_qcmd,
4893 	.target_alloc			= mpi3mr_target_alloc,
4894 	.slave_alloc			= mpi3mr_slave_alloc,
4895 	.slave_configure		= mpi3mr_slave_configure,
4896 	.target_destroy			= mpi3mr_target_destroy,
4897 	.slave_destroy			= mpi3mr_slave_destroy,
4898 	.scan_finished			= mpi3mr_scan_finished,
4899 	.scan_start			= mpi3mr_scan_start,
4900 	.change_queue_depth		= mpi3mr_change_queue_depth,
4901 	.eh_device_reset_handler	= mpi3mr_eh_dev_reset,
4902 	.eh_target_reset_handler	= mpi3mr_eh_target_reset,
4903 	.eh_host_reset_handler		= mpi3mr_eh_host_reset,
4904 	.bios_param			= mpi3mr_bios_param,
4905 	.map_queues			= mpi3mr_map_queues,
4906 	.mq_poll                        = mpi3mr_blk_mq_poll,
4907 	.no_write_same			= 1,
4908 	.can_queue			= 1,
4909 	.this_id			= -1,
4910 	.sg_tablesize			= MPI3MR_DEFAULT_SGL_ENTRIES,
4911 	/* max xfer supported is 1M (2K in 512 byte sized sectors)
4912 	 */
4913 	.max_sectors			= (MPI3MR_DEFAULT_MAX_IO_SIZE / 512),
4914 	.cmd_per_lun			= MPI3MR_MAX_CMDS_LUN,
4915 	.max_segment_size		= 0xffffffff,
4916 	.track_queue_depth		= 1,
4917 	.cmd_size			= sizeof(struct scmd_priv),
4918 	.shost_groups			= mpi3mr_host_groups,
4919 	.sdev_groups			= mpi3mr_dev_groups,
4920 };
4921 
4922 /**
4923  * mpi3mr_init_drv_cmd - Initialize internal command tracker
4924  * @cmdptr: Internal command tracker
4925  * @host_tag: Host tag used for the specific command
4926  *
4927  * Initialize the internal command tracker structure with
4928  * specified host tag.
4929  *
4930  * Return: Nothing.
4931  */
4932 static inline void mpi3mr_init_drv_cmd(struct mpi3mr_drv_cmd *cmdptr,
4933 	u16 host_tag)
4934 {
4935 	mutex_init(&cmdptr->mutex);
4936 	cmdptr->reply = NULL;
4937 	cmdptr->state = MPI3MR_CMD_NOTUSED;
4938 	cmdptr->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
4939 	cmdptr->host_tag = host_tag;
4940 }
4941 
4942 /**
4943  * osintfc_mrioc_security_status -Check controller secure status
4944  * @pdev: PCI device instance
4945  *
4946  * Read the Device Serial Number capability from PCI config
4947  * space and decide whether the controller is secure or not.
4948  *
4949  * Return: 0 on success, non-zero on failure.
4950  */
4951 static int
4952 osintfc_mrioc_security_status(struct pci_dev *pdev)
4953 {
4954 	u32 cap_data;
4955 	int base;
4956 	u32 ctlr_status;
4957 	u32 debug_status;
4958 	int retval = 0;
4959 
4960 	base = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
4961 	if (!base) {
4962 		dev_err(&pdev->dev,
4963 		    "%s: PCI_EXT_CAP_ID_DSN is not supported\n", __func__);
4964 		return -1;
4965 	}
4966 
4967 	pci_read_config_dword(pdev, base + 4, &cap_data);
4968 
4969 	debug_status = cap_data & MPI3MR_CTLR_SECURE_DBG_STATUS_MASK;
4970 	ctlr_status = cap_data & MPI3MR_CTLR_SECURITY_STATUS_MASK;
4971 
4972 	switch (ctlr_status) {
4973 	case MPI3MR_INVALID_DEVICE:
4974 		dev_err(&pdev->dev,
4975 		    "%s: Non secure ctlr (Invalid) is detected: DID: 0x%x: SVID: 0x%x: SDID: 0x%x\n",
4976 		    __func__, pdev->device, pdev->subsystem_vendor,
4977 		    pdev->subsystem_device);
4978 		retval = -1;
4979 		break;
4980 	case MPI3MR_CONFIG_SECURE_DEVICE:
4981 		if (!debug_status)
4982 			dev_info(&pdev->dev,
4983 			    "%s: Config secure ctlr is detected\n",
4984 			    __func__);
4985 		break;
4986 	case MPI3MR_HARD_SECURE_DEVICE:
4987 		break;
4988 	case MPI3MR_TAMPERED_DEVICE:
4989 		dev_err(&pdev->dev,
4990 		    "%s: Non secure ctlr (Tampered) is detected: DID: 0x%x: SVID: 0x%x: SDID: 0x%x\n",
4991 		    __func__, pdev->device, pdev->subsystem_vendor,
4992 		    pdev->subsystem_device);
4993 		retval = -1;
4994 		break;
4995 	default:
4996 		retval = -1;
4997 			break;
4998 	}
4999 
5000 	if (!retval && debug_status) {
5001 		dev_err(&pdev->dev,
5002 		    "%s: Non secure ctlr (Secure Dbg) is detected: DID: 0x%x: SVID: 0x%x: SDID: 0x%x\n",
5003 		    __func__, pdev->device, pdev->subsystem_vendor,
5004 		    pdev->subsystem_device);
5005 		retval = -1;
5006 	}
5007 
5008 	return retval;
5009 }
5010 
5011 /**
5012  * mpi3mr_probe - PCI probe callback
5013  * @pdev: PCI device instance
5014  * @id: PCI device ID details
5015  *
5016  * controller initialization routine. Checks the security status
5017  * of the controller and if it is invalid or tampered return the
5018  * probe without initializing the controller. Otherwise,
5019  * allocate per adapter instance through shost_priv and
5020  * initialize controller specific data structures, initializae
5021  * the controller hardware, add shost to the SCSI subsystem.
5022  *
5023  * Return: 0 on success, non-zero on failure.
5024  */
5025 
5026 static int
5027 mpi3mr_probe(struct pci_dev *pdev, const struct pci_device_id *id)
5028 {
5029 	struct mpi3mr_ioc *mrioc = NULL;
5030 	struct Scsi_Host *shost = NULL;
5031 	int retval = 0, i;
5032 
5033 	if (osintfc_mrioc_security_status(pdev)) {
5034 		warn_non_secure_ctlr = 1;
5035 		return 1; /* For Invalid and Tampered device */
5036 	}
5037 
5038 	shost = scsi_host_alloc(&mpi3mr_driver_template,
5039 	    sizeof(struct mpi3mr_ioc));
5040 	if (!shost) {
5041 		retval = -ENODEV;
5042 		goto shost_failed;
5043 	}
5044 
5045 	mrioc = shost_priv(shost);
5046 	mrioc->id = mrioc_ids++;
5047 	sprintf(mrioc->driver_name, "%s", MPI3MR_DRIVER_NAME);
5048 	sprintf(mrioc->name, "%s%d", mrioc->driver_name, mrioc->id);
5049 	INIT_LIST_HEAD(&mrioc->list);
5050 	spin_lock(&mrioc_list_lock);
5051 	list_add_tail(&mrioc->list, &mrioc_list);
5052 	spin_unlock(&mrioc_list_lock);
5053 
5054 	spin_lock_init(&mrioc->admin_req_lock);
5055 	spin_lock_init(&mrioc->reply_free_queue_lock);
5056 	spin_lock_init(&mrioc->sbq_lock);
5057 	spin_lock_init(&mrioc->fwevt_lock);
5058 	spin_lock_init(&mrioc->tgtdev_lock);
5059 	spin_lock_init(&mrioc->watchdog_lock);
5060 	spin_lock_init(&mrioc->chain_buf_lock);
5061 	spin_lock_init(&mrioc->sas_node_lock);
5062 
5063 	INIT_LIST_HEAD(&mrioc->fwevt_list);
5064 	INIT_LIST_HEAD(&mrioc->tgtdev_list);
5065 	INIT_LIST_HEAD(&mrioc->delayed_rmhs_list);
5066 	INIT_LIST_HEAD(&mrioc->delayed_evtack_cmds_list);
5067 	INIT_LIST_HEAD(&mrioc->sas_expander_list);
5068 	INIT_LIST_HEAD(&mrioc->hba_port_table_list);
5069 	INIT_LIST_HEAD(&mrioc->enclosure_list);
5070 
5071 	mutex_init(&mrioc->reset_mutex);
5072 	mpi3mr_init_drv_cmd(&mrioc->init_cmds, MPI3MR_HOSTTAG_INITCMDS);
5073 	mpi3mr_init_drv_cmd(&mrioc->host_tm_cmds, MPI3MR_HOSTTAG_BLK_TMS);
5074 	mpi3mr_init_drv_cmd(&mrioc->bsg_cmds, MPI3MR_HOSTTAG_BSG_CMDS);
5075 	mpi3mr_init_drv_cmd(&mrioc->cfg_cmds, MPI3MR_HOSTTAG_CFG_CMDS);
5076 	mpi3mr_init_drv_cmd(&mrioc->transport_cmds,
5077 	    MPI3MR_HOSTTAG_TRANSPORT_CMDS);
5078 
5079 	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
5080 		mpi3mr_init_drv_cmd(&mrioc->dev_rmhs_cmds[i],
5081 		    MPI3MR_HOSTTAG_DEVRMCMD_MIN + i);
5082 
5083 	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++)
5084 		mpi3mr_init_drv_cmd(&mrioc->evtack_cmds[i],
5085 				    MPI3MR_HOSTTAG_EVTACKCMD_MIN + i);
5086 
5087 	if (pdev->revision)
5088 		mrioc->enable_segqueue = true;
5089 
5090 	init_waitqueue_head(&mrioc->reset_waitq);
5091 	mrioc->logging_level = logging_level;
5092 	mrioc->shost = shost;
5093 	mrioc->pdev = pdev;
5094 	mrioc->stop_bsgs = 1;
5095 
5096 	mrioc->max_sgl_entries = max_sgl_entries;
5097 	if (max_sgl_entries > MPI3MR_MAX_SGL_ENTRIES)
5098 		mrioc->max_sgl_entries = MPI3MR_MAX_SGL_ENTRIES;
5099 	else if (max_sgl_entries < MPI3MR_DEFAULT_SGL_ENTRIES)
5100 		mrioc->max_sgl_entries = MPI3MR_DEFAULT_SGL_ENTRIES;
5101 	else {
5102 		mrioc->max_sgl_entries /= MPI3MR_DEFAULT_SGL_ENTRIES;
5103 		mrioc->max_sgl_entries *= MPI3MR_DEFAULT_SGL_ENTRIES;
5104 	}
5105 
5106 	/* init shost parameters */
5107 	shost->max_cmd_len = MPI3MR_MAX_CDB_LENGTH;
5108 	shost->max_lun = -1;
5109 	shost->unique_id = mrioc->id;
5110 
5111 	shost->max_channel = 0;
5112 	shost->max_id = 0xFFFFFFFF;
5113 
5114 	shost->host_tagset = 1;
5115 
5116 	if (prot_mask >= 0)
5117 		scsi_host_set_prot(shost, prot_mask);
5118 	else {
5119 		prot_mask = SHOST_DIF_TYPE1_PROTECTION
5120 		    | SHOST_DIF_TYPE2_PROTECTION
5121 		    | SHOST_DIF_TYPE3_PROTECTION;
5122 		scsi_host_set_prot(shost, prot_mask);
5123 	}
5124 
5125 	ioc_info(mrioc,
5126 	    "%s :host protection capabilities enabled %s%s%s%s%s%s%s\n",
5127 	    __func__,
5128 	    (prot_mask & SHOST_DIF_TYPE1_PROTECTION) ? " DIF1" : "",
5129 	    (prot_mask & SHOST_DIF_TYPE2_PROTECTION) ? " DIF2" : "",
5130 	    (prot_mask & SHOST_DIF_TYPE3_PROTECTION) ? " DIF3" : "",
5131 	    (prot_mask & SHOST_DIX_TYPE0_PROTECTION) ? " DIX0" : "",
5132 	    (prot_mask & SHOST_DIX_TYPE1_PROTECTION) ? " DIX1" : "",
5133 	    (prot_mask & SHOST_DIX_TYPE2_PROTECTION) ? " DIX2" : "",
5134 	    (prot_mask & SHOST_DIX_TYPE3_PROTECTION) ? " DIX3" : "");
5135 
5136 	if (prot_guard_mask)
5137 		scsi_host_set_guard(shost, (prot_guard_mask & 3));
5138 	else
5139 		scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
5140 
5141 	snprintf(mrioc->fwevt_worker_name, sizeof(mrioc->fwevt_worker_name),
5142 	    "%s%d_fwevt_wrkr", mrioc->driver_name, mrioc->id);
5143 	mrioc->fwevt_worker_thread = alloc_ordered_workqueue(
5144 	    mrioc->fwevt_worker_name, 0);
5145 	if (!mrioc->fwevt_worker_thread) {
5146 		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
5147 		    __FILE__, __LINE__, __func__);
5148 		retval = -ENODEV;
5149 		goto fwevtthread_failed;
5150 	}
5151 
5152 	mrioc->is_driver_loading = 1;
5153 	mrioc->cpu_count = num_online_cpus();
5154 	if (mpi3mr_setup_resources(mrioc)) {
5155 		ioc_err(mrioc, "setup resources failed\n");
5156 		retval = -ENODEV;
5157 		goto resource_alloc_failed;
5158 	}
5159 	if (mpi3mr_init_ioc(mrioc)) {
5160 		ioc_err(mrioc, "initializing IOC failed\n");
5161 		retval = -ENODEV;
5162 		goto init_ioc_failed;
5163 	}
5164 
5165 	shost->nr_hw_queues = mrioc->num_op_reply_q;
5166 	if (mrioc->active_poll_qcount)
5167 		shost->nr_maps = 3;
5168 
5169 	shost->can_queue = mrioc->max_host_ios;
5170 	shost->sg_tablesize = mrioc->max_sgl_entries;
5171 	shost->max_id = mrioc->facts.max_perids + 1;
5172 
5173 	retval = scsi_add_host(shost, &pdev->dev);
5174 	if (retval) {
5175 		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
5176 		    __FILE__, __LINE__, __func__);
5177 		goto addhost_failed;
5178 	}
5179 
5180 	scsi_scan_host(shost);
5181 	mpi3mr_bsg_init(mrioc);
5182 	return retval;
5183 
5184 addhost_failed:
5185 	mpi3mr_stop_watchdog(mrioc);
5186 	mpi3mr_cleanup_ioc(mrioc);
5187 init_ioc_failed:
5188 	mpi3mr_free_mem(mrioc);
5189 	mpi3mr_cleanup_resources(mrioc);
5190 resource_alloc_failed:
5191 	destroy_workqueue(mrioc->fwevt_worker_thread);
5192 fwevtthread_failed:
5193 	spin_lock(&mrioc_list_lock);
5194 	list_del(&mrioc->list);
5195 	spin_unlock(&mrioc_list_lock);
5196 	scsi_host_put(shost);
5197 shost_failed:
5198 	return retval;
5199 }
5200 
5201 /**
5202  * mpi3mr_remove - PCI remove callback
5203  * @pdev: PCI device instance
5204  *
5205  * Cleanup the IOC by issuing MUR and shutdown notification.
5206  * Free up all memory and resources associated with the
5207  * controllerand target devices, unregister the shost.
5208  *
5209  * Return: Nothing.
5210  */
5211 static void mpi3mr_remove(struct pci_dev *pdev)
5212 {
5213 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5214 	struct mpi3mr_ioc *mrioc;
5215 	struct workqueue_struct	*wq;
5216 	unsigned long flags;
5217 	struct mpi3mr_tgt_dev *tgtdev, *tgtdev_next;
5218 	struct mpi3mr_hba_port *port, *hba_port_next;
5219 	struct mpi3mr_sas_node *sas_expander, *sas_expander_next;
5220 
5221 	if (!shost)
5222 		return;
5223 
5224 	mrioc = shost_priv(shost);
5225 	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
5226 		ssleep(1);
5227 
5228 	if (!pci_device_is_present(mrioc->pdev)) {
5229 		mrioc->unrecoverable = 1;
5230 		mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
5231 	}
5232 
5233 	mpi3mr_bsg_exit(mrioc);
5234 	mrioc->stop_drv_processing = 1;
5235 	mpi3mr_cleanup_fwevt_list(mrioc);
5236 	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
5237 	wq = mrioc->fwevt_worker_thread;
5238 	mrioc->fwevt_worker_thread = NULL;
5239 	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
5240 	if (wq)
5241 		destroy_workqueue(wq);
5242 
5243 	if (mrioc->sas_transport_enabled)
5244 		sas_remove_host(shost);
5245 	else
5246 		scsi_remove_host(shost);
5247 
5248 	list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list,
5249 	    list) {
5250 		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
5251 		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev, true);
5252 		mpi3mr_tgtdev_put(tgtdev);
5253 	}
5254 	mpi3mr_stop_watchdog(mrioc);
5255 	mpi3mr_cleanup_ioc(mrioc);
5256 	mpi3mr_free_mem(mrioc);
5257 	mpi3mr_cleanup_resources(mrioc);
5258 
5259 	spin_lock_irqsave(&mrioc->sas_node_lock, flags);
5260 	list_for_each_entry_safe_reverse(sas_expander, sas_expander_next,
5261 	    &mrioc->sas_expander_list, list) {
5262 		spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
5263 		mpi3mr_expander_node_remove(mrioc, sas_expander);
5264 		spin_lock_irqsave(&mrioc->sas_node_lock, flags);
5265 	}
5266 	list_for_each_entry_safe(port, hba_port_next, &mrioc->hba_port_table_list, list) {
5267 		ioc_info(mrioc,
5268 		    "removing hba_port entry: %p port: %d from hba_port list\n",
5269 		    port, port->port_id);
5270 		list_del(&port->list);
5271 		kfree(port);
5272 	}
5273 	spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
5274 
5275 	if (mrioc->sas_hba.num_phys) {
5276 		kfree(mrioc->sas_hba.phy);
5277 		mrioc->sas_hba.phy = NULL;
5278 		mrioc->sas_hba.num_phys = 0;
5279 	}
5280 
5281 	spin_lock(&mrioc_list_lock);
5282 	list_del(&mrioc->list);
5283 	spin_unlock(&mrioc_list_lock);
5284 
5285 	scsi_host_put(shost);
5286 }
5287 
5288 /**
5289  * mpi3mr_shutdown - PCI shutdown callback
5290  * @pdev: PCI device instance
5291  *
5292  * Free up all memory and resources associated with the
5293  * controller
5294  *
5295  * Return: Nothing.
5296  */
5297 static void mpi3mr_shutdown(struct pci_dev *pdev)
5298 {
5299 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5300 	struct mpi3mr_ioc *mrioc;
5301 	struct workqueue_struct	*wq;
5302 	unsigned long flags;
5303 
5304 	if (!shost)
5305 		return;
5306 
5307 	mrioc = shost_priv(shost);
5308 	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
5309 		ssleep(1);
5310 
5311 	mrioc->stop_drv_processing = 1;
5312 	mpi3mr_cleanup_fwevt_list(mrioc);
5313 	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
5314 	wq = mrioc->fwevt_worker_thread;
5315 	mrioc->fwevt_worker_thread = NULL;
5316 	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
5317 	if (wq)
5318 		destroy_workqueue(wq);
5319 
5320 	mpi3mr_stop_watchdog(mrioc);
5321 	mpi3mr_cleanup_ioc(mrioc);
5322 	mpi3mr_cleanup_resources(mrioc);
5323 }
5324 
5325 /**
5326  * mpi3mr_suspend - PCI power management suspend callback
5327  * @dev: Device struct
5328  *
5329  * Change the power state to the given value and cleanup the IOC
5330  * by issuing MUR and shutdown notification
5331  *
5332  * Return: 0 always.
5333  */
5334 static int __maybe_unused
5335 mpi3mr_suspend(struct device *dev)
5336 {
5337 	struct pci_dev *pdev = to_pci_dev(dev);
5338 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5339 	struct mpi3mr_ioc *mrioc;
5340 
5341 	if (!shost)
5342 		return 0;
5343 
5344 	mrioc = shost_priv(shost);
5345 	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
5346 		ssleep(1);
5347 	mrioc->stop_drv_processing = 1;
5348 	mpi3mr_cleanup_fwevt_list(mrioc);
5349 	scsi_block_requests(shost);
5350 	mpi3mr_stop_watchdog(mrioc);
5351 	mpi3mr_cleanup_ioc(mrioc);
5352 
5353 	ioc_info(mrioc, "pdev=0x%p, slot=%s, entering operating state\n",
5354 	    pdev, pci_name(pdev));
5355 	mpi3mr_cleanup_resources(mrioc);
5356 
5357 	return 0;
5358 }
5359 
5360 /**
5361  * mpi3mr_resume - PCI power management resume callback
5362  * @dev: Device struct
5363  *
5364  * Restore the power state to D0 and reinitialize the controller
5365  * and resume I/O operations to the target devices
5366  *
5367  * Return: 0 on success, non-zero on failure
5368  */
5369 static int __maybe_unused
5370 mpi3mr_resume(struct device *dev)
5371 {
5372 	struct pci_dev *pdev = to_pci_dev(dev);
5373 	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5374 	struct mpi3mr_ioc *mrioc;
5375 	pci_power_t device_state = pdev->current_state;
5376 	int r;
5377 
5378 	if (!shost)
5379 		return 0;
5380 
5381 	mrioc = shost_priv(shost);
5382 
5383 	ioc_info(mrioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",
5384 	    pdev, pci_name(pdev), device_state);
5385 	mrioc->pdev = pdev;
5386 	mrioc->cpu_count = num_online_cpus();
5387 	r = mpi3mr_setup_resources(mrioc);
5388 	if (r) {
5389 		ioc_info(mrioc, "%s: Setup resources failed[%d]\n",
5390 		    __func__, r);
5391 		return r;
5392 	}
5393 
5394 	mrioc->stop_drv_processing = 0;
5395 	mpi3mr_invalidate_devhandles(mrioc);
5396 	mpi3mr_free_enclosure_list(mrioc);
5397 	mpi3mr_memset_buffers(mrioc);
5398 	r = mpi3mr_reinit_ioc(mrioc, 1);
5399 	if (r) {
5400 		ioc_err(mrioc, "resuming controller failed[%d]\n", r);
5401 		return r;
5402 	}
5403 	ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
5404 	scsi_unblock_requests(shost);
5405 	mrioc->device_refresh_on = 0;
5406 	mpi3mr_start_watchdog(mrioc);
5407 
5408 	return 0;
5409 }
5410 
5411 static const struct pci_device_id mpi3mr_pci_id_table[] = {
5412 	{
5413 		PCI_DEVICE_SUB(MPI3_MFGPAGE_VENDORID_BROADCOM,
5414 		    MPI3_MFGPAGE_DEVID_SAS4116, PCI_ANY_ID, PCI_ANY_ID)
5415 	},
5416 	{ 0 }
5417 };
5418 MODULE_DEVICE_TABLE(pci, mpi3mr_pci_id_table);
5419 
5420 static SIMPLE_DEV_PM_OPS(mpi3mr_pm_ops, mpi3mr_suspend, mpi3mr_resume);
5421 
5422 static struct pci_driver mpi3mr_pci_driver = {
5423 	.name = MPI3MR_DRIVER_NAME,
5424 	.id_table = mpi3mr_pci_id_table,
5425 	.probe = mpi3mr_probe,
5426 	.remove = mpi3mr_remove,
5427 	.shutdown = mpi3mr_shutdown,
5428 	.driver.pm = &mpi3mr_pm_ops,
5429 };
5430 
5431 static ssize_t event_counter_show(struct device_driver *dd, char *buf)
5432 {
5433 	return sprintf(buf, "%llu\n", atomic64_read(&event_counter));
5434 }
5435 static DRIVER_ATTR_RO(event_counter);
5436 
5437 static int __init mpi3mr_init(void)
5438 {
5439 	int ret_val;
5440 
5441 	pr_info("Loading %s version %s\n", MPI3MR_DRIVER_NAME,
5442 	    MPI3MR_DRIVER_VERSION);
5443 
5444 	mpi3mr_transport_template =
5445 	    sas_attach_transport(&mpi3mr_transport_functions);
5446 	if (!mpi3mr_transport_template) {
5447 		pr_err("%s failed to load due to sas transport attach failure\n",
5448 		    MPI3MR_DRIVER_NAME);
5449 		return -ENODEV;
5450 	}
5451 
5452 	ret_val = pci_register_driver(&mpi3mr_pci_driver);
5453 	if (ret_val) {
5454 		pr_err("%s failed to load due to pci register driver failure\n",
5455 		    MPI3MR_DRIVER_NAME);
5456 		goto err_pci_reg_fail;
5457 	}
5458 
5459 	ret_val = driver_create_file(&mpi3mr_pci_driver.driver,
5460 				     &driver_attr_event_counter);
5461 	if (ret_val)
5462 		goto err_event_counter;
5463 
5464 	return ret_val;
5465 
5466 err_event_counter:
5467 	pci_unregister_driver(&mpi3mr_pci_driver);
5468 
5469 err_pci_reg_fail:
5470 	sas_release_transport(mpi3mr_transport_template);
5471 	return ret_val;
5472 }
5473 
5474 static void __exit mpi3mr_exit(void)
5475 {
5476 	if (warn_non_secure_ctlr)
5477 		pr_warn(
5478 		    "Unloading %s version %s while managing a non secure controller\n",
5479 		    MPI3MR_DRIVER_NAME, MPI3MR_DRIVER_VERSION);
5480 	else
5481 		pr_info("Unloading %s version %s\n", MPI3MR_DRIVER_NAME,
5482 		    MPI3MR_DRIVER_VERSION);
5483 
5484 	driver_remove_file(&mpi3mr_pci_driver.driver,
5485 			   &driver_attr_event_counter);
5486 	pci_unregister_driver(&mpi3mr_pci_driver);
5487 	sas_release_transport(mpi3mr_transport_template);
5488 }
5489 
5490 module_init(mpi3mr_init);
5491 module_exit(mpi3mr_exit);
5492