1 /*
2  * This is the Fusion MPT base driver providing common API layer interface
3  * for access to MPT (Message Passing Technology) firmware.
4  *
5  * This code is based on drivers/scsi/mpt3sas/mpt3sas_base.c
6  * Copyright (C) 2012-2014  LSI Corporation
7  * Copyright (C) 2013-2014 Avago Technologies
8  *  (mailto: MPT-FusionLinux.pdl@avagotech.com)
9  *
10  * This program is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU General Public License
12  * as published by the Free Software Foundation; either version 2
13  * of the License, or (at your option) any later version.
14  *
15  * This program is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
18  * GNU General Public License for more details.
19  *
20  * NO WARRANTY
21  * THE PROGRAM IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OR
22  * CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT
23  * LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT,
24  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is
25  * solely responsible for determining the appropriateness of using and
26  * distributing the Program and assumes all risks associated with its
27  * exercise of rights under this Agreement, including but not limited to
28  * the risks and costs of program errors, damage to or loss of data,
29  * programs or equipment, and unavailability or interruption of operations.
30 
31  * DISCLAIMER OF LIABILITY
32  * NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY
33  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34  * DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND
35  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
36  * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
37  * USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED
38  * HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES
39 
40  * You should have received a copy of the GNU General Public License
41  * along with this program; if not, write to the Free Software
42  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301,
43  * USA.
44  */
45 
46 #include <linux/kernel.h>
47 #include <linux/module.h>
48 #include <linux/errno.h>
49 #include <linux/init.h>
50 #include <linux/slab.h>
51 #include <linux/types.h>
52 #include <linux/pci.h>
53 #include <linux/kdev_t.h>
54 #include <linux/blkdev.h>
55 #include <linux/delay.h>
56 #include <linux/interrupt.h>
57 #include <linux/dma-mapping.h>
58 #include <linux/io.h>
59 #include <linux/time.h>
60 #include <linux/ktime.h>
61 #include <linux/kthread.h>
62 #include <asm/page.h>        /* To get host page size per arch */
63 #include <linux/aer.h>
64 
65 
66 #include "mpt3sas_base.h"
67 
68 static MPT_CALLBACK	mpt_callbacks[MPT_MAX_CALLBACKS];
69 
70 
71 #define FAULT_POLLING_INTERVAL 1000 /* in milliseconds */
72 
73  /* maximum controller queue depth */
74 #define MAX_HBA_QUEUE_DEPTH	30000
75 #define MAX_CHAIN_DEPTH		100000
76 static int max_queue_depth = -1;
77 module_param(max_queue_depth, int, 0444);
78 MODULE_PARM_DESC(max_queue_depth, " max controller queue depth ");
79 
80 static int max_sgl_entries = -1;
81 module_param(max_sgl_entries, int, 0444);
82 MODULE_PARM_DESC(max_sgl_entries, " max sg entries ");
83 
84 static int msix_disable = -1;
85 module_param(msix_disable, int, 0444);
86 MODULE_PARM_DESC(msix_disable, " disable msix routed interrupts (default=0)");
87 
88 static int smp_affinity_enable = 1;
89 module_param(smp_affinity_enable, int, 0444);
90 MODULE_PARM_DESC(smp_affinity_enable, "SMP affinity feature enable/disable Default: enable(1)");
91 
92 static int max_msix_vectors = -1;
93 module_param(max_msix_vectors, int, 0444);
94 MODULE_PARM_DESC(max_msix_vectors,
95 	" max msix vectors");
96 
97 static int irqpoll_weight = -1;
98 module_param(irqpoll_weight, int, 0444);
99 MODULE_PARM_DESC(irqpoll_weight,
100 	"irq poll weight (default= one fourth of HBA queue depth)");
101 
102 static int mpt3sas_fwfault_debug;
103 MODULE_PARM_DESC(mpt3sas_fwfault_debug,
104 	" enable detection of firmware fault and halt firmware - (default=0)");
105 
106 static int perf_mode = -1;
107 module_param(perf_mode, int, 0444);
108 MODULE_PARM_DESC(perf_mode,
109 	"Performance mode (only for Aero/Sea Generation), options:\n\t\t"
110 	"0 - balanced: high iops mode is enabled &\n\t\t"
111 	"interrupt coalescing is enabled only on high iops queues,\n\t\t"
112 	"1 - iops: high iops mode is disabled &\n\t\t"
113 	"interrupt coalescing is enabled on all queues,\n\t\t"
114 	"2 - latency: high iops mode is disabled &\n\t\t"
115 	"interrupt coalescing is enabled on all queues with timeout value 0xA,\n"
116 	"\t\tdefault - default perf_mode is 'balanced'"
117 	);
118 
119 enum mpt3sas_perf_mode {
120 	MPT_PERF_MODE_DEFAULT	= -1,
121 	MPT_PERF_MODE_BALANCED	= 0,
122 	MPT_PERF_MODE_IOPS	= 1,
123 	MPT_PERF_MODE_LATENCY	= 2,
124 };
125 
126 static int
127 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc,
128 		u32 ioc_state, int timeout);
129 static int
130 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
131 static void
132 _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc);
133 
134 /**
135  * mpt3sas_base_check_cmd_timeout - Function
136  *		to check timeout and command termination due
137  *		to Host reset.
138  *
139  * @ioc:	per adapter object.
140  * @status:	Status of issued command.
141  * @mpi_request:mf request pointer.
142  * @sz:		size of buffer.
143  *
144  * Return: 1/0 Reset to be done or Not
145  */
146 u8
147 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc,
148 		u8 status, void *mpi_request, int sz)
149 {
150 	u8 issue_reset = 0;
151 
152 	if (!(status & MPT3_CMD_RESET))
153 		issue_reset = 1;
154 
155 	ioc_err(ioc, "Command %s\n",
156 		issue_reset == 0 ? "terminated due to Host Reset" : "Timeout");
157 	_debug_dump_mf(mpi_request, sz);
158 
159 	return issue_reset;
160 }
161 
162 /**
163  * _scsih_set_fwfault_debug - global setting of ioc->fwfault_debug.
164  * @val: ?
165  * @kp: ?
166  *
167  * Return: ?
168  */
169 static int
170 _scsih_set_fwfault_debug(const char *val, const struct kernel_param *kp)
171 {
172 	int ret = param_set_int(val, kp);
173 	struct MPT3SAS_ADAPTER *ioc;
174 
175 	if (ret)
176 		return ret;
177 
178 	/* global ioc spinlock to protect controller list on list operations */
179 	pr_info("setting fwfault_debug(%d)\n", mpt3sas_fwfault_debug);
180 	spin_lock(&gioc_lock);
181 	list_for_each_entry(ioc, &mpt3sas_ioc_list, list)
182 		ioc->fwfault_debug = mpt3sas_fwfault_debug;
183 	spin_unlock(&gioc_lock);
184 	return 0;
185 }
186 module_param_call(mpt3sas_fwfault_debug, _scsih_set_fwfault_debug,
187 	param_get_int, &mpt3sas_fwfault_debug, 0644);
188 
189 /**
190  * _base_readl_aero - retry readl for max three times.
191  * @addr: MPT Fusion system interface register address
192  *
193  * Retry the readl() for max three times if it gets zero value
194  * while reading the system interface register.
195  */
196 static inline u32
197 _base_readl_aero(const volatile void __iomem *addr)
198 {
199 	u32 i = 0, ret_val;
200 
201 	do {
202 		ret_val = readl(addr);
203 		i++;
204 	} while (ret_val == 0 && i < 3);
205 
206 	return ret_val;
207 }
208 
209 static inline u32
210 _base_readl(const volatile void __iomem *addr)
211 {
212 	return readl(addr);
213 }
214 
215 /**
216  * _base_clone_reply_to_sys_mem - copies reply to reply free iomem
217  *				  in BAR0 space.
218  *
219  * @ioc: per adapter object
220  * @reply: reply message frame(lower 32bit addr)
221  * @index: System request message index.
222  */
223 static void
224 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply,
225 		u32 index)
226 {
227 	/*
228 	 * 256 is offset within sys register.
229 	 * 256 offset MPI frame starts. Max MPI frame supported is 32.
230 	 * 32 * 128 = 4K. From here, Clone of reply free for mcpu starts
231 	 */
232 	u16 cmd_credit = ioc->facts.RequestCredit + 1;
233 	void __iomem *reply_free_iomem = (void __iomem *)ioc->chip +
234 			MPI_FRAME_START_OFFSET +
235 			(cmd_credit * ioc->request_sz) + (index * sizeof(u32));
236 
237 	writel(reply, reply_free_iomem);
238 }
239 
240 /**
241  * _base_clone_mpi_to_sys_mem - Writes/copies MPI frames
242  *				to system/BAR0 region.
243  *
244  * @dst_iomem: Pointer to the destination location in BAR0 space.
245  * @src: Pointer to the Source data.
246  * @size: Size of data to be copied.
247  */
248 static void
249 _base_clone_mpi_to_sys_mem(void *dst_iomem, void *src, u32 size)
250 {
251 	int i;
252 	u32 *src_virt_mem = (u32 *)src;
253 
254 	for (i = 0; i < size/4; i++)
255 		writel((u32)src_virt_mem[i],
256 				(void __iomem *)dst_iomem + (i * 4));
257 }
258 
259 /**
260  * _base_clone_to_sys_mem - Writes/copies data to system/BAR0 region
261  *
262  * @dst_iomem: Pointer to the destination location in BAR0 space.
263  * @src: Pointer to the Source data.
264  * @size: Size of data to be copied.
265  */
266 static void
267 _base_clone_to_sys_mem(void __iomem *dst_iomem, void *src, u32 size)
268 {
269 	int i;
270 	u32 *src_virt_mem = (u32 *)(src);
271 
272 	for (i = 0; i < size/4; i++)
273 		writel((u32)src_virt_mem[i],
274 			(void __iomem *)dst_iomem + (i * 4));
275 }
276 
277 /**
278  * _base_get_chain - Calculates and Returns virtual chain address
279  *			 for the provided smid in BAR0 space.
280  *
281  * @ioc: per adapter object
282  * @smid: system request message index
283  * @sge_chain_count: Scatter gather chain count.
284  *
285  * Return: the chain address.
286  */
287 static inline void __iomem*
288 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,
289 		u8 sge_chain_count)
290 {
291 	void __iomem *base_chain, *chain_virt;
292 	u16 cmd_credit = ioc->facts.RequestCredit + 1;
293 
294 	base_chain  = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +
295 		(cmd_credit * ioc->request_sz) +
296 		REPLY_FREE_POOL_SIZE;
297 	chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *
298 			ioc->request_sz) + (sge_chain_count * ioc->request_sz);
299 	return chain_virt;
300 }
301 
302 /**
303  * _base_get_chain_phys - Calculates and Returns physical address
304  *			in BAR0 for scatter gather chains, for
305  *			the provided smid.
306  *
307  * @ioc: per adapter object
308  * @smid: system request message index
309  * @sge_chain_count: Scatter gather chain count.
310  *
311  * Return: Physical chain address.
312  */
313 static inline phys_addr_t
314 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,
315 		u8 sge_chain_count)
316 {
317 	phys_addr_t base_chain_phys, chain_phys;
318 	u16 cmd_credit = ioc->facts.RequestCredit + 1;
319 
320 	base_chain_phys  = ioc->chip_phys + MPI_FRAME_START_OFFSET +
321 		(cmd_credit * ioc->request_sz) +
322 		REPLY_FREE_POOL_SIZE;
323 	chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *
324 			ioc->request_sz) + (sge_chain_count * ioc->request_sz);
325 	return chain_phys;
326 }
327 
328 /**
329  * _base_get_buffer_bar0 - Calculates and Returns BAR0 mapped Host
330  *			buffer address for the provided smid.
331  *			(Each smid can have 64K starts from 17024)
332  *
333  * @ioc: per adapter object
334  * @smid: system request message index
335  *
336  * Return: Pointer to buffer location in BAR0.
337  */
338 
339 static void __iomem *
340 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
341 {
342 	u16 cmd_credit = ioc->facts.RequestCredit + 1;
343 	// Added extra 1 to reach end of chain.
344 	void __iomem *chain_end = _base_get_chain(ioc,
345 			cmd_credit + 1,
346 			ioc->facts.MaxChainDepth);
347 	return chain_end + (smid * 64 * 1024);
348 }
349 
350 /**
351  * _base_get_buffer_phys_bar0 - Calculates and Returns BAR0 mapped
352  *		Host buffer Physical address for the provided smid.
353  *		(Each smid can have 64K starts from 17024)
354  *
355  * @ioc: per adapter object
356  * @smid: system request message index
357  *
358  * Return: Pointer to buffer location in BAR0.
359  */
360 static phys_addr_t
361 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)
362 {
363 	u16 cmd_credit = ioc->facts.RequestCredit + 1;
364 	phys_addr_t chain_end_phys = _base_get_chain_phys(ioc,
365 			cmd_credit + 1,
366 			ioc->facts.MaxChainDepth);
367 	return chain_end_phys + (smid * 64 * 1024);
368 }
369 
370 /**
371  * _base_get_chain_buffer_dma_to_chain_buffer - Iterates chain
372  *			lookup list and Provides chain_buffer
373  *			address for the matching dma address.
374  *			(Each smid can have 64K starts from 17024)
375  *
376  * @ioc: per adapter object
377  * @chain_buffer_dma: Chain buffer dma address.
378  *
379  * Return: Pointer to chain buffer. Or Null on Failure.
380  */
381 static void *
382 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,
383 		dma_addr_t chain_buffer_dma)
384 {
385 	u16 index, j;
386 	struct chain_tracker *ct;
387 
388 	for (index = 0; index < ioc->scsiio_depth; index++) {
389 		for (j = 0; j < ioc->chains_needed_per_io; j++) {
390 			ct = &ioc->chain_lookup[index].chains_per_smid[j];
391 			if (ct && ct->chain_buffer_dma == chain_buffer_dma)
392 				return ct->chain_buffer;
393 		}
394 	}
395 	ioc_info(ioc, "Provided chain_buffer_dma address is not in the lookup list\n");
396 	return NULL;
397 }
398 
399 /**
400  * _clone_sg_entries -	MPI EP's scsiio and config requests
401  *			are handled here. Base function for
402  *			double buffering, before submitting
403  *			the requests.
404  *
405  * @ioc: per adapter object.
406  * @mpi_request: mf request pointer.
407  * @smid: system request message index.
408  */
409 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,
410 		void *mpi_request, u16 smid)
411 {
412 	Mpi2SGESimple32_t *sgel, *sgel_next;
413 	u32  sgl_flags, sge_chain_count = 0;
414 	bool is_write = false;
415 	u16 i = 0;
416 	void __iomem *buffer_iomem;
417 	phys_addr_t buffer_iomem_phys;
418 	void __iomem *buff_ptr;
419 	phys_addr_t buff_ptr_phys;
420 	void __iomem *dst_chain_addr[MCPU_MAX_CHAINS_PER_IO];
421 	void *src_chain_addr[MCPU_MAX_CHAINS_PER_IO];
422 	phys_addr_t dst_addr_phys;
423 	MPI2RequestHeader_t *request_hdr;
424 	struct scsi_cmnd *scmd;
425 	struct scatterlist *sg_scmd = NULL;
426 	int is_scsiio_req = 0;
427 
428 	request_hdr = (MPI2RequestHeader_t *) mpi_request;
429 
430 	if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST) {
431 		Mpi25SCSIIORequest_t *scsiio_request =
432 			(Mpi25SCSIIORequest_t *)mpi_request;
433 		sgel = (Mpi2SGESimple32_t *) &scsiio_request->SGL;
434 		is_scsiio_req = 1;
435 	} else if (request_hdr->Function == MPI2_FUNCTION_CONFIG) {
436 		Mpi2ConfigRequest_t  *config_req =
437 			(Mpi2ConfigRequest_t *)mpi_request;
438 		sgel = (Mpi2SGESimple32_t *) &config_req->PageBufferSGE;
439 	} else
440 		return;
441 
442 	/* From smid we can get scsi_cmd, once we have sg_scmd,
443 	 * we just need to get sg_virt and sg_next to get virtual
444 	 * address associated with sgel->Address.
445 	 */
446 
447 	if (is_scsiio_req) {
448 		/* Get scsi_cmd using smid */
449 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
450 		if (scmd == NULL) {
451 			ioc_err(ioc, "scmd is NULL\n");
452 			return;
453 		}
454 
455 		/* Get sg_scmd from scmd provided */
456 		sg_scmd = scsi_sglist(scmd);
457 	}
458 
459 	/*
460 	 * 0 - 255	System register
461 	 * 256 - 4352	MPI Frame. (This is based on maxCredit 32)
462 	 * 4352 - 4864	Reply_free pool (512 byte is reserved
463 	 *		considering maxCredit 32. Reply need extra
464 	 *		room, for mCPU case kept four times of
465 	 *		maxCredit).
466 	 * 4864 - 17152	SGE chain element. (32cmd * 3 chain of
467 	 *		128 byte size = 12288)
468 	 * 17152 - x	Host buffer mapped with smid.
469 	 *		(Each smid can have 64K Max IO.)
470 	 * BAR0+Last 1K MSIX Addr and Data
471 	 * Total size in use 2113664 bytes of 4MB BAR0
472 	 */
473 
474 	buffer_iomem = _base_get_buffer_bar0(ioc, smid);
475 	buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);
476 
477 	buff_ptr = buffer_iomem;
478 	buff_ptr_phys = buffer_iomem_phys;
479 	WARN_ON(buff_ptr_phys > U32_MAX);
480 
481 	if (le32_to_cpu(sgel->FlagsLength) &
482 			(MPI2_SGE_FLAGS_HOST_TO_IOC << MPI2_SGE_FLAGS_SHIFT))
483 		is_write = true;
484 
485 	for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {
486 
487 		sgl_flags =
488 		    (le32_to_cpu(sgel->FlagsLength) >> MPI2_SGE_FLAGS_SHIFT);
489 
490 		switch (sgl_flags & MPI2_SGE_FLAGS_ELEMENT_MASK) {
491 		case MPI2_SGE_FLAGS_CHAIN_ELEMENT:
492 			/*
493 			 * Helper function which on passing
494 			 * chain_buffer_dma returns chain_buffer. Get
495 			 * the virtual address for sgel->Address
496 			 */
497 			sgel_next =
498 				_base_get_chain_buffer_dma_to_chain_buffer(ioc,
499 						le32_to_cpu(sgel->Address));
500 			if (sgel_next == NULL)
501 				return;
502 			/*
503 			 * This is coping 128 byte chain
504 			 * frame (not a host buffer)
505 			 */
506 			dst_chain_addr[sge_chain_count] =
507 				_base_get_chain(ioc,
508 					smid, sge_chain_count);
509 			src_chain_addr[sge_chain_count] =
510 						(void *) sgel_next;
511 			dst_addr_phys = _base_get_chain_phys(ioc,
512 						smid, sge_chain_count);
513 			WARN_ON(dst_addr_phys > U32_MAX);
514 			sgel->Address =
515 				cpu_to_le32(lower_32_bits(dst_addr_phys));
516 			sgel = sgel_next;
517 			sge_chain_count++;
518 			break;
519 		case MPI2_SGE_FLAGS_SIMPLE_ELEMENT:
520 			if (is_write) {
521 				if (is_scsiio_req) {
522 					_base_clone_to_sys_mem(buff_ptr,
523 					    sg_virt(sg_scmd),
524 					    (le32_to_cpu(sgel->FlagsLength) &
525 					    0x00ffffff));
526 					/*
527 					 * FIXME: this relies on a a zero
528 					 * PCI mem_offset.
529 					 */
530 					sgel->Address =
531 					    cpu_to_le32((u32)buff_ptr_phys);
532 				} else {
533 					_base_clone_to_sys_mem(buff_ptr,
534 					    ioc->config_vaddr,
535 					    (le32_to_cpu(sgel->FlagsLength) &
536 					    0x00ffffff));
537 					sgel->Address =
538 					    cpu_to_le32((u32)buff_ptr_phys);
539 				}
540 			}
541 			buff_ptr += (le32_to_cpu(sgel->FlagsLength) &
542 			    0x00ffffff);
543 			buff_ptr_phys += (le32_to_cpu(sgel->FlagsLength) &
544 			    0x00ffffff);
545 			if ((le32_to_cpu(sgel->FlagsLength) &
546 			    (MPI2_SGE_FLAGS_END_OF_BUFFER
547 					<< MPI2_SGE_FLAGS_SHIFT)))
548 				goto eob_clone_chain;
549 			else {
550 				/*
551 				 * Every single element in MPT will have
552 				 * associated sg_next. Better to sanity that
553 				 * sg_next is not NULL, but it will be a bug
554 				 * if it is null.
555 				 */
556 				if (is_scsiio_req) {
557 					sg_scmd = sg_next(sg_scmd);
558 					if (sg_scmd)
559 						sgel++;
560 					else
561 						goto eob_clone_chain;
562 				}
563 			}
564 			break;
565 		}
566 	}
567 
568 eob_clone_chain:
569 	for (i = 0; i < sge_chain_count; i++) {
570 		if (is_scsiio_req)
571 			_base_clone_to_sys_mem(dst_chain_addr[i],
572 				src_chain_addr[i], ioc->request_sz);
573 	}
574 }
575 
576 /**
577  *  mpt3sas_remove_dead_ioc_func - kthread context to remove dead ioc
578  * @arg: input argument, used to derive ioc
579  *
580  * Return:
581  * 0 if controller is removed from pci subsystem.
582  * -1 for other case.
583  */
584 static int mpt3sas_remove_dead_ioc_func(void *arg)
585 {
586 	struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;
587 	struct pci_dev *pdev;
588 
589 	if (!ioc)
590 		return -1;
591 
592 	pdev = ioc->pdev;
593 	if (!pdev)
594 		return -1;
595 	pci_stop_and_remove_bus_device_locked(pdev);
596 	return 0;
597 }
598 
599 /**
600  * _base_sync_drv_fw_timestamp - Sync Drive-Fw TimeStamp.
601  * @ioc: Per Adapter Object
602  *
603  * Return: nothing.
604  */
605 static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc)
606 {
607 	Mpi26IoUnitControlRequest_t *mpi_request;
608 	Mpi26IoUnitControlReply_t *mpi_reply;
609 	u16 smid;
610 	ktime_t current_time;
611 	u64 TimeStamp = 0;
612 	u8 issue_reset = 0;
613 
614 	mutex_lock(&ioc->scsih_cmds.mutex);
615 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {
616 		ioc_err(ioc, "scsih_cmd in use %s\n", __func__);
617 		goto out;
618 	}
619 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;
620 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);
621 	if (!smid) {
622 		ioc_err(ioc, "Failed obtaining a smid %s\n", __func__);
623 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
624 		goto out;
625 	}
626 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
627 	ioc->scsih_cmds.smid = smid;
628 	memset(mpi_request, 0, sizeof(Mpi26IoUnitControlRequest_t));
629 	mpi_request->Function = MPI2_FUNCTION_IO_UNIT_CONTROL;
630 	mpi_request->Operation = MPI26_CTRL_OP_SET_IOC_PARAMETER;
631 	mpi_request->IOCParameter = MPI26_SET_IOC_PARAMETER_SYNC_TIMESTAMP;
632 	current_time = ktime_get_real();
633 	TimeStamp = ktime_to_ms(current_time);
634 	mpi_request->Reserved7 = cpu_to_le32(TimeStamp & 0xFFFFFFFF);
635 	mpi_request->IOCParameterValue = cpu_to_le32(TimeStamp >> 32);
636 	init_completion(&ioc->scsih_cmds.done);
637 	ioc->put_smid_default(ioc, smid);
638 	dinitprintk(ioc, ioc_info(ioc,
639 	    "Io Unit Control Sync TimeStamp (sending), @time %lld ms\n",
640 	    TimeStamp));
641 	wait_for_completion_timeout(&ioc->scsih_cmds.done,
642 		MPT3SAS_TIMESYNC_TIMEOUT_SECONDS*HZ);
643 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {
644 		mpt3sas_check_cmd_timeout(ioc,
645 		    ioc->scsih_cmds.status, mpi_request,
646 		    sizeof(Mpi2SasIoUnitControlRequest_t)/4, issue_reset);
647 		goto issue_host_reset;
648 	}
649 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {
650 		mpi_reply = ioc->scsih_cmds.reply;
651 		dinitprintk(ioc, ioc_info(ioc,
652 		    "Io Unit Control sync timestamp (complete): ioc_status(0x%04x), loginfo(0x%08x)\n",
653 		    le16_to_cpu(mpi_reply->IOCStatus),
654 		    le32_to_cpu(mpi_reply->IOCLogInfo)));
655 	}
656 issue_host_reset:
657 	if (issue_reset)
658 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
659 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
660 out:
661 	mutex_unlock(&ioc->scsih_cmds.mutex);
662 }
663 
664 /**
665  * _base_fault_reset_work - workq handling ioc fault conditions
666  * @work: input argument, used to derive ioc
667  *
668  * Context: sleep.
669  */
670 static void
671 _base_fault_reset_work(struct work_struct *work)
672 {
673 	struct MPT3SAS_ADAPTER *ioc =
674 	    container_of(work, struct MPT3SAS_ADAPTER, fault_reset_work.work);
675 	unsigned long	 flags;
676 	u32 doorbell;
677 	int rc;
678 	struct task_struct *p;
679 
680 
681 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
682 	if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) ||
683 			ioc->pci_error_recovery)
684 		goto rearm_timer;
685 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
686 
687 	doorbell = mpt3sas_base_get_iocstate(ioc, 0);
688 	if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_MASK) {
689 		ioc_err(ioc, "SAS host is non-operational !!!!\n");
690 
691 		/* It may be possible that EEH recovery can resolve some of
692 		 * pci bus failure issues rather removing the dead ioc function
693 		 * by considering controller is in a non-operational state. So
694 		 * here priority is given to the EEH recovery. If it doesn't
695 		 * not resolve this issue, mpt3sas driver will consider this
696 		 * controller to non-operational state and remove the dead ioc
697 		 * function.
698 		 */
699 		if (ioc->non_operational_loop++ < 5) {
700 			spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,
701 							 flags);
702 			goto rearm_timer;
703 		}
704 
705 		/*
706 		 * Call _scsih_flush_pending_cmds callback so that we flush all
707 		 * pending commands back to OS. This call is required to avoid
708 		 * deadlock at block layer. Dead IOC will fail to do diag reset,
709 		 * and this call is safe since dead ioc will never return any
710 		 * command back from HW.
711 		 */
712 		ioc->schedule_dead_ioc_flush_running_cmds(ioc);
713 		/*
714 		 * Set remove_host flag early since kernel thread will
715 		 * take some time to execute.
716 		 */
717 		ioc->remove_host = 1;
718 		/*Remove the Dead Host */
719 		p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,
720 		    "%s_dead_ioc_%d", ioc->driver_name, ioc->id);
721 		if (IS_ERR(p))
722 			ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",
723 				__func__);
724 		else
725 			ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread success !!!!\n",
726 				__func__);
727 		return; /* don't rearm timer */
728 	}
729 
730 	if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) {
731 		u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?
732 		    ioc->manu_pg11.CoreDumpTOSec :
733 		    MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS;
734 
735 		timeout /= (FAULT_POLLING_INTERVAL/1000);
736 
737 		if (ioc->ioc_coredump_loop == 0) {
738 			mpt3sas_print_coredump_info(ioc,
739 			    doorbell & MPI2_DOORBELL_DATA_MASK);
740 			/* do not accept any IOs and disable the interrupts */
741 			spin_lock_irqsave(
742 			    &ioc->ioc_reset_in_progress_lock, flags);
743 			ioc->shost_recovery = 1;
744 			spin_unlock_irqrestore(
745 			    &ioc->ioc_reset_in_progress_lock, flags);
746 			mpt3sas_base_mask_interrupts(ioc);
747 			_base_clear_outstanding_commands(ioc);
748 		}
749 
750 		ioc_info(ioc, "%s: CoreDump loop %d.",
751 		    __func__, ioc->ioc_coredump_loop);
752 
753 		/* Wait until CoreDump completes or times out */
754 		if (ioc->ioc_coredump_loop++ < timeout) {
755 			spin_lock_irqsave(
756 			    &ioc->ioc_reset_in_progress_lock, flags);
757 			goto rearm_timer;
758 		}
759 	}
760 
761 	if (ioc->ioc_coredump_loop) {
762 		if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_COREDUMP)
763 			ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d",
764 			    __func__, ioc->ioc_coredump_loop);
765 		else
766 			ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d",
767 			    __func__, ioc->ioc_coredump_loop);
768 		ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE;
769 	}
770 	ioc->non_operational_loop = 0;
771 	if ((doorbell & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL) {
772 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
773 		ioc_warn(ioc, "%s: hard reset: %s\n",
774 			 __func__, rc == 0 ? "success" : "failed");
775 		doorbell = mpt3sas_base_get_iocstate(ioc, 0);
776 		if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
777 			mpt3sas_print_fault_code(ioc, doorbell &
778 			    MPI2_DOORBELL_DATA_MASK);
779 		} else if ((doorbell & MPI2_IOC_STATE_MASK) ==
780 		    MPI2_IOC_STATE_COREDUMP)
781 			mpt3sas_print_coredump_info(ioc, doorbell &
782 			    MPI2_DOORBELL_DATA_MASK);
783 		if (rc && (doorbell & MPI2_IOC_STATE_MASK) !=
784 		    MPI2_IOC_STATE_OPERATIONAL)
785 			return; /* don't rearm timer */
786 	}
787 	ioc->ioc_coredump_loop = 0;
788 	if (ioc->time_sync_interval &&
789 	    ++ioc->timestamp_update_count >= ioc->time_sync_interval) {
790 		ioc->timestamp_update_count = 0;
791 		_base_sync_drv_fw_timestamp(ioc);
792 	}
793 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
794  rearm_timer:
795 	if (ioc->fault_reset_work_q)
796 		queue_delayed_work(ioc->fault_reset_work_q,
797 		    &ioc->fault_reset_work,
798 		    msecs_to_jiffies(FAULT_POLLING_INTERVAL));
799 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
800 }
801 
802 /**
803  * mpt3sas_base_start_watchdog - start the fault_reset_work_q
804  * @ioc: per adapter object
805  *
806  * Context: sleep.
807  */
808 void
809 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)
810 {
811 	unsigned long	 flags;
812 
813 	if (ioc->fault_reset_work_q)
814 		return;
815 
816 	ioc->timestamp_update_count = 0;
817 	/* initialize fault polling */
818 
819 	INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);
820 	snprintf(ioc->fault_reset_work_q_name,
821 	    sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",
822 	    ioc->driver_name, ioc->id);
823 	ioc->fault_reset_work_q =
824 		create_singlethread_workqueue(ioc->fault_reset_work_q_name);
825 	if (!ioc->fault_reset_work_q) {
826 		ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__);
827 		return;
828 	}
829 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
830 	if (ioc->fault_reset_work_q)
831 		queue_delayed_work(ioc->fault_reset_work_q,
832 		    &ioc->fault_reset_work,
833 		    msecs_to_jiffies(FAULT_POLLING_INTERVAL));
834 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
835 }
836 
837 /**
838  * mpt3sas_base_stop_watchdog - stop the fault_reset_work_q
839  * @ioc: per adapter object
840  *
841  * Context: sleep.
842  */
843 void
844 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)
845 {
846 	unsigned long flags;
847 	struct workqueue_struct *wq;
848 
849 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
850 	wq = ioc->fault_reset_work_q;
851 	ioc->fault_reset_work_q = NULL;
852 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
853 	if (wq) {
854 		if (!cancel_delayed_work_sync(&ioc->fault_reset_work))
855 			flush_workqueue(wq);
856 		destroy_workqueue(wq);
857 	}
858 }
859 
860 /**
861  * mpt3sas_base_fault_info - verbose translation of firmware FAULT code
862  * @ioc: per adapter object
863  * @fault_code: fault code
864  */
865 void
866 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc , u16 fault_code)
867 {
868 	ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code);
869 }
870 
871 /**
872  * mpt3sas_base_coredump_info - verbose translation of firmware CoreDump state
873  * @ioc: per adapter object
874  * @fault_code: fault code
875  *
876  * Return: nothing.
877  */
878 void
879 mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code)
880 {
881 	ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code);
882 }
883 
884 /**
885  * mpt3sas_base_wait_for_coredump_completion - Wait until coredump
886  * completes or times out
887  * @ioc: per adapter object
888  * @caller: caller function name
889  *
890  * Return: 0 for success, non-zero for failure.
891  */
892 int
893 mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc,
894 		const char *caller)
895 {
896 	u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?
897 			ioc->manu_pg11.CoreDumpTOSec :
898 			MPT3SAS_DEFAULT_COREDUMP_TIMEOUT_SECONDS;
899 
900 	int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT,
901 					timeout);
902 
903 	if (ioc_state)
904 		ioc_err(ioc,
905 		    "%s: CoreDump timed out. (ioc_state=0x%x)\n",
906 		    caller, ioc_state);
907 	else
908 		ioc_info(ioc,
909 		    "%s: CoreDump completed. (ioc_state=0x%x)\n",
910 		    caller, ioc_state);
911 
912 	return ioc_state;
913 }
914 
915 /**
916  * mpt3sas_halt_firmware - halt's mpt controller firmware
917  * @ioc: per adapter object
918  *
919  * For debugging timeout related issues.  Writing 0xCOFFEE00
920  * to the doorbell register will halt controller firmware. With
921  * the purpose to stop both driver and firmware, the enduser can
922  * obtain a ring buffer from controller UART.
923  */
924 void
925 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)
926 {
927 	u32 doorbell;
928 
929 	if (!ioc->fwfault_debug)
930 		return;
931 
932 	dump_stack();
933 
934 	doorbell = ioc->base_readl(&ioc->chip->Doorbell);
935 	if ((doorbell & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
936 		mpt3sas_print_fault_code(ioc, doorbell &
937 		    MPI2_DOORBELL_DATA_MASK);
938 	} else if ((doorbell & MPI2_IOC_STATE_MASK) ==
939 	    MPI2_IOC_STATE_COREDUMP) {
940 		mpt3sas_print_coredump_info(ioc, doorbell &
941 		    MPI2_DOORBELL_DATA_MASK);
942 	} else {
943 		writel(0xC0FFEE00, &ioc->chip->Doorbell);
944 		ioc_err(ioc, "Firmware is halted due to command timeout\n");
945 	}
946 
947 	if (ioc->fwfault_debug == 2)
948 		for (;;)
949 			;
950 	else
951 		panic("panic in %s\n", __func__);
952 }
953 
954 /**
955  * _base_sas_ioc_info - verbose translation of the ioc status
956  * @ioc: per adapter object
957  * @mpi_reply: reply mf payload returned from firmware
958  * @request_hdr: request mf
959  */
960 static void
961 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,
962 	MPI2RequestHeader_t *request_hdr)
963 {
964 	u16 ioc_status = le16_to_cpu(mpi_reply->IOCStatus) &
965 	    MPI2_IOCSTATUS_MASK;
966 	char *desc = NULL;
967 	u16 frame_sz;
968 	char *func_str = NULL;
969 
970 	/* SCSI_IO, RAID_PASS are handled from _scsih_scsi_ioc_info */
971 	if (request_hdr->Function == MPI2_FUNCTION_SCSI_IO_REQUEST ||
972 	    request_hdr->Function == MPI2_FUNCTION_RAID_SCSI_IO_PASSTHROUGH ||
973 	    request_hdr->Function == MPI2_FUNCTION_EVENT_NOTIFICATION)
974 		return;
975 
976 	if (ioc_status == MPI2_IOCSTATUS_CONFIG_INVALID_PAGE)
977 		return;
978 	/*
979 	 * Older Firmware version doesn't support driver trigger pages.
980 	 * So, skip displaying 'config invalid type' type
981 	 * of error message.
982 	 */
983 	if (request_hdr->Function == MPI2_FUNCTION_CONFIG) {
984 		Mpi2ConfigRequest_t *rqst = (Mpi2ConfigRequest_t *)request_hdr;
985 
986 		if ((rqst->ExtPageType ==
987 		    MPI2_CONFIG_EXTPAGETYPE_DRIVER_PERSISTENT_TRIGGER) &&
988 		    !(ioc->logging_level & MPT_DEBUG_CONFIG)) {
989 			return;
990 		}
991 	}
992 
993 	switch (ioc_status) {
994 
995 /****************************************************************************
996 *  Common IOCStatus values for all replies
997 ****************************************************************************/
998 
999 	case MPI2_IOCSTATUS_INVALID_FUNCTION:
1000 		desc = "invalid function";
1001 		break;
1002 	case MPI2_IOCSTATUS_BUSY:
1003 		desc = "busy";
1004 		break;
1005 	case MPI2_IOCSTATUS_INVALID_SGL:
1006 		desc = "invalid sgl";
1007 		break;
1008 	case MPI2_IOCSTATUS_INTERNAL_ERROR:
1009 		desc = "internal error";
1010 		break;
1011 	case MPI2_IOCSTATUS_INVALID_VPID:
1012 		desc = "invalid vpid";
1013 		break;
1014 	case MPI2_IOCSTATUS_INSUFFICIENT_RESOURCES:
1015 		desc = "insufficient resources";
1016 		break;
1017 	case MPI2_IOCSTATUS_INSUFFICIENT_POWER:
1018 		desc = "insufficient power";
1019 		break;
1020 	case MPI2_IOCSTATUS_INVALID_FIELD:
1021 		desc = "invalid field";
1022 		break;
1023 	case MPI2_IOCSTATUS_INVALID_STATE:
1024 		desc = "invalid state";
1025 		break;
1026 	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
1027 		desc = "op state not supported";
1028 		break;
1029 
1030 /****************************************************************************
1031 *  Config IOCStatus values
1032 ****************************************************************************/
1033 
1034 	case MPI2_IOCSTATUS_CONFIG_INVALID_ACTION:
1035 		desc = "config invalid action";
1036 		break;
1037 	case MPI2_IOCSTATUS_CONFIG_INVALID_TYPE:
1038 		desc = "config invalid type";
1039 		break;
1040 	case MPI2_IOCSTATUS_CONFIG_INVALID_PAGE:
1041 		desc = "config invalid page";
1042 		break;
1043 	case MPI2_IOCSTATUS_CONFIG_INVALID_DATA:
1044 		desc = "config invalid data";
1045 		break;
1046 	case MPI2_IOCSTATUS_CONFIG_NO_DEFAULTS:
1047 		desc = "config no defaults";
1048 		break;
1049 	case MPI2_IOCSTATUS_CONFIG_CANT_COMMIT:
1050 		desc = "config cant commit";
1051 		break;
1052 
1053 /****************************************************************************
1054 *  SCSI IO Reply
1055 ****************************************************************************/
1056 
1057 	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
1058 	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
1059 	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1060 	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
1061 	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
1062 	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
1063 	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
1064 	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
1065 	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
1066 	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
1067 	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
1068 	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
1069 		break;
1070 
1071 /****************************************************************************
1072 *  For use by SCSI Initiator and SCSI Target end-to-end data protection
1073 ****************************************************************************/
1074 
1075 	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
1076 		desc = "eedp guard error";
1077 		break;
1078 	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
1079 		desc = "eedp ref tag error";
1080 		break;
1081 	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
1082 		desc = "eedp app tag error";
1083 		break;
1084 
1085 /****************************************************************************
1086 *  SCSI Target values
1087 ****************************************************************************/
1088 
1089 	case MPI2_IOCSTATUS_TARGET_INVALID_IO_INDEX:
1090 		desc = "target invalid io index";
1091 		break;
1092 	case MPI2_IOCSTATUS_TARGET_ABORTED:
1093 		desc = "target aborted";
1094 		break;
1095 	case MPI2_IOCSTATUS_TARGET_NO_CONN_RETRYABLE:
1096 		desc = "target no conn retryable";
1097 		break;
1098 	case MPI2_IOCSTATUS_TARGET_NO_CONNECTION:
1099 		desc = "target no connection";
1100 		break;
1101 	case MPI2_IOCSTATUS_TARGET_XFER_COUNT_MISMATCH:
1102 		desc = "target xfer count mismatch";
1103 		break;
1104 	case MPI2_IOCSTATUS_TARGET_DATA_OFFSET_ERROR:
1105 		desc = "target data offset error";
1106 		break;
1107 	case MPI2_IOCSTATUS_TARGET_TOO_MUCH_WRITE_DATA:
1108 		desc = "target too much write data";
1109 		break;
1110 	case MPI2_IOCSTATUS_TARGET_IU_TOO_SHORT:
1111 		desc = "target iu too short";
1112 		break;
1113 	case MPI2_IOCSTATUS_TARGET_ACK_NAK_TIMEOUT:
1114 		desc = "target ack nak timeout";
1115 		break;
1116 	case MPI2_IOCSTATUS_TARGET_NAK_RECEIVED:
1117 		desc = "target nak received";
1118 		break;
1119 
1120 /****************************************************************************
1121 *  Serial Attached SCSI values
1122 ****************************************************************************/
1123 
1124 	case MPI2_IOCSTATUS_SAS_SMP_REQUEST_FAILED:
1125 		desc = "smp request failed";
1126 		break;
1127 	case MPI2_IOCSTATUS_SAS_SMP_DATA_OVERRUN:
1128 		desc = "smp data overrun";
1129 		break;
1130 
1131 /****************************************************************************
1132 *  Diagnostic Buffer Post / Diagnostic Release values
1133 ****************************************************************************/
1134 
1135 	case MPI2_IOCSTATUS_DIAGNOSTIC_RELEASED:
1136 		desc = "diagnostic released";
1137 		break;
1138 	default:
1139 		break;
1140 	}
1141 
1142 	if (!desc)
1143 		return;
1144 
1145 	switch (request_hdr->Function) {
1146 	case MPI2_FUNCTION_CONFIG:
1147 		frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;
1148 		func_str = "config_page";
1149 		break;
1150 	case MPI2_FUNCTION_SCSI_TASK_MGMT:
1151 		frame_sz = sizeof(Mpi2SCSITaskManagementRequest_t);
1152 		func_str = "task_mgmt";
1153 		break;
1154 	case MPI2_FUNCTION_SAS_IO_UNIT_CONTROL:
1155 		frame_sz = sizeof(Mpi2SasIoUnitControlRequest_t);
1156 		func_str = "sas_iounit_ctl";
1157 		break;
1158 	case MPI2_FUNCTION_SCSI_ENCLOSURE_PROCESSOR:
1159 		frame_sz = sizeof(Mpi2SepRequest_t);
1160 		func_str = "enclosure";
1161 		break;
1162 	case MPI2_FUNCTION_IOC_INIT:
1163 		frame_sz = sizeof(Mpi2IOCInitRequest_t);
1164 		func_str = "ioc_init";
1165 		break;
1166 	case MPI2_FUNCTION_PORT_ENABLE:
1167 		frame_sz = sizeof(Mpi2PortEnableRequest_t);
1168 		func_str = "port_enable";
1169 		break;
1170 	case MPI2_FUNCTION_SMP_PASSTHROUGH:
1171 		frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;
1172 		func_str = "smp_passthru";
1173 		break;
1174 	case MPI2_FUNCTION_NVME_ENCAPSULATED:
1175 		frame_sz = sizeof(Mpi26NVMeEncapsulatedRequest_t) +
1176 		    ioc->sge_size;
1177 		func_str = "nvme_encapsulated";
1178 		break;
1179 	default:
1180 		frame_sz = 32;
1181 		func_str = "unknown";
1182 		break;
1183 	}
1184 
1185 	ioc_warn(ioc, "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",
1186 		 desc, ioc_status, request_hdr, func_str);
1187 
1188 	_debug_dump_mf(request_hdr, frame_sz/4);
1189 }
1190 
1191 /**
1192  * _base_display_event_data - verbose translation of firmware asyn events
1193  * @ioc: per adapter object
1194  * @mpi_reply: reply mf payload returned from firmware
1195  */
1196 static void
1197 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,
1198 	Mpi2EventNotificationReply_t *mpi_reply)
1199 {
1200 	char *desc = NULL;
1201 	u16 event;
1202 
1203 	if (!(ioc->logging_level & MPT_DEBUG_EVENTS))
1204 		return;
1205 
1206 	event = le16_to_cpu(mpi_reply->Event);
1207 
1208 	switch (event) {
1209 	case MPI2_EVENT_LOG_DATA:
1210 		desc = "Log Data";
1211 		break;
1212 	case MPI2_EVENT_STATE_CHANGE:
1213 		desc = "Status Change";
1214 		break;
1215 	case MPI2_EVENT_HARD_RESET_RECEIVED:
1216 		desc = "Hard Reset Received";
1217 		break;
1218 	case MPI2_EVENT_EVENT_CHANGE:
1219 		desc = "Event Change";
1220 		break;
1221 	case MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE:
1222 		desc = "Device Status Change";
1223 		break;
1224 	case MPI2_EVENT_IR_OPERATION_STATUS:
1225 		if (!ioc->hide_ir_msg)
1226 			desc = "IR Operation Status";
1227 		break;
1228 	case MPI2_EVENT_SAS_DISCOVERY:
1229 	{
1230 		Mpi2EventDataSasDiscovery_t *event_data =
1231 		    (Mpi2EventDataSasDiscovery_t *)mpi_reply->EventData;
1232 		ioc_info(ioc, "Discovery: (%s)",
1233 			 event_data->ReasonCode == MPI2_EVENT_SAS_DISC_RC_STARTED ?
1234 			 "start" : "stop");
1235 		if (event_data->DiscoveryStatus)
1236 			pr_cont(" discovery_status(0x%08x)",
1237 			    le32_to_cpu(event_data->DiscoveryStatus));
1238 		pr_cont("\n");
1239 		return;
1240 	}
1241 	case MPI2_EVENT_SAS_BROADCAST_PRIMITIVE:
1242 		desc = "SAS Broadcast Primitive";
1243 		break;
1244 	case MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
1245 		desc = "SAS Init Device Status Change";
1246 		break;
1247 	case MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW:
1248 		desc = "SAS Init Table Overflow";
1249 		break;
1250 	case MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
1251 		desc = "SAS Topology Change List";
1252 		break;
1253 	case MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE:
1254 		desc = "SAS Enclosure Device Status Change";
1255 		break;
1256 	case MPI2_EVENT_IR_VOLUME:
1257 		if (!ioc->hide_ir_msg)
1258 			desc = "IR Volume";
1259 		break;
1260 	case MPI2_EVENT_IR_PHYSICAL_DISK:
1261 		if (!ioc->hide_ir_msg)
1262 			desc = "IR Physical Disk";
1263 		break;
1264 	case MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST:
1265 		if (!ioc->hide_ir_msg)
1266 			desc = "IR Configuration Change List";
1267 		break;
1268 	case MPI2_EVENT_LOG_ENTRY_ADDED:
1269 		if (!ioc->hide_ir_msg)
1270 			desc = "Log Entry Added";
1271 		break;
1272 	case MPI2_EVENT_TEMP_THRESHOLD:
1273 		desc = "Temperature Threshold";
1274 		break;
1275 	case MPI2_EVENT_ACTIVE_CABLE_EXCEPTION:
1276 		desc = "Cable Event";
1277 		break;
1278 	case MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
1279 		desc = "SAS Device Discovery Error";
1280 		break;
1281 	case MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE:
1282 		desc = "PCIE Device Status Change";
1283 		break;
1284 	case MPI2_EVENT_PCIE_ENUMERATION:
1285 	{
1286 		Mpi26EventDataPCIeEnumeration_t *event_data =
1287 			(Mpi26EventDataPCIeEnumeration_t *)mpi_reply->EventData;
1288 		ioc_info(ioc, "PCIE Enumeration: (%s)",
1289 			 event_data->ReasonCode == MPI26_EVENT_PCIE_ENUM_RC_STARTED ?
1290 			 "start" : "stop");
1291 		if (event_data->EnumerationStatus)
1292 			pr_cont("enumeration_status(0x%08x)",
1293 				le32_to_cpu(event_data->EnumerationStatus));
1294 		pr_cont("\n");
1295 		return;
1296 	}
1297 	case MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
1298 		desc = "PCIE Topology Change List";
1299 		break;
1300 	}
1301 
1302 	if (!desc)
1303 		return;
1304 
1305 	ioc_info(ioc, "%s\n", desc);
1306 }
1307 
1308 /**
1309  * _base_sas_log_info - verbose translation of firmware log info
1310  * @ioc: per adapter object
1311  * @log_info: log info
1312  */
1313 static void
1314 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc , u32 log_info)
1315 {
1316 	union loginfo_type {
1317 		u32	loginfo;
1318 		struct {
1319 			u32	subcode:16;
1320 			u32	code:8;
1321 			u32	originator:4;
1322 			u32	bus_type:4;
1323 		} dw;
1324 	};
1325 	union loginfo_type sas_loginfo;
1326 	char *originator_str = NULL;
1327 
1328 	sas_loginfo.loginfo = log_info;
1329 	if (sas_loginfo.dw.bus_type != 3 /*SAS*/)
1330 		return;
1331 
1332 	/* each nexus loss loginfo */
1333 	if (log_info == 0x31170000)
1334 		return;
1335 
1336 	/* eat the loginfos associated with task aborts */
1337 	if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==
1338 	    0x31140000 || log_info == 0x31130000))
1339 		return;
1340 
1341 	switch (sas_loginfo.dw.originator) {
1342 	case 0:
1343 		originator_str = "IOP";
1344 		break;
1345 	case 1:
1346 		originator_str = "PL";
1347 		break;
1348 	case 2:
1349 		if (!ioc->hide_ir_msg)
1350 			originator_str = "IR";
1351 		else
1352 			originator_str = "WarpDrive";
1353 		break;
1354 	}
1355 
1356 	ioc_warn(ioc, "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",
1357 		 log_info,
1358 		 originator_str, sas_loginfo.dw.code, sas_loginfo.dw.subcode);
1359 }
1360 
1361 /**
1362  * _base_display_reply_info - handle reply descriptors depending on IOC Status
1363  * @ioc: per adapter object
1364  * @smid: system request message index
1365  * @msix_index: MSIX table index supplied by the OS
1366  * @reply: reply message frame (lower 32bit addr)
1367  */
1368 static void
1369 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1370 	u32 reply)
1371 {
1372 	MPI2DefaultReply_t *mpi_reply;
1373 	u16 ioc_status;
1374 	u32 loginfo = 0;
1375 
1376 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1377 	if (unlikely(!mpi_reply)) {
1378 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",
1379 			__FILE__, __LINE__, __func__);
1380 		return;
1381 	}
1382 	ioc_status = le16_to_cpu(mpi_reply->IOCStatus);
1383 
1384 	if ((ioc_status & MPI2_IOCSTATUS_MASK) &&
1385 	    (ioc->logging_level & MPT_DEBUG_REPLY)) {
1386 		_base_sas_ioc_info(ioc , mpi_reply,
1387 		   mpt3sas_base_get_msg_frame(ioc, smid));
1388 	}
1389 
1390 	if (ioc_status & MPI2_IOCSTATUS_FLAG_LOG_INFO_AVAILABLE) {
1391 		loginfo = le32_to_cpu(mpi_reply->IOCLogInfo);
1392 		_base_sas_log_info(ioc, loginfo);
1393 	}
1394 
1395 	if (ioc_status || loginfo) {
1396 		ioc_status &= MPI2_IOCSTATUS_MASK;
1397 		mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);
1398 	}
1399 }
1400 
1401 /**
1402  * mpt3sas_base_done - base internal command completion routine
1403  * @ioc: per adapter object
1404  * @smid: system request message index
1405  * @msix_index: MSIX table index supplied by the OS
1406  * @reply: reply message frame(lower 32bit addr)
1407  *
1408  * Return:
1409  * 1 meaning mf should be freed from _base_interrupt
1410  * 0 means the mf is freed from this function.
1411  */
1412 u8
1413 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
1414 	u32 reply)
1415 {
1416 	MPI2DefaultReply_t *mpi_reply;
1417 
1418 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1419 	if (mpi_reply && mpi_reply->Function == MPI2_FUNCTION_EVENT_ACK)
1420 		return mpt3sas_check_for_pending_internal_cmds(ioc, smid);
1421 
1422 	if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)
1423 		return 1;
1424 
1425 	ioc->base_cmds.status |= MPT3_CMD_COMPLETE;
1426 	if (mpi_reply) {
1427 		ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;
1428 		memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
1429 	}
1430 	ioc->base_cmds.status &= ~MPT3_CMD_PENDING;
1431 
1432 	complete(&ioc->base_cmds.done);
1433 	return 1;
1434 }
1435 
1436 /**
1437  * _base_async_event - main callback handler for firmware asyn events
1438  * @ioc: per adapter object
1439  * @msix_index: MSIX table index supplied by the OS
1440  * @reply: reply message frame(lower 32bit addr)
1441  *
1442  * Return:
1443  * 1 meaning mf should be freed from _base_interrupt
1444  * 0 means the mf is freed from this function.
1445  */
1446 static u8
1447 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)
1448 {
1449 	Mpi2EventNotificationReply_t *mpi_reply;
1450 	Mpi2EventAckRequest_t *ack_request;
1451 	u16 smid;
1452 	struct _event_ack_list *delayed_event_ack;
1453 
1454 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
1455 	if (!mpi_reply)
1456 		return 1;
1457 	if (mpi_reply->Function != MPI2_FUNCTION_EVENT_NOTIFICATION)
1458 		return 1;
1459 
1460 	_base_display_event_data(ioc, mpi_reply);
1461 
1462 	if (!(mpi_reply->AckRequired & MPI2_EVENT_NOTIFICATION_ACK_REQUIRED))
1463 		goto out;
1464 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
1465 	if (!smid) {
1466 		delayed_event_ack = kzalloc(sizeof(*delayed_event_ack),
1467 					GFP_ATOMIC);
1468 		if (!delayed_event_ack)
1469 			goto out;
1470 		INIT_LIST_HEAD(&delayed_event_ack->list);
1471 		delayed_event_ack->Event = mpi_reply->Event;
1472 		delayed_event_ack->EventContext = mpi_reply->EventContext;
1473 		list_add_tail(&delayed_event_ack->list,
1474 				&ioc->delayed_event_ack_list);
1475 		dewtprintk(ioc,
1476 			   ioc_info(ioc, "DELAYED: EVENT ACK: event (0x%04x)\n",
1477 				    le16_to_cpu(mpi_reply->Event)));
1478 		goto out;
1479 	}
1480 
1481 	ack_request = mpt3sas_base_get_msg_frame(ioc, smid);
1482 	memset(ack_request, 0, sizeof(Mpi2EventAckRequest_t));
1483 	ack_request->Function = MPI2_FUNCTION_EVENT_ACK;
1484 	ack_request->Event = mpi_reply->Event;
1485 	ack_request->EventContext = mpi_reply->EventContext;
1486 	ack_request->VF_ID = 0;  /* TODO */
1487 	ack_request->VP_ID = 0;
1488 	ioc->put_smid_default(ioc, smid);
1489 
1490  out:
1491 
1492 	/* scsih callback handler */
1493 	mpt3sas_scsih_event_callback(ioc, msix_index, reply);
1494 
1495 	/* ctl callback handler */
1496 	mpt3sas_ctl_event_callback(ioc, msix_index, reply);
1497 
1498 	return 1;
1499 }
1500 
1501 static struct scsiio_tracker *
1502 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1503 {
1504 	struct scsi_cmnd *cmd;
1505 
1506 	if (WARN_ON(!smid) ||
1507 	    WARN_ON(smid >= ioc->hi_priority_smid))
1508 		return NULL;
1509 
1510 	cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);
1511 	if (cmd)
1512 		return scsi_cmd_priv(cmd);
1513 
1514 	return NULL;
1515 }
1516 
1517 /**
1518  * _base_get_cb_idx - obtain the callback index
1519  * @ioc: per adapter object
1520  * @smid: system request message index
1521  *
1522  * Return: callback index.
1523  */
1524 static u8
1525 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)
1526 {
1527 	int i;
1528 	u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;
1529 	u8 cb_idx = 0xFF;
1530 
1531 	if (smid < ioc->hi_priority_smid) {
1532 		struct scsiio_tracker *st;
1533 
1534 		if (smid < ctl_smid) {
1535 			st = _get_st_from_smid(ioc, smid);
1536 			if (st)
1537 				cb_idx = st->cb_idx;
1538 		} else if (smid == ctl_smid)
1539 			cb_idx = ioc->ctl_cb_idx;
1540 	} else if (smid < ioc->internal_smid) {
1541 		i = smid - ioc->hi_priority_smid;
1542 		cb_idx = ioc->hpr_lookup[i].cb_idx;
1543 	} else if (smid <= ioc->hba_queue_depth) {
1544 		i = smid - ioc->internal_smid;
1545 		cb_idx = ioc->internal_lookup[i].cb_idx;
1546 	}
1547 	return cb_idx;
1548 }
1549 
1550 /**
1551  * mpt3sas_base_mask_interrupts - disable interrupts
1552  * @ioc: per adapter object
1553  *
1554  * Disabling ResetIRQ, Reply and Doorbell Interrupts
1555  */
1556 void
1557 mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1558 {
1559 	u32 him_register;
1560 
1561 	ioc->mask_interrupts = 1;
1562 	him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);
1563 	him_register |= MPI2_HIM_DIM + MPI2_HIM_RIM + MPI2_HIM_RESET_IRQ_MASK;
1564 	writel(him_register, &ioc->chip->HostInterruptMask);
1565 	ioc->base_readl(&ioc->chip->HostInterruptMask);
1566 }
1567 
1568 /**
1569  * mpt3sas_base_unmask_interrupts - enable interrupts
1570  * @ioc: per adapter object
1571  *
1572  * Enabling only Reply Interrupts
1573  */
1574 void
1575 mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)
1576 {
1577 	u32 him_register;
1578 
1579 	him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);
1580 	him_register &= ~MPI2_HIM_RIM;
1581 	writel(him_register, &ioc->chip->HostInterruptMask);
1582 	ioc->mask_interrupts = 0;
1583 }
1584 
1585 union reply_descriptor {
1586 	u64 word;
1587 	struct {
1588 		u32 low;
1589 		u32 high;
1590 	} u;
1591 };
1592 
1593 static u32 base_mod64(u64 dividend, u32 divisor)
1594 {
1595 	u32 remainder;
1596 
1597 	if (!divisor)
1598 		pr_err("mpt3sas: DIVISOR is zero, in div fn\n");
1599 	remainder = do_div(dividend, divisor);
1600 	return remainder;
1601 }
1602 
1603 /**
1604  * _base_process_reply_queue - Process reply descriptors from reply
1605  *		descriptor post queue.
1606  * @reply_q: per IRQ's reply queue object.
1607  *
1608  * Return: number of reply descriptors processed from reply
1609  *		descriptor queue.
1610  */
1611 static int
1612 _base_process_reply_queue(struct adapter_reply_queue *reply_q)
1613 {
1614 	union reply_descriptor rd;
1615 	u64 completed_cmds;
1616 	u8 request_descript_type;
1617 	u16 smid;
1618 	u8 cb_idx;
1619 	u32 reply;
1620 	u8 msix_index = reply_q->msix_index;
1621 	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1622 	Mpi2ReplyDescriptorsUnion_t *rpf;
1623 	u8 rc;
1624 
1625 	completed_cmds = 0;
1626 	if (!atomic_add_unless(&reply_q->busy, 1, 1))
1627 		return completed_cmds;
1628 
1629 	rpf = &reply_q->reply_post_free[reply_q->reply_post_host_index];
1630 	request_descript_type = rpf->Default.ReplyFlags
1631 	     & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1632 	if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
1633 		atomic_dec(&reply_q->busy);
1634 		return completed_cmds;
1635 	}
1636 
1637 	cb_idx = 0xFF;
1638 	do {
1639 		rd.word = le64_to_cpu(rpf->Words);
1640 		if (rd.u.low == UINT_MAX || rd.u.high == UINT_MAX)
1641 			goto out;
1642 		reply = 0;
1643 		smid = le16_to_cpu(rpf->Default.DescriptorTypeDependent1);
1644 		if (request_descript_type ==
1645 		    MPI25_RPY_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO_SUCCESS ||
1646 		    request_descript_type ==
1647 		    MPI2_RPY_DESCRIPT_FLAGS_SCSI_IO_SUCCESS ||
1648 		    request_descript_type ==
1649 		    MPI26_RPY_DESCRIPT_FLAGS_PCIE_ENCAPSULATED_SUCCESS) {
1650 			cb_idx = _base_get_cb_idx(ioc, smid);
1651 			if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1652 			    (likely(mpt_callbacks[cb_idx] != NULL))) {
1653 				rc = mpt_callbacks[cb_idx](ioc, smid,
1654 				    msix_index, 0);
1655 				if (rc)
1656 					mpt3sas_base_free_smid(ioc, smid);
1657 			}
1658 		} else if (request_descript_type ==
1659 		    MPI2_RPY_DESCRIPT_FLAGS_ADDRESS_REPLY) {
1660 			reply = le32_to_cpu(
1661 			    rpf->AddressReply.ReplyFrameAddress);
1662 			if (reply > ioc->reply_dma_max_address ||
1663 			    reply < ioc->reply_dma_min_address)
1664 				reply = 0;
1665 			if (smid) {
1666 				cb_idx = _base_get_cb_idx(ioc, smid);
1667 				if ((likely(cb_idx < MPT_MAX_CALLBACKS)) &&
1668 				    (likely(mpt_callbacks[cb_idx] != NULL))) {
1669 					rc = mpt_callbacks[cb_idx](ioc, smid,
1670 					    msix_index, reply);
1671 					if (reply)
1672 						_base_display_reply_info(ioc,
1673 						    smid, msix_index, reply);
1674 					if (rc)
1675 						mpt3sas_base_free_smid(ioc,
1676 						    smid);
1677 				}
1678 			} else {
1679 				_base_async_event(ioc, msix_index, reply);
1680 			}
1681 
1682 			/* reply free queue handling */
1683 			if (reply) {
1684 				ioc->reply_free_host_index =
1685 				    (ioc->reply_free_host_index ==
1686 				    (ioc->reply_free_queue_depth - 1)) ?
1687 				    0 : ioc->reply_free_host_index + 1;
1688 				ioc->reply_free[ioc->reply_free_host_index] =
1689 				    cpu_to_le32(reply);
1690 				if (ioc->is_mcpu_endpoint)
1691 					_base_clone_reply_to_sys_mem(ioc,
1692 						reply,
1693 						ioc->reply_free_host_index);
1694 				writel(ioc->reply_free_host_index,
1695 				    &ioc->chip->ReplyFreeHostIndex);
1696 			}
1697 		}
1698 
1699 		rpf->Words = cpu_to_le64(ULLONG_MAX);
1700 		reply_q->reply_post_host_index =
1701 		    (reply_q->reply_post_host_index ==
1702 		    (ioc->reply_post_queue_depth - 1)) ? 0 :
1703 		    reply_q->reply_post_host_index + 1;
1704 		request_descript_type =
1705 		    reply_q->reply_post_free[reply_q->reply_post_host_index].
1706 		    Default.ReplyFlags & MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
1707 		completed_cmds++;
1708 		/* Update the reply post host index after continuously
1709 		 * processing the threshold number of Reply Descriptors.
1710 		 * So that FW can find enough entries to post the Reply
1711 		 * Descriptors in the reply descriptor post queue.
1712 		 */
1713 		if (completed_cmds >= ioc->thresh_hold) {
1714 			if (ioc->combined_reply_queue) {
1715 				writel(reply_q->reply_post_host_index |
1716 						((msix_index  & 7) <<
1717 						 MPI2_RPHI_MSIX_INDEX_SHIFT),
1718 				    ioc->replyPostRegisterIndex[msix_index/8]);
1719 			} else {
1720 				writel(reply_q->reply_post_host_index |
1721 						(msix_index <<
1722 						 MPI2_RPHI_MSIX_INDEX_SHIFT),
1723 						&ioc->chip->ReplyPostHostIndex);
1724 			}
1725 			if (!reply_q->irq_poll_scheduled) {
1726 				reply_q->irq_poll_scheduled = true;
1727 				irq_poll_sched(&reply_q->irqpoll);
1728 			}
1729 			atomic_dec(&reply_q->busy);
1730 			return completed_cmds;
1731 		}
1732 		if (request_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
1733 			goto out;
1734 		if (!reply_q->reply_post_host_index)
1735 			rpf = reply_q->reply_post_free;
1736 		else
1737 			rpf++;
1738 	} while (1);
1739 
1740  out:
1741 
1742 	if (!completed_cmds) {
1743 		atomic_dec(&reply_q->busy);
1744 		return completed_cmds;
1745 	}
1746 
1747 	if (ioc->is_warpdrive) {
1748 		writel(reply_q->reply_post_host_index,
1749 		ioc->reply_post_host_index[msix_index]);
1750 		atomic_dec(&reply_q->busy);
1751 		return completed_cmds;
1752 	}
1753 
1754 	/* Update Reply Post Host Index.
1755 	 * For those HBA's which support combined reply queue feature
1756 	 * 1. Get the correct Supplemental Reply Post Host Index Register.
1757 	 *    i.e. (msix_index / 8)th entry from Supplemental Reply Post Host
1758 	 *    Index Register address bank i.e replyPostRegisterIndex[],
1759 	 * 2. Then update this register with new reply host index value
1760 	 *    in ReplyPostIndex field and the MSIxIndex field with
1761 	 *    msix_index value reduced to a value between 0 and 7,
1762 	 *    using a modulo 8 operation. Since each Supplemental Reply Post
1763 	 *    Host Index Register supports 8 MSI-X vectors.
1764 	 *
1765 	 * For other HBA's just update the Reply Post Host Index register with
1766 	 * new reply host index value in ReplyPostIndex Field and msix_index
1767 	 * value in MSIxIndex field.
1768 	 */
1769 	if (ioc->combined_reply_queue)
1770 		writel(reply_q->reply_post_host_index | ((msix_index  & 7) <<
1771 			MPI2_RPHI_MSIX_INDEX_SHIFT),
1772 			ioc->replyPostRegisterIndex[msix_index/8]);
1773 	else
1774 		writel(reply_q->reply_post_host_index | (msix_index <<
1775 			MPI2_RPHI_MSIX_INDEX_SHIFT),
1776 			&ioc->chip->ReplyPostHostIndex);
1777 	atomic_dec(&reply_q->busy);
1778 	return completed_cmds;
1779 }
1780 
1781 /**
1782  * _base_interrupt - MPT adapter (IOC) specific interrupt handler.
1783  * @irq: irq number (not used)
1784  * @bus_id: bus identifier cookie == pointer to MPT_ADAPTER structure
1785  *
1786  * Return: IRQ_HANDLED if processed, else IRQ_NONE.
1787  */
1788 static irqreturn_t
1789 _base_interrupt(int irq, void *bus_id)
1790 {
1791 	struct adapter_reply_queue *reply_q = bus_id;
1792 	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;
1793 
1794 	if (ioc->mask_interrupts)
1795 		return IRQ_NONE;
1796 	if (reply_q->irq_poll_scheduled)
1797 		return IRQ_HANDLED;
1798 	return ((_base_process_reply_queue(reply_q) > 0) ?
1799 			IRQ_HANDLED : IRQ_NONE);
1800 }
1801 
1802 /**
1803  * _base_irqpoll - IRQ poll callback handler
1804  * @irqpoll: irq_poll object
1805  * @budget: irq poll weight
1806  *
1807  * Return: number of reply descriptors processed
1808  */
1809 static int
1810 _base_irqpoll(struct irq_poll *irqpoll, int budget)
1811 {
1812 	struct adapter_reply_queue *reply_q;
1813 	int num_entries = 0;
1814 
1815 	reply_q = container_of(irqpoll, struct adapter_reply_queue,
1816 			irqpoll);
1817 	if (reply_q->irq_line_enable) {
1818 		disable_irq_nosync(reply_q->os_irq);
1819 		reply_q->irq_line_enable = false;
1820 	}
1821 	num_entries = _base_process_reply_queue(reply_q);
1822 	if (num_entries < budget) {
1823 		irq_poll_complete(irqpoll);
1824 		reply_q->irq_poll_scheduled = false;
1825 		reply_q->irq_line_enable = true;
1826 		enable_irq(reply_q->os_irq);
1827 		/*
1828 		 * Go for one more round of processing the
1829 		 * reply descriptor post queue in case the HBA
1830 		 * Firmware has posted some reply descriptors
1831 		 * while reenabling the IRQ.
1832 		 */
1833 		_base_process_reply_queue(reply_q);
1834 	}
1835 
1836 	return num_entries;
1837 }
1838 
1839 /**
1840  * _base_init_irqpolls - initliaze IRQ polls
1841  * @ioc: per adapter object
1842  *
1843  * Return: nothing
1844  */
1845 static void
1846 _base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc)
1847 {
1848 	struct adapter_reply_queue *reply_q, *next;
1849 
1850 	if (list_empty(&ioc->reply_queue_list))
1851 		return;
1852 
1853 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
1854 		irq_poll_init(&reply_q->irqpoll,
1855 			ioc->hba_queue_depth/4, _base_irqpoll);
1856 		reply_q->irq_poll_scheduled = false;
1857 		reply_q->irq_line_enable = true;
1858 		reply_q->os_irq = pci_irq_vector(ioc->pdev,
1859 		    reply_q->msix_index);
1860 	}
1861 }
1862 
1863 /**
1864  * _base_is_controller_msix_enabled - is controller support muli-reply queues
1865  * @ioc: per adapter object
1866  *
1867  * Return: Whether or not MSI/X is enabled.
1868  */
1869 static inline int
1870 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)
1871 {
1872 	return (ioc->facts.IOCCapabilities &
1873 	    MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;
1874 }
1875 
1876 /**
1877  * mpt3sas_base_sync_reply_irqs - flush pending MSIX interrupts
1878  * @ioc: per adapter object
1879  * @poll: poll over reply descriptor pools incase interrupt for
1880  *		timed-out SCSI command got delayed
1881  * Context: non-ISR context
1882  *
1883  * Called when a Task Management request has completed.
1884  */
1885 void
1886 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll)
1887 {
1888 	struct adapter_reply_queue *reply_q;
1889 
1890 	/* If MSIX capability is turned off
1891 	 * then multi-queues are not enabled
1892 	 */
1893 	if (!_base_is_controller_msix_enabled(ioc))
1894 		return;
1895 
1896 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
1897 		if (ioc->shost_recovery || ioc->remove_host ||
1898 				ioc->pci_error_recovery)
1899 			return;
1900 		/* TMs are on msix_index == 0 */
1901 		if (reply_q->msix_index == 0)
1902 			continue;
1903 		synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));
1904 		if (reply_q->irq_poll_scheduled) {
1905 			/* Calling irq_poll_disable will wait for any pending
1906 			 * callbacks to have completed.
1907 			 */
1908 			irq_poll_disable(&reply_q->irqpoll);
1909 			irq_poll_enable(&reply_q->irqpoll);
1910 			/* check how the scheduled poll has ended,
1911 			 * clean up only if necessary
1912 			 */
1913 			if (reply_q->irq_poll_scheduled) {
1914 				reply_q->irq_poll_scheduled = false;
1915 				reply_q->irq_line_enable = true;
1916 				enable_irq(reply_q->os_irq);
1917 			}
1918 		}
1919 	}
1920 	if (poll)
1921 		_base_process_reply_queue(reply_q);
1922 }
1923 
1924 /**
1925  * mpt3sas_base_release_callback_handler - clear interrupt callback handler
1926  * @cb_idx: callback index
1927  */
1928 void
1929 mpt3sas_base_release_callback_handler(u8 cb_idx)
1930 {
1931 	mpt_callbacks[cb_idx] = NULL;
1932 }
1933 
1934 /**
1935  * mpt3sas_base_register_callback_handler - obtain index for the interrupt callback handler
1936  * @cb_func: callback function
1937  *
1938  * Return: Index of @cb_func.
1939  */
1940 u8
1941 mpt3sas_base_register_callback_handler(MPT_CALLBACK cb_func)
1942 {
1943 	u8 cb_idx;
1944 
1945 	for (cb_idx = MPT_MAX_CALLBACKS-1; cb_idx; cb_idx--)
1946 		if (mpt_callbacks[cb_idx] == NULL)
1947 			break;
1948 
1949 	mpt_callbacks[cb_idx] = cb_func;
1950 	return cb_idx;
1951 }
1952 
1953 /**
1954  * mpt3sas_base_initialize_callback_handler - initialize the interrupt callback handler
1955  */
1956 void
1957 mpt3sas_base_initialize_callback_handler(void)
1958 {
1959 	u8 cb_idx;
1960 
1961 	for (cb_idx = 0; cb_idx < MPT_MAX_CALLBACKS; cb_idx++)
1962 		mpt3sas_base_release_callback_handler(cb_idx);
1963 }
1964 
1965 
1966 /**
1967  * _base_build_zero_len_sge - build zero length sg entry
1968  * @ioc: per adapter object
1969  * @paddr: virtual address for SGE
1970  *
1971  * Create a zero length scatter gather entry to insure the IOCs hardware has
1972  * something to use if the target device goes brain dead and tries
1973  * to send data even when none is asked for.
1974  */
1975 static void
1976 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)
1977 {
1978 	u32 flags_length = (u32)((MPI2_SGE_FLAGS_LAST_ELEMENT |
1979 	    MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST |
1980 	    MPI2_SGE_FLAGS_SIMPLE_ELEMENT) <<
1981 	    MPI2_SGE_FLAGS_SHIFT);
1982 	ioc->base_add_sg_single(paddr, flags_length, -1);
1983 }
1984 
1985 /**
1986  * _base_add_sg_single_32 - Place a simple 32 bit SGE at address pAddr.
1987  * @paddr: virtual address for SGE
1988  * @flags_length: SGE flags and data transfer length
1989  * @dma_addr: Physical address
1990  */
1991 static void
1992 _base_add_sg_single_32(void *paddr, u32 flags_length, dma_addr_t dma_addr)
1993 {
1994 	Mpi2SGESimple32_t *sgel = paddr;
1995 
1996 	flags_length |= (MPI2_SGE_FLAGS_32_BIT_ADDRESSING |
1997 	    MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
1998 	sgel->FlagsLength = cpu_to_le32(flags_length);
1999 	sgel->Address = cpu_to_le32(dma_addr);
2000 }
2001 
2002 
2003 /**
2004  * _base_add_sg_single_64 - Place a simple 64 bit SGE at address pAddr.
2005  * @paddr: virtual address for SGE
2006  * @flags_length: SGE flags and data transfer length
2007  * @dma_addr: Physical address
2008  */
2009 static void
2010 _base_add_sg_single_64(void *paddr, u32 flags_length, dma_addr_t dma_addr)
2011 {
2012 	Mpi2SGESimple64_t *sgel = paddr;
2013 
2014 	flags_length |= (MPI2_SGE_FLAGS_64_BIT_ADDRESSING |
2015 	    MPI2_SGE_FLAGS_SYSTEM_ADDRESS) << MPI2_SGE_FLAGS_SHIFT;
2016 	sgel->FlagsLength = cpu_to_le32(flags_length);
2017 	sgel->Address = cpu_to_le64(dma_addr);
2018 }
2019 
2020 /**
2021  * _base_get_chain_buffer_tracker - obtain chain tracker
2022  * @ioc: per adapter object
2023  * @scmd: SCSI commands of the IO request
2024  *
2025  * Return: chain tracker from chain_lookup table using key as
2026  * smid and smid's chain_offset.
2027  */
2028 static struct chain_tracker *
2029 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,
2030 			       struct scsi_cmnd *scmd)
2031 {
2032 	struct chain_tracker *chain_req;
2033 	struct scsiio_tracker *st = scsi_cmd_priv(scmd);
2034 	u16 smid = st->smid;
2035 	u8 chain_offset =
2036 	   atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);
2037 
2038 	if (chain_offset == ioc->chains_needed_per_io)
2039 		return NULL;
2040 
2041 	chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];
2042 	atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);
2043 	return chain_req;
2044 }
2045 
2046 
2047 /**
2048  * _base_build_sg - build generic sg
2049  * @ioc: per adapter object
2050  * @psge: virtual address for SGE
2051  * @data_out_dma: physical address for WRITES
2052  * @data_out_sz: data xfer size for WRITES
2053  * @data_in_dma: physical address for READS
2054  * @data_in_sz: data xfer size for READS
2055  */
2056 static void
2057 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,
2058 	dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2059 	size_t data_in_sz)
2060 {
2061 	u32 sgl_flags;
2062 
2063 	if (!data_out_sz && !data_in_sz) {
2064 		_base_build_zero_len_sge(ioc, psge);
2065 		return;
2066 	}
2067 
2068 	if (data_out_sz && data_in_sz) {
2069 		/* WRITE sgel first */
2070 		sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2071 		    MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_HOST_TO_IOC);
2072 		sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2073 		ioc->base_add_sg_single(psge, sgl_flags |
2074 		    data_out_sz, data_out_dma);
2075 
2076 		/* incr sgel */
2077 		psge += ioc->sge_size;
2078 
2079 		/* READ sgel last */
2080 		sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2081 		    MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
2082 		    MPI2_SGE_FLAGS_END_OF_LIST);
2083 		sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2084 		ioc->base_add_sg_single(psge, sgl_flags |
2085 		    data_in_sz, data_in_dma);
2086 	} else if (data_out_sz) /* WRITE */ {
2087 		sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2088 		    MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
2089 		    MPI2_SGE_FLAGS_END_OF_LIST | MPI2_SGE_FLAGS_HOST_TO_IOC);
2090 		sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2091 		ioc->base_add_sg_single(psge, sgl_flags |
2092 		    data_out_sz, data_out_dma);
2093 	} else if (data_in_sz) /* READ */ {
2094 		sgl_flags = (MPI2_SGE_FLAGS_SIMPLE_ELEMENT |
2095 		    MPI2_SGE_FLAGS_LAST_ELEMENT | MPI2_SGE_FLAGS_END_OF_BUFFER |
2096 		    MPI2_SGE_FLAGS_END_OF_LIST);
2097 		sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2098 		ioc->base_add_sg_single(psge, sgl_flags |
2099 		    data_in_sz, data_in_dma);
2100 	}
2101 }
2102 
2103 /* IEEE format sgls */
2104 
2105 /**
2106  * _base_build_nvme_prp - This function is called for NVMe end devices to build
2107  *                        a native SGL (NVMe PRP).
2108  * @ioc: per adapter object
2109  * @smid: system request message index for getting asscociated SGL
2110  * @nvme_encap_request: the NVMe request msg frame pointer
2111  * @data_out_dma: physical address for WRITES
2112  * @data_out_sz: data xfer size for WRITES
2113  * @data_in_dma: physical address for READS
2114  * @data_in_sz: data xfer size for READS
2115  *
2116  * The native SGL is built starting in the first PRP
2117  * entry of the NVMe message (PRP1).  If the data buffer is small enough to be
2118  * described entirely using PRP1, then PRP2 is not used.  If needed, PRP2 is
2119  * used to describe a larger data buffer.  If the data buffer is too large to
2120  * describe using the two PRP entriess inside the NVMe message, then PRP1
2121  * describes the first data memory segment, and PRP2 contains a pointer to a PRP
2122  * list located elsewhere in memory to describe the remaining data memory
2123  * segments.  The PRP list will be contiguous.
2124  *
2125  * The native SGL for NVMe devices is a Physical Region Page (PRP).  A PRP
2126  * consists of a list of PRP entries to describe a number of noncontigous
2127  * physical memory segments as a single memory buffer, just as a SGL does.  Note
2128  * however, that this function is only used by the IOCTL call, so the memory
2129  * given will be guaranteed to be contiguous.  There is no need to translate
2130  * non-contiguous SGL into a PRP in this case.  All PRPs will describe
2131  * contiguous space that is one page size each.
2132  *
2133  * Each NVMe message contains two PRP entries.  The first (PRP1) either contains
2134  * a PRP list pointer or a PRP element, depending upon the command.  PRP2
2135  * contains the second PRP element if the memory being described fits within 2
2136  * PRP entries, or a PRP list pointer if the PRP spans more than two entries.
2137  *
2138  * A PRP list pointer contains the address of a PRP list, structured as a linear
2139  * array of PRP entries.  Each PRP entry in this list describes a segment of
2140  * physical memory.
2141  *
2142  * Each 64-bit PRP entry comprises an address and an offset field.  The address
2143  * always points at the beginning of a 4KB physical memory page, and the offset
2144  * describes where within that 4KB page the memory segment begins.  Only the
2145  * first element in a PRP list may contain a non-zero offset, implying that all
2146  * memory segments following the first begin at the start of a 4KB page.
2147  *
2148  * Each PRP element normally describes 4KB of physical memory, with exceptions
2149  * for the first and last elements in the list.  If the memory being described
2150  * by the list begins at a non-zero offset within the first 4KB page, then the
2151  * first PRP element will contain a non-zero offset indicating where the region
2152  * begins within the 4KB page.  The last memory segment may end before the end
2153  * of the 4KB segment, depending upon the overall size of the memory being
2154  * described by the PRP list.
2155  *
2156  * Since PRP entries lack any indication of size, the overall data buffer length
2157  * is used to determine where the end of the data memory buffer is located, and
2158  * how many PRP entries are required to describe it.
2159  */
2160 static void
2161 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,
2162 	Mpi26NVMeEncapsulatedRequest_t *nvme_encap_request,
2163 	dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2164 	size_t data_in_sz)
2165 {
2166 	int		prp_size = NVME_PRP_SIZE;
2167 	__le64		*prp_entry, *prp1_entry, *prp2_entry;
2168 	__le64		*prp_page;
2169 	dma_addr_t	prp_entry_dma, prp_page_dma, dma_addr;
2170 	u32		offset, entry_len;
2171 	u32		page_mask_result, page_mask;
2172 	size_t		length;
2173 	struct mpt3sas_nvme_cmd *nvme_cmd =
2174 		(void *)nvme_encap_request->NVMe_Command;
2175 
2176 	/*
2177 	 * Not all commands require a data transfer. If no data, just return
2178 	 * without constructing any PRP.
2179 	 */
2180 	if (!data_in_sz && !data_out_sz)
2181 		return;
2182 	prp1_entry = &nvme_cmd->prp1;
2183 	prp2_entry = &nvme_cmd->prp2;
2184 	prp_entry = prp1_entry;
2185 	/*
2186 	 * For the PRP entries, use the specially allocated buffer of
2187 	 * contiguous memory.
2188 	 */
2189 	prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);
2190 	prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
2191 
2192 	/*
2193 	 * Check if we are within 1 entry of a page boundary we don't
2194 	 * want our first entry to be a PRP List entry.
2195 	 */
2196 	page_mask = ioc->page_size - 1;
2197 	page_mask_result = (uintptr_t)((u8 *)prp_page + prp_size) & page_mask;
2198 	if (!page_mask_result) {
2199 		/* Bump up to next page boundary. */
2200 		prp_page = (__le64 *)((u8 *)prp_page + prp_size);
2201 		prp_page_dma = prp_page_dma + prp_size;
2202 	}
2203 
2204 	/*
2205 	 * Set PRP physical pointer, which initially points to the current PRP
2206 	 * DMA memory page.
2207 	 */
2208 	prp_entry_dma = prp_page_dma;
2209 
2210 	/* Get physical address and length of the data buffer. */
2211 	if (data_in_sz) {
2212 		dma_addr = data_in_dma;
2213 		length = data_in_sz;
2214 	} else {
2215 		dma_addr = data_out_dma;
2216 		length = data_out_sz;
2217 	}
2218 
2219 	/* Loop while the length is not zero. */
2220 	while (length) {
2221 		/*
2222 		 * Check if we need to put a list pointer here if we are at
2223 		 * page boundary - prp_size (8 bytes).
2224 		 */
2225 		page_mask_result = (prp_entry_dma + prp_size) & page_mask;
2226 		if (!page_mask_result) {
2227 			/*
2228 			 * This is the last entry in a PRP List, so we need to
2229 			 * put a PRP list pointer here.  What this does is:
2230 			 *   - bump the current memory pointer to the next
2231 			 *     address, which will be the next full page.
2232 			 *   - set the PRP Entry to point to that page.  This
2233 			 *     is now the PRP List pointer.
2234 			 *   - bump the PRP Entry pointer the start of the
2235 			 *     next page.  Since all of this PRP memory is
2236 			 *     contiguous, no need to get a new page - it's
2237 			 *     just the next address.
2238 			 */
2239 			prp_entry_dma++;
2240 			*prp_entry = cpu_to_le64(prp_entry_dma);
2241 			prp_entry++;
2242 		}
2243 
2244 		/* Need to handle if entry will be part of a page. */
2245 		offset = dma_addr & page_mask;
2246 		entry_len = ioc->page_size - offset;
2247 
2248 		if (prp_entry == prp1_entry) {
2249 			/*
2250 			 * Must fill in the first PRP pointer (PRP1) before
2251 			 * moving on.
2252 			 */
2253 			*prp1_entry = cpu_to_le64(dma_addr);
2254 
2255 			/*
2256 			 * Now point to the second PRP entry within the
2257 			 * command (PRP2).
2258 			 */
2259 			prp_entry = prp2_entry;
2260 		} else if (prp_entry == prp2_entry) {
2261 			/*
2262 			 * Should the PRP2 entry be a PRP List pointer or just
2263 			 * a regular PRP pointer?  If there is more than one
2264 			 * more page of data, must use a PRP List pointer.
2265 			 */
2266 			if (length > ioc->page_size) {
2267 				/*
2268 				 * PRP2 will contain a PRP List pointer because
2269 				 * more PRP's are needed with this command. The
2270 				 * list will start at the beginning of the
2271 				 * contiguous buffer.
2272 				 */
2273 				*prp2_entry = cpu_to_le64(prp_entry_dma);
2274 
2275 				/*
2276 				 * The next PRP Entry will be the start of the
2277 				 * first PRP List.
2278 				 */
2279 				prp_entry = prp_page;
2280 			} else {
2281 				/*
2282 				 * After this, the PRP Entries are complete.
2283 				 * This command uses 2 PRP's and no PRP list.
2284 				 */
2285 				*prp2_entry = cpu_to_le64(dma_addr);
2286 			}
2287 		} else {
2288 			/*
2289 			 * Put entry in list and bump the addresses.
2290 			 *
2291 			 * After PRP1 and PRP2 are filled in, this will fill in
2292 			 * all remaining PRP entries in a PRP List, one per
2293 			 * each time through the loop.
2294 			 */
2295 			*prp_entry = cpu_to_le64(dma_addr);
2296 			prp_entry++;
2297 			prp_entry_dma++;
2298 		}
2299 
2300 		/*
2301 		 * Bump the phys address of the command's data buffer by the
2302 		 * entry_len.
2303 		 */
2304 		dma_addr += entry_len;
2305 
2306 		/* Decrement length accounting for last partial page. */
2307 		if (entry_len > length)
2308 			length = 0;
2309 		else
2310 			length -= entry_len;
2311 	}
2312 }
2313 
2314 /**
2315  * base_make_prp_nvme - Prepare PRPs (Physical Region Page) -
2316  *			SGLs specific to NVMe drives only
2317  *
2318  * @ioc:		per adapter object
2319  * @scmd:		SCSI command from the mid-layer
2320  * @mpi_request:	mpi request
2321  * @smid:		msg Index
2322  * @sge_count:		scatter gather element count.
2323  *
2324  * Return:		true: PRPs are built
2325  *			false: IEEE SGLs needs to be built
2326  */
2327 static void
2328 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,
2329 		struct scsi_cmnd *scmd,
2330 		Mpi25SCSIIORequest_t *mpi_request,
2331 		u16 smid, int sge_count)
2332 {
2333 	int sge_len, num_prp_in_chain = 0;
2334 	Mpi25IeeeSgeChain64_t *main_chain_element, *ptr_first_sgl;
2335 	__le64 *curr_buff;
2336 	dma_addr_t msg_dma, sge_addr, offset;
2337 	u32 page_mask, page_mask_result;
2338 	struct scatterlist *sg_scmd;
2339 	u32 first_prp_len;
2340 	int data_len = scsi_bufflen(scmd);
2341 	u32 nvme_pg_size;
2342 
2343 	nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);
2344 	/*
2345 	 * Nvme has a very convoluted prp format.  One prp is required
2346 	 * for each page or partial page. Driver need to split up OS sg_list
2347 	 * entries if it is longer than one page or cross a page
2348 	 * boundary.  Driver also have to insert a PRP list pointer entry as
2349 	 * the last entry in each physical page of the PRP list.
2350 	 *
2351 	 * NOTE: The first PRP "entry" is actually placed in the first
2352 	 * SGL entry in the main message as IEEE 64 format.  The 2nd
2353 	 * entry in the main message is the chain element, and the rest
2354 	 * of the PRP entries are built in the contiguous pcie buffer.
2355 	 */
2356 	page_mask = nvme_pg_size - 1;
2357 
2358 	/*
2359 	 * Native SGL is needed.
2360 	 * Put a chain element in main message frame that points to the first
2361 	 * chain buffer.
2362 	 *
2363 	 * NOTE:  The ChainOffset field must be 0 when using a chain pointer to
2364 	 *        a native SGL.
2365 	 */
2366 
2367 	/* Set main message chain element pointer */
2368 	main_chain_element = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2369 	/*
2370 	 * For NVMe the chain element needs to be the 2nd SG entry in the main
2371 	 * message.
2372 	 */
2373 	main_chain_element = (Mpi25IeeeSgeChain64_t *)
2374 		((u8 *)main_chain_element + sizeof(MPI25_IEEE_SGE_CHAIN64));
2375 
2376 	/*
2377 	 * For the PRP entries, use the specially allocated buffer of
2378 	 * contiguous memory.  Normal chain buffers can't be used
2379 	 * because each chain buffer would need to be the size of an OS
2380 	 * page (4k).
2381 	 */
2382 	curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);
2383 	msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);
2384 
2385 	main_chain_element->Address = cpu_to_le64(msg_dma);
2386 	main_chain_element->NextChainOffset = 0;
2387 	main_chain_element->Flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2388 			MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2389 			MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2390 
2391 	/* Build first prp, sge need not to be page aligned*/
2392 	ptr_first_sgl = (pMpi25IeeeSgeChain64_t)&mpi_request->SGL;
2393 	sg_scmd = scsi_sglist(scmd);
2394 	sge_addr = sg_dma_address(sg_scmd);
2395 	sge_len = sg_dma_len(sg_scmd);
2396 
2397 	offset = sge_addr & page_mask;
2398 	first_prp_len = nvme_pg_size - offset;
2399 
2400 	ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2401 	ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2402 
2403 	data_len -= first_prp_len;
2404 
2405 	if (sge_len > first_prp_len) {
2406 		sge_addr += first_prp_len;
2407 		sge_len -= first_prp_len;
2408 	} else if (data_len && (sge_len == first_prp_len)) {
2409 		sg_scmd = sg_next(sg_scmd);
2410 		sge_addr = sg_dma_address(sg_scmd);
2411 		sge_len = sg_dma_len(sg_scmd);
2412 	}
2413 
2414 	for (;;) {
2415 		offset = sge_addr & page_mask;
2416 
2417 		/* Put PRP pointer due to page boundary*/
2418 		page_mask_result = (uintptr_t)(curr_buff + 1) & page_mask;
2419 		if (unlikely(!page_mask_result)) {
2420 			scmd_printk(KERN_NOTICE,
2421 				scmd, "page boundary curr_buff: 0x%p\n",
2422 				curr_buff);
2423 			msg_dma += 8;
2424 			*curr_buff = cpu_to_le64(msg_dma);
2425 			curr_buff++;
2426 			num_prp_in_chain++;
2427 		}
2428 
2429 		*curr_buff = cpu_to_le64(sge_addr);
2430 		curr_buff++;
2431 		msg_dma += 8;
2432 		num_prp_in_chain++;
2433 
2434 		sge_addr += nvme_pg_size;
2435 		sge_len -= nvme_pg_size;
2436 		data_len -= nvme_pg_size;
2437 
2438 		if (data_len <= 0)
2439 			break;
2440 
2441 		if (sge_len > 0)
2442 			continue;
2443 
2444 		sg_scmd = sg_next(sg_scmd);
2445 		sge_addr = sg_dma_address(sg_scmd);
2446 		sge_len = sg_dma_len(sg_scmd);
2447 	}
2448 
2449 	main_chain_element->Length =
2450 		cpu_to_le32(num_prp_in_chain * sizeof(u64));
2451 	return;
2452 }
2453 
2454 static bool
2455 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,
2456 	struct _pcie_device *pcie_device, struct scsi_cmnd *scmd, int sge_count)
2457 {
2458 	u32 data_length = 0;
2459 	bool build_prp = true;
2460 
2461 	data_length = scsi_bufflen(scmd);
2462 	if (pcie_device &&
2463 	    (mpt3sas_scsih_is_pcie_scsi_device(pcie_device->device_info))) {
2464 		build_prp = false;
2465 		return build_prp;
2466 	}
2467 
2468 	/* If Datalenth is <= 16K and number of SGE’s entries are <= 2
2469 	 * we built IEEE SGL
2470 	 */
2471 	if ((data_length <= NVME_PRP_PAGE_SIZE*4) && (sge_count <= 2))
2472 		build_prp = false;
2473 
2474 	return build_prp;
2475 }
2476 
2477 /**
2478  * _base_check_pcie_native_sgl - This function is called for PCIe end devices to
2479  * determine if the driver needs to build a native SGL.  If so, that native
2480  * SGL is built in the special contiguous buffers allocated especially for
2481  * PCIe SGL creation.  If the driver will not build a native SGL, return
2482  * TRUE and a normal IEEE SGL will be built.  Currently this routine
2483  * supports NVMe.
2484  * @ioc: per adapter object
2485  * @mpi_request: mf request pointer
2486  * @smid: system request message index
2487  * @scmd: scsi command
2488  * @pcie_device: points to the PCIe device's info
2489  *
2490  * Return: 0 if native SGL was built, 1 if no SGL was built
2491  */
2492 static int
2493 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,
2494 	Mpi25SCSIIORequest_t *mpi_request, u16 smid, struct scsi_cmnd *scmd,
2495 	struct _pcie_device *pcie_device)
2496 {
2497 	int sges_left;
2498 
2499 	/* Get the SG list pointer and info. */
2500 	sges_left = scsi_dma_map(scmd);
2501 	if (sges_left < 0) {
2502 		sdev_printk(KERN_ERR, scmd->device,
2503 			"scsi_dma_map failed: request for %d bytes!\n",
2504 			scsi_bufflen(scmd));
2505 		return 1;
2506 	}
2507 
2508 	/* Check if we need to build a native SG list. */
2509 	if (!base_is_prp_possible(ioc, pcie_device,
2510 				scmd, sges_left)) {
2511 		/* We built a native SG list, just return. */
2512 		goto out;
2513 	}
2514 
2515 	/*
2516 	 * Build native NVMe PRP.
2517 	 */
2518 	base_make_prp_nvme(ioc, scmd, mpi_request,
2519 			smid, sges_left);
2520 
2521 	return 0;
2522 out:
2523 	scsi_dma_unmap(scmd);
2524 	return 1;
2525 }
2526 
2527 /**
2528  * _base_add_sg_single_ieee - add sg element for IEEE format
2529  * @paddr: virtual address for SGE
2530  * @flags: SGE flags
2531  * @chain_offset: number of 128 byte elements from start of segment
2532  * @length: data transfer length
2533  * @dma_addr: Physical address
2534  */
2535 static void
2536 _base_add_sg_single_ieee(void *paddr, u8 flags, u8 chain_offset, u32 length,
2537 	dma_addr_t dma_addr)
2538 {
2539 	Mpi25IeeeSgeChain64_t *sgel = paddr;
2540 
2541 	sgel->Flags = flags;
2542 	sgel->NextChainOffset = chain_offset;
2543 	sgel->Length = cpu_to_le32(length);
2544 	sgel->Address = cpu_to_le64(dma_addr);
2545 }
2546 
2547 /**
2548  * _base_build_zero_len_sge_ieee - build zero length sg entry for IEEE format
2549  * @ioc: per adapter object
2550  * @paddr: virtual address for SGE
2551  *
2552  * Create a zero length scatter gather entry to insure the IOCs hardware has
2553  * something to use if the target device goes brain dead and tries
2554  * to send data even when none is asked for.
2555  */
2556 static void
2557 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)
2558 {
2559 	u8 sgl_flags = (MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2560 		MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR |
2561 		MPI25_IEEE_SGE_FLAGS_END_OF_LIST);
2562 
2563 	_base_add_sg_single_ieee(paddr, sgl_flags, 0, 0, -1);
2564 }
2565 
2566 /**
2567  * _base_build_sg_scmd - main sg creation routine
2568  *		pcie_device is unused here!
2569  * @ioc: per adapter object
2570  * @scmd: scsi command
2571  * @smid: system request message index
2572  * @unused: unused pcie_device pointer
2573  * Context: none.
2574  *
2575  * The main routine that builds scatter gather table from a given
2576  * scsi request sent via the .queuecommand main handler.
2577  *
2578  * Return: 0 success, anything else error
2579  */
2580 static int
2581 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,
2582 	struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *unused)
2583 {
2584 	Mpi2SCSIIORequest_t *mpi_request;
2585 	dma_addr_t chain_dma;
2586 	struct scatterlist *sg_scmd;
2587 	void *sg_local, *chain;
2588 	u32 chain_offset;
2589 	u32 chain_length;
2590 	u32 chain_flags;
2591 	int sges_left;
2592 	u32 sges_in_segment;
2593 	u32 sgl_flags;
2594 	u32 sgl_flags_last_element;
2595 	u32 sgl_flags_end_buffer;
2596 	struct chain_tracker *chain_req;
2597 
2598 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2599 
2600 	/* init scatter gather flags */
2601 	sgl_flags = MPI2_SGE_FLAGS_SIMPLE_ELEMENT;
2602 	if (scmd->sc_data_direction == DMA_TO_DEVICE)
2603 		sgl_flags |= MPI2_SGE_FLAGS_HOST_TO_IOC;
2604 	sgl_flags_last_element = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT)
2605 	    << MPI2_SGE_FLAGS_SHIFT;
2606 	sgl_flags_end_buffer = (sgl_flags | MPI2_SGE_FLAGS_LAST_ELEMENT |
2607 	    MPI2_SGE_FLAGS_END_OF_BUFFER | MPI2_SGE_FLAGS_END_OF_LIST)
2608 	    << MPI2_SGE_FLAGS_SHIFT;
2609 	sgl_flags = sgl_flags << MPI2_SGE_FLAGS_SHIFT;
2610 
2611 	sg_scmd = scsi_sglist(scmd);
2612 	sges_left = scsi_dma_map(scmd);
2613 	if (sges_left < 0) {
2614 		sdev_printk(KERN_ERR, scmd->device,
2615 		 "scsi_dma_map failed: request for %d bytes!\n",
2616 		 scsi_bufflen(scmd));
2617 		return -ENOMEM;
2618 	}
2619 
2620 	sg_local = &mpi_request->SGL;
2621 	sges_in_segment = ioc->max_sges_in_main_message;
2622 	if (sges_left <= sges_in_segment)
2623 		goto fill_in_last_segment;
2624 
2625 	mpi_request->ChainOffset = (offsetof(Mpi2SCSIIORequest_t, SGL) +
2626 	    (sges_in_segment * ioc->sge_size))/4;
2627 
2628 	/* fill in main message segment when there is a chain following */
2629 	while (sges_in_segment) {
2630 		if (sges_in_segment == 1)
2631 			ioc->base_add_sg_single(sg_local,
2632 			    sgl_flags_last_element | sg_dma_len(sg_scmd),
2633 			    sg_dma_address(sg_scmd));
2634 		else
2635 			ioc->base_add_sg_single(sg_local, sgl_flags |
2636 			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2637 		sg_scmd = sg_next(sg_scmd);
2638 		sg_local += ioc->sge_size;
2639 		sges_left--;
2640 		sges_in_segment--;
2641 	}
2642 
2643 	/* initializing the chain flags and pointers */
2644 	chain_flags = MPI2_SGE_FLAGS_CHAIN_ELEMENT << MPI2_SGE_FLAGS_SHIFT;
2645 	chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2646 	if (!chain_req)
2647 		return -1;
2648 	chain = chain_req->chain_buffer;
2649 	chain_dma = chain_req->chain_buffer_dma;
2650 	do {
2651 		sges_in_segment = (sges_left <=
2652 		    ioc->max_sges_in_chain_message) ? sges_left :
2653 		    ioc->max_sges_in_chain_message;
2654 		chain_offset = (sges_left == sges_in_segment) ?
2655 		    0 : (sges_in_segment * ioc->sge_size)/4;
2656 		chain_length = sges_in_segment * ioc->sge_size;
2657 		if (chain_offset) {
2658 			chain_offset = chain_offset <<
2659 			    MPI2_SGE_CHAIN_OFFSET_SHIFT;
2660 			chain_length += ioc->sge_size;
2661 		}
2662 		ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |
2663 		    chain_length, chain_dma);
2664 		sg_local = chain;
2665 		if (!chain_offset)
2666 			goto fill_in_last_segment;
2667 
2668 		/* fill in chain segments */
2669 		while (sges_in_segment) {
2670 			if (sges_in_segment == 1)
2671 				ioc->base_add_sg_single(sg_local,
2672 				    sgl_flags_last_element |
2673 				    sg_dma_len(sg_scmd),
2674 				    sg_dma_address(sg_scmd));
2675 			else
2676 				ioc->base_add_sg_single(sg_local, sgl_flags |
2677 				    sg_dma_len(sg_scmd),
2678 				    sg_dma_address(sg_scmd));
2679 			sg_scmd = sg_next(sg_scmd);
2680 			sg_local += ioc->sge_size;
2681 			sges_left--;
2682 			sges_in_segment--;
2683 		}
2684 
2685 		chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2686 		if (!chain_req)
2687 			return -1;
2688 		chain = chain_req->chain_buffer;
2689 		chain_dma = chain_req->chain_buffer_dma;
2690 	} while (1);
2691 
2692 
2693  fill_in_last_segment:
2694 
2695 	/* fill the last segment */
2696 	while (sges_left) {
2697 		if (sges_left == 1)
2698 			ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |
2699 			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2700 		else
2701 			ioc->base_add_sg_single(sg_local, sgl_flags |
2702 			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2703 		sg_scmd = sg_next(sg_scmd);
2704 		sg_local += ioc->sge_size;
2705 		sges_left--;
2706 	}
2707 
2708 	return 0;
2709 }
2710 
2711 /**
2712  * _base_build_sg_scmd_ieee - main sg creation routine for IEEE format
2713  * @ioc: per adapter object
2714  * @scmd: scsi command
2715  * @smid: system request message index
2716  * @pcie_device: Pointer to pcie_device. If set, the pcie native sgl will be
2717  * constructed on need.
2718  * Context: none.
2719  *
2720  * The main routine that builds scatter gather table from a given
2721  * scsi request sent via the .queuecommand main handler.
2722  *
2723  * Return: 0 success, anything else error
2724  */
2725 static int
2726 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,
2727 	struct scsi_cmnd *scmd, u16 smid, struct _pcie_device *pcie_device)
2728 {
2729 	Mpi25SCSIIORequest_t *mpi_request;
2730 	dma_addr_t chain_dma;
2731 	struct scatterlist *sg_scmd;
2732 	void *sg_local, *chain;
2733 	u32 chain_offset;
2734 	u32 chain_length;
2735 	int sges_left;
2736 	u32 sges_in_segment;
2737 	u8 simple_sgl_flags;
2738 	u8 simple_sgl_flags_last;
2739 	u8 chain_sgl_flags;
2740 	struct chain_tracker *chain_req;
2741 
2742 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
2743 
2744 	/* init scatter gather flags */
2745 	simple_sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2746 	    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2747 	simple_sgl_flags_last = simple_sgl_flags |
2748 	    MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2749 	chain_sgl_flags = MPI2_IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2750 	    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2751 
2752 	/* Check if we need to build a native SG list. */
2753 	if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,
2754 			smid, scmd, pcie_device) == 0)) {
2755 		/* We built a native SG list, just return. */
2756 		return 0;
2757 	}
2758 
2759 	sg_scmd = scsi_sglist(scmd);
2760 	sges_left = scsi_dma_map(scmd);
2761 	if (sges_left < 0) {
2762 		sdev_printk(KERN_ERR, scmd->device,
2763 			"scsi_dma_map failed: request for %d bytes!\n",
2764 			scsi_bufflen(scmd));
2765 		return -ENOMEM;
2766 	}
2767 
2768 	sg_local = &mpi_request->SGL;
2769 	sges_in_segment = (ioc->request_sz -
2770 		   offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;
2771 	if (sges_left <= sges_in_segment)
2772 		goto fill_in_last_segment;
2773 
2774 	mpi_request->ChainOffset = (sges_in_segment - 1 /* chain element */) +
2775 	    (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);
2776 
2777 	/* fill in main message segment when there is a chain following */
2778 	while (sges_in_segment > 1) {
2779 		_base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2780 		    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2781 		sg_scmd = sg_next(sg_scmd);
2782 		sg_local += ioc->sge_size_ieee;
2783 		sges_left--;
2784 		sges_in_segment--;
2785 	}
2786 
2787 	/* initializing the pointers */
2788 	chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2789 	if (!chain_req)
2790 		return -1;
2791 	chain = chain_req->chain_buffer;
2792 	chain_dma = chain_req->chain_buffer_dma;
2793 	do {
2794 		sges_in_segment = (sges_left <=
2795 		    ioc->max_sges_in_chain_message) ? sges_left :
2796 		    ioc->max_sges_in_chain_message;
2797 		chain_offset = (sges_left == sges_in_segment) ?
2798 		    0 : sges_in_segment;
2799 		chain_length = sges_in_segment * ioc->sge_size_ieee;
2800 		if (chain_offset)
2801 			chain_length += ioc->sge_size_ieee;
2802 		_base_add_sg_single_ieee(sg_local, chain_sgl_flags,
2803 		    chain_offset, chain_length, chain_dma);
2804 
2805 		sg_local = chain;
2806 		if (!chain_offset)
2807 			goto fill_in_last_segment;
2808 
2809 		/* fill in chain segments */
2810 		while (sges_in_segment) {
2811 			_base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2812 			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2813 			sg_scmd = sg_next(sg_scmd);
2814 			sg_local += ioc->sge_size_ieee;
2815 			sges_left--;
2816 			sges_in_segment--;
2817 		}
2818 
2819 		chain_req = _base_get_chain_buffer_tracker(ioc, scmd);
2820 		if (!chain_req)
2821 			return -1;
2822 		chain = chain_req->chain_buffer;
2823 		chain_dma = chain_req->chain_buffer_dma;
2824 	} while (1);
2825 
2826 
2827  fill_in_last_segment:
2828 
2829 	/* fill the last segment */
2830 	while (sges_left > 0) {
2831 		if (sges_left == 1)
2832 			_base_add_sg_single_ieee(sg_local,
2833 			    simple_sgl_flags_last, 0, sg_dma_len(sg_scmd),
2834 			    sg_dma_address(sg_scmd));
2835 		else
2836 			_base_add_sg_single_ieee(sg_local, simple_sgl_flags, 0,
2837 			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
2838 		sg_scmd = sg_next(sg_scmd);
2839 		sg_local += ioc->sge_size_ieee;
2840 		sges_left--;
2841 	}
2842 
2843 	return 0;
2844 }
2845 
2846 /**
2847  * _base_build_sg_ieee - build generic sg for IEEE format
2848  * @ioc: per adapter object
2849  * @psge: virtual address for SGE
2850  * @data_out_dma: physical address for WRITES
2851  * @data_out_sz: data xfer size for WRITES
2852  * @data_in_dma: physical address for READS
2853  * @data_in_sz: data xfer size for READS
2854  */
2855 static void
2856 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,
2857 	dma_addr_t data_out_dma, size_t data_out_sz, dma_addr_t data_in_dma,
2858 	size_t data_in_sz)
2859 {
2860 	u8 sgl_flags;
2861 
2862 	if (!data_out_sz && !data_in_sz) {
2863 		_base_build_zero_len_sge_ieee(ioc, psge);
2864 		return;
2865 	}
2866 
2867 	if (data_out_sz && data_in_sz) {
2868 		/* WRITE sgel first */
2869 		sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2870 		    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2871 		_base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2872 		    data_out_dma);
2873 
2874 		/* incr sgel */
2875 		psge += ioc->sge_size_ieee;
2876 
2877 		/* READ sgel last */
2878 		sgl_flags |= MPI25_IEEE_SGE_FLAGS_END_OF_LIST;
2879 		_base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2880 		    data_in_dma);
2881 	} else if (data_out_sz) /* WRITE */ {
2882 		sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2883 		    MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2884 		    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2885 		_base_add_sg_single_ieee(psge, sgl_flags, 0, data_out_sz,
2886 		    data_out_dma);
2887 	} else if (data_in_sz) /* READ */ {
2888 		sgl_flags = MPI2_IEEE_SGE_FLAGS_SIMPLE_ELEMENT |
2889 		    MPI25_IEEE_SGE_FLAGS_END_OF_LIST |
2890 		    MPI2_IEEE_SGE_FLAGS_SYSTEM_ADDR;
2891 		_base_add_sg_single_ieee(psge, sgl_flags, 0, data_in_sz,
2892 		    data_in_dma);
2893 	}
2894 }
2895 
2896 #define convert_to_kb(x) ((x) << (PAGE_SHIFT - 10))
2897 
2898 /**
2899  * _base_config_dma_addressing - set dma addressing
2900  * @ioc: per adapter object
2901  * @pdev: PCI device struct
2902  *
2903  * Return: 0 for success, non-zero for failure.
2904  */
2905 static int
2906 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)
2907 {
2908 	struct sysinfo s;
2909 
2910 	if (ioc->is_mcpu_endpoint ||
2911 	    sizeof(dma_addr_t) == 4 || ioc->use_32bit_dma ||
2912 	    dma_get_required_mask(&pdev->dev) <= 32)
2913 		ioc->dma_mask = 32;
2914 	/* Set 63 bit DMA mask for all SAS3 and SAS35 controllers */
2915 	else if (ioc->hba_mpi_version_belonged > MPI2_VERSION)
2916 		ioc->dma_mask = 63;
2917 	else
2918 		ioc->dma_mask = 64;
2919 
2920 	if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)) ||
2921 	    dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(ioc->dma_mask)))
2922 		return -ENODEV;
2923 
2924 	if (ioc->dma_mask > 32) {
2925 		ioc->base_add_sg_single = &_base_add_sg_single_64;
2926 		ioc->sge_size = sizeof(Mpi2SGESimple64_t);
2927 	} else {
2928 		ioc->base_add_sg_single = &_base_add_sg_single_32;
2929 		ioc->sge_size = sizeof(Mpi2SGESimple32_t);
2930 	}
2931 
2932 	si_meminfo(&s);
2933 	ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",
2934 		ioc->dma_mask, convert_to_kb(s.totalram));
2935 
2936 	return 0;
2937 }
2938 
2939 /**
2940  * _base_check_enable_msix - checks MSIX capabable.
2941  * @ioc: per adapter object
2942  *
2943  * Check to see if card is capable of MSIX, and set number
2944  * of available msix vectors
2945  */
2946 static int
2947 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)
2948 {
2949 	int base;
2950 	u16 message_control;
2951 
2952 	/* Check whether controller SAS2008 B0 controller,
2953 	 * if it is SAS2008 B0 controller use IO-APIC instead of MSIX
2954 	 */
2955 	if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&
2956 	    ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {
2957 		return -EINVAL;
2958 	}
2959 
2960 	base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);
2961 	if (!base) {
2962 		dfailprintk(ioc, ioc_info(ioc, "msix not supported\n"));
2963 		return -EINVAL;
2964 	}
2965 
2966 	/* get msix vector count */
2967 	/* NUMA_IO not supported for older controllers */
2968 	if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||
2969 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||
2970 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||
2971 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||
2972 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||
2973 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||
2974 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)
2975 		ioc->msix_vector_count = 1;
2976 	else {
2977 		pci_read_config_word(ioc->pdev, base + 2, &message_control);
2978 		ioc->msix_vector_count = (message_control & 0x3FF) + 1;
2979 	}
2980 	dinitprintk(ioc, ioc_info(ioc, "msix is supported, vector_count(%d)\n",
2981 				  ioc->msix_vector_count));
2982 	return 0;
2983 }
2984 
2985 /**
2986  * mpt3sas_base_free_irq - free irq
2987  * @ioc: per adapter object
2988  *
2989  * Freeing respective reply_queue from the list.
2990  */
2991 void
2992 mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc)
2993 {
2994 	struct adapter_reply_queue *reply_q, *next;
2995 
2996 	if (list_empty(&ioc->reply_queue_list))
2997 		return;
2998 
2999 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {
3000 		list_del(&reply_q->list);
3001 		if (ioc->smp_affinity_enable)
3002 			irq_set_affinity_hint(pci_irq_vector(ioc->pdev,
3003 			    reply_q->msix_index), NULL);
3004 		free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),
3005 			 reply_q);
3006 		kfree(reply_q);
3007 	}
3008 }
3009 
3010 /**
3011  * _base_request_irq - request irq
3012  * @ioc: per adapter object
3013  * @index: msix index into vector table
3014  *
3015  * Inserting respective reply_queue into the list.
3016  */
3017 static int
3018 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)
3019 {
3020 	struct pci_dev *pdev = ioc->pdev;
3021 	struct adapter_reply_queue *reply_q;
3022 	int r;
3023 
3024 	reply_q =  kzalloc(sizeof(struct adapter_reply_queue), GFP_KERNEL);
3025 	if (!reply_q) {
3026 		ioc_err(ioc, "unable to allocate memory %zu!\n",
3027 			sizeof(struct adapter_reply_queue));
3028 		return -ENOMEM;
3029 	}
3030 	reply_q->ioc = ioc;
3031 	reply_q->msix_index = index;
3032 
3033 	atomic_set(&reply_q->busy, 0);
3034 	if (ioc->msix_enable)
3035 		snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d-msix%d",
3036 		    ioc->driver_name, ioc->id, index);
3037 	else
3038 		snprintf(reply_q->name, MPT_NAME_LENGTH, "%s%d",
3039 		    ioc->driver_name, ioc->id);
3040 	r = request_irq(pci_irq_vector(pdev, index), _base_interrupt,
3041 			IRQF_SHARED, reply_q->name, reply_q);
3042 	if (r) {
3043 		pr_err("%s: unable to allocate interrupt %d!\n",
3044 		       reply_q->name, pci_irq_vector(pdev, index));
3045 		kfree(reply_q);
3046 		return -EBUSY;
3047 	}
3048 
3049 	INIT_LIST_HEAD(&reply_q->list);
3050 	list_add_tail(&reply_q->list, &ioc->reply_queue_list);
3051 	return 0;
3052 }
3053 
3054 /**
3055  * _base_assign_reply_queues - assigning msix index for each cpu
3056  * @ioc: per adapter object
3057  *
3058  * The enduser would need to set the affinity via /proc/irq/#/smp_affinity
3059  *
3060  * It would nice if we could call irq_set_affinity, however it is not
3061  * an exported symbol
3062  */
3063 static void
3064 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)
3065 {
3066 	unsigned int cpu, nr_cpus, nr_msix, index = 0;
3067 	struct adapter_reply_queue *reply_q;
3068 	int local_numa_node;
3069 
3070 	if (!_base_is_controller_msix_enabled(ioc))
3071 		return;
3072 
3073 	if (ioc->msix_load_balance)
3074 		return;
3075 
3076 	memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);
3077 
3078 	nr_cpus = num_online_cpus();
3079 	nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,
3080 					       ioc->facts.MaxMSIxVectors);
3081 	if (!nr_msix)
3082 		return;
3083 
3084 	if (ioc->smp_affinity_enable) {
3085 
3086 		/*
3087 		 * set irq affinity to local numa node for those irqs
3088 		 * corresponding to high iops queues.
3089 		 */
3090 		if (ioc->high_iops_queues) {
3091 			local_numa_node = dev_to_node(&ioc->pdev->dev);
3092 			for (index = 0; index < ioc->high_iops_queues;
3093 			    index++) {
3094 				irq_set_affinity_hint(pci_irq_vector(ioc->pdev,
3095 				    index), cpumask_of_node(local_numa_node));
3096 			}
3097 		}
3098 
3099 		list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
3100 			const cpumask_t *mask;
3101 
3102 			if (reply_q->msix_index < ioc->high_iops_queues)
3103 				continue;
3104 
3105 			mask = pci_irq_get_affinity(ioc->pdev,
3106 			    reply_q->msix_index);
3107 			if (!mask) {
3108 				ioc_warn(ioc, "no affinity for msi %x\n",
3109 					 reply_q->msix_index);
3110 				goto fall_back;
3111 			}
3112 
3113 			for_each_cpu_and(cpu, mask, cpu_online_mask) {
3114 				if (cpu >= ioc->cpu_msix_table_sz)
3115 					break;
3116 				ioc->cpu_msix_table[cpu] = reply_q->msix_index;
3117 			}
3118 		}
3119 		return;
3120 	}
3121 
3122 fall_back:
3123 	cpu = cpumask_first(cpu_online_mask);
3124 	nr_msix -= ioc->high_iops_queues;
3125 	index = 0;
3126 
3127 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
3128 		unsigned int i, group = nr_cpus / nr_msix;
3129 
3130 		if (reply_q->msix_index < ioc->high_iops_queues)
3131 			continue;
3132 
3133 		if (cpu >= nr_cpus)
3134 			break;
3135 
3136 		if (index < nr_cpus % nr_msix)
3137 			group++;
3138 
3139 		for (i = 0 ; i < group ; i++) {
3140 			ioc->cpu_msix_table[cpu] = reply_q->msix_index;
3141 			cpu = cpumask_next(cpu, cpu_online_mask);
3142 		}
3143 		index++;
3144 	}
3145 }
3146 
3147 /**
3148  * _base_check_and_enable_high_iops_queues - enable high iops mode
3149  * @ioc: per adapter object
3150  * @hba_msix_vector_count: msix vectors supported by HBA
3151  *
3152  * Enable high iops queues only if
3153  *  - HBA is a SEA/AERO controller and
3154  *  - MSI-Xs vector supported by the HBA is 128 and
3155  *  - total CPU count in the system >=16 and
3156  *  - loaded driver with default max_msix_vectors module parameter and
3157  *  - system booted in non kdump mode
3158  *
3159  * Return: nothing.
3160  */
3161 static void
3162 _base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc,
3163 		int hba_msix_vector_count)
3164 {
3165 	u16 lnksta, speed;
3166 
3167 	if (perf_mode == MPT_PERF_MODE_IOPS ||
3168 	    perf_mode == MPT_PERF_MODE_LATENCY) {
3169 		ioc->high_iops_queues = 0;
3170 		return;
3171 	}
3172 
3173 	if (perf_mode == MPT_PERF_MODE_DEFAULT) {
3174 
3175 		pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta);
3176 		speed = lnksta & PCI_EXP_LNKSTA_CLS;
3177 
3178 		if (speed < 0x4) {
3179 			ioc->high_iops_queues = 0;
3180 			return;
3181 		}
3182 	}
3183 
3184 	if (!reset_devices && ioc->is_aero_ioc &&
3185 	    hba_msix_vector_count == MPT3SAS_GEN35_MAX_MSIX_QUEUES &&
3186 	    num_online_cpus() >= MPT3SAS_HIGH_IOPS_REPLY_QUEUES &&
3187 	    max_msix_vectors == -1)
3188 		ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES;
3189 	else
3190 		ioc->high_iops_queues = 0;
3191 }
3192 
3193 /**
3194  * mpt3sas_base_disable_msix - disables msix
3195  * @ioc: per adapter object
3196  *
3197  */
3198 void
3199 mpt3sas_base_disable_msix(struct MPT3SAS_ADAPTER *ioc)
3200 {
3201 	if (!ioc->msix_enable)
3202 		return;
3203 	pci_free_irq_vectors(ioc->pdev);
3204 	ioc->msix_enable = 0;
3205 }
3206 
3207 /**
3208  * _base_alloc_irq_vectors - allocate msix vectors
3209  * @ioc: per adapter object
3210  *
3211  */
3212 static int
3213 _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc)
3214 {
3215 	int i, irq_flags = PCI_IRQ_MSIX;
3216 	struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues };
3217 	struct irq_affinity *descp = &desc;
3218 
3219 	if (ioc->smp_affinity_enable)
3220 		irq_flags |= PCI_IRQ_AFFINITY;
3221 	else
3222 		descp = NULL;
3223 
3224 	ioc_info(ioc, " %d %d\n", ioc->high_iops_queues,
3225 	    ioc->reply_queue_count);
3226 
3227 	i = pci_alloc_irq_vectors_affinity(ioc->pdev,
3228 	    ioc->high_iops_queues,
3229 	    ioc->reply_queue_count, irq_flags, descp);
3230 
3231 	return i;
3232 }
3233 
3234 /**
3235  * _base_enable_msix - enables msix, failback to io_apic
3236  * @ioc: per adapter object
3237  *
3238  */
3239 static int
3240 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)
3241 {
3242 	int r;
3243 	int i, local_max_msix_vectors;
3244 	u8 try_msix = 0;
3245 
3246 	ioc->msix_load_balance = false;
3247 
3248 	if (msix_disable == -1 || msix_disable == 0)
3249 		try_msix = 1;
3250 
3251 	if (!try_msix)
3252 		goto try_ioapic;
3253 
3254 	if (_base_check_enable_msix(ioc) != 0)
3255 		goto try_ioapic;
3256 
3257 	ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count);
3258 	pr_info("\t no of cores: %d, max_msix_vectors: %d\n",
3259 		ioc->cpu_count, max_msix_vectors);
3260 	if (ioc->is_aero_ioc)
3261 		_base_check_and_enable_high_iops_queues(ioc,
3262 			ioc->msix_vector_count);
3263 	ioc->reply_queue_count =
3264 		min_t(int, ioc->cpu_count + ioc->high_iops_queues,
3265 		ioc->msix_vector_count);
3266 
3267 	if (!ioc->rdpq_array_enable && max_msix_vectors == -1)
3268 		local_max_msix_vectors = (reset_devices) ? 1 : 8;
3269 	else
3270 		local_max_msix_vectors = max_msix_vectors;
3271 
3272 	if (local_max_msix_vectors > 0)
3273 		ioc->reply_queue_count = min_t(int, local_max_msix_vectors,
3274 			ioc->reply_queue_count);
3275 	else if (local_max_msix_vectors == 0)
3276 		goto try_ioapic;
3277 
3278 	/*
3279 	 * Enable msix_load_balance only if combined reply queue mode is
3280 	 * disabled on SAS3 & above generation HBA devices.
3281 	 */
3282 	if (!ioc->combined_reply_queue &&
3283 	    ioc->hba_mpi_version_belonged != MPI2_VERSION) {
3284 		ioc_info(ioc,
3285 		    "combined ReplyQueue is off, Enabling msix load balance\n");
3286 		ioc->msix_load_balance = true;
3287 	}
3288 
3289 	/*
3290 	 * smp affinity setting is not need when msix load balance
3291 	 * is enabled.
3292 	 */
3293 	if (ioc->msix_load_balance)
3294 		ioc->smp_affinity_enable = 0;
3295 
3296 	r = _base_alloc_irq_vectors(ioc);
3297 	if (r < 0) {
3298 		ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r);
3299 		goto try_ioapic;
3300 	}
3301 
3302 	ioc->msix_enable = 1;
3303 	ioc->reply_queue_count = r;
3304 	for (i = 0; i < ioc->reply_queue_count; i++) {
3305 		r = _base_request_irq(ioc, i);
3306 		if (r) {
3307 			mpt3sas_base_free_irq(ioc);
3308 			mpt3sas_base_disable_msix(ioc);
3309 			goto try_ioapic;
3310 		}
3311 	}
3312 
3313 	ioc_info(ioc, "High IOPs queues : %s\n",
3314 			ioc->high_iops_queues ? "enabled" : "disabled");
3315 
3316 	return 0;
3317 
3318 /* failback to io_apic interrupt routing */
3319  try_ioapic:
3320 	ioc->high_iops_queues = 0;
3321 	ioc_info(ioc, "High IOPs queues : disabled\n");
3322 	ioc->reply_queue_count = 1;
3323 	r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);
3324 	if (r < 0) {
3325 		dfailprintk(ioc,
3326 			    ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",
3327 				     r));
3328 	} else
3329 		r = _base_request_irq(ioc, 0);
3330 
3331 	return r;
3332 }
3333 
3334 /**
3335  * mpt3sas_base_unmap_resources - free controller resources
3336  * @ioc: per adapter object
3337  */
3338 static void
3339 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)
3340 {
3341 	struct pci_dev *pdev = ioc->pdev;
3342 
3343 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
3344 
3345 	mpt3sas_base_free_irq(ioc);
3346 	mpt3sas_base_disable_msix(ioc);
3347 
3348 	kfree(ioc->replyPostRegisterIndex);
3349 	ioc->replyPostRegisterIndex = NULL;
3350 
3351 
3352 	if (ioc->chip_phys) {
3353 		iounmap(ioc->chip);
3354 		ioc->chip_phys = 0;
3355 	}
3356 
3357 	if (pci_is_enabled(pdev)) {
3358 		pci_release_selected_regions(ioc->pdev, ioc->bars);
3359 		pci_disable_pcie_error_reporting(pdev);
3360 		pci_disable_device(pdev);
3361 	}
3362 }
3363 
3364 static int
3365 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
3366 
3367 /**
3368  * mpt3sas_base_check_for_fault_and_issue_reset - check if IOC is in fault state
3369  *     and if it is in fault state then issue diag reset.
3370  * @ioc: per adapter object
3371  *
3372  * Return: 0 for success, non-zero for failure.
3373  */
3374 int
3375 mpt3sas_base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc)
3376 {
3377 	u32 ioc_state;
3378 	int rc = -EFAULT;
3379 
3380 	dinitprintk(ioc, pr_info("%s\n", __func__));
3381 	if (ioc->pci_error_recovery)
3382 		return 0;
3383 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
3384 	dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state));
3385 
3386 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
3387 		mpt3sas_print_fault_code(ioc, ioc_state &
3388 		    MPI2_DOORBELL_DATA_MASK);
3389 		mpt3sas_base_mask_interrupts(ioc);
3390 		rc = _base_diag_reset(ioc);
3391 	} else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
3392 	    MPI2_IOC_STATE_COREDUMP) {
3393 		mpt3sas_print_coredump_info(ioc, ioc_state &
3394 		     MPI2_DOORBELL_DATA_MASK);
3395 		mpt3sas_base_wait_for_coredump_completion(ioc, __func__);
3396 		mpt3sas_base_mask_interrupts(ioc);
3397 		rc = _base_diag_reset(ioc);
3398 	}
3399 
3400 	return rc;
3401 }
3402 
3403 /**
3404  * mpt3sas_base_map_resources - map in controller resources (io/irq/memap)
3405  * @ioc: per adapter object
3406  *
3407  * Return: 0 for success, non-zero for failure.
3408  */
3409 int
3410 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)
3411 {
3412 	struct pci_dev *pdev = ioc->pdev;
3413 	u32 memap_sz;
3414 	u32 pio_sz;
3415 	int i, r = 0, rc;
3416 	u64 pio_chip = 0;
3417 	phys_addr_t chip_phys = 0;
3418 	struct adapter_reply_queue *reply_q;
3419 
3420 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
3421 
3422 	ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3423 	if (pci_enable_device_mem(pdev)) {
3424 		ioc_warn(ioc, "pci_enable_device_mem: failed\n");
3425 		ioc->bars = 0;
3426 		return -ENODEV;
3427 	}
3428 
3429 
3430 	if (pci_request_selected_regions(pdev, ioc->bars,
3431 	    ioc->driver_name)) {
3432 		ioc_warn(ioc, "pci_request_selected_regions: failed\n");
3433 		ioc->bars = 0;
3434 		r = -ENODEV;
3435 		goto out_fail;
3436 	}
3437 
3438 /* AER (Advanced Error Reporting) hooks */
3439 	pci_enable_pcie_error_reporting(pdev);
3440 
3441 	pci_set_master(pdev);
3442 
3443 
3444 	if (_base_config_dma_addressing(ioc, pdev) != 0) {
3445 		ioc_warn(ioc, "no suitable DMA mask for %s\n", pci_name(pdev));
3446 		r = -ENODEV;
3447 		goto out_fail;
3448 	}
3449 
3450 	for (i = 0, memap_sz = 0, pio_sz = 0; (i < DEVICE_COUNT_RESOURCE) &&
3451 	     (!memap_sz || !pio_sz); i++) {
3452 		if (pci_resource_flags(pdev, i) & IORESOURCE_IO) {
3453 			if (pio_sz)
3454 				continue;
3455 			pio_chip = (u64)pci_resource_start(pdev, i);
3456 			pio_sz = pci_resource_len(pdev, i);
3457 		} else if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3458 			if (memap_sz)
3459 				continue;
3460 			ioc->chip_phys = pci_resource_start(pdev, i);
3461 			chip_phys = ioc->chip_phys;
3462 			memap_sz = pci_resource_len(pdev, i);
3463 			ioc->chip = ioremap(ioc->chip_phys, memap_sz);
3464 		}
3465 	}
3466 
3467 	if (ioc->chip == NULL) {
3468 		ioc_err(ioc,
3469 		    "unable to map adapter memory! or resource not found\n");
3470 		r = -EINVAL;
3471 		goto out_fail;
3472 	}
3473 
3474 	mpt3sas_base_mask_interrupts(ioc);
3475 
3476 	r = _base_get_ioc_facts(ioc);
3477 	if (r) {
3478 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);
3479 		if (rc || (_base_get_ioc_facts(ioc)))
3480 			goto out_fail;
3481 	}
3482 
3483 	if (!ioc->rdpq_array_enable_assigned) {
3484 		ioc->rdpq_array_enable = ioc->rdpq_array_capable;
3485 		ioc->rdpq_array_enable_assigned = 1;
3486 	}
3487 
3488 	r = _base_enable_msix(ioc);
3489 	if (r)
3490 		goto out_fail;
3491 
3492 	if (!ioc->is_driver_loading)
3493 		_base_init_irqpolls(ioc);
3494 	/* Use the Combined reply queue feature only for SAS3 C0 & higher
3495 	 * revision HBAs and also only when reply queue count is greater than 8
3496 	 */
3497 	if (ioc->combined_reply_queue) {
3498 		/* Determine the Supplemental Reply Post Host Index Registers
3499 		 * Addresse. Supplemental Reply Post Host Index Registers
3500 		 * starts at offset MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET and
3501 		 * each register is at offset bytes of
3502 		 * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET from previous one.
3503 		 */
3504 		ioc->replyPostRegisterIndex = kcalloc(
3505 		     ioc->combined_reply_index_count,
3506 		     sizeof(resource_size_t *), GFP_KERNEL);
3507 		if (!ioc->replyPostRegisterIndex) {
3508 			ioc_err(ioc,
3509 			    "allocation for replyPostRegisterIndex failed!\n");
3510 			r = -ENOMEM;
3511 			goto out_fail;
3512 		}
3513 
3514 		for (i = 0; i < ioc->combined_reply_index_count; i++) {
3515 			ioc->replyPostRegisterIndex[i] = (resource_size_t *)
3516 			     ((u8 __force *)&ioc->chip->Doorbell +
3517 			     MPI25_SUP_REPLY_POST_HOST_INDEX_OFFSET +
3518 			     (i * MPT3_SUP_REPLY_POST_HOST_INDEX_REG_OFFSET));
3519 		}
3520 	}
3521 
3522 	if (ioc->is_warpdrive) {
3523 		ioc->reply_post_host_index[0] = (resource_size_t __iomem *)
3524 		    &ioc->chip->ReplyPostHostIndex;
3525 
3526 		for (i = 1; i < ioc->cpu_msix_table_sz; i++)
3527 			ioc->reply_post_host_index[i] =
3528 			(resource_size_t __iomem *)
3529 			((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)
3530 			* 4)));
3531 	}
3532 
3533 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list)
3534 		pr_info("%s: %s enabled: IRQ %d\n",
3535 			reply_q->name,
3536 			ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC",
3537 			pci_irq_vector(ioc->pdev, reply_q->msix_index));
3538 
3539 	ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n",
3540 		 &chip_phys, ioc->chip, memap_sz);
3541 	ioc_info(ioc, "ioport(0x%016llx), size(%d)\n",
3542 		 (unsigned long long)pio_chip, pio_sz);
3543 
3544 	/* Save PCI configuration state for recovery from PCI AER/EEH errors */
3545 	pci_save_state(pdev);
3546 	return 0;
3547 
3548  out_fail:
3549 	mpt3sas_base_unmap_resources(ioc);
3550 	return r;
3551 }
3552 
3553 /**
3554  * mpt3sas_base_get_msg_frame - obtain request mf pointer
3555  * @ioc: per adapter object
3556  * @smid: system request message index(smid zero is invalid)
3557  *
3558  * Return: virt pointer to message frame.
3559  */
3560 void *
3561 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3562 {
3563 	return (void *)(ioc->request + (smid * ioc->request_sz));
3564 }
3565 
3566 /**
3567  * mpt3sas_base_get_sense_buffer - obtain a sense buffer virt addr
3568  * @ioc: per adapter object
3569  * @smid: system request message index
3570  *
3571  * Return: virt pointer to sense buffer.
3572  */
3573 void *
3574 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3575 {
3576 	return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));
3577 }
3578 
3579 /**
3580  * mpt3sas_base_get_sense_buffer_dma - obtain a sense buffer dma addr
3581  * @ioc: per adapter object
3582  * @smid: system request message index
3583  *
3584  * Return: phys pointer to the low 32bit address of the sense buffer.
3585  */
3586 __le32
3587 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3588 {
3589 	return cpu_to_le32(ioc->sense_dma + ((smid - 1) *
3590 	    SCSI_SENSE_BUFFERSIZE));
3591 }
3592 
3593 /**
3594  * mpt3sas_base_get_pcie_sgl - obtain a PCIe SGL virt addr
3595  * @ioc: per adapter object
3596  * @smid: system request message index
3597  *
3598  * Return: virt pointer to a PCIe SGL.
3599  */
3600 void *
3601 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3602 {
3603 	return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);
3604 }
3605 
3606 /**
3607  * mpt3sas_base_get_pcie_sgl_dma - obtain a PCIe SGL dma addr
3608  * @ioc: per adapter object
3609  * @smid: system request message index
3610  *
3611  * Return: phys pointer to the address of the PCIe buffer.
3612  */
3613 dma_addr_t
3614 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3615 {
3616 	return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;
3617 }
3618 
3619 /**
3620  * mpt3sas_base_get_reply_virt_addr - obtain reply frames virt address
3621  * @ioc: per adapter object
3622  * @phys_addr: lower 32 physical addr of the reply
3623  *
3624  * Converts 32bit lower physical addr into a virt address.
3625  */
3626 void *
3627 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)
3628 {
3629 	if (!phys_addr)
3630 		return NULL;
3631 	return ioc->reply + (phys_addr - (u32)ioc->reply_dma);
3632 }
3633 
3634 /**
3635  * _base_get_msix_index - get the msix index
3636  * @ioc: per adapter object
3637  * @scmd: scsi_cmnd object
3638  *
3639  * Return: msix index of general reply queues,
3640  * i.e. reply queue on which IO request's reply
3641  * should be posted by the HBA firmware.
3642  */
3643 static inline u8
3644 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc,
3645 	struct scsi_cmnd *scmd)
3646 {
3647 	/* Enables reply_queue load balancing */
3648 	if (ioc->msix_load_balance)
3649 		return ioc->reply_queue_count ?
3650 		    base_mod64(atomic64_add_return(1,
3651 		    &ioc->total_io_cnt), ioc->reply_queue_count) : 0;
3652 
3653 	if (scmd && ioc->shost->nr_hw_queues > 1) {
3654 		u32 tag = blk_mq_unique_tag(scmd->request);
3655 
3656 		return blk_mq_unique_tag_to_hwq(tag) +
3657 			ioc->high_iops_queues;
3658 	}
3659 
3660 	return ioc->cpu_msix_table[raw_smp_processor_id()];
3661 }
3662 
3663 /**
3664  * _base_get_high_iops_msix_index - get the msix index of
3665  *				high iops queues
3666  * @ioc: per adapter object
3667  * @scmd: scsi_cmnd object
3668  *
3669  * Return: msix index of high iops reply queues.
3670  * i.e. high iops reply queue on which IO request's
3671  * reply should be posted by the HBA firmware.
3672  */
3673 static inline u8
3674 _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc,
3675 	struct scsi_cmnd *scmd)
3676 {
3677 	/**
3678 	 * Round robin the IO interrupts among the high iops
3679 	 * reply queues in terms of batch count 16 when outstanding
3680 	 * IOs on the target device is >=8.
3681 	 */
3682 
3683 	if (scsi_device_busy(scmd->device) > MPT3SAS_DEVICE_HIGH_IOPS_DEPTH)
3684 		return base_mod64((
3685 		    atomic64_add_return(1, &ioc->high_iops_outstanding) /
3686 		    MPT3SAS_HIGH_IOPS_BATCH_COUNT),
3687 		    MPT3SAS_HIGH_IOPS_REPLY_QUEUES);
3688 
3689 	return _base_get_msix_index(ioc, scmd);
3690 }
3691 
3692 /**
3693  * mpt3sas_base_get_smid - obtain a free smid from internal queue
3694  * @ioc: per adapter object
3695  * @cb_idx: callback index
3696  *
3697  * Return: smid (zero is invalid)
3698  */
3699 u16
3700 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3701 {
3702 	unsigned long flags;
3703 	struct request_tracker *request;
3704 	u16 smid;
3705 
3706 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3707 	if (list_empty(&ioc->internal_free_list)) {
3708 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3709 		ioc_err(ioc, "%s: smid not available\n", __func__);
3710 		return 0;
3711 	}
3712 
3713 	request = list_entry(ioc->internal_free_list.next,
3714 	    struct request_tracker, tracker_list);
3715 	request->cb_idx = cb_idx;
3716 	smid = request->smid;
3717 	list_del(&request->tracker_list);
3718 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3719 	return smid;
3720 }
3721 
3722 /**
3723  * mpt3sas_base_get_smid_scsiio - obtain a free smid from scsiio queue
3724  * @ioc: per adapter object
3725  * @cb_idx: callback index
3726  * @scmd: pointer to scsi command object
3727  *
3728  * Return: smid (zero is invalid)
3729  */
3730 u16
3731 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,
3732 	struct scsi_cmnd *scmd)
3733 {
3734 	struct scsiio_tracker *request = scsi_cmd_priv(scmd);
3735 	u16 smid;
3736 	u32 tag, unique_tag;
3737 
3738 	unique_tag = blk_mq_unique_tag(scmd->request);
3739 	tag = blk_mq_unique_tag_to_tag(unique_tag);
3740 
3741 	/*
3742 	 * Store hw queue number corresponding to the tag.
3743 	 * This hw queue number is used later to determine
3744 	 * the unique_tag using the logic below. This unique_tag
3745 	 * is used to retrieve the scmd pointer corresponding
3746 	 * to tag using scsi_host_find_tag() API.
3747 	 *
3748 	 * tag = smid - 1;
3749 	 * unique_tag = ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag;
3750 	 */
3751 	ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag);
3752 
3753 	smid = tag + 1;
3754 	request->cb_idx = cb_idx;
3755 	request->smid = smid;
3756 	request->scmd = scmd;
3757 	INIT_LIST_HEAD(&request->chain_list);
3758 	return smid;
3759 }
3760 
3761 /**
3762  * mpt3sas_base_get_smid_hpr - obtain a free smid from hi-priority queue
3763  * @ioc: per adapter object
3764  * @cb_idx: callback index
3765  *
3766  * Return: smid (zero is invalid)
3767  */
3768 u16
3769 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)
3770 {
3771 	unsigned long flags;
3772 	struct request_tracker *request;
3773 	u16 smid;
3774 
3775 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3776 	if (list_empty(&ioc->hpr_free_list)) {
3777 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3778 		return 0;
3779 	}
3780 
3781 	request = list_entry(ioc->hpr_free_list.next,
3782 	    struct request_tracker, tracker_list);
3783 	request->cb_idx = cb_idx;
3784 	smid = request->smid;
3785 	list_del(&request->tracker_list);
3786 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3787 	return smid;
3788 }
3789 
3790 static void
3791 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc)
3792 {
3793 	/*
3794 	 * See _wait_for_commands_to_complete() call with regards to this code.
3795 	 */
3796 	if (ioc->shost_recovery && ioc->pending_io_count) {
3797 		ioc->pending_io_count = scsi_host_busy(ioc->shost);
3798 		if (ioc->pending_io_count == 0)
3799 			wake_up(&ioc->reset_wq);
3800 	}
3801 }
3802 
3803 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,
3804 			   struct scsiio_tracker *st)
3805 {
3806 	if (WARN_ON(st->smid == 0))
3807 		return;
3808 	st->cb_idx = 0xFF;
3809 	st->direct_io = 0;
3810 	st->scmd = NULL;
3811 	atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);
3812 	st->smid = 0;
3813 }
3814 
3815 /**
3816  * mpt3sas_base_free_smid - put smid back on free_list
3817  * @ioc: per adapter object
3818  * @smid: system request message index
3819  */
3820 void
3821 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3822 {
3823 	unsigned long flags;
3824 	int i;
3825 
3826 	if (smid < ioc->hi_priority_smid) {
3827 		struct scsiio_tracker *st;
3828 		void *request;
3829 
3830 		st = _get_st_from_smid(ioc, smid);
3831 		if (!st) {
3832 			_base_recovery_check(ioc);
3833 			return;
3834 		}
3835 
3836 		/* Clear MPI request frame */
3837 		request = mpt3sas_base_get_msg_frame(ioc, smid);
3838 		memset(request, 0, ioc->request_sz);
3839 
3840 		mpt3sas_base_clear_st(ioc, st);
3841 		_base_recovery_check(ioc);
3842 		ioc->io_queue_num[smid - 1] = 0;
3843 		return;
3844 	}
3845 
3846 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
3847 	if (smid < ioc->internal_smid) {
3848 		/* hi-priority */
3849 		i = smid - ioc->hi_priority_smid;
3850 		ioc->hpr_lookup[i].cb_idx = 0xFF;
3851 		list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);
3852 	} else if (smid <= ioc->hba_queue_depth) {
3853 		/* internal queue */
3854 		i = smid - ioc->internal_smid;
3855 		ioc->internal_lookup[i].cb_idx = 0xFF;
3856 		list_add(&ioc->internal_lookup[i].tracker_list,
3857 		    &ioc->internal_free_list);
3858 	}
3859 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
3860 }
3861 
3862 /**
3863  * _base_mpi_ep_writeq - 32 bit write to MMIO
3864  * @b: data payload
3865  * @addr: address in MMIO space
3866  * @writeq_lock: spin lock
3867  *
3868  * This special handling for MPI EP to take care of 32 bit
3869  * environment where its not quarenteed to send the entire word
3870  * in one transfer.
3871  */
3872 static inline void
3873 _base_mpi_ep_writeq(__u64 b, volatile void __iomem *addr,
3874 					spinlock_t *writeq_lock)
3875 {
3876 	unsigned long flags;
3877 
3878 	spin_lock_irqsave(writeq_lock, flags);
3879 	__raw_writel((u32)(b), addr);
3880 	__raw_writel((u32)(b >> 32), (addr + 4));
3881 	spin_unlock_irqrestore(writeq_lock, flags);
3882 }
3883 
3884 /**
3885  * _base_writeq - 64 bit write to MMIO
3886  * @b: data payload
3887  * @addr: address in MMIO space
3888  * @writeq_lock: spin lock
3889  *
3890  * Glue for handling an atomic 64 bit word to MMIO. This special handling takes
3891  * care of 32 bit environment where its not quarenteed to send the entire word
3892  * in one transfer.
3893  */
3894 #if defined(writeq) && defined(CONFIG_64BIT)
3895 static inline void
3896 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3897 {
3898 	wmb();
3899 	__raw_writeq(b, addr);
3900 	barrier();
3901 }
3902 #else
3903 static inline void
3904 _base_writeq(__u64 b, volatile void __iomem *addr, spinlock_t *writeq_lock)
3905 {
3906 	_base_mpi_ep_writeq(b, addr, writeq_lock);
3907 }
3908 #endif
3909 
3910 /**
3911  * _base_set_and_get_msix_index - get the msix index and assign to msix_io
3912  *                                variable of scsi tracker
3913  * @ioc: per adapter object
3914  * @smid: system request message index
3915  *
3916  * Return: msix index.
3917  */
3918 static u8
3919 _base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid)
3920 {
3921 	struct scsiio_tracker *st = NULL;
3922 
3923 	if (smid < ioc->hi_priority_smid)
3924 		st = _get_st_from_smid(ioc, smid);
3925 
3926 	if (st == NULL)
3927 		return  _base_get_msix_index(ioc, NULL);
3928 
3929 	st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd);
3930 	return st->msix_io;
3931 }
3932 
3933 /**
3934  * _base_put_smid_mpi_ep_scsi_io - send SCSI_IO request to firmware
3935  * @ioc: per adapter object
3936  * @smid: system request message index
3937  * @handle: device handle
3938  */
3939 static void
3940 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc,
3941 	u16 smid, u16 handle)
3942 {
3943 	Mpi2RequestDescriptorUnion_t descriptor;
3944 	u64 *request = (u64 *)&descriptor;
3945 	void *mpi_req_iomem;
3946 	__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
3947 
3948 	_clone_sg_entries(ioc, (void *) mfp, smid);
3949 	mpi_req_iomem = (void __force *)ioc->chip +
3950 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
3951 	_base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
3952 					ioc->request_sz);
3953 	descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3954 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
3955 	descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3956 	descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3957 	descriptor.SCSIIO.LMID = 0;
3958 	_base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3959 	    &ioc->scsi_lookup_lock);
3960 }
3961 
3962 /**
3963  * _base_put_smid_scsi_io - send SCSI_IO request to firmware
3964  * @ioc: per adapter object
3965  * @smid: system request message index
3966  * @handle: device handle
3967  */
3968 static void
3969 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)
3970 {
3971 	Mpi2RequestDescriptorUnion_t descriptor;
3972 	u64 *request = (u64 *)&descriptor;
3973 
3974 
3975 	descriptor.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
3976 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
3977 	descriptor.SCSIIO.SMID = cpu_to_le16(smid);
3978 	descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
3979 	descriptor.SCSIIO.LMID = 0;
3980 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
3981 	    &ioc->scsi_lookup_lock);
3982 }
3983 
3984 /**
3985  * _base_put_smid_fast_path - send fast path request to firmware
3986  * @ioc: per adapter object
3987  * @smid: system request message index
3988  * @handle: device handle
3989  */
3990 static void
3991 _base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,
3992 	u16 handle)
3993 {
3994 	Mpi2RequestDescriptorUnion_t descriptor;
3995 	u64 *request = (u64 *)&descriptor;
3996 
3997 	descriptor.SCSIIO.RequestFlags =
3998 	    MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
3999 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4000 	descriptor.SCSIIO.SMID = cpu_to_le16(smid);
4001 	descriptor.SCSIIO.DevHandle = cpu_to_le16(handle);
4002 	descriptor.SCSIIO.LMID = 0;
4003 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
4004 	    &ioc->scsi_lookup_lock);
4005 }
4006 
4007 /**
4008  * _base_put_smid_hi_priority - send Task Management request to firmware
4009  * @ioc: per adapter object
4010  * @smid: system request message index
4011  * @msix_task: msix_task will be same as msix of IO in case of task abort else 0
4012  */
4013 static void
4014 _base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4015 	u16 msix_task)
4016 {
4017 	Mpi2RequestDescriptorUnion_t descriptor;
4018 	void *mpi_req_iomem;
4019 	u64 *request;
4020 
4021 	if (ioc->is_mcpu_endpoint) {
4022 		__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
4023 
4024 		/* TBD 256 is offset within sys register. */
4025 		mpi_req_iomem = (void __force *)ioc->chip
4026 					+ MPI_FRAME_START_OFFSET
4027 					+ (smid * ioc->request_sz);
4028 		_base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
4029 							ioc->request_sz);
4030 	}
4031 
4032 	request = (u64 *)&descriptor;
4033 
4034 	descriptor.HighPriority.RequestFlags =
4035 	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
4036 	descriptor.HighPriority.MSIxIndex =  msix_task;
4037 	descriptor.HighPriority.SMID = cpu_to_le16(smid);
4038 	descriptor.HighPriority.LMID = 0;
4039 	descriptor.HighPriority.Reserved1 = 0;
4040 	if (ioc->is_mcpu_endpoint)
4041 		_base_mpi_ep_writeq(*request,
4042 				&ioc->chip->RequestDescriptorPostLow,
4043 				&ioc->scsi_lookup_lock);
4044 	else
4045 		_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
4046 		    &ioc->scsi_lookup_lock);
4047 }
4048 
4049 /**
4050  * mpt3sas_base_put_smid_nvme_encap - send NVMe encapsulated request to
4051  *  firmware
4052  * @ioc: per adapter object
4053  * @smid: system request message index
4054  */
4055 void
4056 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4057 {
4058 	Mpi2RequestDescriptorUnion_t descriptor;
4059 	u64 *request = (u64 *)&descriptor;
4060 
4061 	descriptor.Default.RequestFlags =
4062 		MPI26_REQ_DESCRIPT_FLAGS_PCIE_ENCAPSULATED;
4063 	descriptor.Default.MSIxIndex =  _base_set_and_get_msix_index(ioc, smid);
4064 	descriptor.Default.SMID = cpu_to_le16(smid);
4065 	descriptor.Default.LMID = 0;
4066 	descriptor.Default.DescriptorTypeDependent = 0;
4067 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
4068 	    &ioc->scsi_lookup_lock);
4069 }
4070 
4071 /**
4072  * _base_put_smid_default - Default, primarily used for config pages
4073  * @ioc: per adapter object
4074  * @smid: system request message index
4075  */
4076 static void
4077 _base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4078 {
4079 	Mpi2RequestDescriptorUnion_t descriptor;
4080 	void *mpi_req_iomem;
4081 	u64 *request;
4082 
4083 	if (ioc->is_mcpu_endpoint) {
4084 		__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);
4085 
4086 		_clone_sg_entries(ioc, (void *) mfp, smid);
4087 		/* TBD 256 is offset within sys register */
4088 		mpi_req_iomem = (void __force *)ioc->chip +
4089 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);
4090 		_base_clone_mpi_to_sys_mem(mpi_req_iomem, (void *)mfp,
4091 							ioc->request_sz);
4092 	}
4093 	request = (u64 *)&descriptor;
4094 	descriptor.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
4095 	descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4096 	descriptor.Default.SMID = cpu_to_le16(smid);
4097 	descriptor.Default.LMID = 0;
4098 	descriptor.Default.DescriptorTypeDependent = 0;
4099 	if (ioc->is_mcpu_endpoint)
4100 		_base_mpi_ep_writeq(*request,
4101 				&ioc->chip->RequestDescriptorPostLow,
4102 				&ioc->scsi_lookup_lock);
4103 	else
4104 		_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,
4105 				&ioc->scsi_lookup_lock);
4106 }
4107 
4108 /**
4109  * _base_put_smid_scsi_io_atomic - send SCSI_IO request to firmware using
4110  *   Atomic Request Descriptor
4111  * @ioc: per adapter object
4112  * @smid: system request message index
4113  * @handle: device handle, unused in this function, for function type match
4114  *
4115  * Return: nothing.
4116  */
4117 static void
4118 _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4119 	u16 handle)
4120 {
4121 	Mpi26AtomicRequestDescriptor_t descriptor;
4122 	u32 *request = (u32 *)&descriptor;
4123 
4124 	descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
4125 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4126 	descriptor.SMID = cpu_to_le16(smid);
4127 
4128 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4129 }
4130 
4131 /**
4132  * _base_put_smid_fast_path_atomic - send fast path request to firmware
4133  * using Atomic Request Descriptor
4134  * @ioc: per adapter object
4135  * @smid: system request message index
4136  * @handle: device handle, unused in this function, for function type match
4137  * Return: nothing
4138  */
4139 static void
4140 _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4141 	u16 handle)
4142 {
4143 	Mpi26AtomicRequestDescriptor_t descriptor;
4144 	u32 *request = (u32 *)&descriptor;
4145 
4146 	descriptor.RequestFlags = MPI25_REQ_DESCRIPT_FLAGS_FAST_PATH_SCSI_IO;
4147 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4148 	descriptor.SMID = cpu_to_le16(smid);
4149 
4150 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4151 }
4152 
4153 /**
4154  * _base_put_smid_hi_priority_atomic - send Task Management request to
4155  * firmware using Atomic Request Descriptor
4156  * @ioc: per adapter object
4157  * @smid: system request message index
4158  * @msix_task: msix_task will be same as msix of IO in case of task abort else 0
4159  *
4160  * Return: nothing.
4161  */
4162 static void
4163 _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,
4164 	u16 msix_task)
4165 {
4166 	Mpi26AtomicRequestDescriptor_t descriptor;
4167 	u32 *request = (u32 *)&descriptor;
4168 
4169 	descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
4170 	descriptor.MSIxIndex = msix_task;
4171 	descriptor.SMID = cpu_to_le16(smid);
4172 
4173 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4174 }
4175 
4176 /**
4177  * _base_put_smid_default_atomic - Default, primarily used for config pages
4178  * use Atomic Request Descriptor
4179  * @ioc: per adapter object
4180  * @smid: system request message index
4181  *
4182  * Return: nothing.
4183  */
4184 static void
4185 _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)
4186 {
4187 	Mpi26AtomicRequestDescriptor_t descriptor;
4188 	u32 *request = (u32 *)&descriptor;
4189 
4190 	descriptor.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
4191 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);
4192 	descriptor.SMID = cpu_to_le16(smid);
4193 
4194 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);
4195 }
4196 
4197 /**
4198  * _base_display_OEMs_branding - Display branding string
4199  * @ioc: per adapter object
4200  */
4201 static void
4202 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)
4203 {
4204 	if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
4205 		return;
4206 
4207 	switch (ioc->pdev->subsystem_vendor) {
4208 	case PCI_VENDOR_ID_INTEL:
4209 		switch (ioc->pdev->device) {
4210 		case MPI2_MFGPAGE_DEVID_SAS2008:
4211 			switch (ioc->pdev->subsystem_device) {
4212 			case MPT2SAS_INTEL_RMS2LL080_SSDID:
4213 				ioc_info(ioc, "%s\n",
4214 					 MPT2SAS_INTEL_RMS2LL080_BRANDING);
4215 				break;
4216 			case MPT2SAS_INTEL_RMS2LL040_SSDID:
4217 				ioc_info(ioc, "%s\n",
4218 					 MPT2SAS_INTEL_RMS2LL040_BRANDING);
4219 				break;
4220 			case MPT2SAS_INTEL_SSD910_SSDID:
4221 				ioc_info(ioc, "%s\n",
4222 					 MPT2SAS_INTEL_SSD910_BRANDING);
4223 				break;
4224 			default:
4225 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4226 					 ioc->pdev->subsystem_device);
4227 				break;
4228 			}
4229 			break;
4230 		case MPI2_MFGPAGE_DEVID_SAS2308_2:
4231 			switch (ioc->pdev->subsystem_device) {
4232 			case MPT2SAS_INTEL_RS25GB008_SSDID:
4233 				ioc_info(ioc, "%s\n",
4234 					 MPT2SAS_INTEL_RS25GB008_BRANDING);
4235 				break;
4236 			case MPT2SAS_INTEL_RMS25JB080_SSDID:
4237 				ioc_info(ioc, "%s\n",
4238 					 MPT2SAS_INTEL_RMS25JB080_BRANDING);
4239 				break;
4240 			case MPT2SAS_INTEL_RMS25JB040_SSDID:
4241 				ioc_info(ioc, "%s\n",
4242 					 MPT2SAS_INTEL_RMS25JB040_BRANDING);
4243 				break;
4244 			case MPT2SAS_INTEL_RMS25KB080_SSDID:
4245 				ioc_info(ioc, "%s\n",
4246 					 MPT2SAS_INTEL_RMS25KB080_BRANDING);
4247 				break;
4248 			case MPT2SAS_INTEL_RMS25KB040_SSDID:
4249 				ioc_info(ioc, "%s\n",
4250 					 MPT2SAS_INTEL_RMS25KB040_BRANDING);
4251 				break;
4252 			case MPT2SAS_INTEL_RMS25LB040_SSDID:
4253 				ioc_info(ioc, "%s\n",
4254 					 MPT2SAS_INTEL_RMS25LB040_BRANDING);
4255 				break;
4256 			case MPT2SAS_INTEL_RMS25LB080_SSDID:
4257 				ioc_info(ioc, "%s\n",
4258 					 MPT2SAS_INTEL_RMS25LB080_BRANDING);
4259 				break;
4260 			default:
4261 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4262 					 ioc->pdev->subsystem_device);
4263 				break;
4264 			}
4265 			break;
4266 		case MPI25_MFGPAGE_DEVID_SAS3008:
4267 			switch (ioc->pdev->subsystem_device) {
4268 			case MPT3SAS_INTEL_RMS3JC080_SSDID:
4269 				ioc_info(ioc, "%s\n",
4270 					 MPT3SAS_INTEL_RMS3JC080_BRANDING);
4271 				break;
4272 
4273 			case MPT3SAS_INTEL_RS3GC008_SSDID:
4274 				ioc_info(ioc, "%s\n",
4275 					 MPT3SAS_INTEL_RS3GC008_BRANDING);
4276 				break;
4277 			case MPT3SAS_INTEL_RS3FC044_SSDID:
4278 				ioc_info(ioc, "%s\n",
4279 					 MPT3SAS_INTEL_RS3FC044_BRANDING);
4280 				break;
4281 			case MPT3SAS_INTEL_RS3UC080_SSDID:
4282 				ioc_info(ioc, "%s\n",
4283 					 MPT3SAS_INTEL_RS3UC080_BRANDING);
4284 				break;
4285 			default:
4286 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4287 					 ioc->pdev->subsystem_device);
4288 				break;
4289 			}
4290 			break;
4291 		default:
4292 			ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",
4293 				 ioc->pdev->subsystem_device);
4294 			break;
4295 		}
4296 		break;
4297 	case PCI_VENDOR_ID_DELL:
4298 		switch (ioc->pdev->device) {
4299 		case MPI2_MFGPAGE_DEVID_SAS2008:
4300 			switch (ioc->pdev->subsystem_device) {
4301 			case MPT2SAS_DELL_6GBPS_SAS_HBA_SSDID:
4302 				ioc_info(ioc, "%s\n",
4303 					 MPT2SAS_DELL_6GBPS_SAS_HBA_BRANDING);
4304 				break;
4305 			case MPT2SAS_DELL_PERC_H200_ADAPTER_SSDID:
4306 				ioc_info(ioc, "%s\n",
4307 					 MPT2SAS_DELL_PERC_H200_ADAPTER_BRANDING);
4308 				break;
4309 			case MPT2SAS_DELL_PERC_H200_INTEGRATED_SSDID:
4310 				ioc_info(ioc, "%s\n",
4311 					 MPT2SAS_DELL_PERC_H200_INTEGRATED_BRANDING);
4312 				break;
4313 			case MPT2SAS_DELL_PERC_H200_MODULAR_SSDID:
4314 				ioc_info(ioc, "%s\n",
4315 					 MPT2SAS_DELL_PERC_H200_MODULAR_BRANDING);
4316 				break;
4317 			case MPT2SAS_DELL_PERC_H200_EMBEDDED_SSDID:
4318 				ioc_info(ioc, "%s\n",
4319 					 MPT2SAS_DELL_PERC_H200_EMBEDDED_BRANDING);
4320 				break;
4321 			case MPT2SAS_DELL_PERC_H200_SSDID:
4322 				ioc_info(ioc, "%s\n",
4323 					 MPT2SAS_DELL_PERC_H200_BRANDING);
4324 				break;
4325 			case MPT2SAS_DELL_6GBPS_SAS_SSDID:
4326 				ioc_info(ioc, "%s\n",
4327 					 MPT2SAS_DELL_6GBPS_SAS_BRANDING);
4328 				break;
4329 			default:
4330 				ioc_info(ioc, "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",
4331 					 ioc->pdev->subsystem_device);
4332 				break;
4333 			}
4334 			break;
4335 		case MPI25_MFGPAGE_DEVID_SAS3008:
4336 			switch (ioc->pdev->subsystem_device) {
4337 			case MPT3SAS_DELL_12G_HBA_SSDID:
4338 				ioc_info(ioc, "%s\n",
4339 					 MPT3SAS_DELL_12G_HBA_BRANDING);
4340 				break;
4341 			default:
4342 				ioc_info(ioc, "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",
4343 					 ioc->pdev->subsystem_device);
4344 				break;
4345 			}
4346 			break;
4347 		default:
4348 			ioc_info(ioc, "Dell HBA: Subsystem ID: 0x%X\n",
4349 				 ioc->pdev->subsystem_device);
4350 			break;
4351 		}
4352 		break;
4353 	case PCI_VENDOR_ID_CISCO:
4354 		switch (ioc->pdev->device) {
4355 		case MPI25_MFGPAGE_DEVID_SAS3008:
4356 			switch (ioc->pdev->subsystem_device) {
4357 			case MPT3SAS_CISCO_12G_8E_HBA_SSDID:
4358 				ioc_info(ioc, "%s\n",
4359 					 MPT3SAS_CISCO_12G_8E_HBA_BRANDING);
4360 				break;
4361 			case MPT3SAS_CISCO_12G_8I_HBA_SSDID:
4362 				ioc_info(ioc, "%s\n",
4363 					 MPT3SAS_CISCO_12G_8I_HBA_BRANDING);
4364 				break;
4365 			case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
4366 				ioc_info(ioc, "%s\n",
4367 					 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
4368 				break;
4369 			default:
4370 				ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
4371 					 ioc->pdev->subsystem_device);
4372 				break;
4373 			}
4374 			break;
4375 		case MPI25_MFGPAGE_DEVID_SAS3108_1:
4376 			switch (ioc->pdev->subsystem_device) {
4377 			case MPT3SAS_CISCO_12G_AVILA_HBA_SSDID:
4378 				ioc_info(ioc, "%s\n",
4379 					 MPT3SAS_CISCO_12G_AVILA_HBA_BRANDING);
4380 				break;
4381 			case MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_SSDID:
4382 				ioc_info(ioc, "%s\n",
4383 					 MPT3SAS_CISCO_12G_COLUSA_MEZZANINE_HBA_BRANDING);
4384 				break;
4385 			default:
4386 				ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",
4387 					 ioc->pdev->subsystem_device);
4388 				break;
4389 			}
4390 			break;
4391 		default:
4392 			ioc_info(ioc, "Cisco SAS HBA: Subsystem ID: 0x%X\n",
4393 				 ioc->pdev->subsystem_device);
4394 			break;
4395 		}
4396 		break;
4397 	case MPT2SAS_HP_3PAR_SSVID:
4398 		switch (ioc->pdev->device) {
4399 		case MPI2_MFGPAGE_DEVID_SAS2004:
4400 			switch (ioc->pdev->subsystem_device) {
4401 			case MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_SSDID:
4402 				ioc_info(ioc, "%s\n",
4403 					 MPT2SAS_HP_DAUGHTER_2_4_INTERNAL_BRANDING);
4404 				break;
4405 			default:
4406 				ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
4407 					 ioc->pdev->subsystem_device);
4408 				break;
4409 			}
4410 			break;
4411 		case MPI2_MFGPAGE_DEVID_SAS2308_2:
4412 			switch (ioc->pdev->subsystem_device) {
4413 			case MPT2SAS_HP_2_4_INTERNAL_SSDID:
4414 				ioc_info(ioc, "%s\n",
4415 					 MPT2SAS_HP_2_4_INTERNAL_BRANDING);
4416 				break;
4417 			case MPT2SAS_HP_2_4_EXTERNAL_SSDID:
4418 				ioc_info(ioc, "%s\n",
4419 					 MPT2SAS_HP_2_4_EXTERNAL_BRANDING);
4420 				break;
4421 			case MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_SSDID:
4422 				ioc_info(ioc, "%s\n",
4423 					 MPT2SAS_HP_1_4_INTERNAL_1_4_EXTERNAL_BRANDING);
4424 				break;
4425 			case MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_SSDID:
4426 				ioc_info(ioc, "%s\n",
4427 					 MPT2SAS_HP_EMBEDDED_2_4_INTERNAL_BRANDING);
4428 				break;
4429 			default:
4430 				ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",
4431 					 ioc->pdev->subsystem_device);
4432 				break;
4433 			}
4434 			break;
4435 		default:
4436 			ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n",
4437 				 ioc->pdev->subsystem_device);
4438 			break;
4439 		}
4440 		break;
4441 	default:
4442 		break;
4443 	}
4444 }
4445 
4446 /**
4447  * _base_display_fwpkg_version - sends FWUpload request to pull FWPkg
4448  *				version from FW Image Header.
4449  * @ioc: per adapter object
4450  *
4451  * Return: 0 for success, non-zero for failure.
4452  */
4453 	static int
4454 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)
4455 {
4456 	Mpi2FWImageHeader_t *fw_img_hdr;
4457 	Mpi26ComponentImageHeader_t *cmp_img_hdr;
4458 	Mpi25FWUploadRequest_t *mpi_request;
4459 	Mpi2FWUploadReply_t mpi_reply;
4460 	int r = 0, issue_diag_reset = 0;
4461 	u32  package_version = 0;
4462 	void *fwpkg_data = NULL;
4463 	dma_addr_t fwpkg_data_dma;
4464 	u16 smid, ioc_status;
4465 	size_t data_length;
4466 
4467 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
4468 
4469 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
4470 		ioc_err(ioc, "%s: internal command already in use\n", __func__);
4471 		return -EAGAIN;
4472 	}
4473 
4474 	data_length = sizeof(Mpi2FWImageHeader_t);
4475 	fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length,
4476 			&fwpkg_data_dma, GFP_KERNEL);
4477 	if (!fwpkg_data) {
4478 		ioc_err(ioc,
4479 		    "Memory allocation for fwpkg data failed at %s:%d/%s()!\n",
4480 			__FILE__, __LINE__, __func__);
4481 		return -ENOMEM;
4482 	}
4483 
4484 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
4485 	if (!smid) {
4486 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
4487 		r = -EAGAIN;
4488 		goto out;
4489 	}
4490 
4491 	ioc->base_cmds.status = MPT3_CMD_PENDING;
4492 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
4493 	ioc->base_cmds.smid = smid;
4494 	memset(mpi_request, 0, sizeof(Mpi25FWUploadRequest_t));
4495 	mpi_request->Function = MPI2_FUNCTION_FW_UPLOAD;
4496 	mpi_request->ImageType = MPI2_FW_UPLOAD_ITYPE_FW_FLASH;
4497 	mpi_request->ImageSize = cpu_to_le32(data_length);
4498 	ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,
4499 			data_length);
4500 	init_completion(&ioc->base_cmds.done);
4501 	ioc->put_smid_default(ioc, smid);
4502 	/* Wait for 15 seconds */
4503 	wait_for_completion_timeout(&ioc->base_cmds.done,
4504 			FW_IMG_HDR_READ_TIMEOUT*HZ);
4505 	ioc_info(ioc, "%s: complete\n", __func__);
4506 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
4507 		ioc_err(ioc, "%s: timeout\n", __func__);
4508 		_debug_dump_mf(mpi_request,
4509 				sizeof(Mpi25FWUploadRequest_t)/4);
4510 		issue_diag_reset = 1;
4511 	} else {
4512 		memset(&mpi_reply, 0, sizeof(Mpi2FWUploadReply_t));
4513 		if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {
4514 			memcpy(&mpi_reply, ioc->base_cmds.reply,
4515 					sizeof(Mpi2FWUploadReply_t));
4516 			ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4517 						MPI2_IOCSTATUS_MASK;
4518 			if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
4519 				fw_img_hdr = (Mpi2FWImageHeader_t *)fwpkg_data;
4520 				if (le32_to_cpu(fw_img_hdr->Signature) ==
4521 				    MPI26_IMAGE_HEADER_SIGNATURE0_MPI26) {
4522 					cmp_img_hdr =
4523 					    (Mpi26ComponentImageHeader_t *)
4524 					    (fwpkg_data);
4525 					package_version =
4526 					    le32_to_cpu(
4527 					    cmp_img_hdr->ApplicationSpecific);
4528 				} else
4529 					package_version =
4530 					    le32_to_cpu(
4531 					    fw_img_hdr->PackageVersion.Word);
4532 				if (package_version)
4533 					ioc_info(ioc,
4534 					"FW Package Ver(%02d.%02d.%02d.%02d)\n",
4535 					((package_version) & 0xFF000000) >> 24,
4536 					((package_version) & 0x00FF0000) >> 16,
4537 					((package_version) & 0x0000FF00) >> 8,
4538 					(package_version) & 0x000000FF);
4539 			} else {
4540 				_debug_dump_mf(&mpi_reply,
4541 						sizeof(Mpi2FWUploadReply_t)/4);
4542 			}
4543 		}
4544 	}
4545 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
4546 out:
4547 	if (fwpkg_data)
4548 		dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data,
4549 				fwpkg_data_dma);
4550 	if (issue_diag_reset) {
4551 		if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED)
4552 			return -EFAULT;
4553 		if (mpt3sas_base_check_for_fault_and_issue_reset(ioc))
4554 			return -EFAULT;
4555 		r = -EAGAIN;
4556 	}
4557 	return r;
4558 }
4559 
4560 /**
4561  * _base_display_ioc_capabilities - Display IOC's capabilities.
4562  * @ioc: per adapter object
4563  */
4564 static void
4565 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)
4566 {
4567 	int i = 0;
4568 	char desc[16];
4569 	u32 iounit_pg1_flags;
4570 	u32 bios_version;
4571 
4572 	bios_version = le32_to_cpu(ioc->bios_pg3.BiosVersion);
4573 	strncpy(desc, ioc->manu_pg0.ChipName, 16);
4574 	ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x), BiosVersion(%02d.%02d.%02d.%02d)\n",
4575 		 desc,
4576 		 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,
4577 		 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,
4578 		 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,
4579 		 ioc->facts.FWVersion.Word & 0x000000FF,
4580 		 ioc->pdev->revision,
4581 		 (bios_version & 0xFF000000) >> 24,
4582 		 (bios_version & 0x00FF0000) >> 16,
4583 		 (bios_version & 0x0000FF00) >> 8,
4584 		 bios_version & 0x000000FF);
4585 
4586 	_base_display_OEMs_branding(ioc);
4587 
4588 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
4589 		pr_info("%sNVMe", i ? "," : "");
4590 		i++;
4591 	}
4592 
4593 	ioc_info(ioc, "Protocol=(");
4594 
4595 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {
4596 		pr_cont("Initiator");
4597 		i++;
4598 	}
4599 
4600 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {
4601 		pr_cont("%sTarget", i ? "," : "");
4602 		i++;
4603 	}
4604 
4605 	i = 0;
4606 	pr_cont("), Capabilities=(");
4607 
4608 	if (!ioc->hide_ir_msg) {
4609 		if (ioc->facts.IOCCapabilities &
4610 		    MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID) {
4611 			pr_cont("Raid");
4612 			i++;
4613 		}
4614 	}
4615 
4616 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {
4617 		pr_cont("%sTLR", i ? "," : "");
4618 		i++;
4619 	}
4620 
4621 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {
4622 		pr_cont("%sMulticast", i ? "," : "");
4623 		i++;
4624 	}
4625 
4626 	if (ioc->facts.IOCCapabilities &
4627 	    MPI2_IOCFACTS_CAPABILITY_BIDIRECTIONAL_TARGET) {
4628 		pr_cont("%sBIDI Target", i ? "," : "");
4629 		i++;
4630 	}
4631 
4632 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {
4633 		pr_cont("%sEEDP", i ? "," : "");
4634 		i++;
4635 	}
4636 
4637 	if (ioc->facts.IOCCapabilities &
4638 	    MPI2_IOCFACTS_CAPABILITY_SNAPSHOT_BUFFER) {
4639 		pr_cont("%sSnapshot Buffer", i ? "," : "");
4640 		i++;
4641 	}
4642 
4643 	if (ioc->facts.IOCCapabilities &
4644 	    MPI2_IOCFACTS_CAPABILITY_DIAG_TRACE_BUFFER) {
4645 		pr_cont("%sDiag Trace Buffer", i ? "," : "");
4646 		i++;
4647 	}
4648 
4649 	if (ioc->facts.IOCCapabilities &
4650 	    MPI2_IOCFACTS_CAPABILITY_EXTENDED_BUFFER) {
4651 		pr_cont("%sDiag Extended Buffer", i ? "," : "");
4652 		i++;
4653 	}
4654 
4655 	if (ioc->facts.IOCCapabilities &
4656 	    MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING) {
4657 		pr_cont("%sTask Set Full", i ? "," : "");
4658 		i++;
4659 	}
4660 
4661 	iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
4662 	if (!(iounit_pg1_flags & MPI2_IOUNITPAGE1_NATIVE_COMMAND_Q_DISABLE)) {
4663 		pr_cont("%sNCQ", i ? "," : "");
4664 		i++;
4665 	}
4666 
4667 	pr_cont(")\n");
4668 }
4669 
4670 /**
4671  * mpt3sas_base_update_missing_delay - change the missing delay timers
4672  * @ioc: per adapter object
4673  * @device_missing_delay: amount of time till device is reported missing
4674  * @io_missing_delay: interval IO is returned when there is a missing device
4675  *
4676  * Passed on the command line, this function will modify the device missing
4677  * delay, as well as the io missing delay. This should be called at driver
4678  * load time.
4679  */
4680 void
4681 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,
4682 	u16 device_missing_delay, u8 io_missing_delay)
4683 {
4684 	u16 dmd, dmd_new, dmd_orignal;
4685 	u8 io_missing_delay_original;
4686 	u16 sz;
4687 	Mpi2SasIOUnitPage1_t *sas_iounit_pg1 = NULL;
4688 	Mpi2ConfigReply_t mpi_reply;
4689 	u8 num_phys = 0;
4690 	u16 ioc_status;
4691 
4692 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);
4693 	if (!num_phys)
4694 		return;
4695 
4696 	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (num_phys *
4697 	    sizeof(Mpi2SasIOUnit1PhyData_t));
4698 	sas_iounit_pg1 = kzalloc(sz, GFP_KERNEL);
4699 	if (!sas_iounit_pg1) {
4700 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
4701 			__FILE__, __LINE__, __func__);
4702 		goto out;
4703 	}
4704 	if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,
4705 	    sas_iounit_pg1, sz))) {
4706 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
4707 			__FILE__, __LINE__, __func__);
4708 		goto out;
4709 	}
4710 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4711 	    MPI2_IOCSTATUS_MASK;
4712 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4713 		ioc_err(ioc, "failure at %s:%d/%s()!\n",
4714 			__FILE__, __LINE__, __func__);
4715 		goto out;
4716 	}
4717 
4718 	/* device missing delay */
4719 	dmd = sas_iounit_pg1->ReportDeviceMissingDelay;
4720 	if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4721 		dmd = (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4722 	else
4723 		dmd = dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4724 	dmd_orignal = dmd;
4725 	if (device_missing_delay > 0x7F) {
4726 		dmd = (device_missing_delay > 0x7F0) ? 0x7F0 :
4727 		    device_missing_delay;
4728 		dmd = dmd / 16;
4729 		dmd |= MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16;
4730 	} else
4731 		dmd = device_missing_delay;
4732 	sas_iounit_pg1->ReportDeviceMissingDelay = dmd;
4733 
4734 	/* io missing delay */
4735 	io_missing_delay_original = sas_iounit_pg1->IODeviceMissingDelay;
4736 	sas_iounit_pg1->IODeviceMissingDelay = io_missing_delay;
4737 
4738 	if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,
4739 	    sz)) {
4740 		if (dmd & MPI2_SASIOUNIT1_REPORT_MISSING_UNIT_16)
4741 			dmd_new = (dmd &
4742 			    MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK) * 16;
4743 		else
4744 			dmd_new =
4745 		    dmd & MPI2_SASIOUNIT1_REPORT_MISSING_TIMEOUT_MASK;
4746 		ioc_info(ioc, "device_missing_delay: old(%d), new(%d)\n",
4747 			 dmd_orignal, dmd_new);
4748 		ioc_info(ioc, "ioc_missing_delay: old(%d), new(%d)\n",
4749 			 io_missing_delay_original,
4750 			 io_missing_delay);
4751 		ioc->device_missing_delay = dmd_new;
4752 		ioc->io_missing_delay = io_missing_delay;
4753 	}
4754 
4755 out:
4756 	kfree(sas_iounit_pg1);
4757 }
4758 
4759 /**
4760  * _base_update_ioc_page1_inlinewith_perf_mode - Update IOC Page1 fields
4761  *    according to performance mode.
4762  * @ioc : per adapter object
4763  *
4764  * Return: zero on success; otherwise return EAGAIN error code asking the
4765  * caller to retry.
4766  */
4767 static int
4768 _base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc)
4769 {
4770 	Mpi2IOCPage1_t ioc_pg1;
4771 	Mpi2ConfigReply_t mpi_reply;
4772 	int rc;
4773 
4774 	rc = mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy);
4775 	if (rc)
4776 		return rc;
4777 	memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t));
4778 
4779 	switch (perf_mode) {
4780 	case MPT_PERF_MODE_DEFAULT:
4781 	case MPT_PERF_MODE_BALANCED:
4782 		if (ioc->high_iops_queues) {
4783 			ioc_info(ioc,
4784 				"Enable interrupt coalescing only for first\t"
4785 				"%d reply queues\n",
4786 				MPT3SAS_HIGH_IOPS_REPLY_QUEUES);
4787 			/*
4788 			 * If 31st bit is zero then interrupt coalescing is
4789 			 * enabled for all reply descriptor post queues.
4790 			 * If 31st bit is set to one then user can
4791 			 * enable/disable interrupt coalescing on per reply
4792 			 * descriptor post queue group(8) basis. So to enable
4793 			 * interrupt coalescing only on first reply descriptor
4794 			 * post queue group 31st bit and zero th bit is enabled.
4795 			 */
4796 			ioc_pg1.ProductSpecific = cpu_to_le32(0x80000000 |
4797 			    ((1 << MPT3SAS_HIGH_IOPS_REPLY_QUEUES/8) - 1));
4798 			rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);
4799 			if (rc)
4800 				return rc;
4801 			ioc_info(ioc, "performance mode: balanced\n");
4802 			return 0;
4803 		}
4804 		fallthrough;
4805 	case MPT_PERF_MODE_LATENCY:
4806 		/*
4807 		 * Enable interrupt coalescing on all reply queues
4808 		 * with timeout value 0xA
4809 		 */
4810 		ioc_pg1.CoalescingTimeout = cpu_to_le32(0xa);
4811 		ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING);
4812 		ioc_pg1.ProductSpecific = 0;
4813 		rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);
4814 		if (rc)
4815 			return rc;
4816 		ioc_info(ioc, "performance mode: latency\n");
4817 		break;
4818 	case MPT_PERF_MODE_IOPS:
4819 		/*
4820 		 * Enable interrupt coalescing on all reply queues.
4821 		 */
4822 		ioc_info(ioc,
4823 		    "performance mode: iops with coalescing timeout: 0x%x\n",
4824 		    le32_to_cpu(ioc_pg1.CoalescingTimeout));
4825 		ioc_pg1.Flags |= cpu_to_le32(MPI2_IOCPAGE1_REPLY_COALESCING);
4826 		ioc_pg1.ProductSpecific = 0;
4827 		rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);
4828 		if (rc)
4829 			return rc;
4830 		break;
4831 	}
4832 	return 0;
4833 }
4834 
4835 /**
4836  * _base_get_event_diag_triggers - get event diag trigger values from
4837  *				persistent pages
4838  * @ioc : per adapter object
4839  *
4840  * Return: nothing.
4841  */
4842 static int
4843 _base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
4844 {
4845 	Mpi26DriverTriggerPage2_t trigger_pg2;
4846 	struct SL_WH_EVENT_TRIGGER_T *event_tg;
4847 	MPI26_DRIVER_MPI_EVENT_TIGGER_ENTRY *mpi_event_tg;
4848 	Mpi2ConfigReply_t mpi_reply;
4849 	int r = 0, i = 0;
4850 	u16 count = 0;
4851 	u16 ioc_status;
4852 
4853 	r = mpt3sas_config_get_driver_trigger_pg2(ioc, &mpi_reply,
4854 	    &trigger_pg2);
4855 	if (r)
4856 		return r;
4857 
4858 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4859 	    MPI2_IOCSTATUS_MASK;
4860 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4861 		dinitprintk(ioc,
4862 		    ioc_err(ioc,
4863 		    "%s: Failed to get trigger pg2, ioc_status(0x%04x)\n",
4864 		   __func__, ioc_status));
4865 		return 0;
4866 	}
4867 
4868 	if (le16_to_cpu(trigger_pg2.NumMPIEventTrigger)) {
4869 		count = le16_to_cpu(trigger_pg2.NumMPIEventTrigger);
4870 		count = min_t(u16, NUM_VALID_ENTRIES, count);
4871 		ioc->diag_trigger_event.ValidEntries = count;
4872 
4873 		event_tg = &ioc->diag_trigger_event.EventTriggerEntry[0];
4874 		mpi_event_tg = &trigger_pg2.MPIEventTriggers[0];
4875 		for (i = 0; i < count; i++) {
4876 			event_tg->EventValue = le16_to_cpu(
4877 			    mpi_event_tg->MPIEventCode);
4878 			event_tg->LogEntryQualifier = le16_to_cpu(
4879 			    mpi_event_tg->MPIEventCodeSpecific);
4880 			event_tg++;
4881 			mpi_event_tg++;
4882 		}
4883 	}
4884 	return 0;
4885 }
4886 
4887 /**
4888  * _base_get_scsi_diag_triggers - get scsi diag trigger values from
4889  *				persistent pages
4890  * @ioc : per adapter object
4891  *
4892  * Return: 0 on success; otherwise return failure status.
4893  */
4894 static int
4895 _base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
4896 {
4897 	Mpi26DriverTriggerPage3_t trigger_pg3;
4898 	struct SL_WH_SCSI_TRIGGER_T *scsi_tg;
4899 	MPI26_DRIVER_SCSI_SENSE_TIGGER_ENTRY *mpi_scsi_tg;
4900 	Mpi2ConfigReply_t mpi_reply;
4901 	int r = 0, i = 0;
4902 	u16 count = 0;
4903 	u16 ioc_status;
4904 
4905 	r = mpt3sas_config_get_driver_trigger_pg3(ioc, &mpi_reply,
4906 	    &trigger_pg3);
4907 	if (r)
4908 		return r;
4909 
4910 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4911 	    MPI2_IOCSTATUS_MASK;
4912 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4913 		dinitprintk(ioc,
4914 		    ioc_err(ioc,
4915 		    "%s: Failed to get trigger pg3, ioc_status(0x%04x)\n",
4916 		    __func__, ioc_status));
4917 		return 0;
4918 	}
4919 
4920 	if (le16_to_cpu(trigger_pg3.NumSCSISenseTrigger)) {
4921 		count = le16_to_cpu(trigger_pg3.NumSCSISenseTrigger);
4922 		count = min_t(u16, NUM_VALID_ENTRIES, count);
4923 		ioc->diag_trigger_scsi.ValidEntries = count;
4924 
4925 		scsi_tg = &ioc->diag_trigger_scsi.SCSITriggerEntry[0];
4926 		mpi_scsi_tg = &trigger_pg3.SCSISenseTriggers[0];
4927 		for (i = 0; i < count; i++) {
4928 			scsi_tg->ASCQ = mpi_scsi_tg->ASCQ;
4929 			scsi_tg->ASC = mpi_scsi_tg->ASC;
4930 			scsi_tg->SenseKey = mpi_scsi_tg->SenseKey;
4931 
4932 			scsi_tg++;
4933 			mpi_scsi_tg++;
4934 		}
4935 	}
4936 	return 0;
4937 }
4938 
4939 /**
4940  * _base_get_mpi_diag_triggers - get mpi diag trigger values from
4941  *				persistent pages
4942  * @ioc : per adapter object
4943  *
4944  * Return: 0 on success; otherwise return failure status.
4945  */
4946 static int
4947 _base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
4948 {
4949 	Mpi26DriverTriggerPage4_t trigger_pg4;
4950 	struct SL_WH_MPI_TRIGGER_T *status_tg;
4951 	MPI26_DRIVER_IOCSTATUS_LOGINFO_TIGGER_ENTRY *mpi_status_tg;
4952 	Mpi2ConfigReply_t mpi_reply;
4953 	int r = 0, i = 0;
4954 	u16 count = 0;
4955 	u16 ioc_status;
4956 
4957 	r = mpt3sas_config_get_driver_trigger_pg4(ioc, &mpi_reply,
4958 	    &trigger_pg4);
4959 	if (r)
4960 		return r;
4961 
4962 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
4963 	    MPI2_IOCSTATUS_MASK;
4964 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
4965 		dinitprintk(ioc,
4966 		    ioc_err(ioc,
4967 		    "%s: Failed to get trigger pg4, ioc_status(0x%04x)\n",
4968 		    __func__, ioc_status));
4969 		return 0;
4970 	}
4971 
4972 	if (le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger)) {
4973 		count = le16_to_cpu(trigger_pg4.NumIOCStatusLogInfoTrigger);
4974 		count = min_t(u16, NUM_VALID_ENTRIES, count);
4975 		ioc->diag_trigger_mpi.ValidEntries = count;
4976 
4977 		status_tg = &ioc->diag_trigger_mpi.MPITriggerEntry[0];
4978 		mpi_status_tg = &trigger_pg4.IOCStatusLoginfoTriggers[0];
4979 
4980 		for (i = 0; i < count; i++) {
4981 			status_tg->IOCStatus = le16_to_cpu(
4982 			    mpi_status_tg->IOCStatus);
4983 			status_tg->IocLogInfo = le32_to_cpu(
4984 			    mpi_status_tg->LogInfo);
4985 
4986 			status_tg++;
4987 			mpi_status_tg++;
4988 		}
4989 	}
4990 	return 0;
4991 }
4992 
4993 /**
4994  * _base_get_master_diag_triggers - get master diag trigger values from
4995  *				persistent pages
4996  * @ioc : per adapter object
4997  *
4998  * Return: nothing.
4999  */
5000 static int
5001 _base_get_master_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
5002 {
5003 	Mpi26DriverTriggerPage1_t trigger_pg1;
5004 	Mpi2ConfigReply_t mpi_reply;
5005 	int r;
5006 	u16 ioc_status;
5007 
5008 	r = mpt3sas_config_get_driver_trigger_pg1(ioc, &mpi_reply,
5009 	    &trigger_pg1);
5010 	if (r)
5011 		return r;
5012 
5013 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
5014 	    MPI2_IOCSTATUS_MASK;
5015 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
5016 		dinitprintk(ioc,
5017 		    ioc_err(ioc,
5018 		    "%s: Failed to get trigger pg1, ioc_status(0x%04x)\n",
5019 		   __func__, ioc_status));
5020 		return 0;
5021 	}
5022 
5023 	if (le16_to_cpu(trigger_pg1.NumMasterTrigger))
5024 		ioc->diag_trigger_master.MasterData |=
5025 		    le32_to_cpu(
5026 		    trigger_pg1.MasterTriggers[0].MasterTriggerFlags);
5027 	return 0;
5028 }
5029 
5030 /**
5031  * _base_check_for_trigger_pages_support - checks whether HBA FW supports
5032  *					driver trigger pages or not
5033  * @ioc : per adapter object
5034  * @trigger_flags : address where trigger page0's TriggerFlags value is copied
5035  *
5036  * Return: trigger flags mask if HBA FW supports driver trigger pages;
5037  * otherwise returns %-EFAULT if driver trigger pages are not supported by FW or
5038  * return EAGAIN if diag reset occurred due to FW fault and asking the
5039  * caller to retry the command.
5040  *
5041  */
5042 static int
5043 _base_check_for_trigger_pages_support(struct MPT3SAS_ADAPTER *ioc, u32 *trigger_flags)
5044 {
5045 	Mpi26DriverTriggerPage0_t trigger_pg0;
5046 	int r = 0;
5047 	Mpi2ConfigReply_t mpi_reply;
5048 	u16 ioc_status;
5049 
5050 	r = mpt3sas_config_get_driver_trigger_pg0(ioc, &mpi_reply,
5051 	    &trigger_pg0);
5052 	if (r)
5053 		return r;
5054 
5055 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) &
5056 	    MPI2_IOCSTATUS_MASK;
5057 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
5058 		return -EFAULT;
5059 
5060 	*trigger_flags = le16_to_cpu(trigger_pg0.TriggerFlags);
5061 	return 0;
5062 }
5063 
5064 /**
5065  * _base_get_diag_triggers - Retrieve diag trigger values from
5066  *				persistent pages.
5067  * @ioc : per adapter object
5068  *
5069  * Return: zero on success; otherwise return EAGAIN error codes
5070  * asking the caller to retry.
5071  */
5072 static int
5073 _base_get_diag_triggers(struct MPT3SAS_ADAPTER *ioc)
5074 {
5075 	int trigger_flags;
5076 	int r;
5077 
5078 	/*
5079 	 * Default setting of master trigger.
5080 	 */
5081 	ioc->diag_trigger_master.MasterData =
5082 	    (MASTER_TRIGGER_FW_FAULT + MASTER_TRIGGER_ADAPTER_RESET);
5083 
5084 	r = _base_check_for_trigger_pages_support(ioc, &trigger_flags);
5085 	if (r) {
5086 		if (r == -EAGAIN)
5087 			return r;
5088 		/*
5089 		 * Don't go for error handling when FW doesn't support
5090 		 * driver trigger pages.
5091 		 */
5092 		return 0;
5093 	}
5094 
5095 	ioc->supports_trigger_pages = 1;
5096 
5097 	/*
5098 	 * Retrieve master diag trigger values from driver trigger pg1
5099 	 * if master trigger bit enabled in TriggerFlags.
5100 	 */
5101 	if ((u16)trigger_flags &
5102 	    MPI26_DRIVER_TRIGGER0_FLAG_MASTER_TRIGGER_VALID) {
5103 		r = _base_get_master_diag_triggers(ioc);
5104 		if (r)
5105 			return r;
5106 	}
5107 
5108 	/*
5109 	 * Retrieve event diag trigger values from driver trigger pg2
5110 	 * if event trigger bit enabled in TriggerFlags.
5111 	 */
5112 	if ((u16)trigger_flags &
5113 	    MPI26_DRIVER_TRIGGER0_FLAG_MPI_EVENT_TRIGGER_VALID) {
5114 		r = _base_get_event_diag_triggers(ioc);
5115 		if (r)
5116 			return r;
5117 	}
5118 
5119 	/*
5120 	 * Retrieve scsi diag trigger values from driver trigger pg3
5121 	 * if scsi trigger bit enabled in TriggerFlags.
5122 	 */
5123 	if ((u16)trigger_flags &
5124 	    MPI26_DRIVER_TRIGGER0_FLAG_SCSI_SENSE_TRIGGER_VALID) {
5125 		r = _base_get_scsi_diag_triggers(ioc);
5126 		if (r)
5127 			return r;
5128 	}
5129 	/*
5130 	 * Retrieve mpi error diag trigger values from driver trigger pg4
5131 	 * if loginfo trigger bit enabled in TriggerFlags.
5132 	 */
5133 	if ((u16)trigger_flags &
5134 	    MPI26_DRIVER_TRIGGER0_FLAG_LOGINFO_TRIGGER_VALID) {
5135 		r = _base_get_mpi_diag_triggers(ioc);
5136 		if (r)
5137 			return r;
5138 	}
5139 	return 0;
5140 }
5141 
5142 /**
5143  * _base_update_diag_trigger_pages - Update the driver trigger pages after
5144  *			online FW update, in case updated FW supports driver
5145  *			trigger pages.
5146  * @ioc : per adapter object
5147  *
5148  * Return: nothing.
5149  */
5150 static void
5151 _base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc)
5152 {
5153 
5154 	if (ioc->diag_trigger_master.MasterData)
5155 		mpt3sas_config_update_driver_trigger_pg1(ioc,
5156 		    &ioc->diag_trigger_master, 1);
5157 
5158 	if (ioc->diag_trigger_event.ValidEntries)
5159 		mpt3sas_config_update_driver_trigger_pg2(ioc,
5160 		    &ioc->diag_trigger_event, 1);
5161 
5162 	if (ioc->diag_trigger_scsi.ValidEntries)
5163 		mpt3sas_config_update_driver_trigger_pg3(ioc,
5164 		    &ioc->diag_trigger_scsi, 1);
5165 
5166 	if (ioc->diag_trigger_mpi.ValidEntries)
5167 		mpt3sas_config_update_driver_trigger_pg4(ioc,
5168 		    &ioc->diag_trigger_mpi, 1);
5169 }
5170 
5171 /**
5172  * _base_static_config_pages - static start of day config pages
5173  * @ioc: per adapter object
5174  */
5175 static int
5176 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)
5177 {
5178 	Mpi2ConfigReply_t mpi_reply;
5179 	u32 iounit_pg1_flags;
5180 	int tg_flags = 0;
5181 	int rc;
5182 	ioc->nvme_abort_timeout = 30;
5183 
5184 	rc = mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply,
5185 	    &ioc->manu_pg0);
5186 	if (rc)
5187 		return rc;
5188 	if (ioc->ir_firmware) {
5189 		rc = mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,
5190 		    &ioc->manu_pg10);
5191 		if (rc)
5192 			return rc;
5193 	}
5194 	/*
5195 	 * Ensure correct T10 PI operation if vendor left EEDPTagMode
5196 	 * flag unset in NVDATA.
5197 	 */
5198 	rc = mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply,
5199 	    &ioc->manu_pg11);
5200 	if (rc)
5201 		return rc;
5202 	if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) {
5203 		pr_err("%s: overriding NVDATA EEDPTagMode setting\n",
5204 		    ioc->name);
5205 		ioc->manu_pg11.EEDPTagMode &= ~0x3;
5206 		ioc->manu_pg11.EEDPTagMode |= 0x1;
5207 		mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,
5208 		    &ioc->manu_pg11);
5209 	}
5210 	if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)
5211 		ioc->tm_custom_handling = 1;
5212 	else {
5213 		ioc->tm_custom_handling = 0;
5214 		if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)
5215 			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;
5216 		else if (ioc->manu_pg11.NVMeAbortTO >
5217 					NVME_TASK_ABORT_MAX_TIMEOUT)
5218 			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;
5219 		else
5220 			ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;
5221 	}
5222 	ioc->time_sync_interval =
5223 	    ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_MASK;
5224 	if (ioc->time_sync_interval) {
5225 		if (ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_UNIT_MASK)
5226 			ioc->time_sync_interval =
5227 			    ioc->time_sync_interval * SECONDS_PER_HOUR;
5228 		else
5229 			ioc->time_sync_interval =
5230 			    ioc->time_sync_interval * SECONDS_PER_MIN;
5231 		dinitprintk(ioc, ioc_info(ioc,
5232 		    "Driver-FW TimeSync interval is %d seconds. ManuPg11 TimeSync Unit is in %s\n",
5233 		    ioc->time_sync_interval, (ioc->manu_pg11.TimeSyncInterval &
5234 		    MPT3SAS_TIMESYNC_UNIT_MASK) ? "Hour" : "Minute"));
5235 	} else {
5236 		if (ioc->is_gen35_ioc)
5237 			ioc_warn(ioc,
5238 			    "TimeSync Interval in Manuf page-11 is not enabled. Periodic Time-Sync will be disabled\n");
5239 	}
5240 	rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);
5241 	if (rc)
5242 		return rc;
5243 	rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);
5244 	if (rc)
5245 		return rc;
5246 	rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);
5247 	if (rc)
5248 		return rc;
5249 	rc = mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);
5250 	if (rc)
5251 		return rc;
5252 	rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
5253 	if (rc)
5254 		return rc;
5255 	rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);
5256 	if (rc)
5257 		return rc;
5258 	_base_display_ioc_capabilities(ioc);
5259 
5260 	/*
5261 	 * Enable task_set_full handling in iounit_pg1 when the
5262 	 * facts capabilities indicate that its supported.
5263 	 */
5264 	iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);
5265 	if ((ioc->facts.IOCCapabilities &
5266 	    MPI2_IOCFACTS_CAPABILITY_TASK_SET_FULL_HANDLING))
5267 		iounit_pg1_flags &=
5268 		    ~MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
5269 	else
5270 		iounit_pg1_flags |=
5271 		    MPI2_IOUNITPAGE1_DISABLE_TASK_SET_FULL_HANDLING;
5272 	ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);
5273 	rc = mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);
5274 	if (rc)
5275 		return rc;
5276 
5277 	if (ioc->iounit_pg8.NumSensors)
5278 		ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;
5279 	if (ioc->is_aero_ioc) {
5280 		rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc);
5281 		if (rc)
5282 			return rc;
5283 	}
5284 	if (ioc->is_gen35_ioc) {
5285 		if (ioc->is_driver_loading) {
5286 			rc = _base_get_diag_triggers(ioc);
5287 			if (rc)
5288 				return rc;
5289 		} else {
5290 			/*
5291 			 * In case of online HBA FW update operation,
5292 			 * check whether updated FW supports the driver trigger
5293 			 * pages or not.
5294 			 * - If previous FW has not supported driver trigger
5295 			 *   pages and newer FW supports them then update these
5296 			 *   pages with current diag trigger values.
5297 			 * - If previous FW has supported driver trigger pages
5298 			 *   and new FW doesn't support them then disable
5299 			 *   support_trigger_pages flag.
5300 			 */
5301 			_base_check_for_trigger_pages_support(ioc, &tg_flags);
5302 			if (!ioc->supports_trigger_pages && tg_flags != -EFAULT)
5303 				_base_update_diag_trigger_pages(ioc);
5304 			else if (ioc->supports_trigger_pages &&
5305 			    tg_flags == -EFAULT)
5306 				ioc->supports_trigger_pages = 0;
5307 		}
5308 	}
5309 	return 0;
5310 }
5311 
5312 /**
5313  * mpt3sas_free_enclosure_list - release memory
5314  * @ioc: per adapter object
5315  *
5316  * Free memory allocated during enclosure add.
5317  */
5318 void
5319 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)
5320 {
5321 	struct _enclosure_node *enclosure_dev, *enclosure_dev_next;
5322 
5323 	/* Free enclosure list */
5324 	list_for_each_entry_safe(enclosure_dev,
5325 			enclosure_dev_next, &ioc->enclosure_list, list) {
5326 		list_del(&enclosure_dev->list);
5327 		kfree(enclosure_dev);
5328 	}
5329 }
5330 
5331 /**
5332  * _base_release_memory_pools - release memory
5333  * @ioc: per adapter object
5334  *
5335  * Free memory allocated from _base_allocate_memory_pools.
5336  */
5337 static void
5338 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)
5339 {
5340 	int i = 0;
5341 	int j = 0;
5342 	int dma_alloc_count = 0;
5343 	struct chain_tracker *ct;
5344 	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;
5345 
5346 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
5347 
5348 	if (ioc->request) {
5349 		dma_free_coherent(&ioc->pdev->dev, ioc->request_dma_sz,
5350 		    ioc->request,  ioc->request_dma);
5351 		dexitprintk(ioc,
5352 			    ioc_info(ioc, "request_pool(0x%p): free\n",
5353 				     ioc->request));
5354 		ioc->request = NULL;
5355 	}
5356 
5357 	if (ioc->sense) {
5358 		dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);
5359 		dma_pool_destroy(ioc->sense_dma_pool);
5360 		dexitprintk(ioc,
5361 			    ioc_info(ioc, "sense_pool(0x%p): free\n",
5362 				     ioc->sense));
5363 		ioc->sense = NULL;
5364 	}
5365 
5366 	if (ioc->reply) {
5367 		dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);
5368 		dma_pool_destroy(ioc->reply_dma_pool);
5369 		dexitprintk(ioc,
5370 			    ioc_info(ioc, "reply_pool(0x%p): free\n",
5371 				     ioc->reply));
5372 		ioc->reply = NULL;
5373 	}
5374 
5375 	if (ioc->reply_free) {
5376 		dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,
5377 		    ioc->reply_free_dma);
5378 		dma_pool_destroy(ioc->reply_free_dma_pool);
5379 		dexitprintk(ioc,
5380 			    ioc_info(ioc, "reply_free_pool(0x%p): free\n",
5381 				     ioc->reply_free));
5382 		ioc->reply_free = NULL;
5383 	}
5384 
5385 	if (ioc->reply_post) {
5386 		dma_alloc_count = DIV_ROUND_UP(count,
5387 				RDPQ_MAX_INDEX_IN_ONE_CHUNK);
5388 		for (i = 0; i < count; i++) {
5389 			if (i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0
5390 			    && dma_alloc_count) {
5391 				if (ioc->reply_post[i].reply_post_free) {
5392 					dma_pool_free(
5393 					    ioc->reply_post_free_dma_pool,
5394 					    ioc->reply_post[i].reply_post_free,
5395 					ioc->reply_post[i].reply_post_free_dma);
5396 					dexitprintk(ioc, ioc_info(ioc,
5397 					   "reply_post_free_pool(0x%p): free\n",
5398 					   ioc->reply_post[i].reply_post_free));
5399 					ioc->reply_post[i].reply_post_free =
5400 									NULL;
5401 				}
5402 				--dma_alloc_count;
5403 			}
5404 		}
5405 		dma_pool_destroy(ioc->reply_post_free_dma_pool);
5406 		if (ioc->reply_post_free_array &&
5407 			ioc->rdpq_array_enable) {
5408 			dma_pool_free(ioc->reply_post_free_array_dma_pool,
5409 			    ioc->reply_post_free_array,
5410 			    ioc->reply_post_free_array_dma);
5411 			ioc->reply_post_free_array = NULL;
5412 		}
5413 		dma_pool_destroy(ioc->reply_post_free_array_dma_pool);
5414 		kfree(ioc->reply_post);
5415 	}
5416 
5417 	if (ioc->pcie_sgl_dma_pool) {
5418 		for (i = 0; i < ioc->scsiio_depth; i++) {
5419 			dma_pool_free(ioc->pcie_sgl_dma_pool,
5420 					ioc->pcie_sg_lookup[i].pcie_sgl,
5421 					ioc->pcie_sg_lookup[i].pcie_sgl_dma);
5422 			ioc->pcie_sg_lookup[i].pcie_sgl = NULL;
5423 		}
5424 		dma_pool_destroy(ioc->pcie_sgl_dma_pool);
5425 	}
5426 	if (ioc->config_page) {
5427 		dexitprintk(ioc,
5428 			    ioc_info(ioc, "config_page(0x%p): free\n",
5429 				     ioc->config_page));
5430 		dma_free_coherent(&ioc->pdev->dev, ioc->config_page_sz,
5431 		    ioc->config_page, ioc->config_page_dma);
5432 	}
5433 
5434 	kfree(ioc->hpr_lookup);
5435 	ioc->hpr_lookup = NULL;
5436 	kfree(ioc->internal_lookup);
5437 	ioc->internal_lookup = NULL;
5438 	if (ioc->chain_lookup) {
5439 		for (i = 0; i < ioc->scsiio_depth; i++) {
5440 			for (j = ioc->chains_per_prp_buffer;
5441 			    j < ioc->chains_needed_per_io; j++) {
5442 				ct = &ioc->chain_lookup[i].chains_per_smid[j];
5443 				if (ct && ct->chain_buffer)
5444 					dma_pool_free(ioc->chain_dma_pool,
5445 						ct->chain_buffer,
5446 						ct->chain_buffer_dma);
5447 			}
5448 			kfree(ioc->chain_lookup[i].chains_per_smid);
5449 		}
5450 		dma_pool_destroy(ioc->chain_dma_pool);
5451 		kfree(ioc->chain_lookup);
5452 		ioc->chain_lookup = NULL;
5453 	}
5454 
5455 	kfree(ioc->io_queue_num);
5456 	ioc->io_queue_num = NULL;
5457 }
5458 
5459 /**
5460  * mpt3sas_check_same_4gb_region - checks whether all reply queues in a set are
5461  *	having same upper 32bits in their base memory address.
5462  * @reply_pool_start_address: Base address of a reply queue set
5463  * @pool_sz: Size of single Reply Descriptor Post Queues pool size
5464  *
5465  * Return: 1 if reply queues in a set have a same upper 32bits in their base
5466  * memory address, else 0.
5467  */
5468 
5469 static int
5470 mpt3sas_check_same_4gb_region(long reply_pool_start_address, u32 pool_sz)
5471 {
5472 	long reply_pool_end_address;
5473 
5474 	reply_pool_end_address = reply_pool_start_address + pool_sz;
5475 
5476 	if (upper_32_bits(reply_pool_start_address) ==
5477 		upper_32_bits(reply_pool_end_address))
5478 		return 1;
5479 	else
5480 		return 0;
5481 }
5482 
5483 /**
5484  * _base_reduce_hba_queue_depth- Retry with reduced queue depth
5485  * @ioc: Adapter object
5486  *
5487  * Return: 0 for success, non-zero for failure.
5488  **/
5489 static inline int
5490 _base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc)
5491 {
5492 	int reduce_sz = 64;
5493 
5494 	if ((ioc->hba_queue_depth - reduce_sz) >
5495 	    (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) {
5496 		ioc->hba_queue_depth -= reduce_sz;
5497 		return 0;
5498 	} else
5499 		return -ENOMEM;
5500 }
5501 
5502 /**
5503  * _base_allocate_pcie_sgl_pool - Allocating DMA'able memory
5504  *			for pcie sgl pools.
5505  * @ioc: Adapter object
5506  * @sz: DMA Pool size
5507  *
5508  * Return: 0 for success, non-zero for failure.
5509  */
5510 
5511 static int
5512 _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
5513 {
5514 	int i = 0, j = 0;
5515 	struct chain_tracker *ct;
5516 
5517 	ioc->pcie_sgl_dma_pool =
5518 	    dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz,
5519 	    ioc->page_size, 0);
5520 	if (!ioc->pcie_sgl_dma_pool) {
5521 		ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n");
5522 		return -ENOMEM;
5523 	}
5524 
5525 	ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;
5526 	ioc->chains_per_prp_buffer =
5527 	    min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io);
5528 	for (i = 0; i < ioc->scsiio_depth; i++) {
5529 		ioc->pcie_sg_lookup[i].pcie_sgl =
5530 		    dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL,
5531 		    &ioc->pcie_sg_lookup[i].pcie_sgl_dma);
5532 		if (!ioc->pcie_sg_lookup[i].pcie_sgl) {
5533 			ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n");
5534 			return -EAGAIN;
5535 		}
5536 
5537 		if (!mpt3sas_check_same_4gb_region(
5538 		    (long)ioc->pcie_sg_lookup[i].pcie_sgl, sz)) {
5539 			ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n",
5540 			    ioc->pcie_sg_lookup[i].pcie_sgl,
5541 			    (unsigned long long)
5542 			    ioc->pcie_sg_lookup[i].pcie_sgl_dma);
5543 			ioc->use_32bit_dma = true;
5544 			return -EAGAIN;
5545 		}
5546 
5547 		for (j = 0; j < ioc->chains_per_prp_buffer; j++) {
5548 			ct = &ioc->chain_lookup[i].chains_per_smid[j];
5549 			ct->chain_buffer =
5550 			    ioc->pcie_sg_lookup[i].pcie_sgl +
5551 			    (j * ioc->chain_segment_sz);
5552 			ct->chain_buffer_dma =
5553 			    ioc->pcie_sg_lookup[i].pcie_sgl_dma +
5554 			    (j * ioc->chain_segment_sz);
5555 		}
5556 	}
5557 	dinitprintk(ioc, ioc_info(ioc,
5558 	    "PCIe sgl pool depth(%d), element_size(%d), pool_size(%d kB)\n",
5559 	    ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));
5560 	dinitprintk(ioc, ioc_info(ioc,
5561 	    "Number of chains can fit in a PRP page(%d)\n",
5562 	    ioc->chains_per_prp_buffer));
5563 	return 0;
5564 }
5565 
5566 /**
5567  * _base_allocate_chain_dma_pool - Allocating DMA'able memory
5568  *			for chain dma pool.
5569  * @ioc: Adapter object
5570  * @sz: DMA Pool size
5571  *
5572  * Return: 0 for success, non-zero for failure.
5573  */
5574 static int
5575 _base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
5576 {
5577 	int i = 0, j = 0;
5578 	struct chain_tracker *ctr;
5579 
5580 	ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,
5581 	    ioc->chain_segment_sz, 16, 0);
5582 	if (!ioc->chain_dma_pool)
5583 		return -ENOMEM;
5584 
5585 	for (i = 0; i < ioc->scsiio_depth; i++) {
5586 		for (j = ioc->chains_per_prp_buffer;
5587 		    j < ioc->chains_needed_per_io; j++) {
5588 			ctr = &ioc->chain_lookup[i].chains_per_smid[j];
5589 			ctr->chain_buffer = dma_pool_alloc(ioc->chain_dma_pool,
5590 			    GFP_KERNEL, &ctr->chain_buffer_dma);
5591 			if (!ctr->chain_buffer)
5592 				return -EAGAIN;
5593 			if (!mpt3sas_check_same_4gb_region((long)
5594 			    ctr->chain_buffer, ioc->chain_segment_sz)) {
5595 				ioc_err(ioc,
5596 				    "Chain buffers are not in same 4G !!! Chain buff (0x%p) dma = (0x%llx)\n",
5597 				    ctr->chain_buffer,
5598 				    (unsigned long long)ctr->chain_buffer_dma);
5599 				ioc->use_32bit_dma = true;
5600 				return -EAGAIN;
5601 			}
5602 		}
5603 	}
5604 	dinitprintk(ioc, ioc_info(ioc,
5605 	    "chain_lookup depth (%d), frame_size(%d), pool_size(%d kB)\n",
5606 	    ioc->scsiio_depth, ioc->chain_segment_sz, ((ioc->scsiio_depth *
5607 	    (ioc->chains_needed_per_io - ioc->chains_per_prp_buffer) *
5608 	    ioc->chain_segment_sz))/1024));
5609 	return 0;
5610 }
5611 
5612 /**
5613  * _base_allocate_sense_dma_pool - Allocating DMA'able memory
5614  *			for sense dma pool.
5615  * @ioc: Adapter object
5616  * @sz: DMA Pool size
5617  * Return: 0 for success, non-zero for failure.
5618  */
5619 static int
5620 _base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
5621 {
5622 	ioc->sense_dma_pool =
5623 	    dma_pool_create("sense pool", &ioc->pdev->dev, sz, 4, 0);
5624 	if (!ioc->sense_dma_pool)
5625 		return -ENOMEM;
5626 	ioc->sense = dma_pool_alloc(ioc->sense_dma_pool,
5627 	    GFP_KERNEL, &ioc->sense_dma);
5628 	if (!ioc->sense)
5629 		return -EAGAIN;
5630 	if (!mpt3sas_check_same_4gb_region((long)ioc->sense, sz)) {
5631 		dinitprintk(ioc, pr_err(
5632 		    "Bad Sense Pool! sense (0x%p) sense_dma = (0x%llx)\n",
5633 		    ioc->sense, (unsigned long long) ioc->sense_dma));
5634 		ioc->use_32bit_dma = true;
5635 		return -EAGAIN;
5636 	}
5637 	ioc_info(ioc,
5638 	    "sense pool(0x%p) - dma(0x%llx): depth(%d), element_size(%d), pool_size (%d kB)\n",
5639 	    ioc->sense, (unsigned long long)ioc->sense_dma,
5640 	    ioc->scsiio_depth, SCSI_SENSE_BUFFERSIZE, sz/1024);
5641 	return 0;
5642 }
5643 
5644 /**
5645  * _base_allocate_reply_pool - Allocating DMA'able memory
5646  *			for reply pool.
5647  * @ioc: Adapter object
5648  * @sz: DMA Pool size
5649  * Return: 0 for success, non-zero for failure.
5650  */
5651 static int
5652 _base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
5653 {
5654 	/* reply pool, 4 byte align */
5655 	ioc->reply_dma_pool = dma_pool_create("reply pool",
5656 	    &ioc->pdev->dev, sz, 4, 0);
5657 	if (!ioc->reply_dma_pool)
5658 		return -ENOMEM;
5659 	ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,
5660 	    &ioc->reply_dma);
5661 	if (!ioc->reply)
5662 		return -EAGAIN;
5663 	if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) {
5664 		dinitprintk(ioc, pr_err(
5665 		    "Bad Reply Pool! Reply (0x%p) Reply dma = (0x%llx)\n",
5666 		    ioc->reply, (unsigned long long) ioc->reply_dma));
5667 		ioc->use_32bit_dma = true;
5668 		return -EAGAIN;
5669 	}
5670 	ioc->reply_dma_min_address = (u32)(ioc->reply_dma);
5671 	ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;
5672 	ioc_info(ioc,
5673 	    "reply pool(0x%p) - dma(0x%llx): depth(%d), frame_size(%d), pool_size(%d kB)\n",
5674 	    ioc->reply, (unsigned long long)ioc->reply_dma,
5675 	    ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024);
5676 	return 0;
5677 }
5678 
5679 /**
5680  * _base_allocate_reply_free_dma_pool - Allocating DMA'able memory
5681  *			for reply free dma pool.
5682  * @ioc: Adapter object
5683  * @sz: DMA Pool size
5684  * Return: 0 for success, non-zero for failure.
5685  */
5686 static int
5687 _base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)
5688 {
5689 	/* reply free queue, 16 byte align */
5690 	ioc->reply_free_dma_pool = dma_pool_create(
5691 	    "reply_free pool", &ioc->pdev->dev, sz, 16, 0);
5692 	if (!ioc->reply_free_dma_pool)
5693 		return -ENOMEM;
5694 	ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool,
5695 	    GFP_KERNEL, &ioc->reply_free_dma);
5696 	if (!ioc->reply_free)
5697 		return -EAGAIN;
5698 	if (!mpt3sas_check_same_4gb_region((long)ioc->reply_free, sz)) {
5699 		dinitprintk(ioc,
5700 		    pr_err("Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n",
5701 		    ioc->reply_free, (unsigned long long) ioc->reply_free_dma));
5702 		ioc->use_32bit_dma = true;
5703 		return -EAGAIN;
5704 	}
5705 	memset(ioc->reply_free, 0, sz);
5706 	dinitprintk(ioc, ioc_info(ioc,
5707 	    "reply_free pool(0x%p): depth(%d), element_size(%d), pool_size(%d kB)\n",
5708 	    ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));
5709 	dinitprintk(ioc, ioc_info(ioc,
5710 	    "reply_free_dma (0x%llx)\n",
5711 	    (unsigned long long)ioc->reply_free_dma));
5712 	return 0;
5713 }
5714 
5715 /**
5716  * _base_allocate_reply_post_free_array - Allocating DMA'able memory
5717  *			for reply post free array.
5718  * @ioc: Adapter object
5719  * @reply_post_free_array_sz: DMA Pool size
5720  * Return: 0 for success, non-zero for failure.
5721  */
5722 
5723 static int
5724 _base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc,
5725 	u32 reply_post_free_array_sz)
5726 {
5727 	ioc->reply_post_free_array_dma_pool =
5728 	    dma_pool_create("reply_post_free_array pool",
5729 	    &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);
5730 	if (!ioc->reply_post_free_array_dma_pool)
5731 		return -ENOMEM;
5732 	ioc->reply_post_free_array =
5733 	    dma_pool_alloc(ioc->reply_post_free_array_dma_pool,
5734 	    GFP_KERNEL, &ioc->reply_post_free_array_dma);
5735 	if (!ioc->reply_post_free_array)
5736 		return -EAGAIN;
5737 	if (!mpt3sas_check_same_4gb_region((long)ioc->reply_post_free_array,
5738 	    reply_post_free_array_sz)) {
5739 		dinitprintk(ioc, pr_err(
5740 		    "Bad Reply Free Pool! Reply Free (0x%p) Reply Free dma = (0x%llx)\n",
5741 		    ioc->reply_free,
5742 		    (unsigned long long) ioc->reply_free_dma));
5743 		ioc->use_32bit_dma = true;
5744 		return -EAGAIN;
5745 	}
5746 	return 0;
5747 }
5748 /**
5749  * base_alloc_rdpq_dma_pool - Allocating DMA'able memory
5750  *                     for reply queues.
5751  * @ioc: per adapter object
5752  * @sz: DMA Pool size
5753  * Return: 0 for success, non-zero for failure.
5754  */
5755 static int
5756 base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz)
5757 {
5758 	int i = 0;
5759 	u32 dma_alloc_count = 0;
5760 	int reply_post_free_sz = ioc->reply_post_queue_depth *
5761 		sizeof(Mpi2DefaultReplyDescriptor_t);
5762 	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;
5763 
5764 	ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct),
5765 			GFP_KERNEL);
5766 	if (!ioc->reply_post)
5767 		return -ENOMEM;
5768 	/*
5769 	 *  For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
5770 	 *  VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should
5771 	 *  be within 4GB boundary i.e reply queues in a set must have same
5772 	 *  upper 32-bits in their memory address. so here driver is allocating
5773 	 *  the DMA'able memory for reply queues according.
5774 	 *  Driver uses limitation of
5775 	 *  VENTURA_SERIES to manage INVADER_SERIES as well.
5776 	 */
5777 	dma_alloc_count = DIV_ROUND_UP(count,
5778 				RDPQ_MAX_INDEX_IN_ONE_CHUNK);
5779 	ioc->reply_post_free_dma_pool =
5780 		dma_pool_create("reply_post_free pool",
5781 		    &ioc->pdev->dev, sz, 16, 0);
5782 	if (!ioc->reply_post_free_dma_pool)
5783 		return -ENOMEM;
5784 	for (i = 0; i < count; i++) {
5785 		if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) {
5786 			ioc->reply_post[i].reply_post_free =
5787 			    dma_pool_zalloc(ioc->reply_post_free_dma_pool,
5788 				GFP_KERNEL,
5789 				&ioc->reply_post[i].reply_post_free_dma);
5790 			if (!ioc->reply_post[i].reply_post_free)
5791 				return -ENOMEM;
5792 			/*
5793 			 * Each set of RDPQ pool must satisfy 4gb boundary
5794 			 * restriction.
5795 			 * 1) Check if allocated resources for RDPQ pool are in
5796 			 *	the same 4GB range.
5797 			 * 2) If #1 is true, continue with 64 bit DMA.
5798 			 * 3) If #1 is false, return 1. which means free all the
5799 			 * resources and set DMA mask to 32 and allocate.
5800 			 */
5801 			if (!mpt3sas_check_same_4gb_region(
5802 				(long)ioc->reply_post[i].reply_post_free, sz)) {
5803 				dinitprintk(ioc,
5804 				    ioc_err(ioc, "bad Replypost free pool(0x%p)"
5805 				    "reply_post_free_dma = (0x%llx)\n",
5806 				    ioc->reply_post[i].reply_post_free,
5807 				    (unsigned long long)
5808 				    ioc->reply_post[i].reply_post_free_dma));
5809 				return -EAGAIN;
5810 			}
5811 			dma_alloc_count--;
5812 
5813 		} else {
5814 			ioc->reply_post[i].reply_post_free =
5815 			    (Mpi2ReplyDescriptorsUnion_t *)
5816 			    ((long)ioc->reply_post[i-1].reply_post_free
5817 			    + reply_post_free_sz);
5818 			ioc->reply_post[i].reply_post_free_dma =
5819 			    (dma_addr_t)
5820 			    (ioc->reply_post[i-1].reply_post_free_dma +
5821 			    reply_post_free_sz);
5822 		}
5823 	}
5824 	return 0;
5825 }
5826 
5827 /**
5828  * _base_allocate_memory_pools - allocate start of day memory pools
5829  * @ioc: per adapter object
5830  *
5831  * Return: 0 success, anything else error.
5832  */
5833 static int
5834 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)
5835 {
5836 	struct mpt3sas_facts *facts;
5837 	u16 max_sge_elements;
5838 	u16 chains_needed_per_io;
5839 	u32 sz, total_sz, reply_post_free_sz, reply_post_free_array_sz;
5840 	u32 retry_sz;
5841 	u32 rdpq_sz = 0, sense_sz = 0;
5842 	u16 max_request_credit, nvme_blocks_needed;
5843 	unsigned short sg_tablesize;
5844 	u16 sge_size;
5845 	int i;
5846 	int ret = 0, rc = 0;
5847 
5848 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
5849 
5850 
5851 	retry_sz = 0;
5852 	facts = &ioc->facts;
5853 
5854 	/* command line tunables for max sgl entries */
5855 	if (max_sgl_entries != -1)
5856 		sg_tablesize = max_sgl_entries;
5857 	else {
5858 		if (ioc->hba_mpi_version_belonged == MPI2_VERSION)
5859 			sg_tablesize = MPT2SAS_SG_DEPTH;
5860 		else
5861 			sg_tablesize = MPT3SAS_SG_DEPTH;
5862 	}
5863 
5864 	/* max sgl entries <= MPT_KDUMP_MIN_PHYS_SEGMENTS in KDUMP mode */
5865 	if (reset_devices)
5866 		sg_tablesize = min_t(unsigned short, sg_tablesize,
5867 		   MPT_KDUMP_MIN_PHYS_SEGMENTS);
5868 
5869 	if (ioc->is_mcpu_endpoint)
5870 		ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
5871 	else {
5872 		if (sg_tablesize < MPT_MIN_PHYS_SEGMENTS)
5873 			sg_tablesize = MPT_MIN_PHYS_SEGMENTS;
5874 		else if (sg_tablesize > MPT_MAX_PHYS_SEGMENTS) {
5875 			sg_tablesize = min_t(unsigned short, sg_tablesize,
5876 					SG_MAX_SEGMENTS);
5877 			ioc_warn(ioc, "sg_tablesize(%u) is bigger than kernel defined SG_CHUNK_SIZE(%u)\n",
5878 				 sg_tablesize, MPT_MAX_PHYS_SEGMENTS);
5879 		}
5880 		ioc->shost->sg_tablesize = sg_tablesize;
5881 	}
5882 
5883 	ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),
5884 		(facts->RequestCredit / 4));
5885 	if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {
5886 		if (facts->RequestCredit <= (INTERNAL_CMDS_COUNT +
5887 				INTERNAL_SCSIIO_CMDS_COUNT)) {
5888 			ioc_err(ioc, "IOC doesn't have enough Request Credits, it has just %d number of credits\n",
5889 				facts->RequestCredit);
5890 			return -ENOMEM;
5891 		}
5892 		ioc->internal_depth = 10;
5893 	}
5894 
5895 	ioc->hi_priority_depth = ioc->internal_depth - (5);
5896 	/* command line tunables  for max controller queue depth */
5897 	if (max_queue_depth != -1 && max_queue_depth != 0) {
5898 		max_request_credit = min_t(u16, max_queue_depth +
5899 			ioc->internal_depth, facts->RequestCredit);
5900 		if (max_request_credit > MAX_HBA_QUEUE_DEPTH)
5901 			max_request_credit =  MAX_HBA_QUEUE_DEPTH;
5902 	} else if (reset_devices)
5903 		max_request_credit = min_t(u16, facts->RequestCredit,
5904 		    (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));
5905 	else
5906 		max_request_credit = min_t(u16, facts->RequestCredit,
5907 		    MAX_HBA_QUEUE_DEPTH);
5908 
5909 	/* Firmware maintains additional facts->HighPriorityCredit number of
5910 	 * credits for HiPriprity Request messages, so hba queue depth will be
5911 	 * sum of max_request_credit and high priority queue depth.
5912 	 */
5913 	ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;
5914 
5915 	/* request frame size */
5916 	ioc->request_sz = facts->IOCRequestFrameSize * 4;
5917 
5918 	/* reply frame size */
5919 	ioc->reply_sz = facts->ReplyFrameSize * 4;
5920 
5921 	/* chain segment size */
5922 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
5923 		if (facts->IOCMaxChainSegmentSize)
5924 			ioc->chain_segment_sz =
5925 					facts->IOCMaxChainSegmentSize *
5926 					MAX_CHAIN_ELEMT_SZ;
5927 		else
5928 		/* set to 128 bytes size if IOCMaxChainSegmentSize is zero */
5929 			ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *
5930 						    MAX_CHAIN_ELEMT_SZ;
5931 	} else
5932 		ioc->chain_segment_sz = ioc->request_sz;
5933 
5934 	/* calculate the max scatter element size */
5935 	sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);
5936 
5937  retry_allocation:
5938 	total_sz = 0;
5939 	/* calculate number of sg elements left over in the 1st frame */
5940 	max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -
5941 	    sizeof(Mpi2SGEIOUnion_t)) + sge_size);
5942 	ioc->max_sges_in_main_message = max_sge_elements/sge_size;
5943 
5944 	/* now do the same for a chain buffer */
5945 	max_sge_elements = ioc->chain_segment_sz - sge_size;
5946 	ioc->max_sges_in_chain_message = max_sge_elements/sge_size;
5947 
5948 	/*
5949 	 *  MPT3SAS_SG_DEPTH = CONFIG_FUSION_MAX_SGE
5950 	 */
5951 	chains_needed_per_io = ((ioc->shost->sg_tablesize -
5952 	   ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)
5953 	    + 1;
5954 	if (chains_needed_per_io > facts->MaxChainDepth) {
5955 		chains_needed_per_io = facts->MaxChainDepth;
5956 		ioc->shost->sg_tablesize = min_t(u16,
5957 		ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message
5958 		* chains_needed_per_io), ioc->shost->sg_tablesize);
5959 	}
5960 	ioc->chains_needed_per_io = chains_needed_per_io;
5961 
5962 	/* reply free queue sizing - taking into account for 64 FW events */
5963 	ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
5964 
5965 	/* mCPU manage single counters for simplicity */
5966 	if (ioc->is_mcpu_endpoint)
5967 		ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;
5968 	else {
5969 		/* calculate reply descriptor post queue depth */
5970 		ioc->reply_post_queue_depth = ioc->hba_queue_depth +
5971 			ioc->reply_free_queue_depth +  1;
5972 		/* align the reply post queue on the next 16 count boundary */
5973 		if (ioc->reply_post_queue_depth % 16)
5974 			ioc->reply_post_queue_depth += 16 -
5975 				(ioc->reply_post_queue_depth % 16);
5976 	}
5977 
5978 	if (ioc->reply_post_queue_depth >
5979 	    facts->MaxReplyDescriptorPostQueueDepth) {
5980 		ioc->reply_post_queue_depth =
5981 				facts->MaxReplyDescriptorPostQueueDepth -
5982 		    (facts->MaxReplyDescriptorPostQueueDepth % 16);
5983 		ioc->hba_queue_depth =
5984 				((ioc->reply_post_queue_depth - 64) / 2) - 1;
5985 		ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;
5986 	}
5987 
5988 	ioc_info(ioc,
5989 	    "scatter gather: sge_in_main_msg(%d), sge_per_chain(%d), "
5990 	    "sge_per_io(%d), chains_per_io(%d)\n",
5991 	    ioc->max_sges_in_main_message,
5992 	    ioc->max_sges_in_chain_message,
5993 	    ioc->shost->sg_tablesize,
5994 	    ioc->chains_needed_per_io);
5995 
5996 	/* reply post queue, 16 byte align */
5997 	reply_post_free_sz = ioc->reply_post_queue_depth *
5998 	    sizeof(Mpi2DefaultReplyDescriptor_t);
5999 	rdpq_sz = reply_post_free_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
6000 	if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)
6001 	    || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK))
6002 		rdpq_sz = reply_post_free_sz * ioc->reply_queue_count;
6003 	ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz);
6004 	if (ret == -EAGAIN) {
6005 		/*
6006 		 * Free allocated bad RDPQ memory pools.
6007 		 * Change dma coherent mask to 32 bit and reallocate RDPQ
6008 		 */
6009 		_base_release_memory_pools(ioc);
6010 		ioc->use_32bit_dma = true;
6011 		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {
6012 			ioc_err(ioc,
6013 			    "32 DMA mask failed %s\n", pci_name(ioc->pdev));
6014 			return -ENODEV;
6015 		}
6016 		if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz))
6017 			return -ENOMEM;
6018 	} else if (ret == -ENOMEM)
6019 		return -ENOMEM;
6020 	total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 :
6021 	    DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK));
6022 	ioc->scsiio_depth = ioc->hba_queue_depth -
6023 	    ioc->hi_priority_depth - ioc->internal_depth;
6024 
6025 	/* set the scsi host can_queue depth
6026 	 * with some internal commands that could be outstanding
6027 	 */
6028 	ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;
6029 	dinitprintk(ioc,
6030 		    ioc_info(ioc, "scsi host: can_queue depth (%d)\n",
6031 			     ioc->shost->can_queue));
6032 
6033 	/* contiguous pool for request and chains, 16 byte align, one extra "
6034 	 * "frame for smid=0
6035 	 */
6036 	ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;
6037 	sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);
6038 
6039 	/* hi-priority queue */
6040 	sz += (ioc->hi_priority_depth * ioc->request_sz);
6041 
6042 	/* internal queue */
6043 	sz += (ioc->internal_depth * ioc->request_sz);
6044 
6045 	ioc->request_dma_sz = sz;
6046 	ioc->request = dma_alloc_coherent(&ioc->pdev->dev, sz,
6047 			&ioc->request_dma, GFP_KERNEL);
6048 	if (!ioc->request) {
6049 		ioc_err(ioc, "request pool: dma_alloc_coherent failed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kB)\n",
6050 			ioc->hba_queue_depth, ioc->chains_needed_per_io,
6051 			ioc->request_sz, sz / 1024);
6052 		if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)
6053 			goto out;
6054 		retry_sz = 64;
6055 		ioc->hba_queue_depth -= retry_sz;
6056 		_base_release_memory_pools(ioc);
6057 		goto retry_allocation;
6058 	}
6059 
6060 	if (retry_sz)
6061 		ioc_err(ioc, "request pool: dma_alloc_coherent succeed: hba_depth(%d), chains_per_io(%d), frame_sz(%d), total(%d kb)\n",
6062 			ioc->hba_queue_depth, ioc->chains_needed_per_io,
6063 			ioc->request_sz, sz / 1024);
6064 
6065 	/* hi-priority queue */
6066 	ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *
6067 	    ioc->request_sz);
6068 	ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *
6069 	    ioc->request_sz);
6070 
6071 	/* internal queue */
6072 	ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *
6073 	    ioc->request_sz);
6074 	ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *
6075 	    ioc->request_sz);
6076 
6077 	ioc_info(ioc,
6078 	    "request pool(0x%p) - dma(0x%llx): "
6079 	    "depth(%d), frame_size(%d), pool_size(%d kB)\n",
6080 	    ioc->request, (unsigned long long) ioc->request_dma,
6081 	    ioc->hba_queue_depth, ioc->request_sz,
6082 	    (ioc->hba_queue_depth * ioc->request_sz) / 1024);
6083 
6084 	total_sz += sz;
6085 
6086 	dinitprintk(ioc,
6087 		    ioc_info(ioc, "scsiio(0x%p): depth(%d)\n",
6088 			     ioc->request, ioc->scsiio_depth));
6089 
6090 	ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);
6091 	sz = ioc->scsiio_depth * sizeof(struct chain_lookup);
6092 	ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);
6093 	if (!ioc->chain_lookup) {
6094 		ioc_err(ioc, "chain_lookup: __get_free_pages failed\n");
6095 		goto out;
6096 	}
6097 
6098 	sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);
6099 	for (i = 0; i < ioc->scsiio_depth; i++) {
6100 		ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);
6101 		if (!ioc->chain_lookup[i].chains_per_smid) {
6102 			ioc_err(ioc, "chain_lookup: kzalloc failed\n");
6103 			goto out;
6104 		}
6105 	}
6106 
6107 	/* initialize hi-priority queue smid's */
6108 	ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,
6109 	    sizeof(struct request_tracker), GFP_KERNEL);
6110 	if (!ioc->hpr_lookup) {
6111 		ioc_err(ioc, "hpr_lookup: kcalloc failed\n");
6112 		goto out;
6113 	}
6114 	ioc->hi_priority_smid = ioc->scsiio_depth + 1;
6115 	dinitprintk(ioc,
6116 		    ioc_info(ioc, "hi_priority(0x%p): depth(%d), start smid(%d)\n",
6117 			     ioc->hi_priority,
6118 			     ioc->hi_priority_depth, ioc->hi_priority_smid));
6119 
6120 	/* initialize internal queue smid's */
6121 	ioc->internal_lookup = kcalloc(ioc->internal_depth,
6122 	    sizeof(struct request_tracker), GFP_KERNEL);
6123 	if (!ioc->internal_lookup) {
6124 		ioc_err(ioc, "internal_lookup: kcalloc failed\n");
6125 		goto out;
6126 	}
6127 	ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;
6128 	dinitprintk(ioc,
6129 		    ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n",
6130 			     ioc->internal,
6131 			     ioc->internal_depth, ioc->internal_smid));
6132 
6133 	ioc->io_queue_num = kcalloc(ioc->scsiio_depth,
6134 	    sizeof(u16), GFP_KERNEL);
6135 	if (!ioc->io_queue_num)
6136 		goto out;
6137 	/*
6138 	 * The number of NVMe page sized blocks needed is:
6139 	 *     (((sg_tablesize * 8) - 1) / (page_size - 8)) + 1
6140 	 * ((sg_tablesize * 8) - 1) is the max PRP's minus the first PRP entry
6141 	 * that is placed in the main message frame.  8 is the size of each PRP
6142 	 * entry or PRP list pointer entry.  8 is subtracted from page_size
6143 	 * because of the PRP list pointer entry at the end of a page, so this
6144 	 * is not counted as a PRP entry.  The 1 added page is a round up.
6145 	 *
6146 	 * To avoid allocation failures due to the amount of memory that could
6147 	 * be required for NVMe PRP's, only each set of NVMe blocks will be
6148 	 * contiguous, so a new set is allocated for each possible I/O.
6149 	 */
6150 
6151 	ioc->chains_per_prp_buffer = 0;
6152 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {
6153 		nvme_blocks_needed =
6154 			(ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;
6155 		nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);
6156 		nvme_blocks_needed++;
6157 
6158 		sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;
6159 		ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);
6160 		if (!ioc->pcie_sg_lookup) {
6161 			ioc_info(ioc, "PCIe SGL lookup: kzalloc failed\n");
6162 			goto out;
6163 		}
6164 		sz = nvme_blocks_needed * ioc->page_size;
6165 		rc = _base_allocate_pcie_sgl_pool(ioc, sz);
6166 		if (rc == -ENOMEM)
6167 			return -ENOMEM;
6168 		else if (rc == -EAGAIN)
6169 			goto try_32bit_dma;
6170 		total_sz += sz * ioc->scsiio_depth;
6171 	}
6172 
6173 	rc = _base_allocate_chain_dma_pool(ioc, ioc->chain_segment_sz);
6174 	if (rc == -ENOMEM)
6175 		return -ENOMEM;
6176 	else if (rc == -EAGAIN)
6177 		goto try_32bit_dma;
6178 	total_sz += ioc->chain_segment_sz * ((ioc->chains_needed_per_io -
6179 		ioc->chains_per_prp_buffer) * ioc->scsiio_depth);
6180 	dinitprintk(ioc,
6181 	    ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",
6182 	    ioc->chain_depth, ioc->chain_segment_sz,
6183 	    (ioc->chain_depth * ioc->chain_segment_sz) / 1024));
6184 	/* sense buffers, 4 byte align */
6185 	sense_sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;
6186 	rc = _base_allocate_sense_dma_pool(ioc, sense_sz);
6187 	if (rc  == -ENOMEM)
6188 		return -ENOMEM;
6189 	else if (rc == -EAGAIN)
6190 		goto try_32bit_dma;
6191 	total_sz += sense_sz;
6192 	ioc_info(ioc,
6193 	    "sense pool(0x%p)- dma(0x%llx): depth(%d),"
6194 	    "element_size(%d), pool_size(%d kB)\n",
6195 	    ioc->sense, (unsigned long long)ioc->sense_dma, ioc->scsiio_depth,
6196 	    SCSI_SENSE_BUFFERSIZE, sz / 1024);
6197 	/* reply pool, 4 byte align */
6198 	sz = ioc->reply_free_queue_depth * ioc->reply_sz;
6199 	rc = _base_allocate_reply_pool(ioc, sz);
6200 	if (rc == -ENOMEM)
6201 		return -ENOMEM;
6202 	else if (rc == -EAGAIN)
6203 		goto try_32bit_dma;
6204 	total_sz += sz;
6205 
6206 	/* reply free queue, 16 byte align */
6207 	sz = ioc->reply_free_queue_depth * 4;
6208 	rc = _base_allocate_reply_free_dma_pool(ioc, sz);
6209 	if (rc  == -ENOMEM)
6210 		return -ENOMEM;
6211 	else if (rc == -EAGAIN)
6212 		goto try_32bit_dma;
6213 	dinitprintk(ioc,
6214 		    ioc_info(ioc, "reply_free_dma (0x%llx)\n",
6215 			     (unsigned long long)ioc->reply_free_dma));
6216 	total_sz += sz;
6217 	if (ioc->rdpq_array_enable) {
6218 		reply_post_free_array_sz = ioc->reply_queue_count *
6219 		    sizeof(Mpi2IOCInitRDPQArrayEntry);
6220 		rc = _base_allocate_reply_post_free_array(ioc,
6221 		    reply_post_free_array_sz);
6222 		if (rc == -ENOMEM)
6223 			return -ENOMEM;
6224 		else if (rc == -EAGAIN)
6225 			goto try_32bit_dma;
6226 	}
6227 	ioc->config_page_sz = 512;
6228 	ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev,
6229 			ioc->config_page_sz, &ioc->config_page_dma, GFP_KERNEL);
6230 	if (!ioc->config_page) {
6231 		ioc_err(ioc, "config page: dma_pool_alloc failed\n");
6232 		goto out;
6233 	}
6234 
6235 	ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n",
6236 	    ioc->config_page, (unsigned long long)ioc->config_page_dma,
6237 	    ioc->config_page_sz);
6238 	total_sz += ioc->config_page_sz;
6239 
6240 	ioc_info(ioc, "Allocated physical memory: size(%d kB)\n",
6241 		 total_sz / 1024);
6242 	ioc_info(ioc, "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",
6243 		 ioc->shost->can_queue, facts->RequestCredit);
6244 	ioc_info(ioc, "Scatter Gather Elements per IO(%d)\n",
6245 		 ioc->shost->sg_tablesize);
6246 	return 0;
6247 
6248 try_32bit_dma:
6249 	_base_release_memory_pools(ioc);
6250 	if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) {
6251 		/* Change dma coherent mask to 32 bit and reallocate */
6252 		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {
6253 			pr_err("Setting 32 bit coherent DMA mask Failed %s\n",
6254 			    pci_name(ioc->pdev));
6255 			return -ENODEV;
6256 		}
6257 	} else if (_base_reduce_hba_queue_depth(ioc) != 0)
6258 		return -ENOMEM;
6259 	goto retry_allocation;
6260 
6261  out:
6262 	return -ENOMEM;
6263 }
6264 
6265 /**
6266  * mpt3sas_base_get_iocstate - Get the current state of a MPT adapter.
6267  * @ioc: Pointer to MPT_ADAPTER structure
6268  * @cooked: Request raw or cooked IOC state
6269  *
6270  * Return: all IOC Doorbell register bits if cooked==0, else just the
6271  * Doorbell bits in MPI_IOC_STATE_MASK.
6272  */
6273 u32
6274 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)
6275 {
6276 	u32 s, sc;
6277 
6278 	s = ioc->base_readl(&ioc->chip->Doorbell);
6279 	sc = s & MPI2_IOC_STATE_MASK;
6280 	return cooked ? sc : s;
6281 }
6282 
6283 /**
6284  * _base_wait_on_iocstate - waiting on a particular ioc state
6285  * @ioc: ?
6286  * @ioc_state: controller state { READY, OPERATIONAL, or RESET }
6287  * @timeout: timeout in second
6288  *
6289  * Return: 0 for success, non-zero for failure.
6290  */
6291 static int
6292 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)
6293 {
6294 	u32 count, cntdn;
6295 	u32 current_state;
6296 
6297 	count = 0;
6298 	cntdn = 1000 * timeout;
6299 	do {
6300 		current_state = mpt3sas_base_get_iocstate(ioc, 1);
6301 		if (current_state == ioc_state)
6302 			return 0;
6303 		if (count && current_state == MPI2_IOC_STATE_FAULT)
6304 			break;
6305 		if (count && current_state == MPI2_IOC_STATE_COREDUMP)
6306 			break;
6307 
6308 		usleep_range(1000, 1500);
6309 		count++;
6310 	} while (--cntdn);
6311 
6312 	return current_state;
6313 }
6314 
6315 /**
6316  * _base_dump_reg_set -	This function will print hexdump of register set.
6317  * @ioc: per adapter object
6318  *
6319  * Return: nothing.
6320  */
6321 static inline void
6322 _base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc)
6323 {
6324 	unsigned int i, sz = 256;
6325 	u32 __iomem *reg = (u32 __iomem *)ioc->chip;
6326 
6327 	ioc_info(ioc, "System Register set:\n");
6328 	for (i = 0; i < (sz / sizeof(u32)); i++)
6329 		pr_info("%08x: %08x\n", (i * 4), readl(&reg[i]));
6330 }
6331 
6332 /**
6333  * _base_wait_for_doorbell_int - waiting for controller interrupt(generated by
6334  * a write to the doorbell)
6335  * @ioc: per adapter object
6336  * @timeout: timeout in seconds
6337  *
6338  * Return: 0 for success, non-zero for failure.
6339  *
6340  * Notes: MPI2_HIS_IOC2SYS_DB_STATUS - set to one when IOC writes to doorbell.
6341  */
6342 
6343 static int
6344 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
6345 {
6346 	u32 cntdn, count;
6347 	u32 int_status;
6348 
6349 	count = 0;
6350 	cntdn = 1000 * timeout;
6351 	do {
6352 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);
6353 		if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
6354 			dhsprintk(ioc,
6355 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
6356 					   __func__, count, timeout));
6357 			return 0;
6358 		}
6359 
6360 		usleep_range(1000, 1500);
6361 		count++;
6362 	} while (--cntdn);
6363 
6364 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",
6365 		__func__, count, int_status);
6366 	return -EFAULT;
6367 }
6368 
6369 static int
6370 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)
6371 {
6372 	u32 cntdn, count;
6373 	u32 int_status;
6374 
6375 	count = 0;
6376 	cntdn = 2000 * timeout;
6377 	do {
6378 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);
6379 		if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
6380 			dhsprintk(ioc,
6381 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
6382 					   __func__, count, timeout));
6383 			return 0;
6384 		}
6385 
6386 		udelay(500);
6387 		count++;
6388 	} while (--cntdn);
6389 
6390 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",
6391 		__func__, count, int_status);
6392 	return -EFAULT;
6393 
6394 }
6395 
6396 /**
6397  * _base_wait_for_doorbell_ack - waiting for controller to read the doorbell.
6398  * @ioc: per adapter object
6399  * @timeout: timeout in second
6400  *
6401  * Return: 0 for success, non-zero for failure.
6402  *
6403  * Notes: MPI2_HIS_SYS2IOC_DB_STATUS - set to one when host writes to
6404  * doorbell.
6405  */
6406 static int
6407 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)
6408 {
6409 	u32 cntdn, count;
6410 	u32 int_status;
6411 	u32 doorbell;
6412 
6413 	count = 0;
6414 	cntdn = 1000 * timeout;
6415 	do {
6416 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);
6417 		if (!(int_status & MPI2_HIS_SYS2IOC_DB_STATUS)) {
6418 			dhsprintk(ioc,
6419 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
6420 					   __func__, count, timeout));
6421 			return 0;
6422 		} else if (int_status & MPI2_HIS_IOC2SYS_DB_STATUS) {
6423 			doorbell = ioc->base_readl(&ioc->chip->Doorbell);
6424 			if ((doorbell & MPI2_IOC_STATE_MASK) ==
6425 			    MPI2_IOC_STATE_FAULT) {
6426 				mpt3sas_print_fault_code(ioc, doorbell);
6427 				return -EFAULT;
6428 			}
6429 			if ((doorbell & MPI2_IOC_STATE_MASK) ==
6430 			    MPI2_IOC_STATE_COREDUMP) {
6431 				mpt3sas_print_coredump_info(ioc, doorbell);
6432 				return -EFAULT;
6433 			}
6434 		} else if (int_status == 0xFFFFFFFF)
6435 			goto out;
6436 
6437 		usleep_range(1000, 1500);
6438 		count++;
6439 	} while (--cntdn);
6440 
6441  out:
6442 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",
6443 		__func__, count, int_status);
6444 	return -EFAULT;
6445 }
6446 
6447 /**
6448  * _base_wait_for_doorbell_not_used - waiting for doorbell to not be in use
6449  * @ioc: per adapter object
6450  * @timeout: timeout in second
6451  *
6452  * Return: 0 for success, non-zero for failure.
6453  */
6454 static int
6455 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)
6456 {
6457 	u32 cntdn, count;
6458 	u32 doorbell_reg;
6459 
6460 	count = 0;
6461 	cntdn = 1000 * timeout;
6462 	do {
6463 		doorbell_reg = ioc->base_readl(&ioc->chip->Doorbell);
6464 		if (!(doorbell_reg & MPI2_DOORBELL_USED)) {
6465 			dhsprintk(ioc,
6466 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",
6467 					   __func__, count, timeout));
6468 			return 0;
6469 		}
6470 
6471 		usleep_range(1000, 1500);
6472 		count++;
6473 	} while (--cntdn);
6474 
6475 	ioc_err(ioc, "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",
6476 		__func__, count, doorbell_reg);
6477 	return -EFAULT;
6478 }
6479 
6480 /**
6481  * _base_send_ioc_reset - send doorbell reset
6482  * @ioc: per adapter object
6483  * @reset_type: currently only supports: MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET
6484  * @timeout: timeout in second
6485  *
6486  * Return: 0 for success, non-zero for failure.
6487  */
6488 static int
6489 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)
6490 {
6491 	u32 ioc_state;
6492 	int r = 0;
6493 	unsigned long flags;
6494 
6495 	if (reset_type != MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET) {
6496 		ioc_err(ioc, "%s: unknown reset_type\n", __func__);
6497 		return -EFAULT;
6498 	}
6499 
6500 	if (!(ioc->facts.IOCCapabilities &
6501 	   MPI2_IOCFACTS_CAPABILITY_EVENT_REPLAY))
6502 		return -EFAULT;
6503 
6504 	ioc_info(ioc, "sending message unit reset !!\n");
6505 
6506 	writel(reset_type << MPI2_DOORBELL_FUNCTION_SHIFT,
6507 	    &ioc->chip->Doorbell);
6508 	if ((_base_wait_for_doorbell_ack(ioc, 15))) {
6509 		r = -EFAULT;
6510 		goto out;
6511 	}
6512 
6513 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
6514 	if (ioc_state) {
6515 		ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
6516 			__func__, ioc_state);
6517 		r = -EFAULT;
6518 		goto out;
6519 	}
6520  out:
6521 	if (r != 0) {
6522 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6523 		spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
6524 		/*
6525 		 * Wait for IOC state CoreDump to clear only during
6526 		 * HBA initialization & release time.
6527 		 */
6528 		if ((ioc_state & MPI2_IOC_STATE_MASK) ==
6529 		    MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 ||
6530 		    ioc->fault_reset_work_q == NULL)) {
6531 			spin_unlock_irqrestore(
6532 			    &ioc->ioc_reset_in_progress_lock, flags);
6533 			mpt3sas_print_coredump_info(ioc, ioc_state);
6534 			mpt3sas_base_wait_for_coredump_completion(ioc,
6535 			    __func__);
6536 			spin_lock_irqsave(
6537 			    &ioc->ioc_reset_in_progress_lock, flags);
6538 		}
6539 		spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
6540 	}
6541 	ioc_info(ioc, "message unit reset: %s\n",
6542 		 r == 0 ? "SUCCESS" : "FAILED");
6543 	return r;
6544 }
6545 
6546 /**
6547  * mpt3sas_wait_for_ioc - IOC's operational state is checked here.
6548  * @ioc: per adapter object
6549  * @timeout: timeout in seconds
6550  *
6551  * Return: Waits up to timeout seconds for the IOC to
6552  * become operational. Returns 0 if IOC is present
6553  * and operational; otherwise returns %-EFAULT.
6554  */
6555 
6556 int
6557 mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout)
6558 {
6559 	int wait_state_count = 0;
6560 	u32 ioc_state;
6561 
6562 	do {
6563 		ioc_state = mpt3sas_base_get_iocstate(ioc, 1);
6564 		if (ioc_state == MPI2_IOC_STATE_OPERATIONAL)
6565 			break;
6566 
6567 		/*
6568 		 * Watchdog thread will be started after IOC Initialization, so
6569 		 * no need to wait here for IOC state to become operational
6570 		 * when IOC Initialization is on. Instead the driver will
6571 		 * return ETIME status, so that calling function can issue
6572 		 * diag reset operation and retry the command.
6573 		 */
6574 		if (ioc->is_driver_loading)
6575 			return -ETIME;
6576 
6577 		ssleep(1);
6578 		ioc_info(ioc, "%s: waiting for operational state(count=%d)\n",
6579 				__func__, ++wait_state_count);
6580 	} while (--timeout);
6581 	if (!timeout) {
6582 		ioc_err(ioc, "%s: failed due to ioc not operational\n", __func__);
6583 		return -EFAULT;
6584 	}
6585 	if (wait_state_count)
6586 		ioc_info(ioc, "ioc is operational\n");
6587 	return 0;
6588 }
6589 
6590 /**
6591  * _base_handshake_req_reply_wait - send request thru doorbell interface
6592  * @ioc: per adapter object
6593  * @request_bytes: request length
6594  * @request: pointer having request payload
6595  * @reply_bytes: reply length
6596  * @reply: pointer to reply payload
6597  * @timeout: timeout in second
6598  *
6599  * Return: 0 for success, non-zero for failure.
6600  */
6601 static int
6602 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,
6603 	u32 *request, int reply_bytes, u16 *reply, int timeout)
6604 {
6605 	MPI2DefaultReply_t *default_reply = (MPI2DefaultReply_t *)reply;
6606 	int i;
6607 	u8 failed;
6608 	__le32 *mfp;
6609 
6610 	/* make sure doorbell is not in use */
6611 	if ((ioc->base_readl(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {
6612 		ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__);
6613 		return -EFAULT;
6614 	}
6615 
6616 	/* clear pending doorbell interrupts from previous state changes */
6617 	if (ioc->base_readl(&ioc->chip->HostInterruptStatus) &
6618 	    MPI2_HIS_IOC2SYS_DB_STATUS)
6619 		writel(0, &ioc->chip->HostInterruptStatus);
6620 
6621 	/* send message to ioc */
6622 	writel(((MPI2_FUNCTION_HANDSHAKE<<MPI2_DOORBELL_FUNCTION_SHIFT) |
6623 	    ((request_bytes/4)<<MPI2_DOORBELL_ADD_DWORDS_SHIFT)),
6624 	    &ioc->chip->Doorbell);
6625 
6626 	if ((_base_spin_on_doorbell_int(ioc, 5))) {
6627 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6628 			__LINE__);
6629 		return -EFAULT;
6630 	}
6631 	writel(0, &ioc->chip->HostInterruptStatus);
6632 
6633 	if ((_base_wait_for_doorbell_ack(ioc, 5))) {
6634 		ioc_err(ioc, "doorbell handshake ack failed (line=%d)\n",
6635 			__LINE__);
6636 		return -EFAULT;
6637 	}
6638 
6639 	/* send message 32-bits at a time */
6640 	for (i = 0, failed = 0; i < request_bytes/4 && !failed; i++) {
6641 		writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);
6642 		if ((_base_wait_for_doorbell_ack(ioc, 5)))
6643 			failed = 1;
6644 	}
6645 
6646 	if (failed) {
6647 		ioc_err(ioc, "doorbell handshake sending request failed (line=%d)\n",
6648 			__LINE__);
6649 		return -EFAULT;
6650 	}
6651 
6652 	/* now wait for the reply */
6653 	if ((_base_wait_for_doorbell_int(ioc, timeout))) {
6654 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6655 			__LINE__);
6656 		return -EFAULT;
6657 	}
6658 
6659 	/* read the first two 16-bits, it gives the total length of the reply */
6660 	reply[0] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell)
6661 	    & MPI2_DOORBELL_DATA_MASK);
6662 	writel(0, &ioc->chip->HostInterruptStatus);
6663 	if ((_base_wait_for_doorbell_int(ioc, 5))) {
6664 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6665 			__LINE__);
6666 		return -EFAULT;
6667 	}
6668 	reply[1] = le16_to_cpu(ioc->base_readl(&ioc->chip->Doorbell)
6669 	    & MPI2_DOORBELL_DATA_MASK);
6670 	writel(0, &ioc->chip->HostInterruptStatus);
6671 
6672 	for (i = 2; i < default_reply->MsgLength * 2; i++)  {
6673 		if ((_base_wait_for_doorbell_int(ioc, 5))) {
6674 			ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",
6675 				__LINE__);
6676 			return -EFAULT;
6677 		}
6678 		if (i >=  reply_bytes/2) /* overflow case */
6679 			ioc->base_readl(&ioc->chip->Doorbell);
6680 		else
6681 			reply[i] = le16_to_cpu(
6682 			    ioc->base_readl(&ioc->chip->Doorbell)
6683 			    & MPI2_DOORBELL_DATA_MASK);
6684 		writel(0, &ioc->chip->HostInterruptStatus);
6685 	}
6686 
6687 	_base_wait_for_doorbell_int(ioc, 5);
6688 	if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {
6689 		dhsprintk(ioc,
6690 			  ioc_info(ioc, "doorbell is in use (line=%d)\n",
6691 				   __LINE__));
6692 	}
6693 	writel(0, &ioc->chip->HostInterruptStatus);
6694 
6695 	if (ioc->logging_level & MPT_DEBUG_INIT) {
6696 		mfp = (__le32 *)reply;
6697 		pr_info("\toffset:data\n");
6698 		for (i = 0; i < reply_bytes/4; i++)
6699 			ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,
6700 			    le32_to_cpu(mfp[i]));
6701 	}
6702 	return 0;
6703 }
6704 
6705 /**
6706  * mpt3sas_base_sas_iounit_control - send sas iounit control to FW
6707  * @ioc: per adapter object
6708  * @mpi_reply: the reply payload from FW
6709  * @mpi_request: the request payload sent to FW
6710  *
6711  * The SAS IO Unit Control Request message allows the host to perform low-level
6712  * operations, such as resets on the PHYs of the IO Unit, also allows the host
6713  * to obtain the IOC assigned device handles for a device if it has other
6714  * identifying information about the device, in addition allows the host to
6715  * remove IOC resources associated with the device.
6716  *
6717  * Return: 0 for success, non-zero for failure.
6718  */
6719 int
6720 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,
6721 	Mpi2SasIoUnitControlReply_t *mpi_reply,
6722 	Mpi2SasIoUnitControlRequest_t *mpi_request)
6723 {
6724 	u16 smid;
6725 	u8 issue_reset = 0;
6726 	int rc;
6727 	void *request;
6728 
6729 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6730 
6731 	mutex_lock(&ioc->base_cmds.mutex);
6732 
6733 	if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
6734 		ioc_err(ioc, "%s: base_cmd in use\n", __func__);
6735 		rc = -EAGAIN;
6736 		goto out;
6737 	}
6738 
6739 	rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);
6740 	if (rc)
6741 		goto out;
6742 
6743 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
6744 	if (!smid) {
6745 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6746 		rc = -EAGAIN;
6747 		goto out;
6748 	}
6749 
6750 	rc = 0;
6751 	ioc->base_cmds.status = MPT3_CMD_PENDING;
6752 	request = mpt3sas_base_get_msg_frame(ioc, smid);
6753 	ioc->base_cmds.smid = smid;
6754 	memcpy(request, mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t));
6755 	if (mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
6756 	    mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET)
6757 		ioc->ioc_link_reset_in_progress = 1;
6758 	init_completion(&ioc->base_cmds.done);
6759 	ioc->put_smid_default(ioc, smid);
6760 	wait_for_completion_timeout(&ioc->base_cmds.done,
6761 	    msecs_to_jiffies(10000));
6762 	if ((mpi_request->Operation == MPI2_SAS_OP_PHY_HARD_RESET ||
6763 	    mpi_request->Operation == MPI2_SAS_OP_PHY_LINK_RESET) &&
6764 	    ioc->ioc_link_reset_in_progress)
6765 		ioc->ioc_link_reset_in_progress = 0;
6766 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
6767 		mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status,
6768 		    mpi_request, sizeof(Mpi2SasIoUnitControlRequest_t)/4,
6769 		    issue_reset);
6770 		goto issue_host_reset;
6771 	}
6772 	if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
6773 		memcpy(mpi_reply, ioc->base_cmds.reply,
6774 		    sizeof(Mpi2SasIoUnitControlReply_t));
6775 	else
6776 		memset(mpi_reply, 0, sizeof(Mpi2SasIoUnitControlReply_t));
6777 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6778 	goto out;
6779 
6780  issue_host_reset:
6781 	if (issue_reset)
6782 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
6783 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6784 	rc = -EFAULT;
6785  out:
6786 	mutex_unlock(&ioc->base_cmds.mutex);
6787 	return rc;
6788 }
6789 
6790 /**
6791  * mpt3sas_base_scsi_enclosure_processor - sending request to sep device
6792  * @ioc: per adapter object
6793  * @mpi_reply: the reply payload from FW
6794  * @mpi_request: the request payload sent to FW
6795  *
6796  * The SCSI Enclosure Processor request message causes the IOC to
6797  * communicate with SES devices to control LED status signals.
6798  *
6799  * Return: 0 for success, non-zero for failure.
6800  */
6801 int
6802 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,
6803 	Mpi2SepReply_t *mpi_reply, Mpi2SepRequest_t *mpi_request)
6804 {
6805 	u16 smid;
6806 	u8 issue_reset = 0;
6807 	int rc;
6808 	void *request;
6809 
6810 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6811 
6812 	mutex_lock(&ioc->base_cmds.mutex);
6813 
6814 	if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {
6815 		ioc_err(ioc, "%s: base_cmd in use\n", __func__);
6816 		rc = -EAGAIN;
6817 		goto out;
6818 	}
6819 
6820 	rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);
6821 	if (rc)
6822 		goto out;
6823 
6824 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
6825 	if (!smid) {
6826 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
6827 		rc = -EAGAIN;
6828 		goto out;
6829 	}
6830 
6831 	rc = 0;
6832 	ioc->base_cmds.status = MPT3_CMD_PENDING;
6833 	request = mpt3sas_base_get_msg_frame(ioc, smid);
6834 	ioc->base_cmds.smid = smid;
6835 	memset(request, 0, ioc->request_sz);
6836 	memcpy(request, mpi_request, sizeof(Mpi2SepReply_t));
6837 	init_completion(&ioc->base_cmds.done);
6838 	ioc->put_smid_default(ioc, smid);
6839 	wait_for_completion_timeout(&ioc->base_cmds.done,
6840 	    msecs_to_jiffies(10000));
6841 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
6842 		mpt3sas_check_cmd_timeout(ioc,
6843 		    ioc->base_cmds.status, mpi_request,
6844 		    sizeof(Mpi2SepRequest_t)/4, issue_reset);
6845 		goto issue_host_reset;
6846 	}
6847 	if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)
6848 		memcpy(mpi_reply, ioc->base_cmds.reply,
6849 		    sizeof(Mpi2SepReply_t));
6850 	else
6851 		memset(mpi_reply, 0, sizeof(Mpi2SepReply_t));
6852 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6853 	goto out;
6854 
6855  issue_host_reset:
6856 	if (issue_reset)
6857 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);
6858 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
6859 	rc = -EFAULT;
6860  out:
6861 	mutex_unlock(&ioc->base_cmds.mutex);
6862 	return rc;
6863 }
6864 
6865 /**
6866  * _base_get_port_facts - obtain port facts reply and save in ioc
6867  * @ioc: per adapter object
6868  * @port: ?
6869  *
6870  * Return: 0 for success, non-zero for failure.
6871  */
6872 static int
6873 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)
6874 {
6875 	Mpi2PortFactsRequest_t mpi_request;
6876 	Mpi2PortFactsReply_t mpi_reply;
6877 	struct mpt3sas_port_facts *pfacts;
6878 	int mpi_reply_sz, mpi_request_sz, r;
6879 
6880 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6881 
6882 	mpi_reply_sz = sizeof(Mpi2PortFactsReply_t);
6883 	mpi_request_sz = sizeof(Mpi2PortFactsRequest_t);
6884 	memset(&mpi_request, 0, mpi_request_sz);
6885 	mpi_request.Function = MPI2_FUNCTION_PORT_FACTS;
6886 	mpi_request.PortNumber = port;
6887 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
6888 	    (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
6889 
6890 	if (r != 0) {
6891 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);
6892 		return r;
6893 	}
6894 
6895 	pfacts = &ioc->pfacts[port];
6896 	memset(pfacts, 0, sizeof(struct mpt3sas_port_facts));
6897 	pfacts->PortNumber = mpi_reply.PortNumber;
6898 	pfacts->VP_ID = mpi_reply.VP_ID;
6899 	pfacts->VF_ID = mpi_reply.VF_ID;
6900 	pfacts->MaxPostedCmdBuffers =
6901 	    le16_to_cpu(mpi_reply.MaxPostedCmdBuffers);
6902 
6903 	return 0;
6904 }
6905 
6906 /**
6907  * _base_wait_for_iocstate - Wait until the card is in READY or OPERATIONAL
6908  * @ioc: per adapter object
6909  * @timeout:
6910  *
6911  * Return: 0 for success, non-zero for failure.
6912  */
6913 static int
6914 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)
6915 {
6916 	u32 ioc_state;
6917 	int rc;
6918 
6919 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6920 
6921 	if (ioc->pci_error_recovery) {
6922 		dfailprintk(ioc,
6923 			    ioc_info(ioc, "%s: host in pci error recovery\n",
6924 				     __func__));
6925 		return -EFAULT;
6926 	}
6927 
6928 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
6929 	dhsprintk(ioc,
6930 		  ioc_info(ioc, "%s: ioc_state(0x%08x)\n",
6931 			   __func__, ioc_state));
6932 
6933 	if (((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY) ||
6934 	    (ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
6935 		return 0;
6936 
6937 	if (ioc_state & MPI2_DOORBELL_USED) {
6938 		dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));
6939 		goto issue_diag_reset;
6940 	}
6941 
6942 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
6943 		mpt3sas_print_fault_code(ioc, ioc_state &
6944 		    MPI2_DOORBELL_DATA_MASK);
6945 		goto issue_diag_reset;
6946 	} else if ((ioc_state & MPI2_IOC_STATE_MASK) ==
6947 	    MPI2_IOC_STATE_COREDUMP) {
6948 		ioc_info(ioc,
6949 		    "%s: Skipping the diag reset here. (ioc_state=0x%x)\n",
6950 		    __func__, ioc_state);
6951 		return -EFAULT;
6952 	}
6953 
6954 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);
6955 	if (ioc_state) {
6956 		dfailprintk(ioc,
6957 			    ioc_info(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
6958 				     __func__, ioc_state));
6959 		return -EFAULT;
6960 	}
6961 
6962  issue_diag_reset:
6963 	rc = _base_diag_reset(ioc);
6964 	return rc;
6965 }
6966 
6967 /**
6968  * _base_get_ioc_facts - obtain ioc facts reply and save in ioc
6969  * @ioc: per adapter object
6970  *
6971  * Return: 0 for success, non-zero for failure.
6972  */
6973 static int
6974 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)
6975 {
6976 	Mpi2IOCFactsRequest_t mpi_request;
6977 	Mpi2IOCFactsReply_t mpi_reply;
6978 	struct mpt3sas_facts *facts;
6979 	int mpi_reply_sz, mpi_request_sz, r;
6980 
6981 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
6982 
6983 	r = _base_wait_for_iocstate(ioc, 10);
6984 	if (r) {
6985 		dfailprintk(ioc,
6986 			    ioc_info(ioc, "%s: failed getting to correct state\n",
6987 				     __func__));
6988 		return r;
6989 	}
6990 	mpi_reply_sz = sizeof(Mpi2IOCFactsReply_t);
6991 	mpi_request_sz = sizeof(Mpi2IOCFactsRequest_t);
6992 	memset(&mpi_request, 0, mpi_request_sz);
6993 	mpi_request.Function = MPI2_FUNCTION_IOC_FACTS;
6994 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,
6995 	    (u32 *)&mpi_request, mpi_reply_sz, (u16 *)&mpi_reply, 5);
6996 
6997 	if (r != 0) {
6998 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);
6999 		return r;
7000 	}
7001 
7002 	facts = &ioc->facts;
7003 	memset(facts, 0, sizeof(struct mpt3sas_facts));
7004 	facts->MsgVersion = le16_to_cpu(mpi_reply.MsgVersion);
7005 	facts->HeaderVersion = le16_to_cpu(mpi_reply.HeaderVersion);
7006 	facts->VP_ID = mpi_reply.VP_ID;
7007 	facts->VF_ID = mpi_reply.VF_ID;
7008 	facts->IOCExceptions = le16_to_cpu(mpi_reply.IOCExceptions);
7009 	facts->MaxChainDepth = mpi_reply.MaxChainDepth;
7010 	facts->WhoInit = mpi_reply.WhoInit;
7011 	facts->NumberOfPorts = mpi_reply.NumberOfPorts;
7012 	facts->MaxMSIxVectors = mpi_reply.MaxMSIxVectors;
7013 	if (ioc->msix_enable && (facts->MaxMSIxVectors <=
7014 	    MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc)))
7015 		ioc->combined_reply_queue = 0;
7016 	facts->RequestCredit = le16_to_cpu(mpi_reply.RequestCredit);
7017 	facts->MaxReplyDescriptorPostQueueDepth =
7018 	    le16_to_cpu(mpi_reply.MaxReplyDescriptorPostQueueDepth);
7019 	facts->ProductID = le16_to_cpu(mpi_reply.ProductID);
7020 	facts->IOCCapabilities = le32_to_cpu(mpi_reply.IOCCapabilities);
7021 	if ((facts->IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_INTEGRATED_RAID))
7022 		ioc->ir_firmware = 1;
7023 	if ((facts->IOCCapabilities &
7024 	      MPI2_IOCFACTS_CAPABILITY_RDPQ_ARRAY_CAPABLE) && (!reset_devices))
7025 		ioc->rdpq_array_capable = 1;
7026 	if ((facts->IOCCapabilities & MPI26_IOCFACTS_CAPABILITY_ATOMIC_REQ)
7027 	    && ioc->is_aero_ioc)
7028 		ioc->atomic_desc_capable = 1;
7029 	facts->FWVersion.Word = le32_to_cpu(mpi_reply.FWVersion.Word);
7030 	facts->IOCRequestFrameSize =
7031 	    le16_to_cpu(mpi_reply.IOCRequestFrameSize);
7032 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
7033 		facts->IOCMaxChainSegmentSize =
7034 			le16_to_cpu(mpi_reply.IOCMaxChainSegmentSize);
7035 	}
7036 	facts->MaxInitiators = le16_to_cpu(mpi_reply.MaxInitiators);
7037 	facts->MaxTargets = le16_to_cpu(mpi_reply.MaxTargets);
7038 	ioc->shost->max_id = -1;
7039 	facts->MaxSasExpanders = le16_to_cpu(mpi_reply.MaxSasExpanders);
7040 	facts->MaxEnclosures = le16_to_cpu(mpi_reply.MaxEnclosures);
7041 	facts->ProtocolFlags = le16_to_cpu(mpi_reply.ProtocolFlags);
7042 	facts->HighPriorityCredit =
7043 	    le16_to_cpu(mpi_reply.HighPriorityCredit);
7044 	facts->ReplyFrameSize = mpi_reply.ReplyFrameSize;
7045 	facts->MaxDevHandle = le16_to_cpu(mpi_reply.MaxDevHandle);
7046 	facts->CurrentHostPageSize = mpi_reply.CurrentHostPageSize;
7047 
7048 	/*
7049 	 * Get the Page Size from IOC Facts. If it's 0, default to 4k.
7050 	 */
7051 	ioc->page_size = 1 << facts->CurrentHostPageSize;
7052 	if (ioc->page_size == 1) {
7053 		ioc_info(ioc, "CurrentHostPageSize is 0: Setting default host page size to 4k\n");
7054 		ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;
7055 	}
7056 	dinitprintk(ioc,
7057 		    ioc_info(ioc, "CurrentHostPageSize(%d)\n",
7058 			     facts->CurrentHostPageSize));
7059 
7060 	dinitprintk(ioc,
7061 		    ioc_info(ioc, "hba queue depth(%d), max chains per io(%d)\n",
7062 			     facts->RequestCredit, facts->MaxChainDepth));
7063 	dinitprintk(ioc,
7064 		    ioc_info(ioc, "request frame size(%d), reply frame size(%d)\n",
7065 			     facts->IOCRequestFrameSize * 4,
7066 			     facts->ReplyFrameSize * 4));
7067 	return 0;
7068 }
7069 
7070 /**
7071  * _base_send_ioc_init - send ioc_init to firmware
7072  * @ioc: per adapter object
7073  *
7074  * Return: 0 for success, non-zero for failure.
7075  */
7076 static int
7077 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)
7078 {
7079 	Mpi2IOCInitRequest_t mpi_request;
7080 	Mpi2IOCInitReply_t mpi_reply;
7081 	int i, r = 0;
7082 	ktime_t current_time;
7083 	u16 ioc_status;
7084 	u32 reply_post_free_array_sz = 0;
7085 
7086 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7087 
7088 	memset(&mpi_request, 0, sizeof(Mpi2IOCInitRequest_t));
7089 	mpi_request.Function = MPI2_FUNCTION_IOC_INIT;
7090 	mpi_request.WhoInit = MPI2_WHOINIT_HOST_DRIVER;
7091 	mpi_request.VF_ID = 0; /* TODO */
7092 	mpi_request.VP_ID = 0;
7093 	mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);
7094 	mpi_request.HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
7095 	mpi_request.HostPageSize = MPT3SAS_HOST_PAGE_SIZE_4K;
7096 
7097 	if (_base_is_controller_msix_enabled(ioc))
7098 		mpi_request.HostMSIxVectors = ioc->reply_queue_count;
7099 	mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);
7100 	mpi_request.ReplyDescriptorPostQueueDepth =
7101 	    cpu_to_le16(ioc->reply_post_queue_depth);
7102 	mpi_request.ReplyFreeQueueDepth =
7103 	    cpu_to_le16(ioc->reply_free_queue_depth);
7104 
7105 	mpi_request.SenseBufferAddressHigh =
7106 	    cpu_to_le32((u64)ioc->sense_dma >> 32);
7107 	mpi_request.SystemReplyAddressHigh =
7108 	    cpu_to_le32((u64)ioc->reply_dma >> 32);
7109 	mpi_request.SystemRequestFrameBaseAddress =
7110 	    cpu_to_le64((u64)ioc->request_dma);
7111 	mpi_request.ReplyFreeQueueAddress =
7112 	    cpu_to_le64((u64)ioc->reply_free_dma);
7113 
7114 	if (ioc->rdpq_array_enable) {
7115 		reply_post_free_array_sz = ioc->reply_queue_count *
7116 		    sizeof(Mpi2IOCInitRDPQArrayEntry);
7117 		memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);
7118 		for (i = 0; i < ioc->reply_queue_count; i++)
7119 			ioc->reply_post_free_array[i].RDPQBaseAddress =
7120 			    cpu_to_le64(
7121 				(u64)ioc->reply_post[i].reply_post_free_dma);
7122 		mpi_request.MsgFlags = MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE;
7123 		mpi_request.ReplyDescriptorPostQueueAddress =
7124 		    cpu_to_le64((u64)ioc->reply_post_free_array_dma);
7125 	} else {
7126 		mpi_request.ReplyDescriptorPostQueueAddress =
7127 		    cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);
7128 	}
7129 
7130 	/*
7131 	 * Set the flag to enable CoreDump state feature in IOC firmware.
7132 	 */
7133 	mpi_request.ConfigurationFlags |=
7134 	    cpu_to_le16(MPI26_IOCINIT_CFGFLAGS_COREDUMP_ENABLE);
7135 
7136 	/* This time stamp specifies number of milliseconds
7137 	 * since epoch ~ midnight January 1, 1970.
7138 	 */
7139 	current_time = ktime_get_real();
7140 	mpi_request.TimeStamp = cpu_to_le64(ktime_to_ms(current_time));
7141 
7142 	if (ioc->logging_level & MPT_DEBUG_INIT) {
7143 		__le32 *mfp;
7144 		int i;
7145 
7146 		mfp = (__le32 *)&mpi_request;
7147 		ioc_info(ioc, "\toffset:data\n");
7148 		for (i = 0; i < sizeof(Mpi2IOCInitRequest_t)/4; i++)
7149 			ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,
7150 			    le32_to_cpu(mfp[i]));
7151 	}
7152 
7153 	r = _base_handshake_req_reply_wait(ioc,
7154 	    sizeof(Mpi2IOCInitRequest_t), (u32 *)&mpi_request,
7155 	    sizeof(Mpi2IOCInitReply_t), (u16 *)&mpi_reply, 30);
7156 
7157 	if (r != 0) {
7158 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);
7159 		return r;
7160 	}
7161 
7162 	ioc_status = le16_to_cpu(mpi_reply.IOCStatus) & MPI2_IOCSTATUS_MASK;
7163 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS ||
7164 	    mpi_reply.IOCLogInfo) {
7165 		ioc_err(ioc, "%s: failed\n", __func__);
7166 		r = -EIO;
7167 	}
7168 
7169 	/* Reset TimeSync Counter*/
7170 	ioc->timestamp_update_count = 0;
7171 	return r;
7172 }
7173 
7174 /**
7175  * mpt3sas_port_enable_done - command completion routine for port enable
7176  * @ioc: per adapter object
7177  * @smid: system request message index
7178  * @msix_index: MSIX table index supplied by the OS
7179  * @reply: reply message frame(lower 32bit addr)
7180  *
7181  * Return: 1 meaning mf should be freed from _base_interrupt
7182  *          0 means the mf is freed from this function.
7183  */
7184 u8
7185 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,
7186 	u32 reply)
7187 {
7188 	MPI2DefaultReply_t *mpi_reply;
7189 	u16 ioc_status;
7190 
7191 	if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)
7192 		return 1;
7193 
7194 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);
7195 	if (!mpi_reply)
7196 		return 1;
7197 
7198 	if (mpi_reply->Function != MPI2_FUNCTION_PORT_ENABLE)
7199 		return 1;
7200 
7201 	ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;
7202 	ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;
7203 	ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;
7204 	memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);
7205 	ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
7206 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS)
7207 		ioc->port_enable_failed = 1;
7208 
7209 	if (ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE_ASYNC) {
7210 		ioc->port_enable_cmds.status &= ~MPT3_CMD_COMPLETE_ASYNC;
7211 		if (ioc_status == MPI2_IOCSTATUS_SUCCESS) {
7212 			mpt3sas_port_enable_complete(ioc);
7213 			return 1;
7214 		} else {
7215 			ioc->start_scan_failed = ioc_status;
7216 			ioc->start_scan = 0;
7217 			return 1;
7218 		}
7219 	}
7220 	complete(&ioc->port_enable_cmds.done);
7221 	return 1;
7222 }
7223 
7224 /**
7225  * _base_send_port_enable - send port_enable(discovery stuff) to firmware
7226  * @ioc: per adapter object
7227  *
7228  * Return: 0 for success, non-zero for failure.
7229  */
7230 static int
7231 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)
7232 {
7233 	Mpi2PortEnableRequest_t *mpi_request;
7234 	Mpi2PortEnableReply_t *mpi_reply;
7235 	int r = 0;
7236 	u16 smid;
7237 	u16 ioc_status;
7238 
7239 	ioc_info(ioc, "sending port enable !!\n");
7240 
7241 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
7242 		ioc_err(ioc, "%s: internal command already in use\n", __func__);
7243 		return -EAGAIN;
7244 	}
7245 
7246 	smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
7247 	if (!smid) {
7248 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
7249 		return -EAGAIN;
7250 	}
7251 
7252 	ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
7253 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
7254 	ioc->port_enable_cmds.smid = smid;
7255 	memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
7256 	mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
7257 
7258 	init_completion(&ioc->port_enable_cmds.done);
7259 	ioc->put_smid_default(ioc, smid);
7260 	wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);
7261 	if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {
7262 		ioc_err(ioc, "%s: timeout\n", __func__);
7263 		_debug_dump_mf(mpi_request,
7264 		    sizeof(Mpi2PortEnableRequest_t)/4);
7265 		if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)
7266 			r = -EFAULT;
7267 		else
7268 			r = -ETIME;
7269 		goto out;
7270 	}
7271 
7272 	mpi_reply = ioc->port_enable_cmds.reply;
7273 	ioc_status = le16_to_cpu(mpi_reply->IOCStatus) & MPI2_IOCSTATUS_MASK;
7274 	if (ioc_status != MPI2_IOCSTATUS_SUCCESS) {
7275 		ioc_err(ioc, "%s: failed with (ioc_status=0x%08x)\n",
7276 			__func__, ioc_status);
7277 		r = -EFAULT;
7278 		goto out;
7279 	}
7280 
7281  out:
7282 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
7283 	ioc_info(ioc, "port enable: %s\n", r == 0 ? "SUCCESS" : "FAILED");
7284 	return r;
7285 }
7286 
7287 /**
7288  * mpt3sas_port_enable - initiate firmware discovery (don't wait for reply)
7289  * @ioc: per adapter object
7290  *
7291  * Return: 0 for success, non-zero for failure.
7292  */
7293 int
7294 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)
7295 {
7296 	Mpi2PortEnableRequest_t *mpi_request;
7297 	u16 smid;
7298 
7299 	ioc_info(ioc, "sending port enable !!\n");
7300 
7301 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
7302 		ioc_err(ioc, "%s: internal command already in use\n", __func__);
7303 		return -EAGAIN;
7304 	}
7305 
7306 	smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);
7307 	if (!smid) {
7308 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
7309 		return -EAGAIN;
7310 	}
7311 	ioc->drv_internal_flags |= MPT_DRV_INTERNAL_FIRST_PE_ISSUED;
7312 	ioc->port_enable_cmds.status = MPT3_CMD_PENDING;
7313 	ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE_ASYNC;
7314 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
7315 	ioc->port_enable_cmds.smid = smid;
7316 	memset(mpi_request, 0, sizeof(Mpi2PortEnableRequest_t));
7317 	mpi_request->Function = MPI2_FUNCTION_PORT_ENABLE;
7318 
7319 	ioc->put_smid_default(ioc, smid);
7320 	return 0;
7321 }
7322 
7323 /**
7324  * _base_determine_wait_on_discovery - desposition
7325  * @ioc: per adapter object
7326  *
7327  * Decide whether to wait on discovery to complete. Used to either
7328  * locate boot device, or report volumes ahead of physical devices.
7329  *
7330  * Return: 1 for wait, 0 for don't wait.
7331  */
7332 static int
7333 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)
7334 {
7335 	/* We wait for discovery to complete if IR firmware is loaded.
7336 	 * The sas topology events arrive before PD events, so we need time to
7337 	 * turn on the bit in ioc->pd_handles to indicate PD
7338 	 * Also, it maybe required to report Volumes ahead of physical
7339 	 * devices when MPI2_IOCPAGE8_IRFLAGS_LOW_VOLUME_MAPPING is set.
7340 	 */
7341 	if (ioc->ir_firmware)
7342 		return 1;
7343 
7344 	/* if no Bios, then we don't need to wait */
7345 	if (!ioc->bios_pg3.BiosVersion)
7346 		return 0;
7347 
7348 	/* Bios is present, then we drop down here.
7349 	 *
7350 	 * If there any entries in the Bios Page 2, then we wait
7351 	 * for discovery to complete.
7352 	 */
7353 
7354 	/* Current Boot Device */
7355 	if ((ioc->bios_pg2.CurrentBootDeviceForm &
7356 	    MPI2_BIOSPAGE2_FORM_MASK) ==
7357 	    MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
7358 	/* Request Boot Device */
7359 	   (ioc->bios_pg2.ReqBootDeviceForm &
7360 	    MPI2_BIOSPAGE2_FORM_MASK) ==
7361 	    MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED &&
7362 	/* Alternate Request Boot Device */
7363 	   (ioc->bios_pg2.ReqAltBootDeviceForm &
7364 	    MPI2_BIOSPAGE2_FORM_MASK) ==
7365 	    MPI2_BIOSPAGE2_FORM_NO_DEVICE_SPECIFIED)
7366 		return 0;
7367 
7368 	return 1;
7369 }
7370 
7371 /**
7372  * _base_unmask_events - turn on notification for this event
7373  * @ioc: per adapter object
7374  * @event: firmware event
7375  *
7376  * The mask is stored in ioc->event_masks.
7377  */
7378 static void
7379 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)
7380 {
7381 	u32 desired_event;
7382 
7383 	if (event >= 128)
7384 		return;
7385 
7386 	desired_event = (1 << (event % 32));
7387 
7388 	if (event < 32)
7389 		ioc->event_masks[0] &= ~desired_event;
7390 	else if (event < 64)
7391 		ioc->event_masks[1] &= ~desired_event;
7392 	else if (event < 96)
7393 		ioc->event_masks[2] &= ~desired_event;
7394 	else if (event < 128)
7395 		ioc->event_masks[3] &= ~desired_event;
7396 }
7397 
7398 /**
7399  * _base_event_notification - send event notification
7400  * @ioc: per adapter object
7401  *
7402  * Return: 0 for success, non-zero for failure.
7403  */
7404 static int
7405 _base_event_notification(struct MPT3SAS_ADAPTER *ioc)
7406 {
7407 	Mpi2EventNotificationRequest_t *mpi_request;
7408 	u16 smid;
7409 	int r = 0;
7410 	int i, issue_diag_reset = 0;
7411 
7412 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7413 
7414 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
7415 		ioc_err(ioc, "%s: internal command already in use\n", __func__);
7416 		return -EAGAIN;
7417 	}
7418 
7419 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);
7420 	if (!smid) {
7421 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);
7422 		return -EAGAIN;
7423 	}
7424 	ioc->base_cmds.status = MPT3_CMD_PENDING;
7425 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);
7426 	ioc->base_cmds.smid = smid;
7427 	memset(mpi_request, 0, sizeof(Mpi2EventNotificationRequest_t));
7428 	mpi_request->Function = MPI2_FUNCTION_EVENT_NOTIFICATION;
7429 	mpi_request->VF_ID = 0; /* TODO */
7430 	mpi_request->VP_ID = 0;
7431 	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
7432 		mpi_request->EventMasks[i] =
7433 		    cpu_to_le32(ioc->event_masks[i]);
7434 	init_completion(&ioc->base_cmds.done);
7435 	ioc->put_smid_default(ioc, smid);
7436 	wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);
7437 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {
7438 		ioc_err(ioc, "%s: timeout\n", __func__);
7439 		_debug_dump_mf(mpi_request,
7440 		    sizeof(Mpi2EventNotificationRequest_t)/4);
7441 		if (ioc->base_cmds.status & MPT3_CMD_RESET)
7442 			r = -EFAULT;
7443 		else
7444 			issue_diag_reset = 1;
7445 
7446 	} else
7447 		dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__));
7448 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
7449 
7450 	if (issue_diag_reset) {
7451 		if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED)
7452 			return -EFAULT;
7453 		if (mpt3sas_base_check_for_fault_and_issue_reset(ioc))
7454 			return -EFAULT;
7455 		r = -EAGAIN;
7456 	}
7457 	return r;
7458 }
7459 
7460 /**
7461  * mpt3sas_base_validate_event_type - validating event types
7462  * @ioc: per adapter object
7463  * @event_type: firmware event
7464  *
7465  * This will turn on firmware event notification when application
7466  * ask for that event. We don't mask events that are already enabled.
7467  */
7468 void
7469 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)
7470 {
7471 	int i, j;
7472 	u32 event_mask, desired_event;
7473 	u8 send_update_to_fw;
7474 
7475 	for (i = 0, send_update_to_fw = 0; i <
7476 	    MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++) {
7477 		event_mask = ~event_type[i];
7478 		desired_event = 1;
7479 		for (j = 0; j < 32; j++) {
7480 			if (!(event_mask & desired_event) &&
7481 			    (ioc->event_masks[i] & desired_event)) {
7482 				ioc->event_masks[i] &= ~desired_event;
7483 				send_update_to_fw = 1;
7484 			}
7485 			desired_event = (desired_event << 1);
7486 		}
7487 	}
7488 
7489 	if (!send_update_to_fw)
7490 		return;
7491 
7492 	mutex_lock(&ioc->base_cmds.mutex);
7493 	_base_event_notification(ioc);
7494 	mutex_unlock(&ioc->base_cmds.mutex);
7495 }
7496 
7497 /**
7498  * _base_diag_reset - the "big hammer" start of day reset
7499  * @ioc: per adapter object
7500  *
7501  * Return: 0 for success, non-zero for failure.
7502  */
7503 static int
7504 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc)
7505 {
7506 	u32 host_diagnostic;
7507 	u32 ioc_state;
7508 	u32 count;
7509 	u32 hcb_size;
7510 
7511 	ioc_info(ioc, "sending diag reset !!\n");
7512 
7513 	pci_cfg_access_lock(ioc->pdev);
7514 
7515 	drsprintk(ioc, ioc_info(ioc, "clear interrupts\n"));
7516 
7517 	count = 0;
7518 	do {
7519 		/* Write magic sequence to WriteSequence register
7520 		 * Loop until in diagnostic mode
7521 		 */
7522 		drsprintk(ioc, ioc_info(ioc, "write magic sequence\n"));
7523 		writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
7524 		writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);
7525 		writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);
7526 		writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);
7527 		writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);
7528 		writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);
7529 		writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);
7530 
7531 		/* wait 100 msec */
7532 		msleep(100);
7533 
7534 		if (count++ > 20) {
7535 			ioc_info(ioc,
7536 			    "Stop writing magic sequence after 20 retries\n");
7537 			_base_dump_reg_set(ioc);
7538 			goto out;
7539 		}
7540 
7541 		host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic);
7542 		drsprintk(ioc,
7543 			  ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",
7544 				   count, host_diagnostic));
7545 
7546 	} while ((host_diagnostic & MPI2_DIAG_DIAG_WRITE_ENABLE) == 0);
7547 
7548 	hcb_size = ioc->base_readl(&ioc->chip->HCBSize);
7549 
7550 	drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n"));
7551 	writel(host_diagnostic | MPI2_DIAG_RESET_ADAPTER,
7552 	     &ioc->chip->HostDiagnostic);
7553 
7554 	/*This delay allows the chip PCIe hardware time to finish reset tasks*/
7555 	msleep(MPI2_HARD_RESET_PCIE_FIRST_READ_DELAY_MICRO_SEC/1000);
7556 
7557 	/* Approximately 300 second max wait */
7558 	for (count = 0; count < (300000000 /
7559 		MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC); count++) {
7560 
7561 		host_diagnostic = ioc->base_readl(&ioc->chip->HostDiagnostic);
7562 
7563 		if (host_diagnostic == 0xFFFFFFFF) {
7564 			ioc_info(ioc,
7565 			    "Invalid host diagnostic register value\n");
7566 			_base_dump_reg_set(ioc);
7567 			goto out;
7568 		}
7569 		if (!(host_diagnostic & MPI2_DIAG_RESET_ADAPTER))
7570 			break;
7571 
7572 		msleep(MPI2_HARD_RESET_PCIE_SECOND_READ_DELAY_MICRO_SEC / 1000);
7573 	}
7574 
7575 	if (host_diagnostic & MPI2_DIAG_HCB_MODE) {
7576 
7577 		drsprintk(ioc,
7578 			  ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n"));
7579 		host_diagnostic &= ~MPI2_DIAG_BOOT_DEVICE_SELECT_MASK;
7580 		host_diagnostic |= MPI2_DIAG_BOOT_DEVICE_SELECT_HCDW;
7581 		writel(host_diagnostic, &ioc->chip->HostDiagnostic);
7582 
7583 		drsprintk(ioc, ioc_info(ioc, "re-enable the HCDW\n"));
7584 		writel(hcb_size | MPI2_HCB_SIZE_HCB_ENABLE,
7585 		    &ioc->chip->HCBSize);
7586 	}
7587 
7588 	drsprintk(ioc, ioc_info(ioc, "restart the adapter\n"));
7589 	writel(host_diagnostic & ~MPI2_DIAG_HOLD_IOC_RESET,
7590 	    &ioc->chip->HostDiagnostic);
7591 
7592 	drsprintk(ioc,
7593 		  ioc_info(ioc, "disable writes to the diagnostic register\n"));
7594 	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);
7595 
7596 	drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n"));
7597 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);
7598 	if (ioc_state) {
7599 		ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
7600 			__func__, ioc_state);
7601 		_base_dump_reg_set(ioc);
7602 		goto out;
7603 	}
7604 
7605 	pci_cfg_access_unlock(ioc->pdev);
7606 	ioc_info(ioc, "diag reset: SUCCESS\n");
7607 	return 0;
7608 
7609  out:
7610 	pci_cfg_access_unlock(ioc->pdev);
7611 	ioc_err(ioc, "diag reset: FAILED\n");
7612 	return -EFAULT;
7613 }
7614 
7615 /**
7616  * mpt3sas_base_make_ioc_ready - put controller in READY state
7617  * @ioc: per adapter object
7618  * @type: FORCE_BIG_HAMMER or SOFT_RESET
7619  *
7620  * Return: 0 for success, non-zero for failure.
7621  */
7622 int
7623 mpt3sas_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)
7624 {
7625 	u32 ioc_state;
7626 	int rc;
7627 	int count;
7628 
7629 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7630 
7631 	if (ioc->pci_error_recovery)
7632 		return 0;
7633 
7634 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
7635 	dhsprintk(ioc,
7636 		  ioc_info(ioc, "%s: ioc_state(0x%08x)\n",
7637 			   __func__, ioc_state));
7638 
7639 	/* if in RESET state, it should move to READY state shortly */
7640 	count = 0;
7641 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_RESET) {
7642 		while ((ioc_state & MPI2_IOC_STATE_MASK) !=
7643 		    MPI2_IOC_STATE_READY) {
7644 			if (count++ == 10) {
7645 				ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",
7646 					__func__, ioc_state);
7647 				return -EFAULT;
7648 			}
7649 			ssleep(1);
7650 			ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
7651 		}
7652 	}
7653 
7654 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_READY)
7655 		return 0;
7656 
7657 	if (ioc_state & MPI2_DOORBELL_USED) {
7658 		ioc_info(ioc, "unexpected doorbell active!\n");
7659 		goto issue_diag_reset;
7660 	}
7661 
7662 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT) {
7663 		mpt3sas_print_fault_code(ioc, ioc_state &
7664 		    MPI2_DOORBELL_DATA_MASK);
7665 		goto issue_diag_reset;
7666 	}
7667 
7668 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_COREDUMP) {
7669 		/*
7670 		 * if host reset is invoked while watch dog thread is waiting
7671 		 * for IOC state to be changed to Fault state then driver has
7672 		 * to wait here for CoreDump state to clear otherwise reset
7673 		 * will be issued to the FW and FW move the IOC state to
7674 		 * reset state without copying the FW logs to coredump region.
7675 		 */
7676 		if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) {
7677 			mpt3sas_print_coredump_info(ioc, ioc_state &
7678 			    MPI2_DOORBELL_DATA_MASK);
7679 			mpt3sas_base_wait_for_coredump_completion(ioc,
7680 			    __func__);
7681 		}
7682 		goto issue_diag_reset;
7683 	}
7684 
7685 	if (type == FORCE_BIG_HAMMER)
7686 		goto issue_diag_reset;
7687 
7688 	if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_OPERATIONAL)
7689 		if (!(_base_send_ioc_reset(ioc,
7690 		    MPI2_FUNCTION_IOC_MESSAGE_UNIT_RESET, 15))) {
7691 			return 0;
7692 	}
7693 
7694  issue_diag_reset:
7695 	rc = _base_diag_reset(ioc);
7696 	return rc;
7697 }
7698 
7699 /**
7700  * _base_make_ioc_operational - put controller in OPERATIONAL state
7701  * @ioc: per adapter object
7702  *
7703  * Return: 0 for success, non-zero for failure.
7704  */
7705 static int
7706 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)
7707 {
7708 	int r, i, index, rc;
7709 	unsigned long	flags;
7710 	u32 reply_address;
7711 	u16 smid;
7712 	struct _tr_list *delayed_tr, *delayed_tr_next;
7713 	struct _sc_list *delayed_sc, *delayed_sc_next;
7714 	struct _event_ack_list *delayed_event_ack, *delayed_event_ack_next;
7715 	u8 hide_flag;
7716 	struct adapter_reply_queue *reply_q;
7717 	Mpi2ReplyDescriptorsUnion_t *reply_post_free_contig;
7718 
7719 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7720 
7721 	/* clean the delayed target reset list */
7722 	list_for_each_entry_safe(delayed_tr, delayed_tr_next,
7723 	    &ioc->delayed_tr_list, list) {
7724 		list_del(&delayed_tr->list);
7725 		kfree(delayed_tr);
7726 	}
7727 
7728 
7729 	list_for_each_entry_safe(delayed_tr, delayed_tr_next,
7730 	    &ioc->delayed_tr_volume_list, list) {
7731 		list_del(&delayed_tr->list);
7732 		kfree(delayed_tr);
7733 	}
7734 
7735 	list_for_each_entry_safe(delayed_sc, delayed_sc_next,
7736 	    &ioc->delayed_sc_list, list) {
7737 		list_del(&delayed_sc->list);
7738 		kfree(delayed_sc);
7739 	}
7740 
7741 	list_for_each_entry_safe(delayed_event_ack, delayed_event_ack_next,
7742 	    &ioc->delayed_event_ack_list, list) {
7743 		list_del(&delayed_event_ack->list);
7744 		kfree(delayed_event_ack);
7745 	}
7746 
7747 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);
7748 
7749 	/* hi-priority queue */
7750 	INIT_LIST_HEAD(&ioc->hpr_free_list);
7751 	smid = ioc->hi_priority_smid;
7752 	for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {
7753 		ioc->hpr_lookup[i].cb_idx = 0xFF;
7754 		ioc->hpr_lookup[i].smid = smid;
7755 		list_add_tail(&ioc->hpr_lookup[i].tracker_list,
7756 		    &ioc->hpr_free_list);
7757 	}
7758 
7759 	/* internal queue */
7760 	INIT_LIST_HEAD(&ioc->internal_free_list);
7761 	smid = ioc->internal_smid;
7762 	for (i = 0; i < ioc->internal_depth; i++, smid++) {
7763 		ioc->internal_lookup[i].cb_idx = 0xFF;
7764 		ioc->internal_lookup[i].smid = smid;
7765 		list_add_tail(&ioc->internal_lookup[i].tracker_list,
7766 		    &ioc->internal_free_list);
7767 	}
7768 
7769 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);
7770 
7771 	/* initialize Reply Free Queue */
7772 	for (i = 0, reply_address = (u32)ioc->reply_dma ;
7773 	    i < ioc->reply_free_queue_depth ; i++, reply_address +=
7774 	    ioc->reply_sz) {
7775 		ioc->reply_free[i] = cpu_to_le32(reply_address);
7776 		if (ioc->is_mcpu_endpoint)
7777 			_base_clone_reply_to_sys_mem(ioc,
7778 					reply_address, i);
7779 	}
7780 
7781 	/* initialize reply queues */
7782 	if (ioc->is_driver_loading)
7783 		_base_assign_reply_queues(ioc);
7784 
7785 	/* initialize Reply Post Free Queue */
7786 	index = 0;
7787 	reply_post_free_contig = ioc->reply_post[0].reply_post_free;
7788 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
7789 		/*
7790 		 * If RDPQ is enabled, switch to the next allocation.
7791 		 * Otherwise advance within the contiguous region.
7792 		 */
7793 		if (ioc->rdpq_array_enable) {
7794 			reply_q->reply_post_free =
7795 				ioc->reply_post[index++].reply_post_free;
7796 		} else {
7797 			reply_q->reply_post_free = reply_post_free_contig;
7798 			reply_post_free_contig += ioc->reply_post_queue_depth;
7799 		}
7800 
7801 		reply_q->reply_post_host_index = 0;
7802 		for (i = 0; i < ioc->reply_post_queue_depth; i++)
7803 			reply_q->reply_post_free[i].Words =
7804 			    cpu_to_le64(ULLONG_MAX);
7805 		if (!_base_is_controller_msix_enabled(ioc))
7806 			goto skip_init_reply_post_free_queue;
7807 	}
7808  skip_init_reply_post_free_queue:
7809 
7810 	r = _base_send_ioc_init(ioc);
7811 	if (r) {
7812 		/*
7813 		 * No need to check IOC state for fault state & issue
7814 		 * diag reset during host reset. This check is need
7815 		 * only during driver load time.
7816 		 */
7817 		if (!ioc->is_driver_loading)
7818 			return r;
7819 
7820 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);
7821 		if (rc || (_base_send_ioc_init(ioc)))
7822 			return r;
7823 	}
7824 
7825 	/* initialize reply free host index */
7826 	ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;
7827 	writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);
7828 
7829 	/* initialize reply post host index */
7830 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {
7831 		if (ioc->combined_reply_queue)
7832 			writel((reply_q->msix_index & 7)<<
7833 			   MPI2_RPHI_MSIX_INDEX_SHIFT,
7834 			   ioc->replyPostRegisterIndex[reply_q->msix_index/8]);
7835 		else
7836 			writel(reply_q->msix_index <<
7837 				MPI2_RPHI_MSIX_INDEX_SHIFT,
7838 				&ioc->chip->ReplyPostHostIndex);
7839 
7840 		if (!_base_is_controller_msix_enabled(ioc))
7841 			goto skip_init_reply_post_host_index;
7842 	}
7843 
7844  skip_init_reply_post_host_index:
7845 
7846 	mpt3sas_base_unmask_interrupts(ioc);
7847 
7848 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {
7849 		r = _base_display_fwpkg_version(ioc);
7850 		if (r)
7851 			return r;
7852 	}
7853 
7854 	rc = _base_static_config_pages(ioc);
7855 	if (r)
7856 		return r;
7857 
7858 	r = _base_event_notification(ioc);
7859 	if (r)
7860 		return r;
7861 
7862 	if (!ioc->shost_recovery) {
7863 
7864 		if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier
7865 		    == 0x80) {
7866 			hide_flag = (u8) (
7867 			    le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &
7868 			    MFG_PAGE10_HIDE_SSDS_MASK);
7869 			if (hide_flag != MFG_PAGE10_HIDE_SSDS_MASK)
7870 				ioc->mfg_pg10_hide_flag = hide_flag;
7871 		}
7872 
7873 		ioc->wait_for_discovery_to_complete =
7874 		    _base_determine_wait_on_discovery(ioc);
7875 
7876 		return r; /* scan_start and scan_finished support */
7877 	}
7878 
7879 	r = _base_send_port_enable(ioc);
7880 	if (r)
7881 		return r;
7882 
7883 	return r;
7884 }
7885 
7886 /**
7887  * mpt3sas_base_free_resources - free resources controller resources
7888  * @ioc: per adapter object
7889  */
7890 void
7891 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)
7892 {
7893 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7894 
7895 	/* synchronizing freeing resource with pci_access_mutex lock */
7896 	mutex_lock(&ioc->pci_access_mutex);
7897 	if (ioc->chip_phys && ioc->chip) {
7898 		mpt3sas_base_mask_interrupts(ioc);
7899 		ioc->shost_recovery = 1;
7900 		mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);
7901 		ioc->shost_recovery = 0;
7902 	}
7903 
7904 	mpt3sas_base_unmap_resources(ioc);
7905 	mutex_unlock(&ioc->pci_access_mutex);
7906 	return;
7907 }
7908 
7909 /**
7910  * mpt3sas_base_attach - attach controller instance
7911  * @ioc: per adapter object
7912  *
7913  * Return: 0 for success, non-zero for failure.
7914  */
7915 int
7916 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)
7917 {
7918 	int r, i, rc;
7919 	int cpu_id, last_cpu_id = 0;
7920 
7921 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
7922 
7923 	/* setup cpu_msix_table */
7924 	ioc->cpu_count = num_online_cpus();
7925 	for_each_online_cpu(cpu_id)
7926 		last_cpu_id = cpu_id;
7927 	ioc->cpu_msix_table_sz = last_cpu_id + 1;
7928 	ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);
7929 	ioc->reply_queue_count = 1;
7930 	if (!ioc->cpu_msix_table) {
7931 		ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n");
7932 		r = -ENOMEM;
7933 		goto out_free_resources;
7934 	}
7935 
7936 	if (ioc->is_warpdrive) {
7937 		ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,
7938 		    sizeof(resource_size_t *), GFP_KERNEL);
7939 		if (!ioc->reply_post_host_index) {
7940 			ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n");
7941 			r = -ENOMEM;
7942 			goto out_free_resources;
7943 		}
7944 	}
7945 
7946 	ioc->smp_affinity_enable = smp_affinity_enable;
7947 
7948 	ioc->rdpq_array_enable_assigned = 0;
7949 	ioc->use_32bit_dma = false;
7950 	ioc->dma_mask = 64;
7951 	if (ioc->is_aero_ioc)
7952 		ioc->base_readl = &_base_readl_aero;
7953 	else
7954 		ioc->base_readl = &_base_readl;
7955 	r = mpt3sas_base_map_resources(ioc);
7956 	if (r)
7957 		goto out_free_resources;
7958 
7959 	pci_set_drvdata(ioc->pdev, ioc->shost);
7960 	r = _base_get_ioc_facts(ioc);
7961 	if (r) {
7962 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);
7963 		if (rc || (_base_get_ioc_facts(ioc)))
7964 			goto out_free_resources;
7965 	}
7966 
7967 	switch (ioc->hba_mpi_version_belonged) {
7968 	case MPI2_VERSION:
7969 		ioc->build_sg_scmd = &_base_build_sg_scmd;
7970 		ioc->build_sg = &_base_build_sg;
7971 		ioc->build_zero_len_sge = &_base_build_zero_len_sge;
7972 		ioc->get_msix_index_for_smlio = &_base_get_msix_index;
7973 		break;
7974 	case MPI25_VERSION:
7975 	case MPI26_VERSION:
7976 		/*
7977 		 * In SAS3.0,
7978 		 * SCSI_IO, SMP_PASSTHRU, SATA_PASSTHRU, Target Assist, and
7979 		 * Target Status - all require the IEEE formatted scatter gather
7980 		 * elements.
7981 		 */
7982 		ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;
7983 		ioc->build_sg = &_base_build_sg_ieee;
7984 		ioc->build_nvme_prp = &_base_build_nvme_prp;
7985 		ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;
7986 		ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);
7987 		if (ioc->high_iops_queues)
7988 			ioc->get_msix_index_for_smlio =
7989 					&_base_get_high_iops_msix_index;
7990 		else
7991 			ioc->get_msix_index_for_smlio = &_base_get_msix_index;
7992 		break;
7993 	}
7994 	if (ioc->atomic_desc_capable) {
7995 		ioc->put_smid_default = &_base_put_smid_default_atomic;
7996 		ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic;
7997 		ioc->put_smid_fast_path =
7998 				&_base_put_smid_fast_path_atomic;
7999 		ioc->put_smid_hi_priority =
8000 				&_base_put_smid_hi_priority_atomic;
8001 	} else {
8002 		ioc->put_smid_default = &_base_put_smid_default;
8003 		ioc->put_smid_fast_path = &_base_put_smid_fast_path;
8004 		ioc->put_smid_hi_priority = &_base_put_smid_hi_priority;
8005 		if (ioc->is_mcpu_endpoint)
8006 			ioc->put_smid_scsi_io =
8007 				&_base_put_smid_mpi_ep_scsi_io;
8008 		else
8009 			ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;
8010 	}
8011 	/*
8012 	 * These function pointers for other requests that don't
8013 	 * the require IEEE scatter gather elements.
8014 	 *
8015 	 * For example Configuration Pages and SAS IOUNIT Control don't.
8016 	 */
8017 	ioc->build_sg_mpi = &_base_build_sg;
8018 	ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;
8019 
8020 	r = mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);
8021 	if (r)
8022 		goto out_free_resources;
8023 
8024 	ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,
8025 	    sizeof(struct mpt3sas_port_facts), GFP_KERNEL);
8026 	if (!ioc->pfacts) {
8027 		r = -ENOMEM;
8028 		goto out_free_resources;
8029 	}
8030 
8031 	for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {
8032 		r = _base_get_port_facts(ioc, i);
8033 		if (r) {
8034 			rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);
8035 			if (rc || (_base_get_port_facts(ioc, i)))
8036 				goto out_free_resources;
8037 		}
8038 	}
8039 
8040 	r = _base_allocate_memory_pools(ioc);
8041 	if (r)
8042 		goto out_free_resources;
8043 
8044 	if (irqpoll_weight > 0)
8045 		ioc->thresh_hold = irqpoll_weight;
8046 	else
8047 		ioc->thresh_hold = ioc->hba_queue_depth/4;
8048 
8049 	_base_init_irqpolls(ioc);
8050 	init_waitqueue_head(&ioc->reset_wq);
8051 
8052 	/* allocate memory pd handle bitmask list */
8053 	ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
8054 	if (ioc->facts.MaxDevHandle % 8)
8055 		ioc->pd_handles_sz++;
8056 	ioc->pd_handles = kzalloc(ioc->pd_handles_sz,
8057 	    GFP_KERNEL);
8058 	if (!ioc->pd_handles) {
8059 		r = -ENOMEM;
8060 		goto out_free_resources;
8061 	}
8062 	ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,
8063 	    GFP_KERNEL);
8064 	if (!ioc->blocking_handles) {
8065 		r = -ENOMEM;
8066 		goto out_free_resources;
8067 	}
8068 
8069 	/* allocate memory for pending OS device add list */
8070 	ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);
8071 	if (ioc->facts.MaxDevHandle % 8)
8072 		ioc->pend_os_device_add_sz++;
8073 	ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,
8074 	    GFP_KERNEL);
8075 	if (!ioc->pend_os_device_add) {
8076 		r = -ENOMEM;
8077 		goto out_free_resources;
8078 	}
8079 
8080 	ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;
8081 	ioc->device_remove_in_progress =
8082 		kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);
8083 	if (!ioc->device_remove_in_progress) {
8084 		r = -ENOMEM;
8085 		goto out_free_resources;
8086 	}
8087 
8088 	ioc->fwfault_debug = mpt3sas_fwfault_debug;
8089 
8090 	/* base internal command bits */
8091 	mutex_init(&ioc->base_cmds.mutex);
8092 	ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8093 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;
8094 
8095 	/* port_enable command bits */
8096 	ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8097 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;
8098 
8099 	/* transport internal command bits */
8100 	ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8101 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;
8102 	mutex_init(&ioc->transport_cmds.mutex);
8103 
8104 	/* scsih internal command bits */
8105 	ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8106 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;
8107 	mutex_init(&ioc->scsih_cmds.mutex);
8108 
8109 	/* task management internal command bits */
8110 	ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8111 	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;
8112 	mutex_init(&ioc->tm_cmds.mutex);
8113 
8114 	/* config page internal command bits */
8115 	ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8116 	ioc->config_cmds.status = MPT3_CMD_NOT_USED;
8117 	mutex_init(&ioc->config_cmds.mutex);
8118 
8119 	/* ctl module internal command bits */
8120 	ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);
8121 	ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);
8122 	ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;
8123 	mutex_init(&ioc->ctl_cmds.mutex);
8124 
8125 	if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||
8126 	    !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||
8127 	    !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||
8128 	    !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {
8129 		r = -ENOMEM;
8130 		goto out_free_resources;
8131 	}
8132 
8133 	for (i = 0; i < MPI2_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
8134 		ioc->event_masks[i] = -1;
8135 
8136 	/* here we enable the events we care about */
8137 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);
8138 	_base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
8139 	_base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
8140 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
8141 	_base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
8142 	_base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
8143 	_base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);
8144 	_base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);
8145 	_base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);
8146 	_base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);
8147 	_base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);
8148 	_base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);
8149 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
8150 	if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {
8151 		if (ioc->is_gen35_ioc) {
8152 			_base_unmask_events(ioc,
8153 				MPI2_EVENT_PCIE_DEVICE_STATUS_CHANGE);
8154 			_base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);
8155 			_base_unmask_events(ioc,
8156 				MPI2_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
8157 		}
8158 	}
8159 	r = _base_make_ioc_operational(ioc);
8160 	if (r == -EAGAIN) {
8161 		r = _base_make_ioc_operational(ioc);
8162 		if (r)
8163 			goto out_free_resources;
8164 	}
8165 
8166 	/*
8167 	 * Copy current copy of IOCFacts in prev_fw_facts
8168 	 * and it will be used during online firmware upgrade.
8169 	 */
8170 	memcpy(&ioc->prev_fw_facts, &ioc->facts,
8171 	    sizeof(struct mpt3sas_facts));
8172 
8173 	ioc->non_operational_loop = 0;
8174 	ioc->ioc_coredump_loop = 0;
8175 	ioc->got_task_abort_from_ioctl = 0;
8176 	return 0;
8177 
8178  out_free_resources:
8179 
8180 	ioc->remove_host = 1;
8181 
8182 	mpt3sas_base_free_resources(ioc);
8183 	_base_release_memory_pools(ioc);
8184 	pci_set_drvdata(ioc->pdev, NULL);
8185 	kfree(ioc->cpu_msix_table);
8186 	if (ioc->is_warpdrive)
8187 		kfree(ioc->reply_post_host_index);
8188 	kfree(ioc->pd_handles);
8189 	kfree(ioc->blocking_handles);
8190 	kfree(ioc->device_remove_in_progress);
8191 	kfree(ioc->pend_os_device_add);
8192 	kfree(ioc->tm_cmds.reply);
8193 	kfree(ioc->transport_cmds.reply);
8194 	kfree(ioc->scsih_cmds.reply);
8195 	kfree(ioc->config_cmds.reply);
8196 	kfree(ioc->base_cmds.reply);
8197 	kfree(ioc->port_enable_cmds.reply);
8198 	kfree(ioc->ctl_cmds.reply);
8199 	kfree(ioc->ctl_cmds.sense);
8200 	kfree(ioc->pfacts);
8201 	ioc->ctl_cmds.reply = NULL;
8202 	ioc->base_cmds.reply = NULL;
8203 	ioc->tm_cmds.reply = NULL;
8204 	ioc->scsih_cmds.reply = NULL;
8205 	ioc->transport_cmds.reply = NULL;
8206 	ioc->config_cmds.reply = NULL;
8207 	ioc->pfacts = NULL;
8208 	return r;
8209 }
8210 
8211 
8212 /**
8213  * mpt3sas_base_detach - remove controller instance
8214  * @ioc: per adapter object
8215  */
8216 void
8217 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)
8218 {
8219 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));
8220 
8221 	mpt3sas_base_stop_watchdog(ioc);
8222 	mpt3sas_base_free_resources(ioc);
8223 	_base_release_memory_pools(ioc);
8224 	mpt3sas_free_enclosure_list(ioc);
8225 	pci_set_drvdata(ioc->pdev, NULL);
8226 	kfree(ioc->cpu_msix_table);
8227 	if (ioc->is_warpdrive)
8228 		kfree(ioc->reply_post_host_index);
8229 	kfree(ioc->pd_handles);
8230 	kfree(ioc->blocking_handles);
8231 	kfree(ioc->device_remove_in_progress);
8232 	kfree(ioc->pend_os_device_add);
8233 	kfree(ioc->pfacts);
8234 	kfree(ioc->ctl_cmds.reply);
8235 	kfree(ioc->ctl_cmds.sense);
8236 	kfree(ioc->base_cmds.reply);
8237 	kfree(ioc->port_enable_cmds.reply);
8238 	kfree(ioc->tm_cmds.reply);
8239 	kfree(ioc->transport_cmds.reply);
8240 	kfree(ioc->scsih_cmds.reply);
8241 	kfree(ioc->config_cmds.reply);
8242 }
8243 
8244 /**
8245  * _base_pre_reset_handler - pre reset handler
8246  * @ioc: per adapter object
8247  */
8248 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)
8249 {
8250 	mpt3sas_scsih_pre_reset_handler(ioc);
8251 	mpt3sas_ctl_pre_reset_handler(ioc);
8252 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));
8253 }
8254 
8255 /**
8256  * _base_clear_outstanding_mpt_commands - clears outstanding mpt commands
8257  * @ioc: per adapter object
8258  */
8259 static void
8260 _base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc)
8261 {
8262 	dtmprintk(ioc,
8263 	    ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__));
8264 	if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {
8265 		ioc->transport_cmds.status |= MPT3_CMD_RESET;
8266 		mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);
8267 		complete(&ioc->transport_cmds.done);
8268 	}
8269 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {
8270 		ioc->base_cmds.status |= MPT3_CMD_RESET;
8271 		mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);
8272 		complete(&ioc->base_cmds.done);
8273 	}
8274 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {
8275 		ioc->port_enable_failed = 1;
8276 		ioc->port_enable_cmds.status |= MPT3_CMD_RESET;
8277 		mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);
8278 		if (ioc->is_driver_loading) {
8279 			ioc->start_scan_failed =
8280 				MPI2_IOCSTATUS_INTERNAL_ERROR;
8281 			ioc->start_scan = 0;
8282 		} else {
8283 			complete(&ioc->port_enable_cmds.done);
8284 		}
8285 	}
8286 	if (ioc->config_cmds.status & MPT3_CMD_PENDING) {
8287 		ioc->config_cmds.status |= MPT3_CMD_RESET;
8288 		mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);
8289 		ioc->config_cmds.smid = USHRT_MAX;
8290 		complete(&ioc->config_cmds.done);
8291 	}
8292 }
8293 
8294 /**
8295  * _base_clear_outstanding_commands - clear all outstanding commands
8296  * @ioc: per adapter object
8297  */
8298 static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc)
8299 {
8300 	mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc);
8301 	mpt3sas_ctl_clear_outstanding_ioctls(ioc);
8302 	_base_clear_outstanding_mpt_commands(ioc);
8303 }
8304 
8305 /**
8306  * _base_reset_done_handler - reset done handler
8307  * @ioc: per adapter object
8308  */
8309 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)
8310 {
8311 	mpt3sas_scsih_reset_done_handler(ioc);
8312 	mpt3sas_ctl_reset_done_handler(ioc);
8313 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));
8314 }
8315 
8316 /**
8317  * mpt3sas_wait_for_commands_to_complete - reset controller
8318  * @ioc: Pointer to MPT_ADAPTER structure
8319  *
8320  * This function is waiting 10s for all pending commands to complete
8321  * prior to putting controller in reset.
8322  */
8323 void
8324 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)
8325 {
8326 	u32 ioc_state;
8327 
8328 	ioc->pending_io_count = 0;
8329 
8330 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
8331 	if ((ioc_state & MPI2_IOC_STATE_MASK) != MPI2_IOC_STATE_OPERATIONAL)
8332 		return;
8333 
8334 	/* pending command count */
8335 	ioc->pending_io_count = scsi_host_busy(ioc->shost);
8336 
8337 	if (!ioc->pending_io_count)
8338 		return;
8339 
8340 	/* wait for pending commands to complete */
8341 	wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);
8342 }
8343 
8344 /**
8345  * _base_check_ioc_facts_changes - Look for increase/decrease of IOCFacts
8346  *     attributes during online firmware upgrade and update the corresponding
8347  *     IOC variables accordingly.
8348  *
8349  * @ioc: Pointer to MPT_ADAPTER structure
8350  */
8351 static int
8352 _base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc)
8353 {
8354 	u16 pd_handles_sz;
8355 	void *pd_handles = NULL, *blocking_handles = NULL;
8356 	void *pend_os_device_add = NULL, *device_remove_in_progress = NULL;
8357 	struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts;
8358 
8359 	if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) {
8360 		pd_handles_sz = (ioc->facts.MaxDevHandle / 8);
8361 		if (ioc->facts.MaxDevHandle % 8)
8362 			pd_handles_sz++;
8363 
8364 		pd_handles = krealloc(ioc->pd_handles, pd_handles_sz,
8365 		    GFP_KERNEL);
8366 		if (!pd_handles) {
8367 			ioc_info(ioc,
8368 			    "Unable to allocate the memory for pd_handles of sz: %d\n",
8369 			    pd_handles_sz);
8370 			return -ENOMEM;
8371 		}
8372 		memset(pd_handles + ioc->pd_handles_sz, 0,
8373 		    (pd_handles_sz - ioc->pd_handles_sz));
8374 		ioc->pd_handles = pd_handles;
8375 
8376 		blocking_handles = krealloc(ioc->blocking_handles,
8377 		    pd_handles_sz, GFP_KERNEL);
8378 		if (!blocking_handles) {
8379 			ioc_info(ioc,
8380 			    "Unable to allocate the memory for "
8381 			    "blocking_handles of sz: %d\n",
8382 			    pd_handles_sz);
8383 			return -ENOMEM;
8384 		}
8385 		memset(blocking_handles + ioc->pd_handles_sz, 0,
8386 		    (pd_handles_sz - ioc->pd_handles_sz));
8387 		ioc->blocking_handles = blocking_handles;
8388 		ioc->pd_handles_sz = pd_handles_sz;
8389 
8390 		pend_os_device_add = krealloc(ioc->pend_os_device_add,
8391 		    pd_handles_sz, GFP_KERNEL);
8392 		if (!pend_os_device_add) {
8393 			ioc_info(ioc,
8394 			    "Unable to allocate the memory for pend_os_device_add of sz: %d\n",
8395 			    pd_handles_sz);
8396 			return -ENOMEM;
8397 		}
8398 		memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0,
8399 		    (pd_handles_sz - ioc->pend_os_device_add_sz));
8400 		ioc->pend_os_device_add = pend_os_device_add;
8401 		ioc->pend_os_device_add_sz = pd_handles_sz;
8402 
8403 		device_remove_in_progress = krealloc(
8404 		    ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL);
8405 		if (!device_remove_in_progress) {
8406 			ioc_info(ioc,
8407 			    "Unable to allocate the memory for "
8408 			    "device_remove_in_progress of sz: %d\n "
8409 			    , pd_handles_sz);
8410 			return -ENOMEM;
8411 		}
8412 		memset(device_remove_in_progress +
8413 		    ioc->device_remove_in_progress_sz, 0,
8414 		    (pd_handles_sz - ioc->device_remove_in_progress_sz));
8415 		ioc->device_remove_in_progress = device_remove_in_progress;
8416 		ioc->device_remove_in_progress_sz = pd_handles_sz;
8417 	}
8418 
8419 	memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts));
8420 	return 0;
8421 }
8422 
8423 /**
8424  * mpt3sas_base_hard_reset_handler - reset controller
8425  * @ioc: Pointer to MPT_ADAPTER structure
8426  * @type: FORCE_BIG_HAMMER or SOFT_RESET
8427  *
8428  * Return: 0 for success, non-zero for failure.
8429  */
8430 int
8431 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,
8432 	enum reset_type type)
8433 {
8434 	int r;
8435 	unsigned long flags;
8436 	u32 ioc_state;
8437 	u8 is_fault = 0, is_trigger = 0;
8438 
8439 	dtmprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__));
8440 
8441 	if (ioc->pci_error_recovery) {
8442 		ioc_err(ioc, "%s: pci error recovery reset\n", __func__);
8443 		r = 0;
8444 		goto out_unlocked;
8445 	}
8446 
8447 	if (mpt3sas_fwfault_debug)
8448 		mpt3sas_halt_firmware(ioc);
8449 
8450 	/* wait for an active reset in progress to complete */
8451 	mutex_lock(&ioc->reset_in_progress_mutex);
8452 
8453 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
8454 	ioc->shost_recovery = 1;
8455 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
8456 
8457 	if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
8458 	    MPT3_DIAG_BUFFER_IS_REGISTERED) &&
8459 	    (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &
8460 	    MPT3_DIAG_BUFFER_IS_RELEASED))) {
8461 		is_trigger = 1;
8462 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);
8463 		if ((ioc_state & MPI2_IOC_STATE_MASK) == MPI2_IOC_STATE_FAULT ||
8464 		    (ioc_state & MPI2_IOC_STATE_MASK) ==
8465 		    MPI2_IOC_STATE_COREDUMP) {
8466 			is_fault = 1;
8467 			ioc->htb_rel.trigger_info_dwords[1] =
8468 			    (ioc_state & MPI2_DOORBELL_DATA_MASK);
8469 		}
8470 	}
8471 	_base_pre_reset_handler(ioc);
8472 	mpt3sas_wait_for_commands_to_complete(ioc);
8473 	mpt3sas_base_mask_interrupts(ioc);
8474 	r = mpt3sas_base_make_ioc_ready(ioc, type);
8475 	if (r)
8476 		goto out;
8477 	_base_clear_outstanding_commands(ioc);
8478 
8479 	/* If this hard reset is called while port enable is active, then
8480 	 * there is no reason to call make_ioc_operational
8481 	 */
8482 	if (ioc->is_driver_loading && ioc->port_enable_failed) {
8483 		ioc->remove_host = 1;
8484 		r = -EFAULT;
8485 		goto out;
8486 	}
8487 	r = _base_get_ioc_facts(ioc);
8488 	if (r)
8489 		goto out;
8490 
8491 	r = _base_check_ioc_facts_changes(ioc);
8492 	if (r) {
8493 		ioc_info(ioc,
8494 		    "Some of the parameters got changed in this new firmware"
8495 		    " image and it requires system reboot\n");
8496 		goto out;
8497 	}
8498 	if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)
8499 		panic("%s: Issue occurred with flashing controller firmware."
8500 		      "Please reboot the system and ensure that the correct"
8501 		      " firmware version is running\n", ioc->name);
8502 
8503 	r = _base_make_ioc_operational(ioc);
8504 	if (!r)
8505 		_base_reset_done_handler(ioc);
8506 
8507  out:
8508 	ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED");
8509 
8510 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);
8511 	ioc->shost_recovery = 0;
8512 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);
8513 	ioc->ioc_reset_count++;
8514 	mutex_unlock(&ioc->reset_in_progress_mutex);
8515 
8516  out_unlocked:
8517 	if ((r == 0) && is_trigger) {
8518 		if (is_fault)
8519 			mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);
8520 		else
8521 			mpt3sas_trigger_master(ioc,
8522 			    MASTER_TRIGGER_ADAPTER_RESET);
8523 	}
8524 	dtmprintk(ioc, ioc_info(ioc, "%s: exit\n", __func__));
8525 	return r;
8526 }
8527