1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Linux MegaRAID driver for SAS based RAID controllers
4  *
5  *  Copyright (c) 2009-2013  LSI Corporation
6  *  Copyright (c) 2013-2016  Avago Technologies
7  *  Copyright (c) 2016-2018  Broadcom Inc.
8  *
9  *  FILE: megaraid_sas_fusion.c
10  *
11  *  Authors: Broadcom Inc.
12  *           Sumant Patro
13  *           Adam Radford
14  *           Kashyap Desai <kashyap.desai@broadcom.com>
15  *           Sumit Saxena <sumit.saxena@broadcom.com>
16  *
17  *  Send feedback to: megaraidlinux.pdl@broadcom.com
18  */
19 
20 #include <linux/kernel.h>
21 #include <linux/types.h>
22 #include <linux/pci.h>
23 #include <linux/list.h>
24 #include <linux/moduleparam.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/interrupt.h>
28 #include <linux/delay.h>
29 #include <linux/uio.h>
30 #include <linux/uaccess.h>
31 #include <linux/fs.h>
32 #include <linux/compat.h>
33 #include <linux/blkdev.h>
34 #include <linux/mutex.h>
35 #include <linux/poll.h>
36 #include <linux/vmalloc.h>
37 #include <linux/workqueue.h>
38 #include <linux/irq_poll.h>
39 
40 #include <scsi/scsi.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <scsi/scsi_device.h>
43 #include <scsi/scsi_host.h>
44 #include <scsi/scsi_dbg.h>
45 #include <linux/dmi.h>
46 
47 #include "megaraid_sas_fusion.h"
48 #include "megaraid_sas.h"
49 
50 
51 extern void
52 megasas_complete_cmd(struct megasas_instance *instance,
53 		     struct megasas_cmd *cmd, u8 alt_status);
54 int
55 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
56 	      int seconds);
57 
58 int
59 megasas_clear_intr_fusion(struct megasas_instance *instance);
60 
61 int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
62 
63 extern u32 megasas_dbg_lvl;
64 int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
65 				  int initial);
66 extern struct megasas_mgmt_info megasas_mgmt_info;
67 extern unsigned int resetwaittime;
68 extern unsigned int dual_qdepth_disable;
69 static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
70 static void megasas_free_reply_fusion(struct megasas_instance *instance);
71 static inline
72 void megasas_configure_queue_sizes(struct megasas_instance *instance);
73 static void megasas_fusion_crash_dump(struct megasas_instance *instance);
74 
75 /**
76  * megasas_adp_reset_wait_for_ready -	initiate chip reset and wait for
77  *					controller to come to ready state
78  * @instance:				adapter's soft state
79  * @do_adp_reset:			If true, do a chip reset
80  * @ocr_context:			If called from OCR context this will
81  *					be set to 1, else 0
82  *
83  * This function initiates a chip reset followed by a wait for controller to
84  * transition to ready state.
85  * During this, driver will block all access to PCI config space from userspace
86  */
87 int
88 megasas_adp_reset_wait_for_ready(struct megasas_instance *instance,
89 				 bool do_adp_reset,
90 				 int ocr_context)
91 {
92 	int ret = FAILED;
93 
94 	/*
95 	 * Block access to PCI config space from userspace
96 	 * when diag reset is initiated from driver
97 	 */
98 	if (megasas_dbg_lvl & OCR_DEBUG)
99 		dev_info(&instance->pdev->dev,
100 			 "Block access to PCI config space %s %d\n",
101 			 __func__, __LINE__);
102 
103 	pci_cfg_access_lock(instance->pdev);
104 
105 	if (do_adp_reset) {
106 		if (instance->instancet->adp_reset
107 			(instance, instance->reg_set))
108 			goto out;
109 	}
110 
111 	/* Wait for FW to become ready */
112 	if (megasas_transition_to_ready(instance, ocr_context)) {
113 		dev_warn(&instance->pdev->dev,
114 			 "Failed to transition controller to ready for scsi%d.\n",
115 			 instance->host->host_no);
116 		goto out;
117 	}
118 
119 	ret = SUCCESS;
120 out:
121 	if (megasas_dbg_lvl & OCR_DEBUG)
122 		dev_info(&instance->pdev->dev,
123 			 "Unlock access to PCI config space %s %d\n",
124 			 __func__, __LINE__);
125 
126 	pci_cfg_access_unlock(instance->pdev);
127 
128 	return ret;
129 }
130 
131 /**
132  * megasas_check_same_4gb_region -	check if allocation
133  *					crosses same 4GB boundary or not
134  * @instance:				adapter's soft instance
135  * @start_addr:				start address of DMA allocation
136  * @size:				size of allocation in bytes
137  * @return:				true : allocation does not cross same
138  *					4GB boundary
139  *					false: allocation crosses same
140  *					4GB boundary
141  */
142 static inline bool megasas_check_same_4gb_region
143 	(struct megasas_instance *instance, dma_addr_t start_addr, size_t size)
144 {
145 	dma_addr_t end_addr;
146 
147 	end_addr = start_addr + size;
148 
149 	if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {
150 		dev_err(&instance->pdev->dev,
151 			"Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",
152 			(unsigned long long)start_addr,
153 			(unsigned long long)end_addr);
154 		return false;
155 	}
156 
157 	return true;
158 }
159 
160 /**
161  * megasas_enable_intr_fusion -	Enables interrupts
162  * @instance:	adapter's soft instance
163  */
164 static void
165 megasas_enable_intr_fusion(struct megasas_instance *instance)
166 {
167 	struct megasas_register_set __iomem *regs;
168 	regs = instance->reg_set;
169 
170 	instance->mask_interrupts = 0;
171 	/* For Thunderbolt/Invader also clear intr on enable */
172 	writel(~0, &regs->outbound_intr_status);
173 	readl(&regs->outbound_intr_status);
174 
175 	writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);
176 
177 	/* Dummy readl to force pci flush */
178 	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
179 		 __func__, readl(&regs->outbound_intr_mask));
180 }
181 
182 /**
183  * megasas_disable_intr_fusion - Disables interrupt
184  * @instance:	adapter's soft instance
185  */
186 static void
187 megasas_disable_intr_fusion(struct megasas_instance *instance)
188 {
189 	u32 mask = 0xFFFFFFFF;
190 	struct megasas_register_set __iomem *regs;
191 	regs = instance->reg_set;
192 	instance->mask_interrupts = 1;
193 
194 	writel(mask, &regs->outbound_intr_mask);
195 	/* Dummy readl to force pci flush */
196 	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
197 		 __func__, readl(&regs->outbound_intr_mask));
198 }
199 
200 int
201 megasas_clear_intr_fusion(struct megasas_instance *instance)
202 {
203 	u32 status;
204 	struct megasas_register_set __iomem *regs;
205 	regs = instance->reg_set;
206 	/*
207 	 * Check if it is our interrupt
208 	 */
209 	status = megasas_readl(instance,
210 			       &regs->outbound_intr_status);
211 
212 	if (status & 1) {
213 		writel(status, &regs->outbound_intr_status);
214 		readl(&regs->outbound_intr_status);
215 		return 1;
216 	}
217 	if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
218 		return 0;
219 
220 	return 1;
221 }
222 
223 static inline void
224 megasas_sdev_busy_inc(struct megasas_instance *instance,
225 		      struct scsi_cmnd *scmd)
226 {
227 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
228 		struct MR_PRIV_DEVICE *mr_device_priv_data =
229 			scmd->device->hostdata;
230 		atomic_inc(&mr_device_priv_data->sdev_priv_busy);
231 	}
232 }
233 
234 static inline void
235 megasas_sdev_busy_dec(struct megasas_instance *instance,
236 		      struct scsi_cmnd *scmd)
237 {
238 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
239 		struct MR_PRIV_DEVICE *mr_device_priv_data =
240 			scmd->device->hostdata;
241 		atomic_dec(&mr_device_priv_data->sdev_priv_busy);
242 	}
243 }
244 
245 static inline int
246 megasas_sdev_busy_read(struct megasas_instance *instance,
247 		       struct scsi_cmnd *scmd)
248 {
249 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
250 		struct MR_PRIV_DEVICE *mr_device_priv_data =
251 			scmd->device->hostdata;
252 		return atomic_read(&mr_device_priv_data->sdev_priv_busy);
253 	}
254 	return 0;
255 }
256 
257 /**
258  * megasas_get_cmd_fusion -	Get a command from the free pool
259  * @instance:		Adapter soft state
260  * @blk_tag:		Command tag
261  *
262  * Returns a blk_tag indexed mpt frame
263  */
264 inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
265 						  *instance, u32 blk_tag)
266 {
267 	struct fusion_context *fusion;
268 
269 	fusion = instance->ctrl_context;
270 	return fusion->cmd_list[blk_tag];
271 }
272 
273 /**
274  * megasas_return_cmd_fusion -	Return a cmd to free command pool
275  * @instance:		Adapter soft state
276  * @cmd:		Command packet to be returned to free command pool
277  */
278 inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
279 	struct megasas_cmd_fusion *cmd)
280 {
281 	cmd->scmd = NULL;
282 	memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
283 	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
284 	cmd->cmd_completed = false;
285 }
286 
287 /**
288  * megasas_write_64bit_req_desc -	PCI writes 64bit request descriptor
289  * @instance:				Adapter soft state
290  * @req_desc:				64bit Request descriptor
291  */
292 static void
293 megasas_write_64bit_req_desc(struct megasas_instance *instance,
294 		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
295 {
296 #if defined(writeq) && defined(CONFIG_64BIT)
297 	u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
298 		le32_to_cpu(req_desc->u.low));
299 	writeq(req_data, &instance->reg_set->inbound_low_queue_port);
300 #else
301 	unsigned long flags;
302 	spin_lock_irqsave(&instance->hba_lock, flags);
303 	writel(le32_to_cpu(req_desc->u.low),
304 		&instance->reg_set->inbound_low_queue_port);
305 	writel(le32_to_cpu(req_desc->u.high),
306 		&instance->reg_set->inbound_high_queue_port);
307 	spin_unlock_irqrestore(&instance->hba_lock, flags);
308 #endif
309 }
310 
311 /**
312  * megasas_fire_cmd_fusion -	Sends command to the FW
313  * @instance:			Adapter soft state
314  * @req_desc:			32bit or 64bit Request descriptor
315  *
316  * Perform PCI Write. AERO SERIES supports 32 bit Descriptor.
317  * Prior to AERO_SERIES support 64 bit Descriptor.
318  */
319 static void
320 megasas_fire_cmd_fusion(struct megasas_instance *instance,
321 		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
322 {
323 	if (instance->atomic_desc_support)
324 		writel(le32_to_cpu(req_desc->u.low),
325 			&instance->reg_set->inbound_single_queue_port);
326 	else
327 		megasas_write_64bit_req_desc(instance, req_desc);
328 }
329 
330 /**
331  * megasas_fusion_update_can_queue -	Do all Adapter Queue depth related calculations here
332  * @instance:		Adapter soft state
333  * @fw_boot_context:	Whether this function called during probe or after OCR
334  *
335  * This function is only for fusion controllers.
336  * Update host can queue, if firmware downgrade max supported firmware commands.
337  * Firmware upgrade case will be skipped because underlying firmware has
338  * more resource than exposed to the OS.
339  *
340  */
341 static void
342 megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
343 {
344 	u16 cur_max_fw_cmds = 0;
345 	u16 ldio_threshold = 0;
346 
347 	/* ventura FW does not fill outbound_scratch_pad_2 with queue depth */
348 	if (instance->adapter_type < VENTURA_SERIES)
349 		cur_max_fw_cmds =
350 		megasas_readl(instance,
351 			      &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF;
352 
353 	if (dual_qdepth_disable || !cur_max_fw_cmds)
354 		cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF;
355 	else
356 		ldio_threshold =
357 			(instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
358 
359 	dev_info(&instance->pdev->dev,
360 		 "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",
361 		 cur_max_fw_cmds, ldio_threshold);
362 
363 	if (fw_boot_context == OCR_CONTEXT) {
364 		cur_max_fw_cmds = cur_max_fw_cmds - 1;
365 		if (cur_max_fw_cmds < instance->max_fw_cmds) {
366 			instance->cur_can_queue =
367 				cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
368 						MEGASAS_FUSION_IOCTL_CMDS);
369 			instance->host->can_queue = instance->cur_can_queue;
370 			instance->ldio_threshold = ldio_threshold;
371 		}
372 	} else {
373 		instance->max_fw_cmds = cur_max_fw_cmds;
374 		instance->ldio_threshold = ldio_threshold;
375 
376 		if (reset_devices)
377 			instance->max_fw_cmds = min(instance->max_fw_cmds,
378 						(u16)MEGASAS_KDUMP_QUEUE_DEPTH);
379 		/*
380 		* Reduce the max supported cmds by 1. This is to ensure that the
381 		* reply_q_sz (1 more than the max cmd that driver may send)
382 		* does not exceed max cmds that the FW can support
383 		*/
384 		instance->max_fw_cmds = instance->max_fw_cmds-1;
385 	}
386 }
387 
388 static inline void
389 megasas_get_msix_index(struct megasas_instance *instance,
390 		       struct scsi_cmnd *scmd,
391 		       struct megasas_cmd_fusion *cmd,
392 		       u8 data_arms)
393 {
394 	if (instance->perf_mode == MR_BALANCED_PERF_MODE &&
395 	    (megasas_sdev_busy_read(instance, scmd) >
396 	     (data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) {
397 		cmd->request_desc->SCSIIO.MSIxIndex =
398 			mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) /
399 					MR_HIGH_IOPS_BATCH_COUNT), instance->low_latency_index_start);
400 	} else if (instance->msix_load_balance) {
401 		cmd->request_desc->SCSIIO.MSIxIndex =
402 			(mega_mod64(atomic64_add_return(1, &instance->total_io_count),
403 				instance->msix_vectors));
404 	} else if (instance->host->nr_hw_queues > 1) {
405 		u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
406 
407 		cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(tag) +
408 			instance->low_latency_index_start;
409 	} else {
410 		cmd->request_desc->SCSIIO.MSIxIndex =
411 			instance->reply_map[raw_smp_processor_id()];
412 	}
413 }
414 
415 /**
416  * megasas_free_cmds_fusion -	Free all the cmds in the free cmd pool
417  * @instance:		Adapter soft state
418  */
419 void
420 megasas_free_cmds_fusion(struct megasas_instance *instance)
421 {
422 	int i;
423 	struct fusion_context *fusion = instance->ctrl_context;
424 	struct megasas_cmd_fusion *cmd;
425 
426 	if (fusion->sense)
427 		dma_pool_free(fusion->sense_dma_pool, fusion->sense,
428 			      fusion->sense_phys_addr);
429 
430 	/* SG */
431 	if (fusion->cmd_list) {
432 		for (i = 0; i < instance->max_mpt_cmds; i++) {
433 			cmd = fusion->cmd_list[i];
434 			if (cmd) {
435 				if (cmd->sg_frame)
436 					dma_pool_free(fusion->sg_dma_pool,
437 						      cmd->sg_frame,
438 						      cmd->sg_frame_phys_addr);
439 			}
440 			kfree(cmd);
441 		}
442 		kfree(fusion->cmd_list);
443 	}
444 
445 	if (fusion->sg_dma_pool) {
446 		dma_pool_destroy(fusion->sg_dma_pool);
447 		fusion->sg_dma_pool = NULL;
448 	}
449 	if (fusion->sense_dma_pool) {
450 		dma_pool_destroy(fusion->sense_dma_pool);
451 		fusion->sense_dma_pool = NULL;
452 	}
453 
454 
455 	/* Reply Frame, Desc*/
456 	if (instance->is_rdpq)
457 		megasas_free_rdpq_fusion(instance);
458 	else
459 		megasas_free_reply_fusion(instance);
460 
461 	/* Request Frame, Desc*/
462 	if (fusion->req_frames_desc)
463 		dma_free_coherent(&instance->pdev->dev,
464 			fusion->request_alloc_sz, fusion->req_frames_desc,
465 			fusion->req_frames_desc_phys);
466 	if (fusion->io_request_frames)
467 		dma_pool_free(fusion->io_request_frames_pool,
468 			fusion->io_request_frames,
469 			fusion->io_request_frames_phys);
470 	if (fusion->io_request_frames_pool) {
471 		dma_pool_destroy(fusion->io_request_frames_pool);
472 		fusion->io_request_frames_pool = NULL;
473 	}
474 }
475 
476 /**
477  * megasas_create_sg_sense_fusion -	Creates DMA pool for cmd frames
478  * @instance:			Adapter soft state
479  *
480  */
481 static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
482 {
483 	int i;
484 	u16 max_cmd;
485 	struct fusion_context *fusion;
486 	struct megasas_cmd_fusion *cmd;
487 	int sense_sz;
488 	u32 offset;
489 
490 	fusion = instance->ctrl_context;
491 	max_cmd = instance->max_fw_cmds;
492 	sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;
493 
494 	fusion->sg_dma_pool =
495 			dma_pool_create("mr_sg", &instance->pdev->dev,
496 				instance->max_chain_frame_sz,
497 				MR_DEFAULT_NVME_PAGE_SIZE, 0);
498 	/* SCSI_SENSE_BUFFERSIZE  = 96 bytes */
499 	fusion->sense_dma_pool =
500 			dma_pool_create("mr_sense", &instance->pdev->dev,
501 				sense_sz, 64, 0);
502 
503 	if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
504 		dev_err(&instance->pdev->dev,
505 			"Failed from %s %d\n",  __func__, __LINE__);
506 		return -ENOMEM;
507 	}
508 
509 	fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
510 				       GFP_KERNEL, &fusion->sense_phys_addr);
511 	if (!fusion->sense) {
512 		dev_err(&instance->pdev->dev,
513 			"failed from %s %d\n",  __func__, __LINE__);
514 		return -ENOMEM;
515 	}
516 
517 	/* sense buffer, request frame and reply desc pool requires to be in
518 	 * same 4 gb region. Below function will check this.
519 	 * In case of failure, new pci pool will be created with updated
520 	 * alignment.
521 	 * Older allocation and pool will be destroyed.
522 	 * Alignment will be used such a way that next allocation if success,
523 	 * will always meet same 4gb region requirement.
524 	 * Actual requirement is not alignment, but we need start and end of
525 	 * DMA address must have same upper 32 bit address.
526 	 */
527 
528 	if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr,
529 					   sense_sz)) {
530 		dma_pool_free(fusion->sense_dma_pool, fusion->sense,
531 			      fusion->sense_phys_addr);
532 		fusion->sense = NULL;
533 		dma_pool_destroy(fusion->sense_dma_pool);
534 
535 		fusion->sense_dma_pool =
536 			dma_pool_create("mr_sense_align", &instance->pdev->dev,
537 					sense_sz, roundup_pow_of_two(sense_sz),
538 					0);
539 		if (!fusion->sense_dma_pool) {
540 			dev_err(&instance->pdev->dev,
541 				"Failed from %s %d\n",  __func__, __LINE__);
542 			return -ENOMEM;
543 		}
544 		fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
545 					       GFP_KERNEL,
546 					       &fusion->sense_phys_addr);
547 		if (!fusion->sense) {
548 			dev_err(&instance->pdev->dev,
549 				"failed from %s %d\n",  __func__, __LINE__);
550 			return -ENOMEM;
551 		}
552 	}
553 
554 	/*
555 	 * Allocate and attach a frame to each of the commands in cmd_list
556 	 */
557 	for (i = 0; i < max_cmd; i++) {
558 		cmd = fusion->cmd_list[i];
559 		cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,
560 					GFP_KERNEL, &cmd->sg_frame_phys_addr);
561 
562 		offset = SCSI_SENSE_BUFFERSIZE * i;
563 		cmd->sense = (u8 *)fusion->sense + offset;
564 		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
565 
566 		if (!cmd->sg_frame) {
567 			dev_err(&instance->pdev->dev,
568 				"Failed from %s %d\n",  __func__, __LINE__);
569 			return -ENOMEM;
570 		}
571 	}
572 
573 	/* create sense buffer for the raid 1/10 fp */
574 	for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
575 		cmd = fusion->cmd_list[i];
576 		offset = SCSI_SENSE_BUFFERSIZE * i;
577 		cmd->sense = (u8 *)fusion->sense + offset;
578 		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
579 
580 	}
581 
582 	return 0;
583 }
584 
585 static int
586 megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
587 {
588 	u32 max_mpt_cmd, i, j;
589 	struct fusion_context *fusion;
590 
591 	fusion = instance->ctrl_context;
592 
593 	max_mpt_cmd = instance->max_mpt_cmds;
594 
595 	/*
596 	 * fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.
597 	 * Allocate the dynamic array first and then allocate individual
598 	 * commands.
599 	 */
600 	fusion->cmd_list =
601 		kcalloc(max_mpt_cmd, sizeof(struct megasas_cmd_fusion *),
602 			GFP_KERNEL);
603 	if (!fusion->cmd_list) {
604 		dev_err(&instance->pdev->dev,
605 			"Failed from %s %d\n",  __func__, __LINE__);
606 		return -ENOMEM;
607 	}
608 
609 	for (i = 0; i < max_mpt_cmd; i++) {
610 		fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),
611 					      GFP_KERNEL);
612 		if (!fusion->cmd_list[i]) {
613 			for (j = 0; j < i; j++)
614 				kfree(fusion->cmd_list[j]);
615 			kfree(fusion->cmd_list);
616 			dev_err(&instance->pdev->dev,
617 				"Failed from %s %d\n",  __func__, __LINE__);
618 			return -ENOMEM;
619 		}
620 	}
621 
622 	return 0;
623 }
624 
625 static int
626 megasas_alloc_request_fusion(struct megasas_instance *instance)
627 {
628 	struct fusion_context *fusion;
629 
630 	fusion = instance->ctrl_context;
631 
632 retry_alloc:
633 	fusion->io_request_frames_pool =
634 			dma_pool_create("mr_ioreq", &instance->pdev->dev,
635 				fusion->io_frames_alloc_sz, 16, 0);
636 
637 	if (!fusion->io_request_frames_pool) {
638 		dev_err(&instance->pdev->dev,
639 			"Failed from %s %d\n",  __func__, __LINE__);
640 		return -ENOMEM;
641 	}
642 
643 	fusion->io_request_frames =
644 			dma_pool_alloc(fusion->io_request_frames_pool,
645 				GFP_KERNEL | __GFP_NOWARN,
646 				&fusion->io_request_frames_phys);
647 	if (!fusion->io_request_frames) {
648 		if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {
649 			instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;
650 			dma_pool_destroy(fusion->io_request_frames_pool);
651 			megasas_configure_queue_sizes(instance);
652 			goto retry_alloc;
653 		} else {
654 			dev_err(&instance->pdev->dev,
655 				"Failed from %s %d\n",  __func__, __LINE__);
656 			return -ENOMEM;
657 		}
658 	}
659 
660 	if (!megasas_check_same_4gb_region(instance,
661 					   fusion->io_request_frames_phys,
662 					   fusion->io_frames_alloc_sz)) {
663 		dma_pool_free(fusion->io_request_frames_pool,
664 			      fusion->io_request_frames,
665 			      fusion->io_request_frames_phys);
666 		fusion->io_request_frames = NULL;
667 		dma_pool_destroy(fusion->io_request_frames_pool);
668 
669 		fusion->io_request_frames_pool =
670 			dma_pool_create("mr_ioreq_align",
671 					&instance->pdev->dev,
672 					fusion->io_frames_alloc_sz,
673 					roundup_pow_of_two(fusion->io_frames_alloc_sz),
674 					0);
675 
676 		if (!fusion->io_request_frames_pool) {
677 			dev_err(&instance->pdev->dev,
678 				"Failed from %s %d\n",  __func__, __LINE__);
679 			return -ENOMEM;
680 		}
681 
682 		fusion->io_request_frames =
683 			dma_pool_alloc(fusion->io_request_frames_pool,
684 				       GFP_KERNEL | __GFP_NOWARN,
685 				       &fusion->io_request_frames_phys);
686 
687 		if (!fusion->io_request_frames) {
688 			dev_err(&instance->pdev->dev,
689 				"Failed from %s %d\n",  __func__, __LINE__);
690 			return -ENOMEM;
691 		}
692 	}
693 
694 	fusion->req_frames_desc =
695 		dma_alloc_coherent(&instance->pdev->dev,
696 				   fusion->request_alloc_sz,
697 				   &fusion->req_frames_desc_phys, GFP_KERNEL);
698 	if (!fusion->req_frames_desc) {
699 		dev_err(&instance->pdev->dev,
700 			"Failed from %s %d\n",  __func__, __LINE__);
701 		return -ENOMEM;
702 	}
703 
704 	return 0;
705 }
706 
707 static int
708 megasas_alloc_reply_fusion(struct megasas_instance *instance)
709 {
710 	int i, count;
711 	struct fusion_context *fusion;
712 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
713 	fusion = instance->ctrl_context;
714 
715 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
716 	count += instance->iopoll_q_count;
717 
718 	fusion->reply_frames_desc_pool =
719 			dma_pool_create("mr_reply", &instance->pdev->dev,
720 				fusion->reply_alloc_sz * count, 16, 0);
721 
722 	if (!fusion->reply_frames_desc_pool) {
723 		dev_err(&instance->pdev->dev,
724 			"Failed from %s %d\n",  __func__, __LINE__);
725 		return -ENOMEM;
726 	}
727 
728 	fusion->reply_frames_desc[0] =
729 		dma_pool_alloc(fusion->reply_frames_desc_pool,
730 			GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
731 	if (!fusion->reply_frames_desc[0]) {
732 		dev_err(&instance->pdev->dev,
733 			"Failed from %s %d\n",  __func__, __LINE__);
734 		return -ENOMEM;
735 	}
736 
737 	if (!megasas_check_same_4gb_region(instance,
738 					   fusion->reply_frames_desc_phys[0],
739 					   (fusion->reply_alloc_sz * count))) {
740 		dma_pool_free(fusion->reply_frames_desc_pool,
741 			      fusion->reply_frames_desc[0],
742 			      fusion->reply_frames_desc_phys[0]);
743 		fusion->reply_frames_desc[0] = NULL;
744 		dma_pool_destroy(fusion->reply_frames_desc_pool);
745 
746 		fusion->reply_frames_desc_pool =
747 			dma_pool_create("mr_reply_align",
748 					&instance->pdev->dev,
749 					fusion->reply_alloc_sz * count,
750 					roundup_pow_of_two(fusion->reply_alloc_sz * count),
751 					0);
752 
753 		if (!fusion->reply_frames_desc_pool) {
754 			dev_err(&instance->pdev->dev,
755 				"Failed from %s %d\n",  __func__, __LINE__);
756 			return -ENOMEM;
757 		}
758 
759 		fusion->reply_frames_desc[0] =
760 			dma_pool_alloc(fusion->reply_frames_desc_pool,
761 				       GFP_KERNEL,
762 				       &fusion->reply_frames_desc_phys[0]);
763 
764 		if (!fusion->reply_frames_desc[0]) {
765 			dev_err(&instance->pdev->dev,
766 				"Failed from %s %d\n",  __func__, __LINE__);
767 			return -ENOMEM;
768 		}
769 	}
770 
771 	reply_desc = fusion->reply_frames_desc[0];
772 	for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
773 		reply_desc->Words = cpu_to_le64(ULLONG_MAX);
774 
775 	/* This is not a rdpq mode, but driver still populate
776 	 * reply_frame_desc array to use same msix index in ISR path.
777 	 */
778 	for (i = 0; i < (count - 1); i++)
779 		fusion->reply_frames_desc[i + 1] =
780 			fusion->reply_frames_desc[i] +
781 			(fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
782 
783 	return 0;
784 }
785 
786 static int
787 megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
788 {
789 	int i, j, k, msix_count;
790 	struct fusion_context *fusion;
791 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
792 	union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];
793 	dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];
794 	u8 dma_alloc_count, abs_index;
795 	u32 chunk_size, array_size, offset;
796 
797 	fusion = instance->ctrl_context;
798 	chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
799 	array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
800 		     MAX_MSIX_QUEUES_FUSION;
801 
802 	fusion->rdpq_virt = dma_alloc_coherent(&instance->pdev->dev,
803 					       array_size, &fusion->rdpq_phys,
804 					       GFP_KERNEL);
805 	if (!fusion->rdpq_virt) {
806 		dev_err(&instance->pdev->dev,
807 			"Failed from %s %d\n",  __func__, __LINE__);
808 		return -ENOMEM;
809 	}
810 
811 	msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
812 	msix_count += instance->iopoll_q_count;
813 
814 	fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
815 							 &instance->pdev->dev,
816 							 chunk_size, 16, 0);
817 	fusion->reply_frames_desc_pool_align =
818 				dma_pool_create("mr_rdpq_align",
819 						&instance->pdev->dev,
820 						chunk_size,
821 						roundup_pow_of_two(chunk_size),
822 						0);
823 
824 	if (!fusion->reply_frames_desc_pool ||
825 	    !fusion->reply_frames_desc_pool_align) {
826 		dev_err(&instance->pdev->dev,
827 			"Failed from %s %d\n",  __func__, __LINE__);
828 		return -ENOMEM;
829 	}
830 
831 /*
832  * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
833  * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be
834  * within 4GB boundary and also reply queues in a set must have same
835  * upper 32-bits in their memory address. so here driver is allocating the
836  * DMA'able memory for reply queues according. Driver uses limitation of
837  * VENTURA_SERIES to manage INVADER_SERIES as well.
838  */
839 	dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);
840 
841 	for (i = 0; i < dma_alloc_count; i++) {
842 		rdpq_chunk_virt[i] =
843 			dma_pool_alloc(fusion->reply_frames_desc_pool,
844 				       GFP_KERNEL, &rdpq_chunk_phys[i]);
845 		if (!rdpq_chunk_virt[i]) {
846 			dev_err(&instance->pdev->dev,
847 				"Failed from %s %d\n",  __func__, __LINE__);
848 			return -ENOMEM;
849 		}
850 		/* reply desc pool requires to be in same 4 gb region.
851 		 * Below function will check this.
852 		 * In case of failure, new pci pool will be created with updated
853 		 * alignment.
854 		 * For RDPQ buffers, driver always allocate two separate pci pool.
855 		 * Alignment will be used such a way that next allocation if
856 		 * success, will always meet same 4gb region requirement.
857 		 * rdpq_tracker keep track of each buffer's physical,
858 		 * virtual address and pci pool descriptor. It will help driver
859 		 * while freeing the resources.
860 		 *
861 		 */
862 		if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i],
863 						   chunk_size)) {
864 			dma_pool_free(fusion->reply_frames_desc_pool,
865 				      rdpq_chunk_virt[i],
866 				      rdpq_chunk_phys[i]);
867 
868 			rdpq_chunk_virt[i] =
869 				dma_pool_alloc(fusion->reply_frames_desc_pool_align,
870 					       GFP_KERNEL, &rdpq_chunk_phys[i]);
871 			if (!rdpq_chunk_virt[i]) {
872 				dev_err(&instance->pdev->dev,
873 					"Failed from %s %d\n",
874 					__func__, __LINE__);
875 				return -ENOMEM;
876 			}
877 			fusion->rdpq_tracker[i].dma_pool_ptr =
878 					fusion->reply_frames_desc_pool_align;
879 		} else {
880 			fusion->rdpq_tracker[i].dma_pool_ptr =
881 					fusion->reply_frames_desc_pool;
882 		}
883 
884 		fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];
885 		fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];
886 	}
887 
888 	for (k = 0; k < dma_alloc_count; k++) {
889 		for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {
890 			abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;
891 
892 			if (abs_index == msix_count)
893 				break;
894 			offset = fusion->reply_alloc_sz * i;
895 			fusion->rdpq_virt[abs_index].RDPQBaseAddress =
896 					cpu_to_le64(rdpq_chunk_phys[k] + offset);
897 			fusion->reply_frames_desc_phys[abs_index] =
898 					rdpq_chunk_phys[k] + offset;
899 			fusion->reply_frames_desc[abs_index] =
900 					(union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);
901 
902 			reply_desc = fusion->reply_frames_desc[abs_index];
903 			for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
904 				reply_desc->Words = ULLONG_MAX;
905 		}
906 	}
907 
908 	return 0;
909 }
910 
911 static void
912 megasas_free_rdpq_fusion(struct megasas_instance *instance) {
913 
914 	int i;
915 	struct fusion_context *fusion;
916 
917 	fusion = instance->ctrl_context;
918 
919 	for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {
920 		if (fusion->rdpq_tracker[i].pool_entry_virt)
921 			dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr,
922 				      fusion->rdpq_tracker[i].pool_entry_virt,
923 				      fusion->rdpq_tracker[i].pool_entry_phys);
924 
925 	}
926 
927 	dma_pool_destroy(fusion->reply_frames_desc_pool);
928 	dma_pool_destroy(fusion->reply_frames_desc_pool_align);
929 
930 	if (fusion->rdpq_virt)
931 		dma_free_coherent(&instance->pdev->dev,
932 			sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
933 			fusion->rdpq_virt, fusion->rdpq_phys);
934 }
935 
936 static void
937 megasas_free_reply_fusion(struct megasas_instance *instance) {
938 
939 	struct fusion_context *fusion;
940 
941 	fusion = instance->ctrl_context;
942 
943 	if (fusion->reply_frames_desc[0])
944 		dma_pool_free(fusion->reply_frames_desc_pool,
945 			fusion->reply_frames_desc[0],
946 			fusion->reply_frames_desc_phys[0]);
947 
948 	dma_pool_destroy(fusion->reply_frames_desc_pool);
949 
950 }
951 
952 
953 /**
954  * megasas_alloc_cmds_fusion -	Allocates the command packets
955  * @instance:		Adapter soft state
956  *
957  *
958  * Each frame has a 32-bit field called context. This context is used to get
959  * back the megasas_cmd_fusion from the frame when a frame gets completed
960  * In this driver, the 32 bit values are the indices into an array cmd_list.
961  * This array is used only to look up the megasas_cmd_fusion given the context.
962  * The free commands themselves are maintained in a linked list called cmd_pool.
963  *
964  * cmds are formed in the io_request and sg_frame members of the
965  * megasas_cmd_fusion. The context field is used to get a request descriptor
966  * and is used as SMID of the cmd.
967  * SMID value range is from 1 to max_fw_cmds.
968  */
969 static int
970 megasas_alloc_cmds_fusion(struct megasas_instance *instance)
971 {
972 	int i;
973 	struct fusion_context *fusion;
974 	struct megasas_cmd_fusion *cmd;
975 	u32 offset;
976 	dma_addr_t io_req_base_phys;
977 	u8 *io_req_base;
978 
979 
980 	fusion = instance->ctrl_context;
981 
982 	if (megasas_alloc_request_fusion(instance))
983 		goto fail_exit;
984 
985 	if (instance->is_rdpq) {
986 		if (megasas_alloc_rdpq_fusion(instance))
987 			goto fail_exit;
988 	} else
989 		if (megasas_alloc_reply_fusion(instance))
990 			goto fail_exit;
991 
992 	if (megasas_alloc_cmdlist_fusion(instance))
993 		goto fail_exit;
994 
995 	/* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */
996 	io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
997 	io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
998 
999 	/*
1000 	 * Add all the commands to command pool (fusion->cmd_pool)
1001 	 */
1002 
1003 	/* SMID 0 is reserved. Set SMID/index from 1 */
1004 	for (i = 0; i < instance->max_mpt_cmds; i++) {
1005 		cmd = fusion->cmd_list[i];
1006 		offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
1007 		memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
1008 		cmd->index = i + 1;
1009 		cmd->scmd = NULL;
1010 		cmd->sync_cmd_idx =
1011 		(i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
1012 				(i - instance->max_scsi_cmds) :
1013 				(u32)ULONG_MAX; /* Set to Invalid */
1014 		cmd->instance = instance;
1015 		cmd->io_request =
1016 			(struct MPI2_RAID_SCSI_IO_REQUEST *)
1017 		  (io_req_base + offset);
1018 		memset(cmd->io_request, 0,
1019 		       sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
1020 		cmd->io_request_phys_addr = io_req_base_phys + offset;
1021 		cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1022 	}
1023 
1024 	if (megasas_create_sg_sense_fusion(instance))
1025 		goto fail_exit;
1026 
1027 	return 0;
1028 
1029 fail_exit:
1030 	megasas_free_cmds_fusion(instance);
1031 	return -ENOMEM;
1032 }
1033 
1034 /**
1035  * wait_and_poll -	Issues a polling command
1036  * @instance:			Adapter soft state
1037  * @cmd:			Command packet to be issued
1038  * @seconds:			Maximum poll time
1039  *
1040  * For polling, MFI requires the cmd_status to be set to 0xFF before posting.
1041  */
1042 int
1043 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
1044 	int seconds)
1045 {
1046 	int i;
1047 	struct megasas_header *frame_hdr = &cmd->frame->hdr;
1048 	u32 status_reg;
1049 
1050 	u32 msecs = seconds * 1000;
1051 
1052 	/*
1053 	 * Wait for cmd_status to change
1054 	 */
1055 	for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
1056 		rmb();
1057 		msleep(20);
1058 		if (!(i % 5000)) {
1059 			status_reg = instance->instancet->read_fw_status_reg(instance)
1060 					& MFI_STATE_MASK;
1061 			if (status_reg == MFI_STATE_FAULT)
1062 				break;
1063 		}
1064 	}
1065 
1066 	if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
1067 		return DCMD_TIMEOUT;
1068 	else if (frame_hdr->cmd_status == MFI_STAT_OK)
1069 		return DCMD_SUCCESS;
1070 	else
1071 		return DCMD_FAILED;
1072 }
1073 
1074 /**
1075  * megasas_ioc_init_fusion -	Initializes the FW
1076  * @instance:		Adapter soft state
1077  *
1078  * Issues the IOC Init cmd
1079  */
1080 int
1081 megasas_ioc_init_fusion(struct megasas_instance *instance)
1082 {
1083 	struct megasas_init_frame *init_frame;
1084 	struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
1085 	dma_addr_t	ioc_init_handle;
1086 	struct megasas_cmd *cmd;
1087 	u8 ret, cur_rdpq_mode;
1088 	struct fusion_context *fusion;
1089 	union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
1090 	int i;
1091 	struct megasas_header *frame_hdr;
1092 	const char *sys_info;
1093 	MFI_CAPABILITIES *drv_ops;
1094 	u32 scratch_pad_1;
1095 	ktime_t time;
1096 	bool cur_fw_64bit_dma_capable;
1097 	bool cur_intr_coalescing;
1098 
1099 	fusion = instance->ctrl_context;
1100 
1101 	ioc_init_handle = fusion->ioc_init_request_phys;
1102 	IOCInitMessage = fusion->ioc_init_request;
1103 
1104 	cmd = fusion->ioc_init_cmd;
1105 
1106 	scratch_pad_1 = megasas_readl
1107 		(instance, &instance->reg_set->outbound_scratch_pad_1);
1108 
1109 	cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
1110 
1111 	if (instance->adapter_type == INVADER_SERIES) {
1112 		cur_fw_64bit_dma_capable =
1113 			(scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;
1114 
1115 		if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {
1116 			dev_err(&instance->pdev->dev, "Driver was operating on 64bit "
1117 				"DMA mask, but upcoming FW does not support 64bit DMA mask\n");
1118 			megaraid_sas_kill_hba(instance);
1119 			ret = 1;
1120 			goto fail_fw_init;
1121 		}
1122 	}
1123 
1124 	if (instance->is_rdpq && !cur_rdpq_mode) {
1125 		dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
1126 			" from RDPQ mode to non RDPQ mode\n");
1127 		ret = 1;
1128 		goto fail_fw_init;
1129 	}
1130 
1131 	cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ?
1132 							true : false;
1133 
1134 	if ((instance->low_latency_index_start ==
1135 		MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)
1136 		instance->perf_mode = MR_BALANCED_PERF_MODE;
1137 
1138 	dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n",
1139 		MEGASAS_PERF_MODE_2STR(instance->perf_mode),
1140 		instance->low_latency_index_start);
1141 
1142 	instance->fw_sync_cache_support = (scratch_pad_1 &
1143 		MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
1144 	dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
1145 		 instance->fw_sync_cache_support ? "Yes" : "No");
1146 
1147 	memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
1148 
1149 	IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
1150 	IOCInitMessage->WhoInit	= MPI2_WHOINIT_HOST_DRIVER;
1151 	IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
1152 	IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
1153 	IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
1154 
1155 	IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
1156 	IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
1157 			cpu_to_le64(fusion->rdpq_phys) :
1158 			cpu_to_le64(fusion->reply_frames_desc_phys[0]);
1159 	IOCInitMessage->MsgFlags = instance->is_rdpq ?
1160 			MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
1161 	IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
1162 	IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));
1163 	IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count;
1164 	IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
1165 
1166 	time = ktime_get_real();
1167 	/* Convert to milliseconds as per FW requirement */
1168 	IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));
1169 
1170 	init_frame = (struct megasas_init_frame *)cmd->frame;
1171 	memset(init_frame, 0, IOC_INIT_FRAME_SIZE);
1172 
1173 	frame_hdr = &cmd->frame->hdr;
1174 	frame_hdr->cmd_status = 0xFF;
1175 	frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
1176 
1177 	init_frame->cmd	= MFI_CMD_INIT;
1178 	init_frame->cmd_status = 0xFF;
1179 
1180 	drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
1181 
1182 	/* driver support Extended MSIX */
1183 	if (instance->adapter_type >= INVADER_SERIES)
1184 		drv_ops->mfi_capabilities.support_additional_msix = 1;
1185 	/* driver supports HA / Remote LUN over Fast Path interface */
1186 	drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
1187 
1188 	drv_ops->mfi_capabilities.support_max_255lds = 1;
1189 	drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
1190 	drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
1191 
1192 	if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
1193 		drv_ops->mfi_capabilities.support_ext_io_size = 1;
1194 
1195 	drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
1196 	if (!dual_qdepth_disable)
1197 		drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
1198 
1199 	drv_ops->mfi_capabilities.support_qd_throttling = 1;
1200 	drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
1201 	drv_ops->mfi_capabilities.support_nvme_passthru = 1;
1202 	drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1;
1203 
1204 	if (instance->consistent_mask_64bit)
1205 		drv_ops->mfi_capabilities.support_64bit_mode = 1;
1206 
1207 	/* Convert capability to LE32 */
1208 	cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
1209 
1210 	sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);
1211 	if (instance->system_info_buf && sys_info) {
1212 		memcpy(instance->system_info_buf->systemId, sys_info,
1213 			strlen(sys_info) > 64 ? 64 : strlen(sys_info));
1214 		instance->system_info_buf->systemIdLength =
1215 			strlen(sys_info) > 64 ? 64 : strlen(sys_info);
1216 		init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));
1217 		init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));
1218 	}
1219 
1220 	init_frame->queue_info_new_phys_addr_hi =
1221 		cpu_to_le32(upper_32_bits(ioc_init_handle));
1222 	init_frame->queue_info_new_phys_addr_lo =
1223 		cpu_to_le32(lower_32_bits(ioc_init_handle));
1224 	init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
1225 
1226 	/*
1227 	 * Each bit in replyqueue_mask represents one group of MSI-x vectors
1228 	 * (each group has 8 vectors)
1229 	 */
1230 	switch (instance->perf_mode) {
1231 	case MR_BALANCED_PERF_MODE:
1232 		init_frame->replyqueue_mask =
1233 		       cpu_to_le16(~(~0 << instance->low_latency_index_start/8));
1234 		break;
1235 	case MR_IOPS_PERF_MODE:
1236 		init_frame->replyqueue_mask =
1237 		       cpu_to_le16(~(~0 << instance->msix_vectors/8));
1238 		break;
1239 	}
1240 
1241 
1242 	req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
1243 	req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
1244 	req_desc.MFAIo.RequestFlags =
1245 		(MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
1246 		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1247 
1248 	/*
1249 	 * disable the intr before firing the init frame
1250 	 */
1251 	instance->instancet->disable_intr(instance);
1252 
1253 	for (i = 0; i < (10 * 1000); i += 20) {
1254 		if (megasas_readl(instance, &instance->reg_set->doorbell) & 1)
1255 			msleep(20);
1256 		else
1257 			break;
1258 	}
1259 
1260 	/* For AERO also, IOC_INIT requires 64 bit descriptor write */
1261 	megasas_write_64bit_req_desc(instance, &req_desc);
1262 
1263 	wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS);
1264 
1265 	frame_hdr = &cmd->frame->hdr;
1266 	if (frame_hdr->cmd_status != 0) {
1267 		ret = 1;
1268 		goto fail_fw_init;
1269 	}
1270 
1271 	if (instance->adapter_type >= AERO_SERIES) {
1272 		scratch_pad_1 = megasas_readl
1273 			(instance, &instance->reg_set->outbound_scratch_pad_1);
1274 
1275 		instance->atomic_desc_support =
1276 			(scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0;
1277 
1278 		dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n",
1279 			instance->atomic_desc_support ? "Yes" : "No");
1280 	}
1281 
1282 	return 0;
1283 
1284 fail_fw_init:
1285 	dev_err(&instance->pdev->dev,
1286 		"Init cmd return status FAILED for SCSI host %d\n",
1287 		instance->host->host_no);
1288 
1289 	return ret;
1290 }
1291 
1292 /**
1293  * megasas_sync_pd_seq_num -	JBOD SEQ MAP
1294  * @instance:		Adapter soft state
1295  * @pend:		set to 1, if it is pended jbod map.
1296  *
1297  * Issue Jbod map to the firmware. If it is pended command,
1298  * issue command and return. If it is first instance of jbod map
1299  * issue and receive command.
1300  */
1301 int
1302 megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
1303 	int ret = 0;
1304 	size_t pd_seq_map_sz;
1305 	struct megasas_cmd *cmd;
1306 	struct megasas_dcmd_frame *dcmd;
1307 	struct fusion_context *fusion = instance->ctrl_context;
1308 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1309 	dma_addr_t pd_seq_h;
1310 
1311 	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
1312 	pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
1313 	pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES);
1314 
1315 	cmd = megasas_get_cmd(instance);
1316 	if (!cmd) {
1317 		dev_err(&instance->pdev->dev,
1318 			"Could not get mfi cmd. Fail from %s %d\n",
1319 			__func__, __LINE__);
1320 		return -ENOMEM;
1321 	}
1322 
1323 	dcmd = &cmd->frame->dcmd;
1324 
1325 	memset(pd_sync, 0, pd_seq_map_sz);
1326 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1327 
1328 	if (pend) {
1329 		dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1330 		dcmd->flags = MFI_FRAME_DIR_WRITE;
1331 		instance->jbod_seq_cmd = cmd;
1332 	} else {
1333 		dcmd->flags = MFI_FRAME_DIR_READ;
1334 	}
1335 
1336 	dcmd->cmd = MFI_CMD_DCMD;
1337 	dcmd->cmd_status = 0xFF;
1338 	dcmd->sge_count = 1;
1339 	dcmd->timeout = 0;
1340 	dcmd->pad_0 = 0;
1341 	dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
1342 	dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
1343 
1344 	megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz);
1345 
1346 	if (pend) {
1347 		instance->instancet->issue_dcmd(instance, cmd);
1348 		return 0;
1349 	}
1350 
1351 	/* Below code is only for non pended DCMD */
1352 	if (!instance->mask_interrupts)
1353 		ret = megasas_issue_blocked_cmd(instance, cmd,
1354 			MFI_IO_TIMEOUT_SECS);
1355 	else
1356 		ret = megasas_issue_polled(instance, cmd);
1357 
1358 	if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
1359 		dev_warn(&instance->pdev->dev,
1360 			"driver supports max %d JBOD, but FW reports %d\n",
1361 			MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
1362 		ret = -EINVAL;
1363 	}
1364 
1365 	if (ret == DCMD_TIMEOUT)
1366 		dev_warn(&instance->pdev->dev,
1367 			 "%s DCMD timed out, continue without JBOD sequence map\n",
1368 			 __func__);
1369 
1370 	if (ret == DCMD_SUCCESS)
1371 		instance->pd_seq_map_id++;
1372 
1373 	megasas_return_cmd(instance, cmd);
1374 	return ret;
1375 }
1376 
1377 /*
1378  * megasas_get_ld_map_info -	Returns FW's ld_map structure
1379  * @instance:				Adapter soft state
1380  * @pend:				Pend the command or not
1381  * Issues an internal command (DCMD) to get the FW's controller PD
1382  * list structure.  This information is mainly used to find out SYSTEM
1383  * supported by the FW.
1384  * dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO
1385  * dcmd.mbox.b[0]	- number of LDs being sync'd
1386  * dcmd.mbox.b[1]	- 0 - complete command immediately.
1387  *			- 1 - pend till config change
1388  * dcmd.mbox.b[2]	- 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP
1389  *			- 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and
1390  *				uses extended struct MR_FW_RAID_MAP_EXT
1391  */
1392 static int
1393 megasas_get_ld_map_info(struct megasas_instance *instance)
1394 {
1395 	int ret = 0;
1396 	struct megasas_cmd *cmd;
1397 	struct megasas_dcmd_frame *dcmd;
1398 	void *ci;
1399 	dma_addr_t ci_h = 0;
1400 	u32 size_map_info;
1401 	struct fusion_context *fusion;
1402 
1403 	cmd = megasas_get_cmd(instance);
1404 
1405 	if (!cmd) {
1406 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1407 		return -ENOMEM;
1408 	}
1409 
1410 	fusion = instance->ctrl_context;
1411 
1412 	if (!fusion) {
1413 		megasas_return_cmd(instance, cmd);
1414 		return -ENXIO;
1415 	}
1416 
1417 	dcmd = &cmd->frame->dcmd;
1418 
1419 	size_map_info = fusion->current_map_sz;
1420 
1421 	ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1422 	ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1423 
1424 	if (!ci) {
1425 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1426 		megasas_return_cmd(instance, cmd);
1427 		return -ENOMEM;
1428 	}
1429 
1430 	memset(ci, 0, fusion->max_map_sz);
1431 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1432 	dcmd->cmd = MFI_CMD_DCMD;
1433 	dcmd->cmd_status = 0xFF;
1434 	dcmd->sge_count = 1;
1435 	dcmd->flags = MFI_FRAME_DIR_READ;
1436 	dcmd->timeout = 0;
1437 	dcmd->pad_0 = 0;
1438 	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1439 	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1440 
1441 	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1442 
1443 	if (!instance->mask_interrupts)
1444 		ret = megasas_issue_blocked_cmd(instance, cmd,
1445 			MFI_IO_TIMEOUT_SECS);
1446 	else
1447 		ret = megasas_issue_polled(instance, cmd);
1448 
1449 	if (ret == DCMD_TIMEOUT)
1450 		dev_warn(&instance->pdev->dev,
1451 			 "%s DCMD timed out, RAID map is disabled\n",
1452 			 __func__);
1453 
1454 	megasas_return_cmd(instance, cmd);
1455 
1456 	return ret;
1457 }
1458 
1459 u8
1460 megasas_get_map_info(struct megasas_instance *instance)
1461 {
1462 	struct fusion_context *fusion = instance->ctrl_context;
1463 
1464 	fusion->fast_path_io = 0;
1465 	if (!megasas_get_ld_map_info(instance)) {
1466 		if (MR_ValidateMapInfo(instance, instance->map_id)) {
1467 			fusion->fast_path_io = 1;
1468 			return 0;
1469 		}
1470 	}
1471 	return 1;
1472 }
1473 
1474 /*
1475  * megasas_sync_map_info -	Returns FW's ld_map structure
1476  * @instance:				Adapter soft state
1477  *
1478  * Issues an internal command (DCMD) to get the FW's controller PD
1479  * list structure.  This information is mainly used to find out SYSTEM
1480  * supported by the FW.
1481  */
1482 int
1483 megasas_sync_map_info(struct megasas_instance *instance)
1484 {
1485 	int i;
1486 	struct megasas_cmd *cmd;
1487 	struct megasas_dcmd_frame *dcmd;
1488 	u16 num_lds;
1489 	struct fusion_context *fusion;
1490 	struct MR_LD_TARGET_SYNC *ci = NULL;
1491 	struct MR_DRV_RAID_MAP_ALL *map;
1492 	struct MR_LD_RAID  *raid;
1493 	struct MR_LD_TARGET_SYNC *ld_sync;
1494 	dma_addr_t ci_h = 0;
1495 	u32 size_map_info;
1496 
1497 	cmd = megasas_get_cmd(instance);
1498 
1499 	if (!cmd) {
1500 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1501 		return -ENOMEM;
1502 	}
1503 
1504 	fusion = instance->ctrl_context;
1505 
1506 	if (!fusion) {
1507 		megasas_return_cmd(instance, cmd);
1508 		return 1;
1509 	}
1510 
1511 	map = fusion->ld_drv_map[instance->map_id & 1];
1512 
1513 	num_lds = le16_to_cpu(map->raidMap.ldCount);
1514 
1515 	dcmd = &cmd->frame->dcmd;
1516 
1517 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1518 
1519 	ci = (struct MR_LD_TARGET_SYNC *)
1520 	  fusion->ld_map[(instance->map_id - 1) & 1];
1521 	memset(ci, 0, fusion->max_map_sz);
1522 
1523 	ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1524 
1525 	ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1526 
1527 	for (i = 0; i < num_lds; i++, ld_sync++) {
1528 		raid = MR_LdRaidGet(i, map);
1529 		ld_sync->targetId = MR_GetLDTgtId(i, map);
1530 		ld_sync->seqNum = raid->seqNum;
1531 	}
1532 
1533 	size_map_info = fusion->current_map_sz;
1534 
1535 	dcmd->cmd = MFI_CMD_DCMD;
1536 	dcmd->cmd_status = 0xFF;
1537 	dcmd->sge_count = 1;
1538 	dcmd->flags = MFI_FRAME_DIR_WRITE;
1539 	dcmd->timeout = 0;
1540 	dcmd->pad_0 = 0;
1541 	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1542 	dcmd->mbox.b[0] = num_lds;
1543 	dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1544 	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1545 
1546 	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1547 
1548 	instance->map_update_cmd = cmd;
1549 
1550 	instance->instancet->issue_dcmd(instance, cmd);
1551 
1552 	return 0;
1553 }
1554 
1555 /*
1556  * meagasas_display_intel_branding - Display branding string
1557  * @instance: per adapter object
1558  *
1559  * Return nothing.
1560  */
1561 static void
1562 megasas_display_intel_branding(struct megasas_instance *instance)
1563 {
1564 	if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1565 		return;
1566 
1567 	switch (instance->pdev->device) {
1568 	case PCI_DEVICE_ID_LSI_INVADER:
1569 		switch (instance->pdev->subsystem_device) {
1570 		case MEGARAID_INTEL_RS3DC080_SSDID:
1571 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1572 				instance->host->host_no,
1573 				MEGARAID_INTEL_RS3DC080_BRANDING);
1574 			break;
1575 		case MEGARAID_INTEL_RS3DC040_SSDID:
1576 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1577 				instance->host->host_no,
1578 				MEGARAID_INTEL_RS3DC040_BRANDING);
1579 			break;
1580 		case MEGARAID_INTEL_RS3SC008_SSDID:
1581 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1582 				instance->host->host_no,
1583 				MEGARAID_INTEL_RS3SC008_BRANDING);
1584 			break;
1585 		case MEGARAID_INTEL_RS3MC044_SSDID:
1586 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1587 				instance->host->host_no,
1588 				MEGARAID_INTEL_RS3MC044_BRANDING);
1589 			break;
1590 		default:
1591 			break;
1592 		}
1593 		break;
1594 	case PCI_DEVICE_ID_LSI_FURY:
1595 		switch (instance->pdev->subsystem_device) {
1596 		case MEGARAID_INTEL_RS3WC080_SSDID:
1597 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1598 				instance->host->host_no,
1599 				MEGARAID_INTEL_RS3WC080_BRANDING);
1600 			break;
1601 		case MEGARAID_INTEL_RS3WC040_SSDID:
1602 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1603 				instance->host->host_no,
1604 				MEGARAID_INTEL_RS3WC040_BRANDING);
1605 			break;
1606 		default:
1607 			break;
1608 		}
1609 		break;
1610 	case PCI_DEVICE_ID_LSI_CUTLASS_52:
1611 	case PCI_DEVICE_ID_LSI_CUTLASS_53:
1612 		switch (instance->pdev->subsystem_device) {
1613 		case MEGARAID_INTEL_RMS3BC160_SSDID:
1614 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1615 				instance->host->host_no,
1616 				MEGARAID_INTEL_RMS3BC160_BRANDING);
1617 			break;
1618 		default:
1619 			break;
1620 		}
1621 		break;
1622 	default:
1623 		break;
1624 	}
1625 }
1626 
1627 /**
1628  * megasas_allocate_raid_maps -	Allocate memory for RAID maps
1629  * @instance:				Adapter soft state
1630  *
1631  * return:				if success: return 0
1632  *					failed:  return -ENOMEM
1633  */
1634 static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
1635 {
1636 	struct fusion_context *fusion;
1637 	int i = 0;
1638 
1639 	fusion = instance->ctrl_context;
1640 
1641 	fusion->drv_map_pages = get_order(fusion->drv_map_sz);
1642 
1643 	for (i = 0; i < 2; i++) {
1644 		fusion->ld_map[i] = NULL;
1645 
1646 		fusion->ld_drv_map[i] = (void *)
1647 			__get_free_pages(__GFP_ZERO | GFP_KERNEL,
1648 					 fusion->drv_map_pages);
1649 
1650 		if (!fusion->ld_drv_map[i]) {
1651 			fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);
1652 
1653 			if (!fusion->ld_drv_map[i]) {
1654 				dev_err(&instance->pdev->dev,
1655 					"Could not allocate memory for local map"
1656 					" size requested: %d\n",
1657 					fusion->drv_map_sz);
1658 				goto ld_drv_map_alloc_fail;
1659 			}
1660 		}
1661 	}
1662 
1663 	for (i = 0; i < 2; i++) {
1664 		fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
1665 						       fusion->max_map_sz,
1666 						       &fusion->ld_map_phys[i],
1667 						       GFP_KERNEL);
1668 		if (!fusion->ld_map[i]) {
1669 			dev_err(&instance->pdev->dev,
1670 				"Could not allocate memory for map info %s:%d\n",
1671 				__func__, __LINE__);
1672 			goto ld_map_alloc_fail;
1673 		}
1674 	}
1675 
1676 	return 0;
1677 
1678 ld_map_alloc_fail:
1679 	for (i = 0; i < 2; i++) {
1680 		if (fusion->ld_map[i])
1681 			dma_free_coherent(&instance->pdev->dev,
1682 					  fusion->max_map_sz,
1683 					  fusion->ld_map[i],
1684 					  fusion->ld_map_phys[i]);
1685 	}
1686 
1687 ld_drv_map_alloc_fail:
1688 	for (i = 0; i < 2; i++) {
1689 		if (fusion->ld_drv_map[i]) {
1690 			if (is_vmalloc_addr(fusion->ld_drv_map[i]))
1691 				vfree(fusion->ld_drv_map[i]);
1692 			else
1693 				free_pages((ulong)fusion->ld_drv_map[i],
1694 					   fusion->drv_map_pages);
1695 		}
1696 	}
1697 
1698 	return -ENOMEM;
1699 }
1700 
1701 /**
1702  * megasas_configure_queue_sizes -	Calculate size of request desc queue,
1703  *					reply desc queue,
1704  *					IO request frame queue, set can_queue.
1705  * @instance:				Adapter soft state
1706  * @return:				void
1707  */
1708 static inline
1709 void megasas_configure_queue_sizes(struct megasas_instance *instance)
1710 {
1711 	struct fusion_context *fusion;
1712 	u16 max_cmd;
1713 
1714 	fusion = instance->ctrl_context;
1715 	max_cmd = instance->max_fw_cmds;
1716 
1717 	if (instance->adapter_type >= VENTURA_SERIES)
1718 		instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;
1719 	else
1720 		instance->max_mpt_cmds = instance->max_fw_cmds;
1721 
1722 	instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds;
1723 	instance->cur_can_queue = instance->max_scsi_cmds;
1724 	instance->host->can_queue = instance->cur_can_queue;
1725 
1726 	fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;
1727 
1728 	fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *
1729 					  instance->max_mpt_cmds;
1730 	fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *
1731 					(fusion->reply_q_depth);
1732 	fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1733 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1734 		 * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */
1735 }
1736 
1737 static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)
1738 {
1739 	struct fusion_context *fusion;
1740 	struct megasas_cmd *cmd;
1741 
1742 	fusion = instance->ctrl_context;
1743 
1744 	cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL);
1745 
1746 	if (!cmd) {
1747 		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1748 			__func__, __LINE__);
1749 		return -ENOMEM;
1750 	}
1751 
1752 	cmd->frame = dma_alloc_coherent(&instance->pdev->dev,
1753 					IOC_INIT_FRAME_SIZE,
1754 					&cmd->frame_phys_addr, GFP_KERNEL);
1755 
1756 	if (!cmd->frame) {
1757 		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1758 			__func__, __LINE__);
1759 		kfree(cmd);
1760 		return -ENOMEM;
1761 	}
1762 
1763 	fusion->ioc_init_cmd = cmd;
1764 	return 0;
1765 }
1766 
1767 /**
1768  * megasas_free_ioc_init_cmd -	Free IOC INIT command frame
1769  * @instance:		Adapter soft state
1770  */
1771 static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)
1772 {
1773 	struct fusion_context *fusion;
1774 
1775 	fusion = instance->ctrl_context;
1776 
1777 	if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)
1778 		dma_free_coherent(&instance->pdev->dev,
1779 				  IOC_INIT_FRAME_SIZE,
1780 				  fusion->ioc_init_cmd->frame,
1781 				  fusion->ioc_init_cmd->frame_phys_addr);
1782 
1783 	kfree(fusion->ioc_init_cmd);
1784 }
1785 
1786 /**
1787  * megasas_init_adapter_fusion -	Initializes the FW
1788  * @instance:		Adapter soft state
1789  *
1790  * This is the main function for initializing firmware.
1791  */
1792 static u32
1793 megasas_init_adapter_fusion(struct megasas_instance *instance)
1794 {
1795 	struct fusion_context *fusion;
1796 	u32 scratch_pad_1;
1797 	int i = 0, count;
1798 	u32 status_reg;
1799 
1800 	fusion = instance->ctrl_context;
1801 
1802 	megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);
1803 
1804 	/*
1805 	 * Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames
1806 	 */
1807 	instance->max_mfi_cmds =
1808 		MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1809 
1810 	megasas_configure_queue_sizes(instance);
1811 
1812 	scratch_pad_1 = megasas_readl(instance,
1813 				      &instance->reg_set->outbound_scratch_pad_1);
1814 	/* If scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,
1815 	 * Firmware support extended IO chain frame which is 4 times more than
1816 	 * legacy Firmware.
1817 	 * Legacy Firmware - Frame size is (8 * 128) = 1K
1818 	 * 1M IO Firmware  - Frame size is (8 * 128 * 4)  = 4K
1819 	 */
1820 	if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1821 		instance->max_chain_frame_sz =
1822 			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1823 			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1824 	else
1825 		instance->max_chain_frame_sz =
1826 			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1827 			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1828 
1829 	if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1830 		dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1831 			instance->max_chain_frame_sz,
1832 			MEGASAS_CHAIN_FRAME_SZ_MIN);
1833 		instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1834 	}
1835 
1836 	fusion->max_sge_in_main_msg =
1837 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1838 			- offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1839 
1840 	fusion->max_sge_in_chain =
1841 		instance->max_chain_frame_sz
1842 			/ sizeof(union MPI2_SGE_IO_UNION);
1843 
1844 	instance->max_num_sge =
1845 		rounddown_pow_of_two(fusion->max_sge_in_main_msg
1846 			+ fusion->max_sge_in_chain - 2);
1847 
1848 	/* Used for pass thru MFI frame (DCMD) */
1849 	fusion->chain_offset_mfi_pthru =
1850 		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1851 
1852 	fusion->chain_offset_io_request =
1853 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1854 		 sizeof(union MPI2_SGE_IO_UNION))/16;
1855 
1856 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1857 	count += instance->iopoll_q_count;
1858 
1859 	for (i = 0 ; i < count; i++)
1860 		fusion->last_reply_idx[i] = 0;
1861 
1862 	/*
1863 	 * For fusion adapters, 3 commands for IOCTL and 8 commands
1864 	 * for driver's internal DCMDs.
1865 	 */
1866 	instance->max_scsi_cmds = instance->max_fw_cmds -
1867 				(MEGASAS_FUSION_INTERNAL_CMDS +
1868 				MEGASAS_FUSION_IOCTL_CMDS);
1869 	sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1870 
1871 	for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++)
1872 		atomic_set(&fusion->busy_mq_poll[i], 0);
1873 
1874 	if (megasas_alloc_ioc_init_frame(instance))
1875 		return 1;
1876 
1877 	/*
1878 	 * Allocate memory for descriptors
1879 	 * Create a pool of commands
1880 	 */
1881 	if (megasas_alloc_cmds(instance))
1882 		goto fail_alloc_mfi_cmds;
1883 	if (megasas_alloc_cmds_fusion(instance))
1884 		goto fail_alloc_cmds;
1885 
1886 	if (megasas_ioc_init_fusion(instance)) {
1887 		status_reg = instance->instancet->read_fw_status_reg(instance);
1888 		if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) &&
1889 		    (status_reg & MFI_RESET_ADAPTER)) {
1890 			/* Do a chip reset and then retry IOC INIT once */
1891 			if (megasas_adp_reset_wait_for_ready
1892 				(instance, true, 0) == FAILED)
1893 				goto fail_ioc_init;
1894 
1895 			if (megasas_ioc_init_fusion(instance))
1896 				goto fail_ioc_init;
1897 		} else {
1898 			goto fail_ioc_init;
1899 		}
1900 	}
1901 
1902 	megasas_display_intel_branding(instance);
1903 	if (megasas_get_ctrl_info(instance)) {
1904 		dev_err(&instance->pdev->dev,
1905 			"Could not get controller info. Fail from %s %d\n",
1906 			__func__, __LINE__);
1907 		goto fail_ioc_init;
1908 	}
1909 
1910 	instance->flag_ieee = 1;
1911 	instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;
1912 	instance->threshold_reply_count = instance->max_fw_cmds / 4;
1913 	fusion->fast_path_io = 0;
1914 
1915 	if (megasas_allocate_raid_maps(instance))
1916 		goto fail_ioc_init;
1917 
1918 	if (!megasas_get_map_info(instance))
1919 		megasas_sync_map_info(instance);
1920 
1921 	return 0;
1922 
1923 fail_ioc_init:
1924 	megasas_free_cmds_fusion(instance);
1925 fail_alloc_cmds:
1926 	megasas_free_cmds(instance);
1927 fail_alloc_mfi_cmds:
1928 	megasas_free_ioc_init_cmd(instance);
1929 	return 1;
1930 }
1931 
1932 /**
1933  * megasas_fault_detect_work	-	Worker function of
1934  *					FW fault handling workqueue.
1935  * @work:	FW fault work struct
1936  */
1937 static void
1938 megasas_fault_detect_work(struct work_struct *work)
1939 {
1940 	struct megasas_instance *instance =
1941 		container_of(work, struct megasas_instance,
1942 			     fw_fault_work.work);
1943 	u32 fw_state, dma_state, status;
1944 
1945 	/* Check the fw state */
1946 	fw_state = instance->instancet->read_fw_status_reg(instance) &
1947 			MFI_STATE_MASK;
1948 
1949 	if (fw_state == MFI_STATE_FAULT) {
1950 		dma_state = instance->instancet->read_fw_status_reg(instance) &
1951 				MFI_STATE_DMADONE;
1952 		/* Start collecting crash, if DMA bit is done */
1953 		if (instance->crash_dump_drv_support &&
1954 		    instance->crash_dump_app_support && dma_state) {
1955 			megasas_fusion_crash_dump(instance);
1956 		} else {
1957 			if (instance->unload == 0) {
1958 				status = megasas_reset_fusion(instance->host, 0);
1959 				if (status != SUCCESS) {
1960 					dev_err(&instance->pdev->dev,
1961 						"Failed from %s %d, do not re-arm timer\n",
1962 						__func__, __LINE__);
1963 					return;
1964 				}
1965 			}
1966 		}
1967 	}
1968 
1969 	if (instance->fw_fault_work_q)
1970 		queue_delayed_work(instance->fw_fault_work_q,
1971 			&instance->fw_fault_work,
1972 			msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1973 }
1974 
1975 int
1976 megasas_fusion_start_watchdog(struct megasas_instance *instance)
1977 {
1978 	/* Check if the Fault WQ is already started */
1979 	if (instance->fw_fault_work_q)
1980 		return SUCCESS;
1981 
1982 	INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work);
1983 
1984 	snprintf(instance->fault_handler_work_q_name,
1985 		 sizeof(instance->fault_handler_work_q_name),
1986 		 "poll_megasas%d_status", instance->host->host_no);
1987 
1988 	instance->fw_fault_work_q =
1989 		create_singlethread_workqueue(instance->fault_handler_work_q_name);
1990 	if (!instance->fw_fault_work_q) {
1991 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
1992 			__func__, __LINE__);
1993 		return FAILED;
1994 	}
1995 
1996 	queue_delayed_work(instance->fw_fault_work_q,
1997 			   &instance->fw_fault_work,
1998 			   msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1999 
2000 	return SUCCESS;
2001 }
2002 
2003 void
2004 megasas_fusion_stop_watchdog(struct megasas_instance *instance)
2005 {
2006 	struct workqueue_struct *wq;
2007 
2008 	if (instance->fw_fault_work_q) {
2009 		wq = instance->fw_fault_work_q;
2010 		instance->fw_fault_work_q = NULL;
2011 		if (!cancel_delayed_work_sync(&instance->fw_fault_work))
2012 			flush_workqueue(wq);
2013 		destroy_workqueue(wq);
2014 	}
2015 }
2016 
2017 /**
2018  * map_cmd_status -	Maps FW cmd status to OS cmd status
2019  * @fusion:		fusion context
2020  * @scmd:		Pointer to cmd
2021  * @status:		status of cmd returned by FW
2022  * @ext_status:		ext status of cmd returned by FW
2023  * @data_length:	command data length
2024  * @sense:		command sense data
2025  */
2026 static void
2027 map_cmd_status(struct fusion_context *fusion,
2028 		struct scsi_cmnd *scmd, u8 status, u8 ext_status,
2029 		u32 data_length, u8 *sense)
2030 {
2031 	u8 cmd_type;
2032 	int resid;
2033 
2034 	cmd_type = megasas_cmd_type(scmd);
2035 	switch (status) {
2036 
2037 	case MFI_STAT_OK:
2038 		scmd->result = DID_OK << 16;
2039 		break;
2040 
2041 	case MFI_STAT_SCSI_IO_FAILED:
2042 	case MFI_STAT_LD_INIT_IN_PROGRESS:
2043 		scmd->result = (DID_ERROR << 16) | ext_status;
2044 		break;
2045 
2046 	case MFI_STAT_SCSI_DONE_WITH_ERROR:
2047 
2048 		scmd->result = (DID_OK << 16) | ext_status;
2049 		if (ext_status == SAM_STAT_CHECK_CONDITION) {
2050 			memcpy(scmd->sense_buffer, sense,
2051 			       SCSI_SENSE_BUFFERSIZE);
2052 		}
2053 
2054 		/*
2055 		 * If the  IO request is partially completed, then MR FW will
2056 		 * update "io_request->DataLength" field with actual number of
2057 		 * bytes transferred.Driver will set residual bytes count in
2058 		 * SCSI command structure.
2059 		 */
2060 		resid = (scsi_bufflen(scmd) - data_length);
2061 		scsi_set_resid(scmd, resid);
2062 
2063 		if (resid &&
2064 			((cmd_type == READ_WRITE_LDIO) ||
2065 			(cmd_type == READ_WRITE_SYSPDIO)))
2066 			scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
2067 				" requested/completed 0x%x/0x%x\n",
2068 				status, scsi_bufflen(scmd), data_length);
2069 		break;
2070 
2071 	case MFI_STAT_LD_OFFLINE:
2072 	case MFI_STAT_DEVICE_NOT_FOUND:
2073 		scmd->result = DID_BAD_TARGET << 16;
2074 		break;
2075 	case MFI_STAT_CONFIG_SEQ_MISMATCH:
2076 		scmd->result = DID_IMM_RETRY << 16;
2077 		break;
2078 	default:
2079 		scmd->result = DID_ERROR << 16;
2080 		break;
2081 	}
2082 }
2083 
2084 /**
2085  * megasas_is_prp_possible -
2086  * Checks if native NVMe PRPs can be built for the IO
2087  *
2088  * @instance:		Adapter soft state
2089  * @scmd:		SCSI command from the mid-layer
2090  * @sge_count:		scatter gather element count.
2091  *
2092  * Returns:		true: PRPs can be built
2093  *			false: IEEE SGLs needs to be built
2094  */
2095 static bool
2096 megasas_is_prp_possible(struct megasas_instance *instance,
2097 			struct scsi_cmnd *scmd, int sge_count)
2098 {
2099 	u32 data_length = 0;
2100 	struct scatterlist *sg_scmd;
2101 	bool build_prp = false;
2102 	u32 mr_nvme_pg_size;
2103 
2104 	mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2105 				MR_DEFAULT_NVME_PAGE_SIZE);
2106 	data_length = scsi_bufflen(scmd);
2107 	sg_scmd = scsi_sglist(scmd);
2108 
2109 	/*
2110 	 * NVMe uses one PRP for each page (or part of a page)
2111 	 * look at the data length - if 4 pages or less then IEEE is OK
2112 	 * if  > 5 pages then we need to build a native SGL
2113 	 * if > 4 and <= 5 pages, then check physical address of 1st SG entry
2114 	 * if this first size in the page is >= the residual beyond 4 pages
2115 	 * then use IEEE, otherwise use native SGL
2116 	 */
2117 
2118 	if (data_length > (mr_nvme_pg_size * 5)) {
2119 		build_prp = true;
2120 	} else if ((data_length > (mr_nvme_pg_size * 4)) &&
2121 			(data_length <= (mr_nvme_pg_size * 5)))  {
2122 		/* check if 1st SG entry size is < residual beyond 4 pages */
2123 		if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
2124 			build_prp = true;
2125 	}
2126 
2127 	return build_prp;
2128 }
2129 
2130 /**
2131  * megasas_make_prp_nvme -
2132  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
2133  *
2134  * @instance:		Adapter soft state
2135  * @scmd:		SCSI command from the mid-layer
2136  * @sgl_ptr:		SGL to be filled in
2137  * @cmd:		Fusion command frame
2138  * @sge_count:		scatter gather element count.
2139  *
2140  * Returns:		true: PRPs are built
2141  *			false: IEEE SGLs needs to be built
2142  */
2143 static bool
2144 megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
2145 		      struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2146 		      struct megasas_cmd_fusion *cmd, int sge_count)
2147 {
2148 	int sge_len, offset, num_prp_in_chain = 0;
2149 	struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
2150 	u64 *ptr_sgl;
2151 	dma_addr_t ptr_sgl_phys;
2152 	u64 sge_addr;
2153 	u32 page_mask, page_mask_result;
2154 	struct scatterlist *sg_scmd;
2155 	u32 first_prp_len;
2156 	bool build_prp = false;
2157 	int data_len = scsi_bufflen(scmd);
2158 	u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2159 					MR_DEFAULT_NVME_PAGE_SIZE);
2160 
2161 	build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
2162 
2163 	if (!build_prp)
2164 		return false;
2165 
2166 	/*
2167 	 * Nvme has a very convoluted prp format.  One prp is required
2168 	 * for each page or partial page. Driver need to split up OS sg_list
2169 	 * entries if it is longer than one page or cross a page
2170 	 * boundary.  Driver also have to insert a PRP list pointer entry as
2171 	 * the last entry in each physical page of the PRP list.
2172 	 *
2173 	 * NOTE: The first PRP "entry" is actually placed in the first
2174 	 * SGL entry in the main message as IEEE 64 format.  The 2nd
2175 	 * entry in the main message is the chain element, and the rest
2176 	 * of the PRP entries are built in the contiguous pcie buffer.
2177 	 */
2178 	page_mask = mr_nvme_pg_size - 1;
2179 	ptr_sgl = (u64 *)cmd->sg_frame;
2180 	ptr_sgl_phys = cmd->sg_frame_phys_addr;
2181 	memset(ptr_sgl, 0, instance->max_chain_frame_sz);
2182 
2183 	/* Build chain frame element which holds all prps except first*/
2184 	main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
2185 	    ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
2186 
2187 	main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
2188 	main_chain_element->NextChainOffset = 0;
2189 	main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2190 					IEEE_SGE_FLAGS_SYSTEM_ADDR |
2191 					MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2192 
2193 	/* Build first prp, sge need not to be page aligned*/
2194 	ptr_first_sgl = sgl_ptr;
2195 	sg_scmd = scsi_sglist(scmd);
2196 	sge_addr = sg_dma_address(sg_scmd);
2197 	sge_len = sg_dma_len(sg_scmd);
2198 
2199 	offset = (u32)(sge_addr & page_mask);
2200 	first_prp_len = mr_nvme_pg_size - offset;
2201 
2202 	ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2203 	ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2204 
2205 	data_len -= first_prp_len;
2206 
2207 	if (sge_len > first_prp_len) {
2208 		sge_addr += first_prp_len;
2209 		sge_len -= first_prp_len;
2210 	} else if (sge_len == first_prp_len) {
2211 		sg_scmd = sg_next(sg_scmd);
2212 		sge_addr = sg_dma_address(sg_scmd);
2213 		sge_len = sg_dma_len(sg_scmd);
2214 	}
2215 
2216 	for (;;) {
2217 		offset = (u32)(sge_addr & page_mask);
2218 
2219 		/* Put PRP pointer due to page boundary*/
2220 		page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
2221 		if (unlikely(!page_mask_result)) {
2222 			scmd_printk(KERN_NOTICE,
2223 				    scmd, "page boundary ptr_sgl: 0x%p\n",
2224 				    ptr_sgl);
2225 			ptr_sgl_phys += 8;
2226 			*ptr_sgl = cpu_to_le64(ptr_sgl_phys);
2227 			ptr_sgl++;
2228 			num_prp_in_chain++;
2229 		}
2230 
2231 		*ptr_sgl = cpu_to_le64(sge_addr);
2232 		ptr_sgl++;
2233 		ptr_sgl_phys += 8;
2234 		num_prp_in_chain++;
2235 
2236 		sge_addr += mr_nvme_pg_size;
2237 		sge_len -= mr_nvme_pg_size;
2238 		data_len -= mr_nvme_pg_size;
2239 
2240 		if (data_len <= 0)
2241 			break;
2242 
2243 		if (sge_len > 0)
2244 			continue;
2245 
2246 		sg_scmd = sg_next(sg_scmd);
2247 		sge_addr = sg_dma_address(sg_scmd);
2248 		sge_len = sg_dma_len(sg_scmd);
2249 	}
2250 
2251 	main_chain_element->Length =
2252 			cpu_to_le32(num_prp_in_chain * sizeof(u64));
2253 
2254 	return build_prp;
2255 }
2256 
2257 /**
2258  * megasas_make_sgl_fusion -	Prepares 32-bit SGL
2259  * @instance:		Adapter soft state
2260  * @scp:		SCSI command from the mid-layer
2261  * @sgl_ptr:		SGL to be filled in
2262  * @cmd:		cmd we are working on
2263  * @sge_count:		sge count
2264  *
2265  */
2266 static void
2267 megasas_make_sgl_fusion(struct megasas_instance *instance,
2268 			struct scsi_cmnd *scp,
2269 			struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2270 			struct megasas_cmd_fusion *cmd, int sge_count)
2271 {
2272 	int i, sg_processed;
2273 	struct scatterlist *os_sgl;
2274 	struct fusion_context *fusion;
2275 
2276 	fusion = instance->ctrl_context;
2277 
2278 	if (instance->adapter_type >= INVADER_SERIES) {
2279 		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
2280 		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
2281 		sgl_ptr_end->Flags = 0;
2282 	}
2283 
2284 	scsi_for_each_sg(scp, os_sgl, sge_count, i) {
2285 		sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
2286 		sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
2287 		sgl_ptr->Flags = 0;
2288 		if (instance->adapter_type >= INVADER_SERIES)
2289 			if (i == sge_count - 1)
2290 				sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
2291 		sgl_ptr++;
2292 		sg_processed = i + 1;
2293 
2294 		if ((sg_processed ==  (fusion->max_sge_in_main_msg - 1)) &&
2295 		    (sge_count > fusion->max_sge_in_main_msg)) {
2296 
2297 			struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
2298 			if (instance->adapter_type >= INVADER_SERIES) {
2299 				if ((le16_to_cpu(cmd->io_request->IoFlags) &
2300 					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
2301 					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
2302 					cmd->io_request->ChainOffset =
2303 						fusion->
2304 						chain_offset_io_request;
2305 				else
2306 					cmd->io_request->ChainOffset = 0;
2307 			} else
2308 				cmd->io_request->ChainOffset =
2309 					fusion->chain_offset_io_request;
2310 
2311 			sg_chain = sgl_ptr;
2312 			/* Prepare chain element */
2313 			sg_chain->NextChainOffset = 0;
2314 			if (instance->adapter_type >= INVADER_SERIES)
2315 				sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
2316 			else
2317 				sg_chain->Flags =
2318 					(IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2319 					 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
2320 			sg_chain->Length =  cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
2321 			sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
2322 
2323 			sgl_ptr =
2324 			  (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
2325 			memset(sgl_ptr, 0, instance->max_chain_frame_sz);
2326 		}
2327 	}
2328 }
2329 
2330 /**
2331  * megasas_make_sgl -	Build Scatter Gather List(SGLs)
2332  * @scp:		SCSI command pointer
2333  * @instance:		Soft instance of controller
2334  * @cmd:		Fusion command pointer
2335  *
2336  * This function will build sgls based on device type.
2337  * For nvme drives, there is different way of building sgls in nvme native
2338  * format- PRPs(Physical Region Page).
2339  *
2340  * Returns the number of sg lists actually used, zero if the sg lists
2341  * is NULL, or -ENOMEM if the mapping failed
2342  */
2343 static
2344 int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
2345 		     struct megasas_cmd_fusion *cmd)
2346 {
2347 	int sge_count;
2348 	bool build_prp = false;
2349 	struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
2350 
2351 	sge_count = scsi_dma_map(scp);
2352 
2353 	if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
2354 		return sge_count;
2355 
2356 	sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
2357 	if ((le16_to_cpu(cmd->io_request->IoFlags) &
2358 	    MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
2359 	    (cmd->pd_interface == NVME_PD))
2360 		build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,
2361 						  cmd, sge_count);
2362 
2363 	if (!build_prp)
2364 		megasas_make_sgl_fusion(instance, scp, sgl_chain64,
2365 					cmd, sge_count);
2366 
2367 	return sge_count;
2368 }
2369 
2370 /**
2371  * megasas_set_pd_lba -	Sets PD LBA
2372  * @io_request:		IO request
2373  * @cdb_len:		cdb length
2374  * @io_info:		IO information
2375  * @scp:		SCSI command
2376  * @local_map_ptr:	Raid map
2377  * @ref_tag:		Primary reference tag
2378  *
2379  * Used to set the PD LBA in CDB for FP IOs
2380  */
2381 static void
2382 megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
2383 		   struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
2384 		   struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
2385 {
2386 	struct MR_LD_RAID *raid;
2387 	u16 ld;
2388 	u64 start_blk = io_info->pdBlock;
2389 	u8 *cdb = io_request->CDB.CDB32;
2390 	u32 num_blocks = io_info->numBlocks;
2391 	u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
2392 
2393 	/* Check if T10 PI (DIF) is enabled for this LD */
2394 	ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);
2395 	raid = MR_LdRaidGet(ld, local_map_ptr);
2396 	if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
2397 		memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2398 		cdb[0] =  MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
2399 		cdb[7] =  MEGASAS_SCSI_ADDL_CDB_LEN;
2400 
2401 		if (scp->sc_data_direction == DMA_FROM_DEVICE)
2402 			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
2403 		else
2404 			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
2405 		cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
2406 
2407 		/* LBA */
2408 		cdb[12] = (u8)((start_blk >> 56) & 0xff);
2409 		cdb[13] = (u8)((start_blk >> 48) & 0xff);
2410 		cdb[14] = (u8)((start_blk >> 40) & 0xff);
2411 		cdb[15] = (u8)((start_blk >> 32) & 0xff);
2412 		cdb[16] = (u8)((start_blk >> 24) & 0xff);
2413 		cdb[17] = (u8)((start_blk >> 16) & 0xff);
2414 		cdb[18] = (u8)((start_blk >> 8) & 0xff);
2415 		cdb[19] = (u8)(start_blk & 0xff);
2416 
2417 		/* Logical block reference tag */
2418 		io_request->CDB.EEDP32.PrimaryReferenceTag =
2419 			cpu_to_be32(ref_tag);
2420 		io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
2421 		io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */
2422 
2423 		/* Transfer length */
2424 		cdb[28] = (u8)((num_blocks >> 24) & 0xff);
2425 		cdb[29] = (u8)((num_blocks >> 16) & 0xff);
2426 		cdb[30] = (u8)((num_blocks >> 8) & 0xff);
2427 		cdb[31] = (u8)(num_blocks & 0xff);
2428 
2429 		/* set SCSI IO EEDPFlags */
2430 		if (scp->sc_data_direction == DMA_FROM_DEVICE) {
2431 			io_request->EEDPFlags = cpu_to_le16(
2432 				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG  |
2433 				MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2434 				MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
2435 				MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
2436 				MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
2437 				MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
2438 		} else {
2439 			io_request->EEDPFlags = cpu_to_le16(
2440 				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2441 				MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
2442 		}
2443 		io_request->Control |= cpu_to_le32((0x4 << 26));
2444 		io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
2445 	} else {
2446 		/* Some drives don't support 16/12 byte CDB's, convert to 10 */
2447 		if (((cdb_len == 12) || (cdb_len == 16)) &&
2448 		    (start_blk <= 0xffffffff)) {
2449 			if (cdb_len == 16) {
2450 				opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
2451 				flagvals = cdb[1];
2452 				groupnum = cdb[14];
2453 				control = cdb[15];
2454 			} else {
2455 				opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
2456 				flagvals = cdb[1];
2457 				groupnum = cdb[10];
2458 				control = cdb[11];
2459 			}
2460 
2461 			memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2462 
2463 			cdb[0] = opcode;
2464 			cdb[1] = flagvals;
2465 			cdb[6] = groupnum;
2466 			cdb[9] = control;
2467 
2468 			/* Transfer length */
2469 			cdb[8] = (u8)(num_blocks & 0xff);
2470 			cdb[7] = (u8)((num_blocks >> 8) & 0xff);
2471 
2472 			io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */
2473 			cdb_len = 10;
2474 		} else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
2475 			/* Convert to 16 byte CDB for large LBA's */
2476 			switch (cdb_len) {
2477 			case 6:
2478 				opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
2479 				control = cdb[5];
2480 				break;
2481 			case 10:
2482 				opcode =
2483 					cdb[0] == READ_10 ? READ_16 : WRITE_16;
2484 				flagvals = cdb[1];
2485 				groupnum = cdb[6];
2486 				control = cdb[9];
2487 				break;
2488 			case 12:
2489 				opcode =
2490 					cdb[0] == READ_12 ? READ_16 : WRITE_16;
2491 				flagvals = cdb[1];
2492 				groupnum = cdb[10];
2493 				control = cdb[11];
2494 				break;
2495 			}
2496 
2497 			memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2498 
2499 			cdb[0] = opcode;
2500 			cdb[1] = flagvals;
2501 			cdb[14] = groupnum;
2502 			cdb[15] = control;
2503 
2504 			/* Transfer length */
2505 			cdb[13] = (u8)(num_blocks & 0xff);
2506 			cdb[12] = (u8)((num_blocks >> 8) & 0xff);
2507 			cdb[11] = (u8)((num_blocks >> 16) & 0xff);
2508 			cdb[10] = (u8)((num_blocks >> 24) & 0xff);
2509 
2510 			io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */
2511 			cdb_len = 16;
2512 		}
2513 
2514 		/* Normal case, just load LBA here */
2515 		switch (cdb_len) {
2516 		case 6:
2517 		{
2518 			u8 val = cdb[1] & 0xE0;
2519 			cdb[3] = (u8)(start_blk & 0xff);
2520 			cdb[2] = (u8)((start_blk >> 8) & 0xff);
2521 			cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
2522 			break;
2523 		}
2524 		case 10:
2525 			cdb[5] = (u8)(start_blk & 0xff);
2526 			cdb[4] = (u8)((start_blk >> 8) & 0xff);
2527 			cdb[3] = (u8)((start_blk >> 16) & 0xff);
2528 			cdb[2] = (u8)((start_blk >> 24) & 0xff);
2529 			break;
2530 		case 12:
2531 			cdb[5]    = (u8)(start_blk & 0xff);
2532 			cdb[4]    = (u8)((start_blk >> 8) & 0xff);
2533 			cdb[3]    = (u8)((start_blk >> 16) & 0xff);
2534 			cdb[2]    = (u8)((start_blk >> 24) & 0xff);
2535 			break;
2536 		case 16:
2537 			cdb[9]    = (u8)(start_blk & 0xff);
2538 			cdb[8]    = (u8)((start_blk >> 8) & 0xff);
2539 			cdb[7]    = (u8)((start_blk >> 16) & 0xff);
2540 			cdb[6]    = (u8)((start_blk >> 24) & 0xff);
2541 			cdb[5]    = (u8)((start_blk >> 32) & 0xff);
2542 			cdb[4]    = (u8)((start_blk >> 40) & 0xff);
2543 			cdb[3]    = (u8)((start_blk >> 48) & 0xff);
2544 			cdb[2]    = (u8)((start_blk >> 56) & 0xff);
2545 			break;
2546 		}
2547 	}
2548 }
2549 
2550 /**
2551  * megasas_stream_detect -	stream detection on read and and write IOs
2552  * @instance:		Adapter soft state
2553  * @cmd:		    Command to be prepared
2554  * @io_info:		IO Request info
2555  *
2556  */
2557 
2558 /** stream detection on read and and write IOs */
2559 static void megasas_stream_detect(struct megasas_instance *instance,
2560 				  struct megasas_cmd_fusion *cmd,
2561 				  struct IO_REQUEST_INFO *io_info)
2562 {
2563 	struct fusion_context *fusion = instance->ctrl_context;
2564 	u32 device_id = io_info->ldTgtId;
2565 	struct LD_STREAM_DETECT *current_ld_sd
2566 		= fusion->stream_detect_by_ld[device_id];
2567 	u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;
2568 	u32 shifted_values, unshifted_values;
2569 	u32 index_value_mask, shifted_values_mask;
2570 	int i;
2571 	bool is_read_ahead = false;
2572 	struct STREAM_DETECT *current_sd;
2573 	/* find possible stream */
2574 	for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2575 		stream_num = (*track_stream >>
2576 			(i * BITS_PER_INDEX_STREAM)) &
2577 			STREAM_MASK;
2578 		current_sd = &current_ld_sd->stream_track[stream_num];
2579 		/* if we found a stream, update the raid
2580 		 *  context and also update the mruBitMap
2581 		 */
2582 		/*	boundary condition */
2583 		if ((current_sd->next_seq_lba) &&
2584 		    (io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2585 		    (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2586 		    (current_sd->is_read == io_info->isRead)) {
2587 
2588 			if ((io_info->ldStartBlock != current_sd->next_seq_lba)	&&
2589 			    ((!io_info->isRead) || (!is_read_ahead)))
2590 				/*
2591 				 * Once the API is available we need to change this.
2592 				 * At this point we are not allowing any gap
2593 				 */
2594 				continue;
2595 
2596 			SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2597 			current_sd->next_seq_lba =
2598 			io_info->ldStartBlock + io_info->numBlocks;
2599 			/*
2600 			 *	update the mruBitMap LRU
2601 			 */
2602 			shifted_values_mask =
2603 				(1 <<  i * BITS_PER_INDEX_STREAM) - 1;
2604 			shifted_values = ((*track_stream & shifted_values_mask)
2605 						<< BITS_PER_INDEX_STREAM);
2606 			index_value_mask =
2607 				STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2608 			unshifted_values =
2609 				*track_stream & ~(shifted_values_mask |
2610 				index_value_mask);
2611 			*track_stream =
2612 				unshifted_values | shifted_values | stream_num;
2613 			return;
2614 		}
2615 	}
2616 	/*
2617 	 * if we did not find any stream, create a new one
2618 	 * from the least recently used
2619 	 */
2620 	stream_num = (*track_stream >>
2621 		((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2622 		STREAM_MASK;
2623 	current_sd = &current_ld_sd->stream_track[stream_num];
2624 	current_sd->is_read = io_info->isRead;
2625 	current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2626 	*track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2627 	return;
2628 }
2629 
2630 /**
2631  * megasas_set_raidflag_cpu_affinity - This function sets the cpu
2632  * affinity (cpu of the controller) and raid_flags in the raid context
2633  * based on IO type.
2634  *
2635  * @fusion:		Fusion context
2636  * @praid_context:	IO RAID context
2637  * @raid:		LD raid map
2638  * @fp_possible:	Is fast path possible?
2639  * @is_read:		Is read IO?
2640  * @scsi_buff_len:	SCSI command buffer length
2641  *
2642  */
2643 static void
2644 megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion,
2645 				union RAID_CONTEXT_UNION *praid_context,
2646 				struct MR_LD_RAID *raid, bool fp_possible,
2647 				u8 is_read, u32 scsi_buff_len)
2648 {
2649 	u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2650 	struct RAID_CONTEXT_G35 *rctx_g35;
2651 
2652 	rctx_g35 = &praid_context->raid_context_g35;
2653 	if (fp_possible) {
2654 		if (is_read) {
2655 			if ((raid->cpuAffinity.pdRead.cpu0) &&
2656 			    (raid->cpuAffinity.pdRead.cpu1))
2657 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2658 			else if (raid->cpuAffinity.pdRead.cpu1)
2659 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2660 		} else {
2661 			if ((raid->cpuAffinity.pdWrite.cpu0) &&
2662 			    (raid->cpuAffinity.pdWrite.cpu1))
2663 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2664 			else if (raid->cpuAffinity.pdWrite.cpu1)
2665 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2666 			/* Fast path cache by pass capable R0/R1 VD */
2667 			if ((raid->level <= 1) &&
2668 			    (raid->capability.fp_cache_bypass_capable)) {
2669 				rctx_g35->routing_flags |=
2670 					(1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2671 				rctx_g35->raid_flags =
2672 					(MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2673 					<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2674 			}
2675 		}
2676 	} else {
2677 		if (is_read) {
2678 			if ((raid->cpuAffinity.ldRead.cpu0) &&
2679 			    (raid->cpuAffinity.ldRead.cpu1))
2680 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2681 			else if (raid->cpuAffinity.ldRead.cpu1)
2682 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2683 		} else {
2684 			if ((raid->cpuAffinity.ldWrite.cpu0) &&
2685 			    (raid->cpuAffinity.ldWrite.cpu1))
2686 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2687 			else if (raid->cpuAffinity.ldWrite.cpu1)
2688 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2689 
2690 			if (is_stream_detected(rctx_g35) &&
2691 			    ((raid->level == 5) || (raid->level == 6)) &&
2692 			    (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2693 			    (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2694 				cpu_sel = MR_RAID_CTX_CPUSEL_0;
2695 		}
2696 	}
2697 
2698 	rctx_g35->routing_flags |=
2699 		(cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2700 
2701 	/* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2702 	 * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.
2703 	 * IO Subtype is not bitmap.
2704 	 */
2705 	if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) &&
2706 			(scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) {
2707 		praid_context->raid_context_g35.raid_flags =
2708 			(MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2709 			<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2710 	}
2711 }
2712 
2713 /**
2714  * megasas_build_ldio_fusion -	Prepares IOs to devices
2715  * @instance:		Adapter soft state
2716  * @scp:		SCSI command
2717  * @cmd:		Command to be prepared
2718  *
2719  * Prepares the io_request and chain elements (sg_frame) for IO
2720  * The IO can be for PD (Fast Path) or LD
2721  */
2722 static void
2723 megasas_build_ldio_fusion(struct megasas_instance *instance,
2724 			  struct scsi_cmnd *scp,
2725 			  struct megasas_cmd_fusion *cmd)
2726 {
2727 	bool fp_possible;
2728 	u16 ld;
2729 	u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2730 	u32 scsi_buff_len;
2731 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2732 	struct IO_REQUEST_INFO io_info;
2733 	struct fusion_context *fusion;
2734 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2735 	u8 *raidLUN;
2736 	unsigned long spinlock_flags;
2737 	struct MR_LD_RAID *raid = NULL;
2738 	struct MR_PRIV_DEVICE *mrdev_priv;
2739 	struct RAID_CONTEXT *rctx;
2740 	struct RAID_CONTEXT_G35 *rctx_g35;
2741 
2742 	device_id = MEGASAS_DEV_INDEX(scp);
2743 
2744 	fusion = instance->ctrl_context;
2745 
2746 	io_request = cmd->io_request;
2747 	rctx = &io_request->RaidContext.raid_context;
2748 	rctx_g35 = &io_request->RaidContext.raid_context_g35;
2749 
2750 	rctx->virtual_disk_tgt_id = cpu_to_le16(device_id);
2751 	rctx->status = 0;
2752 	rctx->ex_status = 0;
2753 
2754 	start_lba_lo = 0;
2755 	start_lba_hi = 0;
2756 	fp_possible = false;
2757 
2758 	/*
2759 	 * 6-byte READ(0x08) or WRITE(0x0A) cdb
2760 	 */
2761 	if (scp->cmd_len == 6) {
2762 		datalength = (u32) scp->cmnd[4];
2763 		start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2764 			((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2765 
2766 		start_lba_lo &= 0x1FFFFF;
2767 	}
2768 
2769 	/*
2770 	 * 10-byte READ(0x28) or WRITE(0x2A) cdb
2771 	 */
2772 	else if (scp->cmd_len == 10) {
2773 		datalength = (u32) scp->cmnd[8] |
2774 			((u32) scp->cmnd[7] << 8);
2775 		start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2776 			((u32) scp->cmnd[3] << 16) |
2777 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2778 	}
2779 
2780 	/*
2781 	 * 12-byte READ(0xA8) or WRITE(0xAA) cdb
2782 	 */
2783 	else if (scp->cmd_len == 12) {
2784 		datalength = ((u32) scp->cmnd[6] << 24) |
2785 			((u32) scp->cmnd[7] << 16) |
2786 			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2787 		start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2788 			((u32) scp->cmnd[3] << 16) |
2789 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2790 	}
2791 
2792 	/*
2793 	 * 16-byte READ(0x88) or WRITE(0x8A) cdb
2794 	 */
2795 	else if (scp->cmd_len == 16) {
2796 		datalength = ((u32) scp->cmnd[10] << 24) |
2797 			((u32) scp->cmnd[11] << 16) |
2798 			((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2799 		start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2800 			((u32) scp->cmnd[7] << 16) |
2801 			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2802 
2803 		start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2804 			((u32) scp->cmnd[3] << 16) |
2805 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2806 	}
2807 
2808 	memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2809 	io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2810 	io_info.numBlocks = datalength;
2811 	io_info.ldTgtId = device_id;
2812 	io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2813 	scsi_buff_len = scsi_bufflen(scp);
2814 	io_request->DataLength = cpu_to_le32(scsi_buff_len);
2815 	io_info.data_arms = 1;
2816 
2817 	if (scp->sc_data_direction == DMA_FROM_DEVICE)
2818 		io_info.isRead = 1;
2819 
2820 	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2821 	ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2822 
2823 	if (ld < instance->fw_supported_vd_count)
2824 		raid = MR_LdRaidGet(ld, local_map_ptr);
2825 
2826 	if (!raid || (!fusion->fast_path_io)) {
2827 		rctx->reg_lock_flags  = 0;
2828 		fp_possible = false;
2829 	} else {
2830 		if (MR_BuildRaidContext(instance, &io_info, rctx,
2831 					local_map_ptr, &raidLUN))
2832 			fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2833 	}
2834 
2835 	megasas_get_msix_index(instance, scp, cmd, io_info.data_arms);
2836 
2837 	if (instance->adapter_type >= VENTURA_SERIES) {
2838 		/* FP for Optimal raid level 1.
2839 		 * All large RAID-1 writes (> 32 KiB, both WT and WB modes)
2840 		 * are built by the driver as LD I/Os.
2841 		 * All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os
2842 		 * (there is never a reason to process these as buffered writes)
2843 		 * All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os
2844 		 * with the SLD bit asserted.
2845 		 */
2846 		if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2847 			mrdev_priv = scp->device->hostdata;
2848 
2849 			if (atomic_inc_return(&instance->fw_outstanding) >
2850 				(instance->host->can_queue)) {
2851 				fp_possible = false;
2852 				atomic_dec(&instance->fw_outstanding);
2853 			} else if (fusion->pcie_bw_limitation &&
2854 				((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2855 				   (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0))) {
2856 				fp_possible = false;
2857 				atomic_dec(&instance->fw_outstanding);
2858 				if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2859 					atomic_set(&mrdev_priv->r1_ldio_hint,
2860 						   instance->r1_ldio_hint_default);
2861 			}
2862 		}
2863 
2864 		if (!fp_possible ||
2865 		    (io_info.isRead && io_info.ra_capable)) {
2866 			spin_lock_irqsave(&instance->stream_lock,
2867 					  spinlock_flags);
2868 			megasas_stream_detect(instance, cmd, &io_info);
2869 			spin_unlock_irqrestore(&instance->stream_lock,
2870 					       spinlock_flags);
2871 			/* In ventura if stream detected for a read and it is
2872 			 * read ahead capable make this IO as LDIO
2873 			 */
2874 			if (is_stream_detected(rctx_g35))
2875 				fp_possible = false;
2876 		}
2877 
2878 		/* If raid is NULL, set CPU affinity to default CPU0 */
2879 		if (raid)
2880 			megasas_set_raidflag_cpu_affinity(fusion, &io_request->RaidContext,
2881 				raid, fp_possible, io_info.isRead,
2882 				scsi_buff_len);
2883 		else
2884 			rctx_g35->routing_flags |=
2885 				(MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2886 	}
2887 
2888 	if (fp_possible) {
2889 		megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
2890 				   local_map_ptr, start_lba_lo);
2891 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2892 		cmd->request_desc->SCSIIO.RequestFlags =
2893 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2894 			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2895 		if (instance->adapter_type == INVADER_SERIES) {
2896 			rctx->type = MPI2_TYPE_CUDA;
2897 			rctx->nseg = 0x1;
2898 			io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2899 			rctx->reg_lock_flags |=
2900 			  (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2901 			   MR_RL_FLAGS_SEQ_NUM_ENABLE);
2902 		} else if (instance->adapter_type >= VENTURA_SERIES) {
2903 			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2904 			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2905 			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2906 			io_request->IoFlags |=
2907 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2908 		}
2909 		if (fusion->load_balance_info &&
2910 			(fusion->load_balance_info[device_id].loadBalanceFlag) &&
2911 			(io_info.isRead)) {
2912 			io_info.devHandle =
2913 				get_updated_dev_handle(instance,
2914 					&fusion->load_balance_info[device_id],
2915 					&io_info, local_map_ptr);
2916 			megasas_priv(scp)->status |= MEGASAS_LOAD_BALANCE_FLAG;
2917 			cmd->pd_r1_lb = io_info.pd_after_lb;
2918 			if (instance->adapter_type >= VENTURA_SERIES)
2919 				rctx_g35->span_arm = io_info.span_arm;
2920 			else
2921 				rctx->span_arm = io_info.span_arm;
2922 
2923 		} else
2924 			megasas_priv(scp)->status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2925 
2926 		if (instance->adapter_type >= VENTURA_SERIES)
2927 			cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2928 		else
2929 			cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2930 
2931 		if ((raidLUN[0] == 1) &&
2932 			(local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2933 			instance->dev_handle = !(instance->dev_handle);
2934 			io_info.devHandle =
2935 				local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2936 		}
2937 
2938 		cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2939 		io_request->DevHandle = io_info.devHandle;
2940 		cmd->pd_interface = io_info.pd_interface;
2941 		/* populate the LUN field */
2942 		memcpy(io_request->LUN, raidLUN, 8);
2943 	} else {
2944 		rctx->timeout_value =
2945 			cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2946 		cmd->request_desc->SCSIIO.RequestFlags =
2947 			(MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2948 			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2949 		if (instance->adapter_type == INVADER_SERIES) {
2950 			if (io_info.do_fp_rlbypass ||
2951 			(rctx->reg_lock_flags == REGION_TYPE_UNUSED))
2952 				cmd->request_desc->SCSIIO.RequestFlags =
2953 					(MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2954 					MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2955 			rctx->type = MPI2_TYPE_CUDA;
2956 			rctx->reg_lock_flags |=
2957 				(MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2958 					MR_RL_FLAGS_SEQ_NUM_ENABLE);
2959 			rctx->nseg = 0x1;
2960 		} else if (instance->adapter_type >= VENTURA_SERIES) {
2961 			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2962 			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2963 			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2964 		}
2965 		io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2966 		io_request->DevHandle = cpu_to_le16(device_id);
2967 
2968 	} /* Not FP */
2969 }
2970 
2971 /**
2972  * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
2973  * @instance:		Adapter soft state
2974  * @scmd:		SCSI command
2975  * @cmd:		Command to be prepared
2976  *
2977  * Prepares the io_request frame for non-rw io cmds for vd.
2978  */
2979 static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2980 			  struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2981 {
2982 	u32 device_id;
2983 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2984 	u16 ld;
2985 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2986 	struct fusion_context *fusion = instance->ctrl_context;
2987 	u8                          span, physArm;
2988 	__le16                      devHandle;
2989 	u32                         arRef, pd;
2990 	struct MR_LD_RAID                  *raid;
2991 	struct RAID_CONTEXT                *pRAID_Context;
2992 	u8 fp_possible = 1;
2993 
2994 	io_request = cmd->io_request;
2995 	device_id = MEGASAS_DEV_INDEX(scmd);
2996 	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2997 	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
2998 	/* get RAID_Context pointer */
2999 	pRAID_Context = &io_request->RaidContext.raid_context;
3000 	/* Check with FW team */
3001 	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3002 	pRAID_Context->reg_lock_row_lba    = 0;
3003 	pRAID_Context->reg_lock_length    = 0;
3004 
3005 	if (fusion->fast_path_io && (
3006 		device_id < instance->fw_supported_vd_count)) {
3007 
3008 		ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
3009 		if (ld >= instance->fw_supported_vd_count - 1)
3010 			fp_possible = 0;
3011 		else {
3012 			raid = MR_LdRaidGet(ld, local_map_ptr);
3013 			if (!(raid->capability.fpNonRWCapable))
3014 				fp_possible = 0;
3015 		}
3016 	} else
3017 		fp_possible = 0;
3018 
3019 	if (!fp_possible) {
3020 		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3021 		io_request->DevHandle = cpu_to_le16(device_id);
3022 		io_request->LUN[1] = scmd->device->lun;
3023 		pRAID_Context->timeout_value =
3024 			cpu_to_le16(scsi_cmd_to_rq(scmd)->timeout / HZ);
3025 		cmd->request_desc->SCSIIO.RequestFlags =
3026 			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3027 			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3028 	} else {
3029 
3030 		/* set RAID context values */
3031 		pRAID_Context->config_seq_num = raid->seqNum;
3032 		if (instance->adapter_type < VENTURA_SERIES)
3033 			pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
3034 		pRAID_Context->timeout_value =
3035 			cpu_to_le16(raid->fpIoTimeoutForLd);
3036 
3037 		/* get the DevHandle for the PD (since this is
3038 		   fpNonRWCapable, this is a single disk RAID0) */
3039 		span = physArm = 0;
3040 		arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);
3041 		pd = MR_ArPdGet(arRef, physArm, local_map_ptr);
3042 		devHandle = MR_PdDevHandleGet(pd, local_map_ptr);
3043 
3044 		/* build request descriptor */
3045 		cmd->request_desc->SCSIIO.RequestFlags =
3046 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3047 			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3048 		cmd->request_desc->SCSIIO.DevHandle = devHandle;
3049 
3050 		/* populate the LUN field */
3051 		memcpy(io_request->LUN, raid->LUN, 8);
3052 
3053 		/* build the raidScsiIO structure */
3054 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3055 		io_request->DevHandle = devHandle;
3056 	}
3057 }
3058 
3059 /**
3060  * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
3061  * @instance:		Adapter soft state
3062  * @scmd:		SCSI command
3063  * @cmd:		Command to be prepared
3064  * @fp_possible:	parameter to detect fast path or firmware path io.
3065  *
3066  * Prepares the io_request frame for rw/non-rw io cmds for syspds
3067  */
3068 static void
3069 megasas_build_syspd_fusion(struct megasas_instance *instance,
3070 	struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
3071 	bool fp_possible)
3072 {
3073 	u32 device_id;
3074 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
3075 	u16 pd_index = 0;
3076 	u16 os_timeout_value;
3077 	u16 timeout_limit;
3078 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
3079 	struct RAID_CONTEXT	*pRAID_Context;
3080 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
3081 	struct MR_PRIV_DEVICE *mr_device_priv_data;
3082 	struct fusion_context *fusion = instance->ctrl_context;
3083 	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
3084 
3085 	device_id = MEGASAS_DEV_INDEX(scmd);
3086 	pd_index = MEGASAS_PD_INDEX(scmd);
3087 	os_timeout_value = scsi_cmd_to_rq(scmd)->timeout / HZ;
3088 	mr_device_priv_data = scmd->device->hostdata;
3089 	cmd->pd_interface = mr_device_priv_data->interface_type;
3090 
3091 	io_request = cmd->io_request;
3092 	/* get RAID_Context pointer */
3093 	pRAID_Context = &io_request->RaidContext.raid_context;
3094 	pRAID_Context->reg_lock_flags = 0;
3095 	pRAID_Context->reg_lock_row_lba = 0;
3096 	pRAID_Context->reg_lock_length = 0;
3097 	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3098 	io_request->LUN[1] = scmd->device->lun;
3099 	pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
3100 		<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
3101 
3102 	/* If FW supports PD sequence number */
3103 	if (instance->support_seqnum_jbod_fp) {
3104 		if (instance->use_seqnum_jbod_fp &&
3105 			instance->pd_list[pd_index].driveType == TYPE_DISK) {
3106 
3107 			/* More than 256 PD/JBOD support for Ventura */
3108 			if (instance->support_morethan256jbod)
3109 				pRAID_Context->virtual_disk_tgt_id =
3110 					pd_sync->seq[pd_index].pd_target_id;
3111 			else
3112 				pRAID_Context->virtual_disk_tgt_id =
3113 					cpu_to_le16(device_id +
3114 					(MAX_PHYSICAL_DEVICES - 1));
3115 			pRAID_Context->config_seq_num =
3116 				pd_sync->seq[pd_index].seqNum;
3117 			io_request->DevHandle =
3118 				pd_sync->seq[pd_index].devHandle;
3119 			if (instance->adapter_type >= VENTURA_SERIES) {
3120 				io_request->RaidContext.raid_context_g35.routing_flags |=
3121 					(1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
3122 				io_request->RaidContext.raid_context_g35.nseg_type |=
3123 					(1 << RAID_CONTEXT_NSEG_SHIFT);
3124 				io_request->RaidContext.raid_context_g35.nseg_type |=
3125 					(MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
3126 			} else {
3127 				pRAID_Context->type = MPI2_TYPE_CUDA;
3128 				pRAID_Context->nseg = 0x1;
3129 				pRAID_Context->reg_lock_flags |=
3130 					(MR_RL_FLAGS_SEQ_NUM_ENABLE |
3131 					 MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
3132 			}
3133 		} else {
3134 			pRAID_Context->virtual_disk_tgt_id =
3135 				cpu_to_le16(device_id +
3136 				(MAX_PHYSICAL_DEVICES - 1));
3137 			pRAID_Context->config_seq_num = 0;
3138 			io_request->DevHandle = cpu_to_le16(0xFFFF);
3139 		}
3140 	} else {
3141 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3142 		pRAID_Context->config_seq_num = 0;
3143 
3144 		if (fusion->fast_path_io) {
3145 			local_map_ptr =
3146 				fusion->ld_drv_map[(instance->map_id & 1)];
3147 			io_request->DevHandle =
3148 				local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
3149 		} else {
3150 			io_request->DevHandle = cpu_to_le16(0xFFFF);
3151 		}
3152 	}
3153 
3154 	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
3155 
3156 	megasas_get_msix_index(instance, scmd, cmd, 1);
3157 
3158 	if (!fp_possible) {
3159 		/* system pd firmware path */
3160 		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3161 		cmd->request_desc->SCSIIO.RequestFlags =
3162 			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3163 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3164 		pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
3165 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3166 	} else {
3167 		if (os_timeout_value)
3168 			os_timeout_value++;
3169 
3170 		/* system pd Fast Path */
3171 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3172 		timeout_limit = (scmd->device->type == TYPE_DISK) ?
3173 				255 : 0xFFFF;
3174 		pRAID_Context->timeout_value =
3175 			cpu_to_le16((os_timeout_value > timeout_limit) ?
3176 			timeout_limit : os_timeout_value);
3177 		if (instance->adapter_type >= INVADER_SERIES)
3178 			io_request->IoFlags |=
3179 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
3180 
3181 		cmd->request_desc->SCSIIO.RequestFlags =
3182 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3183 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3184 	}
3185 }
3186 
3187 /**
3188  * megasas_build_io_fusion -	Prepares IOs to devices
3189  * @instance:		Adapter soft state
3190  * @scp:		SCSI command
3191  * @cmd:		Command to be prepared
3192  *
3193  * Invokes helper functions to prepare request frames
3194  * and sets flags appropriate for IO/Non-IO cmd
3195  */
3196 static int
3197 megasas_build_io_fusion(struct megasas_instance *instance,
3198 			struct scsi_cmnd *scp,
3199 			struct megasas_cmd_fusion *cmd)
3200 {
3201 	int sge_count;
3202 	u16 pd_index = 0;
3203 	u8 drive_type = 0;
3204 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
3205 	struct MR_PRIV_DEVICE *mr_device_priv_data;
3206 	mr_device_priv_data = scp->device->hostdata;
3207 
3208 	/* Zero out some fields so they don't get reused */
3209 	memset(io_request->LUN, 0x0, 8);
3210 	io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
3211 	io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
3212 	io_request->EEDPFlags = 0;
3213 	io_request->Control = 0;
3214 	io_request->EEDPBlockSize = 0;
3215 	io_request->ChainOffset = 0;
3216 	io_request->RaidContext.raid_context.raid_flags = 0;
3217 	io_request->RaidContext.raid_context.type = 0;
3218 	io_request->RaidContext.raid_context.nseg = 0;
3219 
3220 	memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
3221 	/*
3222 	 * Just the CDB length,rest of the Flags are zero
3223 	 * This will be modified for FP in build_ldio_fusion
3224 	 */
3225 	io_request->IoFlags = cpu_to_le16(scp->cmd_len);
3226 
3227 	switch (megasas_cmd_type(scp)) {
3228 	case READ_WRITE_LDIO:
3229 		megasas_build_ldio_fusion(instance, scp, cmd);
3230 		break;
3231 	case NON_READ_WRITE_LDIO:
3232 		megasas_build_ld_nonrw_fusion(instance, scp, cmd);
3233 		break;
3234 	case READ_WRITE_SYSPDIO:
3235 		megasas_build_syspd_fusion(instance, scp, cmd, true);
3236 		break;
3237 	case NON_READ_WRITE_SYSPDIO:
3238 		pd_index = MEGASAS_PD_INDEX(scp);
3239 		drive_type = instance->pd_list[pd_index].driveType;
3240 		if ((instance->secure_jbod_support ||
3241 		     mr_device_priv_data->is_tm_capable) ||
3242 		     (instance->adapter_type >= VENTURA_SERIES &&
3243 		     drive_type == TYPE_ENCLOSURE))
3244 			megasas_build_syspd_fusion(instance, scp, cmd, false);
3245 		else
3246 			megasas_build_syspd_fusion(instance, scp, cmd, true);
3247 		break;
3248 	default:
3249 		break;
3250 	}
3251 
3252 	/*
3253 	 * Construct SGL
3254 	 */
3255 
3256 	sge_count = megasas_make_sgl(instance, scp, cmd);
3257 
3258 	if (sge_count > instance->max_num_sge || (sge_count < 0)) {
3259 		dev_err(&instance->pdev->dev,
3260 			"%s %d sge_count (%d) is out of range. Range is:  0-%d\n",
3261 			__func__, __LINE__, sge_count, instance->max_num_sge);
3262 		return 1;
3263 	}
3264 
3265 	if (instance->adapter_type >= VENTURA_SERIES) {
3266 		set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);
3267 		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
3268 		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
3269 	} else {
3270 		/* numSGE store lower 8 bit of sge_count.
3271 		 * numSGEExt store higher 8 bit of sge_count
3272 		 */
3273 		io_request->RaidContext.raid_context.num_sge = sge_count;
3274 		io_request->RaidContext.raid_context.num_sge_ext =
3275 			(u8)(sge_count >> 8);
3276 	}
3277 
3278 	io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
3279 
3280 	if (scp->sc_data_direction == DMA_TO_DEVICE)
3281 		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
3282 	else if (scp->sc_data_direction == DMA_FROM_DEVICE)
3283 		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
3284 
3285 	io_request->SGLOffset0 =
3286 		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
3287 
3288 	io_request->SenseBufferLowAddress =
3289 		cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));
3290 	io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
3291 
3292 	cmd->scmd = scp;
3293 	megasas_priv(scp)->cmd_priv = cmd;
3294 
3295 	return 0;
3296 }
3297 
3298 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3299 megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
3300 {
3301 	u8 *p;
3302 	struct fusion_context *fusion;
3303 
3304 	fusion = instance->ctrl_context;
3305 	p = fusion->req_frames_desc +
3306 		sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
3307 
3308 	return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
3309 }
3310 
3311 
3312 /* megasas_prepate_secondRaid1_IO
3313  *  It prepares the raid 1 second IO
3314  */
3315 static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
3316 					   struct megasas_cmd_fusion *cmd,
3317 					   struct megasas_cmd_fusion *r1_cmd)
3318 {
3319 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
3320 	struct fusion_context *fusion;
3321 	fusion = instance->ctrl_context;
3322 	req_desc = cmd->request_desc;
3323 	/* copy the io request frame as well as 8 SGEs data for r1 command*/
3324 	memcpy(r1_cmd->io_request, cmd->io_request,
3325 	       (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
3326 	memcpy(r1_cmd->io_request->SGLs, cmd->io_request->SGLs,
3327 	       (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
3328 	/*sense buffer is different for r1 command*/
3329 	r1_cmd->io_request->SenseBufferLowAddress =
3330 			cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));
3331 	r1_cmd->scmd = cmd->scmd;
3332 	req_desc2 = megasas_get_request_descriptor(instance,
3333 						   (r1_cmd->index - 1));
3334 	req_desc2->Words = 0;
3335 	r1_cmd->request_desc = req_desc2;
3336 	req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
3337 	req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
3338 	r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
3339 	r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
3340 	r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
3341 	cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3342 			cpu_to_le16(r1_cmd->index);
3343 	r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3344 			cpu_to_le16(cmd->index);
3345 	/*MSIxIndex of both commands request descriptors should be same*/
3346 	r1_cmd->request_desc->SCSIIO.MSIxIndex =
3347 			cmd->request_desc->SCSIIO.MSIxIndex;
3348 	/*span arm is different for r1 cmd*/
3349 	r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
3350 			cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
3351 }
3352 
3353 /**
3354  * megasas_build_and_issue_cmd_fusion -Main routine for building and
3355  *                                     issuing non IOCTL cmd
3356  * @instance:			Adapter soft state
3357  * @scmd:			pointer to scsi cmd from OS
3358  */
3359 static u32
3360 megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
3361 				   struct scsi_cmnd *scmd)
3362 {
3363 	struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
3364 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3365 	u32 index;
3366 
3367 	if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&
3368 		instance->ldio_threshold &&
3369 		(atomic_inc_return(&instance->ldio_outstanding) >
3370 		instance->ldio_threshold)) {
3371 		atomic_dec(&instance->ldio_outstanding);
3372 		return SCSI_MLQUEUE_DEVICE_BUSY;
3373 	}
3374 
3375 	if (atomic_inc_return(&instance->fw_outstanding) >
3376 			instance->host->can_queue) {
3377 		atomic_dec(&instance->fw_outstanding);
3378 		return SCSI_MLQUEUE_HOST_BUSY;
3379 	}
3380 
3381 	cmd = megasas_get_cmd_fusion(instance, scsi_cmd_to_rq(scmd)->tag);
3382 
3383 	if (!cmd) {
3384 		atomic_dec(&instance->fw_outstanding);
3385 		return SCSI_MLQUEUE_HOST_BUSY;
3386 	}
3387 
3388 	index = cmd->index;
3389 
3390 	req_desc = megasas_get_request_descriptor(instance, index-1);
3391 
3392 	req_desc->Words = 0;
3393 	cmd->request_desc = req_desc;
3394 
3395 	if (megasas_build_io_fusion(instance, scmd, cmd)) {
3396 		megasas_return_cmd_fusion(instance, cmd);
3397 		dev_err(&instance->pdev->dev, "Error building command\n");
3398 		cmd->request_desc = NULL;
3399 		atomic_dec(&instance->fw_outstanding);
3400 		return SCSI_MLQUEUE_HOST_BUSY;
3401 	}
3402 
3403 	req_desc = cmd->request_desc;
3404 	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3405 
3406 	if (cmd->io_request->ChainOffset != 0 &&
3407 	    cmd->io_request->ChainOffset != 0xF)
3408 		dev_err(&instance->pdev->dev, "The chain offset value is not "
3409 		       "correct : %x\n", cmd->io_request->ChainOffset);
3410 	/*
3411 	 *	if it is raid 1/10 fp write capable.
3412 	 *	try to get second command from pool and construct it.
3413 	 *	From FW, it has confirmed that lba values of two PDs
3414 	 *	corresponds to single R1/10 LD are always same
3415 	 *
3416 	 */
3417 	/*	driver side count always should be less than max_fw_cmds
3418 	 *	to get new command
3419 	 */
3420 	if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
3421 		r1_cmd = megasas_get_cmd_fusion(instance,
3422 				scsi_cmd_to_rq(scmd)->tag + instance->max_fw_cmds);
3423 		megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
3424 	}
3425 
3426 
3427 	/*
3428 	 * Issue the command to the FW
3429 	 */
3430 
3431 	megasas_sdev_busy_inc(instance, scmd);
3432 	megasas_fire_cmd_fusion(instance, req_desc);
3433 
3434 	if (r1_cmd)
3435 		megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);
3436 
3437 
3438 	return 0;
3439 }
3440 
3441 /**
3442  * megasas_complete_r1_command -
3443  * completes R1 FP write commands which has valid peer smid
3444  * @instance:			Adapter soft state
3445  * @cmd:			MPT command frame
3446  *
3447  */
3448 static inline void
3449 megasas_complete_r1_command(struct megasas_instance *instance,
3450 			    struct megasas_cmd_fusion *cmd)
3451 {
3452 	u8 *sense, status, ex_status;
3453 	u32 data_length;
3454 	u16 peer_smid;
3455 	struct fusion_context *fusion;
3456 	struct megasas_cmd_fusion *r1_cmd = NULL;
3457 	struct scsi_cmnd *scmd_local = NULL;
3458 	struct RAID_CONTEXT_G35 *rctx_g35;
3459 
3460 	rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
3461 	fusion = instance->ctrl_context;
3462 	peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid);
3463 
3464 	r1_cmd = fusion->cmd_list[peer_smid - 1];
3465 	scmd_local = cmd->scmd;
3466 	status = rctx_g35->status;
3467 	ex_status = rctx_g35->ex_status;
3468 	data_length = cmd->io_request->DataLength;
3469 	sense = cmd->sense;
3470 
3471 	cmd->cmd_completed = true;
3472 
3473 	/* Check if peer command is completed or not*/
3474 	if (r1_cmd->cmd_completed) {
3475 		rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
3476 		if (rctx_g35->status != MFI_STAT_OK) {
3477 			status = rctx_g35->status;
3478 			ex_status = rctx_g35->ex_status;
3479 			data_length = r1_cmd->io_request->DataLength;
3480 			sense = r1_cmd->sense;
3481 		}
3482 
3483 		megasas_return_cmd_fusion(instance, r1_cmd);
3484 		map_cmd_status(fusion, scmd_local, status, ex_status,
3485 			       le32_to_cpu(data_length), sense);
3486 		if (instance->ldio_threshold &&
3487 		    megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
3488 			atomic_dec(&instance->ldio_outstanding);
3489 		megasas_priv(scmd_local)->cmd_priv = NULL;
3490 		megasas_return_cmd_fusion(instance, cmd);
3491 		scsi_dma_unmap(scmd_local);
3492 		megasas_sdev_busy_dec(instance, scmd_local);
3493 		scsi_done(scmd_local);
3494 	}
3495 }
3496 
3497 /**
3498  * access_irq_context:		Access to reply processing
3499  * @irq_context:		IRQ context
3500  *
3501  * Synchronize access to reply processing.
3502  *
3503  * Return:  true on success, false on failure.
3504  */
3505 static inline
3506 bool access_irq_context(struct megasas_irq_context  *irq_context)
3507 {
3508 	if (!irq_context)
3509 		return true;
3510 
3511 	if (atomic_add_unless(&irq_context->in_used, 1, 1))
3512 		return true;
3513 
3514 	return false;
3515 }
3516 
3517 /**
3518  * release_irq_context:		Release reply processing
3519  * @irq_context:		IRQ context
3520  *
3521  * Release access of reply processing.
3522  *
3523  * Return: Nothing.
3524  */
3525 static inline
3526 void release_irq_context(struct megasas_irq_context  *irq_context)
3527 {
3528 	if (irq_context)
3529 		atomic_dec(&irq_context->in_used);
3530 }
3531 
3532 /**
3533  * complete_cmd_fusion -	Completes command
3534  * @instance:			Adapter soft state
3535  * @MSIxIndex:			MSI number
3536  * @irq_context:		IRQ context
3537  *
3538  * Completes all commands that is in reply descriptor queue
3539  */
3540 static int
3541 complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
3542 		    struct megasas_irq_context *irq_context)
3543 {
3544 	union MPI2_REPLY_DESCRIPTORS_UNION *desc;
3545 	struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
3546 	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
3547 	struct fusion_context *fusion;
3548 	struct megasas_cmd *cmd_mfi;
3549 	struct megasas_cmd_fusion *cmd_fusion;
3550 	u16 smid, num_completed;
3551 	u8 reply_descript_type, *sense, status, extStatus;
3552 	u32 device_id, data_length;
3553 	union desc_value d_val;
3554 	struct LD_LOAD_BALANCE_INFO *lbinfo;
3555 	int threshold_reply_count = 0;
3556 	struct scsi_cmnd *scmd_local = NULL;
3557 	struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
3558 	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
3559 
3560 	fusion = instance->ctrl_context;
3561 
3562 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3563 		return IRQ_HANDLED;
3564 
3565 	if (!access_irq_context(irq_context))
3566 		return 0;
3567 
3568 	desc = fusion->reply_frames_desc[MSIxIndex] +
3569 				fusion->last_reply_idx[MSIxIndex];
3570 
3571 	reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3572 
3573 	d_val.word = desc->Words;
3574 
3575 	reply_descript_type = reply_desc->ReplyFlags &
3576 		MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3577 
3578 	if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
3579 		release_irq_context(irq_context);
3580 		return IRQ_NONE;
3581 	}
3582 
3583 	num_completed = 0;
3584 
3585 	while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3586 	       d_val.u.high != cpu_to_le32(UINT_MAX)) {
3587 
3588 		smid = le16_to_cpu(reply_desc->SMID);
3589 		cmd_fusion = fusion->cmd_list[smid - 1];
3590 		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3591 						cmd_fusion->io_request;
3592 
3593 		scmd_local = cmd_fusion->scmd;
3594 		status = scsi_io_req->RaidContext.raid_context.status;
3595 		extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3596 		sense = cmd_fusion->sense;
3597 		data_length = scsi_io_req->DataLength;
3598 
3599 		switch (scsi_io_req->Function) {
3600 		case MPI2_FUNCTION_SCSI_TASK_MGMT:
3601 			mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3602 						cmd_fusion->io_request;
3603 			mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3604 						&mr_tm_req->TmRequest;
3605 			dev_dbg(&instance->pdev->dev, "TM completion:"
3606 				"type: 0x%x TaskMID: 0x%x\n",
3607 				mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3608 			complete(&cmd_fusion->done);
3609 			break;
3610 		case MPI2_FUNCTION_SCSI_IO_REQUEST:  /*Fast Path IO.*/
3611 			/* Update load balancing info */
3612 			if (fusion->load_balance_info &&
3613 			    (megasas_priv(cmd_fusion->scmd)->status &
3614 			    MEGASAS_LOAD_BALANCE_FLAG)) {
3615 				device_id = MEGASAS_DEV_INDEX(scmd_local);
3616 				lbinfo = &fusion->load_balance_info[device_id];
3617 				atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3618 				megasas_priv(cmd_fusion->scmd)->status &=
3619 					~MEGASAS_LOAD_BALANCE_FLAG;
3620 			}
3621 			fallthrough;	/* and complete IO */
3622 		case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */
3623 			atomic_dec(&instance->fw_outstanding);
3624 			if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3625 				map_cmd_status(fusion, scmd_local, status,
3626 					       extStatus, le32_to_cpu(data_length),
3627 					       sense);
3628 				if (instance->ldio_threshold &&
3629 				    (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))
3630 					atomic_dec(&instance->ldio_outstanding);
3631 				megasas_priv(scmd_local)->cmd_priv = NULL;
3632 				megasas_return_cmd_fusion(instance, cmd_fusion);
3633 				scsi_dma_unmap(scmd_local);
3634 				megasas_sdev_busy_dec(instance, scmd_local);
3635 				scsi_done(scmd_local);
3636 			} else	/* Optimal VD - R1 FP command completion. */
3637 				megasas_complete_r1_command(instance, cmd_fusion);
3638 			break;
3639 		case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */
3640 			cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3641 			/* Poll mode. Dummy free.
3642 			 * In case of Interrupt mode, caller has reverse check.
3643 			 */
3644 			if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3645 				cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3646 				megasas_return_cmd(instance, cmd_mfi);
3647 			} else
3648 				megasas_complete_cmd(instance, cmd_mfi, DID_OK);
3649 			break;
3650 		}
3651 
3652 		fusion->last_reply_idx[MSIxIndex]++;
3653 		if (fusion->last_reply_idx[MSIxIndex] >=
3654 		    fusion->reply_q_depth)
3655 			fusion->last_reply_idx[MSIxIndex] = 0;
3656 
3657 		desc->Words = cpu_to_le64(ULLONG_MAX);
3658 		num_completed++;
3659 		threshold_reply_count++;
3660 
3661 		/* Get the next reply descriptor */
3662 		if (!fusion->last_reply_idx[MSIxIndex])
3663 			desc = fusion->reply_frames_desc[MSIxIndex];
3664 		else
3665 			desc++;
3666 
3667 		reply_desc =
3668 		  (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3669 
3670 		d_val.word = desc->Words;
3671 
3672 		reply_descript_type = reply_desc->ReplyFlags &
3673 			MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3674 
3675 		if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3676 			break;
3677 		/*
3678 		 * Write to reply post host index register after completing threshold
3679 		 * number of reply counts and still there are more replies in reply queue
3680 		 * pending to be completed
3681 		 */
3682 		if (threshold_reply_count >= instance->threshold_reply_count) {
3683 			if (instance->msix_combined)
3684 				writel(((MSIxIndex & 0x7) << 24) |
3685 					fusion->last_reply_idx[MSIxIndex],
3686 					instance->reply_post_host_index_addr[MSIxIndex/8]);
3687 			else
3688 				writel((MSIxIndex << 24) |
3689 					fusion->last_reply_idx[MSIxIndex],
3690 					instance->reply_post_host_index_addr[0]);
3691 			threshold_reply_count = 0;
3692 			if (irq_context) {
3693 				if (!irq_context->irq_poll_scheduled) {
3694 					irq_context->irq_poll_scheduled = true;
3695 					irq_context->irq_line_enable = true;
3696 					irq_poll_sched(&irq_context->irqpoll);
3697 				}
3698 				release_irq_context(irq_context);
3699 				return num_completed;
3700 			}
3701 		}
3702 	}
3703 
3704 	if (num_completed) {
3705 		wmb();
3706 		if (instance->msix_combined)
3707 			writel(((MSIxIndex & 0x7) << 24) |
3708 				fusion->last_reply_idx[MSIxIndex],
3709 				instance->reply_post_host_index_addr[MSIxIndex/8]);
3710 		else
3711 			writel((MSIxIndex << 24) |
3712 				fusion->last_reply_idx[MSIxIndex],
3713 				instance->reply_post_host_index_addr[0]);
3714 		megasas_check_and_restore_queue_depth(instance);
3715 	}
3716 
3717 	release_irq_context(irq_context);
3718 
3719 	return num_completed;
3720 }
3721 
3722 int megasas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
3723 {
3724 
3725 	struct megasas_instance *instance;
3726 	int num_entries = 0;
3727 	struct fusion_context *fusion;
3728 
3729 	instance = (struct megasas_instance *)shost->hostdata;
3730 
3731 	fusion = instance->ctrl_context;
3732 
3733 	queue_num = queue_num + instance->low_latency_index_start;
3734 
3735 	if (!atomic_add_unless(&fusion->busy_mq_poll[queue_num], 1, 1))
3736 		return 0;
3737 
3738 	num_entries = complete_cmd_fusion(instance, queue_num, NULL);
3739 	atomic_dec(&fusion->busy_mq_poll[queue_num]);
3740 
3741 	return num_entries;
3742 }
3743 
3744 /**
3745  * megasas_enable_irq_poll() - enable irqpoll
3746  * @instance:			Adapter soft state
3747  */
3748 static void megasas_enable_irq_poll(struct megasas_instance *instance)
3749 {
3750 	u32 count, i;
3751 	struct megasas_irq_context *irq_ctx;
3752 
3753 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3754 
3755 	for (i = 0; i < count; i++) {
3756 		irq_ctx = &instance->irq_context[i];
3757 		irq_poll_enable(&irq_ctx->irqpoll);
3758 	}
3759 }
3760 
3761 /**
3762  * megasas_sync_irqs -	Synchronizes all IRQs owned by adapter
3763  * @instance_addr:			Adapter soft state address
3764  */
3765 static void megasas_sync_irqs(unsigned long instance_addr)
3766 {
3767 	u32 count, i;
3768 	struct megasas_instance *instance =
3769 		(struct megasas_instance *)instance_addr;
3770 	struct megasas_irq_context *irq_ctx;
3771 
3772 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3773 
3774 	for (i = 0; i < count; i++) {
3775 		synchronize_irq(pci_irq_vector(instance->pdev, i));
3776 		irq_ctx = &instance->irq_context[i];
3777 		irq_poll_disable(&irq_ctx->irqpoll);
3778 		if (irq_ctx->irq_poll_scheduled) {
3779 			irq_ctx->irq_poll_scheduled = false;
3780 			enable_irq(irq_ctx->os_irq);
3781 			complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3782 		}
3783 	}
3784 }
3785 
3786 /**
3787  * megasas_irqpoll() - process a queue for completed reply descriptors
3788  * @irqpoll:	IRQ poll structure associated with queue to poll.
3789  * @budget:	Threshold of reply descriptors to process per poll.
3790  *
3791  * Return: The number of entries processed.
3792  */
3793 
3794 int megasas_irqpoll(struct irq_poll *irqpoll, int budget)
3795 {
3796 	struct megasas_irq_context *irq_ctx;
3797 	struct megasas_instance *instance;
3798 	int num_entries;
3799 
3800 	irq_ctx = container_of(irqpoll, struct megasas_irq_context, irqpoll);
3801 	instance = irq_ctx->instance;
3802 
3803 	if (irq_ctx->irq_line_enable) {
3804 		disable_irq_nosync(irq_ctx->os_irq);
3805 		irq_ctx->irq_line_enable = false;
3806 	}
3807 
3808 	num_entries = complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3809 	if (num_entries < budget) {
3810 		irq_poll_complete(irqpoll);
3811 		irq_ctx->irq_poll_scheduled = false;
3812 		enable_irq(irq_ctx->os_irq);
3813 		complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3814 	}
3815 
3816 	return num_entries;
3817 }
3818 
3819 /**
3820  * megasas_complete_cmd_dpc_fusion -	Completes command
3821  * @instance_addr:			Adapter soft state address
3822  *
3823  * Tasklet to complete cmds
3824  */
3825 static void
3826 megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
3827 {
3828 	struct megasas_instance *instance =
3829 		(struct megasas_instance *)instance_addr;
3830 	struct megasas_irq_context *irq_ctx = NULL;
3831 	u32 count, MSIxIndex;
3832 
3833 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3834 
3835 	/* If we have already declared adapter dead, donot complete cmds */
3836 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3837 		return;
3838 
3839 	for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) {
3840 		irq_ctx = &instance->irq_context[MSIxIndex];
3841 		complete_cmd_fusion(instance, MSIxIndex, irq_ctx);
3842 	}
3843 }
3844 
3845 /**
3846  * megasas_isr_fusion - isr entry point
3847  * @irq:	IRQ number
3848  * @devp:	IRQ context
3849  */
3850 static irqreturn_t megasas_isr_fusion(int irq, void *devp)
3851 {
3852 	struct megasas_irq_context *irq_context = devp;
3853 	struct megasas_instance *instance = irq_context->instance;
3854 	u32 mfiStatus;
3855 
3856 	if (instance->mask_interrupts)
3857 		return IRQ_NONE;
3858 
3859 	if (irq_context->irq_poll_scheduled)
3860 		return IRQ_HANDLED;
3861 
3862 	if (!instance->msix_vectors) {
3863 		mfiStatus = instance->instancet->clear_intr(instance);
3864 		if (!mfiStatus)
3865 			return IRQ_NONE;
3866 	}
3867 
3868 	/* If we are resetting, bail */
3869 	if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {
3870 		instance->instancet->clear_intr(instance);
3871 		return IRQ_HANDLED;
3872 	}
3873 
3874 	return complete_cmd_fusion(instance, irq_context->MSIxIndex, irq_context)
3875 			? IRQ_HANDLED : IRQ_NONE;
3876 }
3877 
3878 /**
3879  * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
3880  * @instance:			Adapter soft state
3881  * @mfi_cmd:			megasas_cmd pointer
3882  *
3883  */
3884 static void
3885 build_mpt_mfi_pass_thru(struct megasas_instance *instance,
3886 			struct megasas_cmd *mfi_cmd)
3887 {
3888 	struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;
3889 	struct MPI2_RAID_SCSI_IO_REQUEST *io_req;
3890 	struct megasas_cmd_fusion *cmd;
3891 	struct fusion_context *fusion;
3892 	struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;
3893 
3894 	fusion = instance->ctrl_context;
3895 
3896 	cmd = megasas_get_cmd_fusion(instance,
3897 			instance->max_scsi_cmds + mfi_cmd->index);
3898 
3899 	/*  Save the smid. To be used for returning the cmd */
3900 	mfi_cmd->context.smid = cmd->index;
3901 
3902 	/*
3903 	 * For cmds where the flag is set, store the flag and check
3904 	 * on completion. For cmds with this flag, don't call
3905 	 * megasas_complete_cmd
3906 	 */
3907 
3908 	if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))
3909 		mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;
3910 
3911 	io_req = cmd->io_request;
3912 
3913 	if (instance->adapter_type >= INVADER_SERIES) {
3914 		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =
3915 			(struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;
3916 		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
3917 		sgl_ptr_end->Flags = 0;
3918 	}
3919 
3920 	mpi25_ieee_chain =
3921 	  (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;
3922 
3923 	io_req->Function    = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
3924 	io_req->SGLOffset0  = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,
3925 				       SGL) / 4;
3926 	io_req->ChainOffset = fusion->chain_offset_mfi_pthru;
3927 
3928 	mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);
3929 
3930 	mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
3931 		MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
3932 
3933 	mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);
3934 }
3935 
3936 /**
3937  * build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd
3938  * @instance:			Adapter soft state
3939  * @cmd:			mfi cmd to build
3940  *
3941  */
3942 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3943 build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
3944 {
3945 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;
3946 	u16 index;
3947 
3948 	build_mpt_mfi_pass_thru(instance, cmd);
3949 	index = cmd->context.smid;
3950 
3951 	req_desc = megasas_get_request_descriptor(instance, index - 1);
3952 
3953 	req_desc->Words = 0;
3954 	req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3955 					 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3956 
3957 	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3958 
3959 	return req_desc;
3960 }
3961 
3962 /**
3963  * megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd
3964  * @instance:			Adapter soft state
3965  * @cmd:			mfi cmd pointer
3966  *
3967  */
3968 static void
3969 megasas_issue_dcmd_fusion(struct megasas_instance *instance,
3970 			  struct megasas_cmd *cmd)
3971 {
3972 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3973 
3974 	req_desc = build_mpt_cmd(instance, cmd);
3975 
3976 	megasas_fire_cmd_fusion(instance, req_desc);
3977 	return;
3978 }
3979 
3980 /**
3981  * megasas_release_fusion -	Reverses the FW initialization
3982  * @instance:			Adapter soft state
3983  */
3984 void
3985 megasas_release_fusion(struct megasas_instance *instance)
3986 {
3987 	megasas_free_ioc_init_cmd(instance);
3988 	megasas_free_cmds(instance);
3989 	megasas_free_cmds_fusion(instance);
3990 
3991 	iounmap(instance->reg_set);
3992 
3993 	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
3994 }
3995 
3996 /**
3997  * megasas_read_fw_status_reg_fusion - returns the current FW status value
3998  * @instance:			Adapter soft state
3999  */
4000 static u32
4001 megasas_read_fw_status_reg_fusion(struct megasas_instance *instance)
4002 {
4003 	return megasas_readl(instance, &instance->reg_set->outbound_scratch_pad_0);
4004 }
4005 
4006 /**
4007  * megasas_alloc_host_crash_buffer -	Host buffers for Crash dump collection from Firmware
4008  * @instance:				Controller's soft instance
4009  * @return:			        Number of allocated host crash buffers
4010  */
4011 static void
4012 megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
4013 {
4014 	unsigned int i;
4015 
4016 	for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
4017 		instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);
4018 		if (!instance->crash_buf[i]) {
4019 			dev_info(&instance->pdev->dev, "Firmware crash dump "
4020 				"memory allocation failed at index %d\n", i);
4021 			break;
4022 		}
4023 	}
4024 	instance->drv_buf_alloc = i;
4025 }
4026 
4027 /**
4028  * megasas_free_host_crash_buffer -	Host buffers for Crash dump collection from Firmware
4029  * @instance:				Controller's soft instance
4030  */
4031 void
4032 megasas_free_host_crash_buffer(struct megasas_instance *instance)
4033 {
4034 	unsigned int i;
4035 	for (i = 0; i < instance->drv_buf_alloc; i++) {
4036 		vfree(instance->crash_buf[i]);
4037 	}
4038 	instance->drv_buf_index = 0;
4039 	instance->drv_buf_alloc = 0;
4040 	instance->fw_crash_state = UNAVAILABLE;
4041 	instance->fw_crash_buffer_size = 0;
4042 }
4043 
4044 /**
4045  * megasas_adp_reset_fusion -	For controller reset
4046  * @instance:				Controller's soft instance
4047  * @regs:				MFI register set
4048  */
4049 static int
4050 megasas_adp_reset_fusion(struct megasas_instance *instance,
4051 			 struct megasas_register_set __iomem *regs)
4052 {
4053 	u32 host_diag, abs_state, retry;
4054 
4055 	/* Now try to reset the chip */
4056 	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4057 	writel(MPI2_WRSEQ_1ST_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4058 	writel(MPI2_WRSEQ_2ND_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4059 	writel(MPI2_WRSEQ_3RD_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4060 	writel(MPI2_WRSEQ_4TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4061 	writel(MPI2_WRSEQ_5TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4062 	writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4063 
4064 	/* Check that the diag write enable (DRWE) bit is on */
4065 	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);
4066 	retry = 0;
4067 	while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {
4068 		msleep(100);
4069 		host_diag = megasas_readl(instance,
4070 					  &instance->reg_set->fusion_host_diag);
4071 		if (retry++ == 100) {
4072 			dev_warn(&instance->pdev->dev,
4073 				"Host diag unlock failed from %s %d\n",
4074 				__func__, __LINE__);
4075 			break;
4076 		}
4077 	}
4078 	if (!(host_diag & HOST_DIAG_WRITE_ENABLE))
4079 		return -1;
4080 
4081 	/* Send chip reset command */
4082 	writel(host_diag | HOST_DIAG_RESET_ADAPTER,
4083 		&instance->reg_set->fusion_host_diag);
4084 	msleep(3000);
4085 
4086 	/* Make sure reset adapter bit is cleared */
4087 	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);
4088 	retry = 0;
4089 	while (host_diag & HOST_DIAG_RESET_ADAPTER) {
4090 		msleep(100);
4091 		host_diag = megasas_readl(instance,
4092 					  &instance->reg_set->fusion_host_diag);
4093 		if (retry++ == 1000) {
4094 			dev_warn(&instance->pdev->dev,
4095 				"Diag reset adapter never cleared %s %d\n",
4096 				__func__, __LINE__);
4097 			break;
4098 		}
4099 	}
4100 	if (host_diag & HOST_DIAG_RESET_ADAPTER)
4101 		return -1;
4102 
4103 	abs_state = instance->instancet->read_fw_status_reg(instance)
4104 			& MFI_STATE_MASK;
4105 	retry = 0;
4106 
4107 	while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {
4108 		msleep(100);
4109 		abs_state = instance->instancet->
4110 			read_fw_status_reg(instance) & MFI_STATE_MASK;
4111 	}
4112 	if (abs_state <= MFI_STATE_FW_INIT) {
4113 		dev_warn(&instance->pdev->dev,
4114 			"fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",
4115 			abs_state, __func__, __LINE__);
4116 		return -1;
4117 	}
4118 
4119 	return 0;
4120 }
4121 
4122 /**
4123  * megasas_check_reset_fusion -	For controller reset check
4124  * @instance:				Controller's soft instance
4125  * @regs:				MFI register set
4126  */
4127 static int
4128 megasas_check_reset_fusion(struct megasas_instance *instance,
4129 			   struct megasas_register_set __iomem *regs)
4130 {
4131 	return 0;
4132 }
4133 
4134 /**
4135  * megasas_trigger_snap_dump -	Trigger snap dump in FW
4136  * @instance:			Soft instance of adapter
4137  */
4138 static inline void megasas_trigger_snap_dump(struct megasas_instance *instance)
4139 {
4140 	int j;
4141 	u32 fw_state, abs_state;
4142 
4143 	if (!instance->disableOnlineCtrlReset) {
4144 		dev_info(&instance->pdev->dev, "Trigger snap dump\n");
4145 		writel(MFI_ADP_TRIGGER_SNAP_DUMP,
4146 		       &instance->reg_set->doorbell);
4147 		readl(&instance->reg_set->doorbell);
4148 	}
4149 
4150 	for (j = 0; j < instance->snapdump_wait_time; j++) {
4151 		abs_state = instance->instancet->read_fw_status_reg(instance);
4152 		fw_state = abs_state & MFI_STATE_MASK;
4153 		if (fw_state == MFI_STATE_FAULT) {
4154 			dev_printk(KERN_ERR, &instance->pdev->dev,
4155 				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4156 				   abs_state & MFI_STATE_FAULT_CODE,
4157 				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4158 			return;
4159 		}
4160 		msleep(1000);
4161 	}
4162 }
4163 
4164 /* This function waits for outstanding commands on fusion to complete */
4165 static int
4166 megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
4167 				    int reason, int *convert)
4168 {
4169 	int i, outstanding, retval = 0, hb_seconds_missed = 0;
4170 	u32 fw_state, abs_state;
4171 	u32 waittime_for_io_completion;
4172 
4173 	waittime_for_io_completion =
4174 		min_t(u32, resetwaittime,
4175 			(resetwaittime - instance->snapdump_wait_time));
4176 
4177 	if (reason == MFI_IO_TIMEOUT_OCR) {
4178 		dev_info(&instance->pdev->dev,
4179 			"MFI command is timed out\n");
4180 		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4181 		if (instance->snapdump_wait_time)
4182 			megasas_trigger_snap_dump(instance);
4183 		retval = 1;
4184 		goto out;
4185 	}
4186 
4187 	for (i = 0; i < waittime_for_io_completion; i++) {
4188 		/* Check if firmware is in fault state */
4189 		abs_state = instance->instancet->read_fw_status_reg(instance);
4190 		fw_state = abs_state & MFI_STATE_MASK;
4191 		if (fw_state == MFI_STATE_FAULT) {
4192 			dev_printk(KERN_ERR, &instance->pdev->dev,
4193 				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4194 				   abs_state & MFI_STATE_FAULT_CODE,
4195 				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4196 			megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4197 			if (instance->requestorId && reason) {
4198 				dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"
4199 				" state while polling during"
4200 				" I/O timeout handling for %d\n",
4201 				instance->host->host_no);
4202 				*convert = 1;
4203 			}
4204 
4205 			retval = 1;
4206 			goto out;
4207 		}
4208 
4209 
4210 		/* If SR-IOV VF mode & heartbeat timeout, don't wait */
4211 		if (instance->requestorId && !reason) {
4212 			retval = 1;
4213 			goto out;
4214 		}
4215 
4216 		/* If SR-IOV VF mode & I/O timeout, check for HB timeout */
4217 		if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {
4218 			if (instance->hb_host_mem->HB.fwCounter !=
4219 			    instance->hb_host_mem->HB.driverCounter) {
4220 				instance->hb_host_mem->HB.driverCounter =
4221 					instance->hb_host_mem->HB.fwCounter;
4222 				hb_seconds_missed = 0;
4223 			} else {
4224 				hb_seconds_missed++;
4225 				if (hb_seconds_missed ==
4226 				    (MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {
4227 					dev_warn(&instance->pdev->dev, "SR-IOV:"
4228 					       " Heartbeat never completed "
4229 					       " while polling during I/O "
4230 					       " timeout handling for "
4231 					       "scsi%d.\n",
4232 					       instance->host->host_no);
4233 					       *convert = 1;
4234 					       retval = 1;
4235 					       goto out;
4236 				}
4237 			}
4238 		}
4239 
4240 		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4241 		outstanding = atomic_read(&instance->fw_outstanding);
4242 		if (!outstanding)
4243 			goto out;
4244 
4245 		if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {
4246 			dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
4247 			       "commands to complete for scsi%d\n", i,
4248 			       outstanding, instance->host->host_no);
4249 		}
4250 		msleep(1000);
4251 	}
4252 
4253 	if (instance->snapdump_wait_time) {
4254 		megasas_trigger_snap_dump(instance);
4255 		retval = 1;
4256 		goto out;
4257 	}
4258 
4259 	if (atomic_read(&instance->fw_outstanding)) {
4260 		dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
4261 		       "will reset adapter scsi%d.\n",
4262 		       instance->host->host_no);
4263 		*convert = 1;
4264 		retval = 1;
4265 	}
4266 
4267 out:
4268 	return retval;
4269 }
4270 
4271 void  megasas_reset_reply_desc(struct megasas_instance *instance)
4272 {
4273 	int i, j, count;
4274 	struct fusion_context *fusion;
4275 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
4276 
4277 	fusion = instance->ctrl_context;
4278 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
4279 	count += instance->iopoll_q_count;
4280 
4281 	for (i = 0 ; i < count ; i++) {
4282 		fusion->last_reply_idx[i] = 0;
4283 		reply_desc = fusion->reply_frames_desc[i];
4284 		for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)
4285 			reply_desc->Words = cpu_to_le64(ULLONG_MAX);
4286 	}
4287 }
4288 
4289 /*
4290  * megasas_refire_mgmt_cmd :	Re-fire management commands
4291  * @instance:				Controller's soft instance
4292 */
4293 static void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
4294 			     bool return_ioctl)
4295 {
4296 	int j;
4297 	struct megasas_cmd_fusion *cmd_fusion;
4298 	struct fusion_context *fusion;
4299 	struct megasas_cmd *cmd_mfi;
4300 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4301 	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
4302 	u16 smid;
4303 	bool refire_cmd = false;
4304 	u8 result;
4305 	u32 opcode = 0;
4306 
4307 	fusion = instance->ctrl_context;
4308 
4309 	/* Re-fire management commands.
4310 	 * Do not traverse complet MPT frame pool. Start from max_scsi_cmds.
4311 	 */
4312 	for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {
4313 		cmd_fusion = fusion->cmd_list[j];
4314 		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4315 		smid = le16_to_cpu(cmd_mfi->context.smid);
4316 		result = REFIRE_CMD;
4317 
4318 		if (!smid)
4319 			continue;
4320 
4321 		req_desc = megasas_get_request_descriptor(instance, smid - 1);
4322 
4323 		switch (cmd_mfi->frame->hdr.cmd) {
4324 		case MFI_CMD_DCMD:
4325 			opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode);
4326 			 /* Do not refire shutdown command */
4327 			if (opcode == MR_DCMD_CTRL_SHUTDOWN) {
4328 				cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;
4329 				result = COMPLETE_CMD;
4330 				break;
4331 			}
4332 
4333 			refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) &&
4334 				      (opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&
4335 				      !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);
4336 
4337 			if (!refire_cmd)
4338 				result = RETURN_CMD;
4339 
4340 			break;
4341 		case MFI_CMD_NVME:
4342 			if (!instance->support_nvme_passthru) {
4343 				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4344 				result = COMPLETE_CMD;
4345 			}
4346 
4347 			break;
4348 		case MFI_CMD_TOOLBOX:
4349 			if (!instance->support_pci_lane_margining) {
4350 				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4351 				result = COMPLETE_CMD;
4352 			}
4353 
4354 			break;
4355 		default:
4356 			break;
4357 		}
4358 
4359 		if (return_ioctl && cmd_mfi->sync_cmd &&
4360 		    cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT) {
4361 			dev_err(&instance->pdev->dev,
4362 				"return -EBUSY from %s %d cmd 0x%x opcode 0x%x\n",
4363 				__func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4364 				le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4365 			cmd_mfi->cmd_status_drv = DCMD_BUSY;
4366 			result = COMPLETE_CMD;
4367 		}
4368 
4369 		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
4370 				cmd_fusion->io_request;
4371 		if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT)
4372 			result = RETURN_CMD;
4373 
4374 		switch (result) {
4375 		case REFIRE_CMD:
4376 			megasas_fire_cmd_fusion(instance, req_desc);
4377 			break;
4378 		case RETURN_CMD:
4379 			megasas_return_cmd(instance, cmd_mfi);
4380 			break;
4381 		case COMPLETE_CMD:
4382 			megasas_complete_cmd(instance, cmd_mfi, DID_OK);
4383 			break;
4384 		}
4385 	}
4386 }
4387 
4388 /*
4389  * megasas_return_polled_cmds: Return polled mode commands back to the pool
4390  *			       before initiating an OCR.
4391  * @instance:                  Controller's soft instance
4392  */
4393 static void
4394 megasas_return_polled_cmds(struct megasas_instance *instance)
4395 {
4396 	int i;
4397 	struct megasas_cmd_fusion *cmd_fusion;
4398 	struct fusion_context *fusion;
4399 	struct megasas_cmd *cmd_mfi;
4400 
4401 	fusion = instance->ctrl_context;
4402 
4403 	for (i = instance->max_scsi_cmds; i < instance->max_fw_cmds; i++) {
4404 		cmd_fusion = fusion->cmd_list[i];
4405 		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4406 
4407 		if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
4408 			if (megasas_dbg_lvl & OCR_DEBUG)
4409 				dev_info(&instance->pdev->dev,
4410 					 "%s %d return cmd 0x%x opcode 0x%x\n",
4411 					 __func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4412 					 le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4413 			cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
4414 			megasas_return_cmd(instance, cmd_mfi);
4415 		}
4416 	}
4417 }
4418 
4419 /*
4420  * megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device
4421  * @instance: per adapter struct
4422  * @channel: the channel assigned by the OS
4423  * @id: the id assigned by the OS
4424  *
4425  * Returns SUCCESS if no IOs pending to SCSI device, else return FAILED
4426  */
4427 
4428 static int megasas_track_scsiio(struct megasas_instance *instance,
4429 		int id, int channel)
4430 {
4431 	int i, found = 0;
4432 	struct megasas_cmd_fusion *cmd_fusion;
4433 	struct fusion_context *fusion;
4434 	fusion = instance->ctrl_context;
4435 
4436 	for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4437 		cmd_fusion = fusion->cmd_list[i];
4438 		if (cmd_fusion->scmd &&
4439 			(cmd_fusion->scmd->device->id == id &&
4440 			cmd_fusion->scmd->device->channel == channel)) {
4441 			dev_info(&instance->pdev->dev,
4442 				"SCSI commands pending to target"
4443 				"channel %d id %d \tSMID: 0x%x\n",
4444 				channel, id, cmd_fusion->index);
4445 			scsi_print_command(cmd_fusion->scmd);
4446 			found = 1;
4447 			break;
4448 		}
4449 	}
4450 
4451 	return found ? FAILED : SUCCESS;
4452 }
4453 
4454 /**
4455  * megasas_tm_response_code - translation of device response code
4456  * @instance:	Controller's soft instance
4457  * @mpi_reply:	MPI reply returned by firmware
4458  *
4459  * Return nothing.
4460  */
4461 static void
4462 megasas_tm_response_code(struct megasas_instance *instance,
4463 		struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
4464 {
4465 	char *desc;
4466 
4467 	switch (mpi_reply->ResponseCode) {
4468 	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
4469 		desc = "task management request completed";
4470 		break;
4471 	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
4472 		desc = "invalid frame";
4473 		break;
4474 	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
4475 		desc = "task management request not supported";
4476 		break;
4477 	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
4478 		desc = "task management request failed";
4479 		break;
4480 	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
4481 		desc = "task management request succeeded";
4482 		break;
4483 	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
4484 		desc = "invalid lun";
4485 		break;
4486 	case 0xA:
4487 		desc = "overlapped tag attempted";
4488 		break;
4489 	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
4490 		desc = "task queued, however not sent to target";
4491 		break;
4492 	default:
4493 		desc = "unknown";
4494 		break;
4495 	}
4496 	dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",
4497 		mpi_reply->ResponseCode, desc);
4498 	dev_dbg(&instance->pdev->dev,
4499 		"TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"
4500 		" 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
4501 		mpi_reply->TerminationCount, mpi_reply->DevHandle,
4502 		mpi_reply->Function, mpi_reply->TaskType,
4503 		mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
4504 }
4505 
4506 /**
4507  * megasas_issue_tm - main routine for sending tm requests
4508  * @instance: per adapter struct
4509  * @device_handle: device handle
4510  * @channel: the channel assigned by the OS
4511  * @id: the id assigned by the OS
4512  * @smid_task: smid assigned to the task
4513  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
4514  * @mr_device_priv_data: private data
4515  * Context: user
4516  *
4517  * MegaRaid use MPT interface for Task Magement request.
4518  * A generic API for sending task management requests to firmware.
4519  *
4520  * Return SUCCESS or FAILED.
4521  */
4522 static int
4523 megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
4524 	uint channel, uint id, u16 smid_task, u8 type,
4525 	struct MR_PRIV_DEVICE *mr_device_priv_data)
4526 {
4527 	struct MR_TASK_MANAGE_REQUEST *mr_request;
4528 	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;
4529 	unsigned long timeleft;
4530 	struct megasas_cmd_fusion *cmd_fusion;
4531 	struct megasas_cmd *cmd_mfi;
4532 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4533 	struct fusion_context *fusion = NULL;
4534 	struct megasas_cmd_fusion *scsi_lookup;
4535 	int rc;
4536 	int timeout = MEGASAS_DEFAULT_TM_TIMEOUT;
4537 	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
4538 
4539 	fusion = instance->ctrl_context;
4540 
4541 	cmd_mfi = megasas_get_cmd(instance);
4542 
4543 	if (!cmd_mfi) {
4544 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4545 			__func__, __LINE__);
4546 		return -ENOMEM;
4547 	}
4548 
4549 	cmd_fusion = megasas_get_cmd_fusion(instance,
4550 			instance->max_scsi_cmds + cmd_mfi->index);
4551 
4552 	/*  Save the smid. To be used for returning the cmd */
4553 	cmd_mfi->context.smid = cmd_fusion->index;
4554 
4555 	req_desc = megasas_get_request_descriptor(instance,
4556 			(cmd_fusion->index - 1));
4557 
4558 	cmd_fusion->request_desc = req_desc;
4559 	req_desc->Words = 0;
4560 
4561 	mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
4562 	memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
4563 	mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
4564 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4565 	mpi_request->DevHandle = cpu_to_le16(device_handle);
4566 	mpi_request->TaskType = type;
4567 	mpi_request->TaskMID = cpu_to_le16(smid_task);
4568 	mpi_request->LUN[1] = 0;
4569 
4570 
4571 	req_desc = cmd_fusion->request_desc;
4572 	req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);
4573 	req_desc->HighPriority.RequestFlags =
4574 		(MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
4575 		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
4576 	req_desc->HighPriority.MSIxIndex =  0;
4577 	req_desc->HighPriority.LMID = 0;
4578 	req_desc->HighPriority.Reserved1 = 0;
4579 
4580 	if (channel < MEGASAS_MAX_PD_CHANNELS)
4581 		mr_request->tmReqFlags.isTMForPD = 1;
4582 	else
4583 		mr_request->tmReqFlags.isTMForLD = 1;
4584 
4585 	init_completion(&cmd_fusion->done);
4586 	megasas_fire_cmd_fusion(instance, req_desc);
4587 
4588 	switch (type) {
4589 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4590 		timeout = mr_device_priv_data->task_abort_tmo;
4591 		break;
4592 	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4593 		timeout = mr_device_priv_data->target_reset_tmo;
4594 		break;
4595 	}
4596 
4597 	timeleft = wait_for_completion_timeout(&cmd_fusion->done, timeout * HZ);
4598 
4599 	if (!timeleft) {
4600 		dev_err(&instance->pdev->dev,
4601 			"task mgmt type 0x%x timed out\n", type);
4602 		mutex_unlock(&instance->reset_mutex);
4603 		rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);
4604 		mutex_lock(&instance->reset_mutex);
4605 		return rc;
4606 	}
4607 
4608 	mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;
4609 	megasas_tm_response_code(instance, mpi_reply);
4610 
4611 	megasas_return_cmd(instance, cmd_mfi);
4612 	rc = SUCCESS;
4613 	switch (type) {
4614 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4615 		scsi_lookup = fusion->cmd_list[smid_task - 1];
4616 
4617 		if (scsi_lookup->scmd == NULL)
4618 			break;
4619 		else {
4620 			instance->instancet->disable_intr(instance);
4621 			megasas_sync_irqs((unsigned long)instance);
4622 			instance->instancet->enable_intr(instance);
4623 			megasas_enable_irq_poll(instance);
4624 			if (scsi_lookup->scmd == NULL)
4625 				break;
4626 		}
4627 		rc = FAILED;
4628 		break;
4629 
4630 	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4631 		if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
4632 			break;
4633 		instance->instancet->disable_intr(instance);
4634 		megasas_sync_irqs((unsigned long)instance);
4635 		rc = megasas_track_scsiio(instance, id, channel);
4636 		instance->instancet->enable_intr(instance);
4637 		megasas_enable_irq_poll(instance);
4638 
4639 		break;
4640 	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
4641 	case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
4642 		break;
4643 	default:
4644 		rc = FAILED;
4645 		break;
4646 	}
4647 
4648 	return rc;
4649 
4650 }
4651 
4652 /*
4653  * megasas_fusion_smid_lookup : Look for fusion command corresponding to SCSI
4654  * @instance: per adapter struct
4655  *
4656  * Return Non Zero index, if SMID found in outstanding commands
4657  */
4658 static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)
4659 {
4660 	int i, ret = 0;
4661 	struct megasas_instance *instance;
4662 	struct megasas_cmd_fusion *cmd_fusion;
4663 	struct fusion_context *fusion;
4664 
4665 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4666 
4667 	fusion = instance->ctrl_context;
4668 
4669 	for (i = 0; i < instance->max_scsi_cmds; i++) {
4670 		cmd_fusion = fusion->cmd_list[i];
4671 		if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {
4672 			scmd_printk(KERN_NOTICE, scmd, "Abort request is for"
4673 				" SMID: %d\n", cmd_fusion->index);
4674 			ret = cmd_fusion->index;
4675 			break;
4676 		}
4677 	}
4678 
4679 	return ret;
4680 }
4681 
4682 /*
4683 * megasas_get_tm_devhandle - Get devhandle for TM request
4684 * @sdev-		     OS provided scsi device
4685 *
4686 * Returns-		     devhandle/targetID of SCSI device
4687 */
4688 static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)
4689 {
4690 	u16 pd_index = 0;
4691 	u32 device_id;
4692 	struct megasas_instance *instance;
4693 	struct fusion_context *fusion;
4694 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
4695 	u16 devhandle = (u16)ULONG_MAX;
4696 
4697 	instance = (struct megasas_instance *)sdev->host->hostdata;
4698 	fusion = instance->ctrl_context;
4699 
4700 	if (!MEGASAS_IS_LOGICAL(sdev)) {
4701 		if (instance->use_seqnum_jbod_fp) {
4702 			pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)
4703 				    + sdev->id;
4704 			pd_sync = (void *)fusion->pd_seq_sync
4705 					[(instance->pd_seq_map_id - 1) & 1];
4706 			devhandle = pd_sync->seq[pd_index].devHandle;
4707 		} else
4708 			sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"
4709 				" without JBOD MAP support from %s %d\n", __func__, __LINE__);
4710 	} else {
4711 		device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)
4712 				+ sdev->id;
4713 		devhandle = device_id;
4714 	}
4715 
4716 	return devhandle;
4717 }
4718 
4719 /*
4720  * megasas_task_abort_fusion : SCSI task abort function for fusion adapters
4721  * @scmd : pointer to scsi command object
4722  *
4723  * Return SUCCESS, if command aborted else FAILED
4724  */
4725 
4726 int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
4727 {
4728 	struct megasas_instance *instance;
4729 	u16 smid, devhandle;
4730 	int ret;
4731 	struct MR_PRIV_DEVICE *mr_device_priv_data;
4732 	mr_device_priv_data = scmd->device->hostdata;
4733 
4734 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4735 
4736 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4737 		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4738 		"SCSI host:%d\n", instance->host->host_no);
4739 		ret = FAILED;
4740 		return ret;
4741 	}
4742 
4743 	if (!mr_device_priv_data) {
4744 		sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4745 			"scmd(%p)\n", scmd);
4746 		scmd->result = DID_NO_CONNECT << 16;
4747 		ret = SUCCESS;
4748 		goto out;
4749 	}
4750 
4751 	if (!mr_device_priv_data->is_tm_capable) {
4752 		ret = FAILED;
4753 		goto out;
4754 	}
4755 
4756 	mutex_lock(&instance->reset_mutex);
4757 
4758 	smid = megasas_fusion_smid_lookup(scmd);
4759 
4760 	if (!smid) {
4761 		ret = SUCCESS;
4762 		scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
4763 			" issued is not found in outstanding commands\n");
4764 		mutex_unlock(&instance->reset_mutex);
4765 		goto out;
4766 	}
4767 
4768 	devhandle = megasas_get_tm_devhandle(scmd->device);
4769 
4770 	if (devhandle == (u16)ULONG_MAX) {
4771 		ret = SUCCESS;
4772 		sdev_printk(KERN_INFO, scmd->device,
4773 			"task abort issued for invalid devhandle\n");
4774 		mutex_unlock(&instance->reset_mutex);
4775 		goto out;
4776 	}
4777 	sdev_printk(KERN_INFO, scmd->device,
4778 		"attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n",
4779 		scmd, devhandle);
4780 
4781 	mr_device_priv_data->tm_busy = true;
4782 	ret = megasas_issue_tm(instance, devhandle,
4783 			scmd->device->channel, scmd->device->id, smid,
4784 			MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
4785 			mr_device_priv_data);
4786 	mr_device_priv_data->tm_busy = false;
4787 
4788 	mutex_unlock(&instance->reset_mutex);
4789 	scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n",
4790 			((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4791 out:
4792 	scsi_print_command(scmd);
4793 	if (megasas_dbg_lvl & TM_DEBUG)
4794 		megasas_dump_fusion_io(scmd);
4795 
4796 	return ret;
4797 }
4798 
4799 /*
4800  * megasas_reset_target_fusion : target reset function for fusion adapters
4801  * scmd: SCSI command pointer
4802  *
4803  * Returns SUCCESS if all commands associated with target aborted else FAILED
4804  */
4805 
4806 int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
4807 {
4808 
4809 	struct megasas_instance *instance;
4810 	int ret = FAILED;
4811 	u16 devhandle;
4812 	struct MR_PRIV_DEVICE *mr_device_priv_data;
4813 	mr_device_priv_data = scmd->device->hostdata;
4814 
4815 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4816 
4817 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4818 		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4819 		"SCSI host:%d\n", instance->host->host_no);
4820 		ret = FAILED;
4821 		return ret;
4822 	}
4823 
4824 	if (!mr_device_priv_data) {
4825 		sdev_printk(KERN_INFO, scmd->device,
4826 			    "device been deleted! scmd: (0x%p)\n", scmd);
4827 		scmd->result = DID_NO_CONNECT << 16;
4828 		ret = SUCCESS;
4829 		goto out;
4830 	}
4831 
4832 	if (!mr_device_priv_data->is_tm_capable) {
4833 		ret = FAILED;
4834 		goto out;
4835 	}
4836 
4837 	mutex_lock(&instance->reset_mutex);
4838 	devhandle = megasas_get_tm_devhandle(scmd->device);
4839 
4840 	if (devhandle == (u16)ULONG_MAX) {
4841 		ret = SUCCESS;
4842 		sdev_printk(KERN_INFO, scmd->device,
4843 			"target reset issued for invalid devhandle\n");
4844 		mutex_unlock(&instance->reset_mutex);
4845 		goto out;
4846 	}
4847 
4848 	sdev_printk(KERN_INFO, scmd->device,
4849 		"attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n",
4850 		scmd, devhandle);
4851 	mr_device_priv_data->tm_busy = true;
4852 	ret = megasas_issue_tm(instance, devhandle,
4853 			scmd->device->channel, scmd->device->id, 0,
4854 			MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET,
4855 			mr_device_priv_data);
4856 	mr_device_priv_data->tm_busy = false;
4857 	mutex_unlock(&instance->reset_mutex);
4858 	scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n",
4859 		(ret == SUCCESS) ? "SUCCESS" : "FAILED");
4860 
4861 out:
4862 	return ret;
4863 }
4864 
4865 /*SRIOV get other instance in cluster if any*/
4866 static struct
4867 megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)
4868 {
4869 	int i;
4870 
4871 	for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {
4872 		if (megasas_mgmt_info.instance[i] &&
4873 			(megasas_mgmt_info.instance[i] != instance) &&
4874 			 megasas_mgmt_info.instance[i]->requestorId &&
4875 			 megasas_mgmt_info.instance[i]->peerIsPresent &&
4876 			(memcmp((megasas_mgmt_info.instance[i]->clusterId),
4877 			instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))
4878 			return megasas_mgmt_info.instance[i];
4879 	}
4880 	return NULL;
4881 }
4882 
4883 /* Check for a second path that is currently UP */
4884 int megasas_check_mpio_paths(struct megasas_instance *instance,
4885 	struct scsi_cmnd *scmd)
4886 {
4887 	struct megasas_instance *peer_instance = NULL;
4888 	int retval = (DID_REQUEUE << 16);
4889 
4890 	if (instance->peerIsPresent) {
4891 		peer_instance = megasas_get_peer_instance(instance);
4892 		if ((peer_instance) &&
4893 			(atomic_read(&peer_instance->adprecovery) ==
4894 			MEGASAS_HBA_OPERATIONAL))
4895 			retval = (DID_NO_CONNECT << 16);
4896 	}
4897 	return retval;
4898 }
4899 
4900 /* Core fusion reset function */
4901 int megasas_reset_fusion(struct Scsi_Host *shost, int reason)
4902 {
4903 	int retval = SUCCESS, i, j, convert = 0;
4904 	struct megasas_instance *instance;
4905 	struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;
4906 	struct fusion_context *fusion;
4907 	u32 abs_state, status_reg, reset_adapter, fpio_count = 0;
4908 	u32 io_timeout_in_crash_mode = 0;
4909 	struct scsi_cmnd *scmd_local = NULL;
4910 	struct scsi_device *sdev;
4911 	int ret_target_prop = DCMD_FAILED;
4912 	bool is_target_prop = false;
4913 	bool do_adp_reset = true;
4914 	int max_reset_tries = MEGASAS_FUSION_MAX_RESET_TRIES;
4915 
4916 	instance = (struct megasas_instance *)shost->hostdata;
4917 	fusion = instance->ctrl_context;
4918 
4919 	mutex_lock(&instance->reset_mutex);
4920 
4921 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
4922 		dev_warn(&instance->pdev->dev, "Hardware critical error, "
4923 		       "returning FAILED for scsi%d.\n",
4924 			instance->host->host_no);
4925 		mutex_unlock(&instance->reset_mutex);
4926 		return FAILED;
4927 	}
4928 	status_reg = instance->instancet->read_fw_status_reg(instance);
4929 	abs_state = status_reg & MFI_STATE_MASK;
4930 
4931 	/* IO timeout detected, forcibly put FW in FAULT state */
4932 	if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&
4933 		instance->crash_dump_app_support && reason) {
4934 		dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "
4935 			"forcibly FAULT Firmware\n");
4936 		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4937 		status_reg = megasas_readl(instance, &instance->reg_set->doorbell);
4938 		writel(status_reg | MFI_STATE_FORCE_OCR,
4939 			&instance->reg_set->doorbell);
4940 		readl(&instance->reg_set->doorbell);
4941 		mutex_unlock(&instance->reset_mutex);
4942 		do {
4943 			ssleep(3);
4944 			io_timeout_in_crash_mode++;
4945 			dev_dbg(&instance->pdev->dev, "waiting for [%d] "
4946 				"seconds for crash dump collection and OCR "
4947 				"to be done\n", (io_timeout_in_crash_mode * 3));
4948 		} while ((atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&
4949 			(io_timeout_in_crash_mode < 80));
4950 
4951 		if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {
4952 			dev_info(&instance->pdev->dev, "OCR done for IO "
4953 				"timeout case\n");
4954 			retval = SUCCESS;
4955 		} else {
4956 			dev_info(&instance->pdev->dev, "Controller is not "
4957 				"operational after 240 seconds wait for IO "
4958 				"timeout case in FW crash dump mode\n do "
4959 				"OCR/kill adapter\n");
4960 			retval = megasas_reset_fusion(shost, 0);
4961 		}
4962 		return retval;
4963 	}
4964 
4965 	if (instance->requestorId && !instance->skip_heartbeat_timer_del)
4966 		del_timer_sync(&instance->sriov_heartbeat_timer);
4967 	set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4968 	set_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);
4969 	atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING);
4970 	instance->instancet->disable_intr(instance);
4971 	megasas_sync_irqs((unsigned long)instance);
4972 
4973 	/* First try waiting for commands to complete */
4974 	if (megasas_wait_for_outstanding_fusion(instance, reason,
4975 						&convert)) {
4976 		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4977 		dev_warn(&instance->pdev->dev, "resetting fusion "
4978 		       "adapter scsi%d.\n", instance->host->host_no);
4979 		if (convert)
4980 			reason = 0;
4981 
4982 		if (megasas_dbg_lvl & OCR_DEBUG)
4983 			dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");
4984 
4985 		/* Now return commands back to the OS */
4986 		for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4987 			cmd_fusion = fusion->cmd_list[i];
4988 			/*check for extra commands issued by driver*/
4989 			if (instance->adapter_type >= VENTURA_SERIES) {
4990 				r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];
4991 				megasas_return_cmd_fusion(instance, r1_cmd);
4992 			}
4993 			scmd_local = cmd_fusion->scmd;
4994 			if (cmd_fusion->scmd) {
4995 				if (megasas_dbg_lvl & OCR_DEBUG) {
4996 					sdev_printk(KERN_INFO,
4997 						cmd_fusion->scmd->device, "SMID: 0x%x\n",
4998 						cmd_fusion->index);
4999 					megasas_dump_fusion_io(cmd_fusion->scmd);
5000 				}
5001 
5002 				if (cmd_fusion->io_request->Function ==
5003 					MPI2_FUNCTION_SCSI_IO_REQUEST)
5004 					fpio_count++;
5005 
5006 				scmd_local->result =
5007 					megasas_check_mpio_paths(instance,
5008 							scmd_local);
5009 				if (instance->ldio_threshold &&
5010 					megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
5011 					atomic_dec(&instance->ldio_outstanding);
5012 				megasas_return_cmd_fusion(instance, cmd_fusion);
5013 				scsi_dma_unmap(scmd_local);
5014 				scsi_done(scmd_local);
5015 			}
5016 		}
5017 
5018 		dev_info(&instance->pdev->dev, "Outstanding fastpath IOs: %d\n",
5019 			fpio_count);
5020 
5021 		atomic_set(&instance->fw_outstanding, 0);
5022 
5023 		status_reg = instance->instancet->read_fw_status_reg(instance);
5024 		abs_state = status_reg & MFI_STATE_MASK;
5025 		reset_adapter = status_reg & MFI_RESET_ADAPTER;
5026 		if (instance->disableOnlineCtrlReset ||
5027 		    (abs_state == MFI_STATE_FAULT && !reset_adapter)) {
5028 			/* Reset not supported, kill adapter */
5029 			dev_warn(&instance->pdev->dev, "Reset not supported"
5030 			       ", killing adapter scsi%d.\n",
5031 				instance->host->host_no);
5032 			goto kill_hba;
5033 		}
5034 
5035 		/* Let SR-IOV VF & PF sync up if there was a HB failure */
5036 		if (instance->requestorId && !reason) {
5037 			msleep(MEGASAS_OCR_SETTLE_TIME_VF);
5038 			do_adp_reset = false;
5039 			max_reset_tries = MEGASAS_SRIOV_MAX_RESET_TRIES_VF;
5040 		}
5041 
5042 		/* Now try to reset the chip */
5043 		for (i = 0; i < max_reset_tries; i++) {
5044 			/*
5045 			 * Do adp reset and wait for
5046 			 * controller to transition to ready
5047 			 */
5048 			if (megasas_adp_reset_wait_for_ready(instance,
5049 				do_adp_reset, 1) == FAILED)
5050 				continue;
5051 
5052 			/* Wait for FW to become ready */
5053 			if (megasas_transition_to_ready(instance, 1)) {
5054 				dev_warn(&instance->pdev->dev,
5055 					"Failed to transition controller to ready for "
5056 					"scsi%d.\n", instance->host->host_no);
5057 				continue;
5058 			}
5059 			megasas_reset_reply_desc(instance);
5060 			megasas_fusion_update_can_queue(instance, OCR_CONTEXT);
5061 
5062 			if (megasas_ioc_init_fusion(instance)) {
5063 				continue;
5064 			}
5065 
5066 			if (megasas_get_ctrl_info(instance)) {
5067 				dev_info(&instance->pdev->dev,
5068 					"Failed from %s %d\n",
5069 					__func__, __LINE__);
5070 				goto kill_hba;
5071 			}
5072 
5073 			megasas_refire_mgmt_cmd(instance,
5074 						(i == (MEGASAS_FUSION_MAX_RESET_TRIES - 1)
5075 							? 1 : 0));
5076 
5077 			/* Reset load balance info */
5078 			if (fusion->load_balance_info)
5079 				memset(fusion->load_balance_info, 0,
5080 				       (sizeof(struct LD_LOAD_BALANCE_INFO) *
5081 				       MAX_LOGICAL_DRIVES_EXT));
5082 
5083 			if (!megasas_get_map_info(instance)) {
5084 				megasas_sync_map_info(instance);
5085 			} else {
5086 				/*
5087 				 * Return pending polled mode cmds before
5088 				 * retrying OCR
5089 				 */
5090 				megasas_return_polled_cmds(instance);
5091 				continue;
5092 			}
5093 
5094 			megasas_setup_jbod_map(instance);
5095 
5096 			/* reset stream detection array */
5097 			if (instance->adapter_type >= VENTURA_SERIES) {
5098 				for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {
5099 					memset(fusion->stream_detect_by_ld[j],
5100 					       0, sizeof(struct LD_STREAM_DETECT));
5101 					fusion->stream_detect_by_ld[j]->mru_bit_map
5102 						= MR_STREAM_BITMAP;
5103 				}
5104 			}
5105 
5106 			clear_bit(MEGASAS_FUSION_IN_RESET,
5107 				  &instance->reset_flags);
5108 			instance->instancet->enable_intr(instance);
5109 			megasas_enable_irq_poll(instance);
5110 			shost_for_each_device(sdev, shost) {
5111 				if ((instance->tgt_prop) &&
5112 				    (instance->nvme_page_size))
5113 					ret_target_prop = megasas_get_target_prop(instance, sdev);
5114 
5115 				is_target_prop = (ret_target_prop == DCMD_SUCCESS) ? true : false;
5116 				megasas_set_dynamic_target_properties(sdev, is_target_prop);
5117 			}
5118 
5119 			status_reg = instance->instancet->read_fw_status_reg
5120 					(instance);
5121 			abs_state = status_reg & MFI_STATE_MASK;
5122 			if (abs_state != MFI_STATE_OPERATIONAL) {
5123 				dev_info(&instance->pdev->dev,
5124 					 "Adapter is not OPERATIONAL, state 0x%x for scsi:%d\n",
5125 					 abs_state, instance->host->host_no);
5126 				goto out;
5127 			}
5128 			atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
5129 
5130 			dev_info(&instance->pdev->dev,
5131 				 "Adapter is OPERATIONAL for scsi:%d\n",
5132 				 instance->host->host_no);
5133 
5134 			/* Restart SR-IOV heartbeat */
5135 			if (instance->requestorId) {
5136 				if (!megasas_sriov_start_heartbeat(instance, 0))
5137 					megasas_start_timer(instance);
5138 				else
5139 					instance->skip_heartbeat_timer_del = 1;
5140 			}
5141 
5142 			if (instance->crash_dump_drv_support &&
5143 				instance->crash_dump_app_support)
5144 				megasas_set_crash_dump_params(instance,
5145 					MR_CRASH_BUF_TURN_ON);
5146 			else
5147 				megasas_set_crash_dump_params(instance,
5148 					MR_CRASH_BUF_TURN_OFF);
5149 
5150 			if (instance->snapdump_wait_time) {
5151 				megasas_get_snapdump_properties(instance);
5152 				dev_info(&instance->pdev->dev,
5153 					 "Snap dump wait time\t: %d\n",
5154 					 instance->snapdump_wait_time);
5155 			}
5156 
5157 			retval = SUCCESS;
5158 
5159 			/* Adapter reset completed successfully */
5160 			dev_warn(&instance->pdev->dev,
5161 				 "Reset successful for scsi%d.\n",
5162 				 instance->host->host_no);
5163 
5164 			goto out;
5165 		}
5166 		/* Reset failed, kill the adapter */
5167 		dev_warn(&instance->pdev->dev, "Reset failed, killing "
5168 		       "adapter scsi%d.\n", instance->host->host_no);
5169 		goto kill_hba;
5170 	} else {
5171 		/* For VF: Restart HB timer if we didn't OCR */
5172 		if (instance->requestorId) {
5173 			megasas_start_timer(instance);
5174 		}
5175 		clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
5176 		instance->instancet->enable_intr(instance);
5177 		megasas_enable_irq_poll(instance);
5178 		atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
5179 		goto out;
5180 	}
5181 kill_hba:
5182 	megaraid_sas_kill_hba(instance);
5183 	megasas_enable_irq_poll(instance);
5184 	instance->skip_heartbeat_timer_del = 1;
5185 	retval = FAILED;
5186 out:
5187 	clear_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);
5188 	mutex_unlock(&instance->reset_mutex);
5189 	return retval;
5190 }
5191 
5192 /* Fusion Crash dump collection */
5193 static void  megasas_fusion_crash_dump(struct megasas_instance *instance)
5194 {
5195 	u32 status_reg;
5196 	u8 partial_copy = 0;
5197 	int wait = 0;
5198 
5199 
5200 	status_reg = instance->instancet->read_fw_status_reg(instance);
5201 
5202 	/*
5203 	 * Allocate host crash buffers to copy data from 1 MB DMA crash buffer
5204 	 * to host crash buffers
5205 	 */
5206 	if (instance->drv_buf_index == 0) {
5207 		/* Buffer is already allocated for old Crash dump.
5208 		 * Do OCR and do not wait for crash dump collection
5209 		 */
5210 		if (instance->drv_buf_alloc) {
5211 			dev_info(&instance->pdev->dev, "earlier crash dump is "
5212 				"not yet copied by application, ignoring this "
5213 				"crash dump and initiating OCR\n");
5214 			status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5215 			writel(status_reg,
5216 				&instance->reg_set->outbound_scratch_pad_0);
5217 			readl(&instance->reg_set->outbound_scratch_pad_0);
5218 			return;
5219 		}
5220 		megasas_alloc_host_crash_buffer(instance);
5221 		dev_info(&instance->pdev->dev, "Number of host crash buffers "
5222 			"allocated: %d\n", instance->drv_buf_alloc);
5223 	}
5224 
5225 	while (!(status_reg & MFI_STATE_CRASH_DUMP_DONE) &&
5226 	       (wait < MEGASAS_WATCHDOG_WAIT_COUNT)) {
5227 		if (!(status_reg & MFI_STATE_DMADONE)) {
5228 			/*
5229 			 * Next crash dump buffer is not yet DMA'd by FW
5230 			 * Check after 10ms. Wait for 1 second for FW to
5231 			 * post the next buffer. If not bail out.
5232 			 */
5233 			wait++;
5234 			msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5235 			status_reg = instance->instancet->read_fw_status_reg(
5236 					instance);
5237 			continue;
5238 		}
5239 
5240 		wait = 0;
5241 		if (instance->drv_buf_index >= instance->drv_buf_alloc) {
5242 			dev_info(&instance->pdev->dev,
5243 				 "Driver is done copying the buffer: %d\n",
5244 				 instance->drv_buf_alloc);
5245 			status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5246 			partial_copy = 1;
5247 			break;
5248 		} else {
5249 			memcpy(instance->crash_buf[instance->drv_buf_index],
5250 			       instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);
5251 			instance->drv_buf_index++;
5252 			status_reg &= ~MFI_STATE_DMADONE;
5253 		}
5254 
5255 		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);
5256 		readl(&instance->reg_set->outbound_scratch_pad_0);
5257 
5258 		msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5259 		status_reg = instance->instancet->read_fw_status_reg(instance);
5260 	}
5261 
5262 	if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {
5263 		dev_info(&instance->pdev->dev, "Crash Dump is available,number "
5264 			"of copied buffers: %d\n", instance->drv_buf_index);
5265 		instance->fw_crash_buffer_size =  instance->drv_buf_index;
5266 		instance->fw_crash_state = AVAILABLE;
5267 		instance->drv_buf_index = 0;
5268 		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);
5269 		readl(&instance->reg_set->outbound_scratch_pad_0);
5270 		if (!partial_copy)
5271 			megasas_reset_fusion(instance->host, 0);
5272 	}
5273 }
5274 
5275 
5276 /* Fusion OCR work queue */
5277 void megasas_fusion_ocr_wq(struct work_struct *work)
5278 {
5279 	struct megasas_instance *instance =
5280 		container_of(work, struct megasas_instance, work_init);
5281 
5282 	megasas_reset_fusion(instance->host, 0);
5283 }
5284 
5285 /* Allocate fusion context */
5286 int
5287 megasas_alloc_fusion_context(struct megasas_instance *instance)
5288 {
5289 	struct fusion_context *fusion;
5290 
5291 	instance->ctrl_context = kzalloc(sizeof(struct fusion_context),
5292 					 GFP_KERNEL);
5293 	if (!instance->ctrl_context) {
5294 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5295 			__func__, __LINE__);
5296 		return -ENOMEM;
5297 	}
5298 
5299 	fusion = instance->ctrl_context;
5300 
5301 	fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5302 					      sizeof(LD_SPAN_INFO));
5303 	fusion->log_to_span =
5304 		(PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5305 						fusion->log_to_span_pages);
5306 	if (!fusion->log_to_span) {
5307 		fusion->log_to_span =
5308 			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5309 					   sizeof(LD_SPAN_INFO)));
5310 		if (!fusion->log_to_span) {
5311 			dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5312 				__func__, __LINE__);
5313 			return -ENOMEM;
5314 		}
5315 	}
5316 
5317 	fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5318 		sizeof(struct LD_LOAD_BALANCE_INFO));
5319 	fusion->load_balance_info =
5320 		(struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5321 		fusion->load_balance_info_pages);
5322 	if (!fusion->load_balance_info) {
5323 		fusion->load_balance_info =
5324 			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5325 					   sizeof(struct LD_LOAD_BALANCE_INFO)));
5326 		if (!fusion->load_balance_info)
5327 			dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "
5328 				"continuing without Load Balance support\n");
5329 	}
5330 
5331 	return 0;
5332 }
5333 
5334 void
5335 megasas_free_fusion_context(struct megasas_instance *instance)
5336 {
5337 	struct fusion_context *fusion = instance->ctrl_context;
5338 
5339 	if (fusion) {
5340 		if (fusion->load_balance_info) {
5341 			if (is_vmalloc_addr(fusion->load_balance_info))
5342 				vfree(fusion->load_balance_info);
5343 			else
5344 				free_pages((ulong)fusion->load_balance_info,
5345 					fusion->load_balance_info_pages);
5346 		}
5347 
5348 		if (fusion->log_to_span) {
5349 			if (is_vmalloc_addr(fusion->log_to_span))
5350 				vfree(fusion->log_to_span);
5351 			else
5352 				free_pages((ulong)fusion->log_to_span,
5353 					   fusion->log_to_span_pages);
5354 		}
5355 
5356 		kfree(fusion);
5357 	}
5358 }
5359 
5360 struct megasas_instance_template megasas_instance_template_fusion = {
5361 	.enable_intr = megasas_enable_intr_fusion,
5362 	.disable_intr = megasas_disable_intr_fusion,
5363 	.clear_intr = megasas_clear_intr_fusion,
5364 	.read_fw_status_reg = megasas_read_fw_status_reg_fusion,
5365 	.adp_reset = megasas_adp_reset_fusion,
5366 	.check_reset = megasas_check_reset_fusion,
5367 	.service_isr = megasas_isr_fusion,
5368 	.tasklet = megasas_complete_cmd_dpc_fusion,
5369 	.init_adapter = megasas_init_adapter_fusion,
5370 	.build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,
5371 	.issue_dcmd = megasas_issue_dcmd_fusion,
5372 };
5373