1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Linux MegaRAID driver for SAS based RAID controllers
4  *
5  *  Copyright (c) 2009-2013  LSI Corporation
6  *  Copyright (c) 2013-2016  Avago Technologies
7  *  Copyright (c) 2016-2018  Broadcom Inc.
8  *
9  *  FILE: megaraid_sas_fusion.c
10  *
11  *  Authors: Broadcom Inc.
12  *           Sumant Patro
13  *           Adam Radford
14  *           Kashyap Desai <kashyap.desai@broadcom.com>
15  *           Sumit Saxena <sumit.saxena@broadcom.com>
16  *
17  *  Send feedback to: megaraidlinux.pdl@broadcom.com
18  */
19 
20 #include <linux/kernel.h>
21 #include <linux/types.h>
22 #include <linux/pci.h>
23 #include <linux/list.h>
24 #include <linux/moduleparam.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/interrupt.h>
28 #include <linux/delay.h>
29 #include <linux/uio.h>
30 #include <linux/uaccess.h>
31 #include <linux/fs.h>
32 #include <linux/compat.h>
33 #include <linux/blkdev.h>
34 #include <linux/mutex.h>
35 #include <linux/poll.h>
36 #include <linux/vmalloc.h>
37 #include <linux/workqueue.h>
38 #include <linux/irq_poll.h>
39 
40 #include <scsi/scsi.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <scsi/scsi_device.h>
43 #include <scsi/scsi_host.h>
44 #include <scsi/scsi_dbg.h>
45 #include <linux/dmi.h>
46 
47 #include "megaraid_sas_fusion.h"
48 #include "megaraid_sas.h"
49 
50 
51 extern void
52 megasas_complete_cmd(struct megasas_instance *instance,
53 		     struct megasas_cmd *cmd, u8 alt_status);
54 int
55 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
56 	      int seconds);
57 
58 int
59 megasas_clear_intr_fusion(struct megasas_instance *instance);
60 
61 int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
62 
63 extern u32 megasas_dbg_lvl;
64 int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
65 				  int initial);
66 extern struct megasas_mgmt_info megasas_mgmt_info;
67 extern unsigned int resetwaittime;
68 extern unsigned int dual_qdepth_disable;
69 static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
70 static void megasas_free_reply_fusion(struct megasas_instance *instance);
71 static inline
72 void megasas_configure_queue_sizes(struct megasas_instance *instance);
73 static void megasas_fusion_crash_dump(struct megasas_instance *instance);
74 
75 /**
76  * megasas_adp_reset_wait_for_ready -	initiate chip reset and wait for
77  *					controller to come to ready state
78  * @instance:				adapter's soft state
79  * @do_adp_reset:			If true, do a chip reset
80  * @ocr_context:			If called from OCR context this will
81  *					be set to 1, else 0
82  *
83  * This function initiates a chip reset followed by a wait for controller to
84  * transition to ready state.
85  * During this, driver will block all access to PCI config space from userspace
86  */
87 int
88 megasas_adp_reset_wait_for_ready(struct megasas_instance *instance,
89 				 bool do_adp_reset,
90 				 int ocr_context)
91 {
92 	int ret = FAILED;
93 
94 	/*
95 	 * Block access to PCI config space from userspace
96 	 * when diag reset is initiated from driver
97 	 */
98 	if (megasas_dbg_lvl & OCR_DEBUG)
99 		dev_info(&instance->pdev->dev,
100 			 "Block access to PCI config space %s %d\n",
101 			 __func__, __LINE__);
102 
103 	pci_cfg_access_lock(instance->pdev);
104 
105 	if (do_adp_reset) {
106 		if (instance->instancet->adp_reset
107 			(instance, instance->reg_set))
108 			goto out;
109 	}
110 
111 	/* Wait for FW to become ready */
112 	if (megasas_transition_to_ready(instance, ocr_context)) {
113 		dev_warn(&instance->pdev->dev,
114 			 "Failed to transition controller to ready for scsi%d.\n",
115 			 instance->host->host_no);
116 		goto out;
117 	}
118 
119 	ret = SUCCESS;
120 out:
121 	if (megasas_dbg_lvl & OCR_DEBUG)
122 		dev_info(&instance->pdev->dev,
123 			 "Unlock access to PCI config space %s %d\n",
124 			 __func__, __LINE__);
125 
126 	pci_cfg_access_unlock(instance->pdev);
127 
128 	return ret;
129 }
130 
131 /**
132  * megasas_check_same_4gb_region -	check if allocation
133  *					crosses same 4GB boundary or not
134  * @instance:				adapter's soft instance
135  * @start_addr:				start address of DMA allocation
136  * @size:				size of allocation in bytes
137  * @return:				true : allocation does not cross same
138  *					4GB boundary
139  *					false: allocation crosses same
140  *					4GB boundary
141  */
142 static inline bool megasas_check_same_4gb_region
143 	(struct megasas_instance *instance, dma_addr_t start_addr, size_t size)
144 {
145 	dma_addr_t end_addr;
146 
147 	end_addr = start_addr + size;
148 
149 	if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {
150 		dev_err(&instance->pdev->dev,
151 			"Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",
152 			(unsigned long long)start_addr,
153 			(unsigned long long)end_addr);
154 		return false;
155 	}
156 
157 	return true;
158 }
159 
160 /**
161  * megasas_enable_intr_fusion -	Enables interrupts
162  * @instance:	adapter's soft instance
163  */
164 static void
165 megasas_enable_intr_fusion(struct megasas_instance *instance)
166 {
167 	struct megasas_register_set __iomem *regs;
168 	regs = instance->reg_set;
169 
170 	instance->mask_interrupts = 0;
171 	/* For Thunderbolt/Invader also clear intr on enable */
172 	writel(~0, &regs->outbound_intr_status);
173 	readl(&regs->outbound_intr_status);
174 
175 	writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);
176 
177 	/* Dummy readl to force pci flush */
178 	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
179 		 __func__, readl(&regs->outbound_intr_mask));
180 }
181 
182 /**
183  * megasas_disable_intr_fusion - Disables interrupt
184  * @instance:	adapter's soft instance
185  */
186 static void
187 megasas_disable_intr_fusion(struct megasas_instance *instance)
188 {
189 	u32 mask = 0xFFFFFFFF;
190 	struct megasas_register_set __iomem *regs;
191 	regs = instance->reg_set;
192 	instance->mask_interrupts = 1;
193 
194 	writel(mask, &regs->outbound_intr_mask);
195 	/* Dummy readl to force pci flush */
196 	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
197 		 __func__, readl(&regs->outbound_intr_mask));
198 }
199 
200 int
201 megasas_clear_intr_fusion(struct megasas_instance *instance)
202 {
203 	u32 status;
204 	struct megasas_register_set __iomem *regs;
205 	regs = instance->reg_set;
206 	/*
207 	 * Check if it is our interrupt
208 	 */
209 	status = megasas_readl(instance,
210 			       &regs->outbound_intr_status);
211 
212 	if (status & 1) {
213 		writel(status, &regs->outbound_intr_status);
214 		readl(&regs->outbound_intr_status);
215 		return 1;
216 	}
217 	if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
218 		return 0;
219 
220 	return 1;
221 }
222 
223 static inline void
224 megasas_sdev_busy_inc(struct megasas_instance *instance,
225 		      struct scsi_cmnd *scmd)
226 {
227 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
228 		struct MR_PRIV_DEVICE *mr_device_priv_data =
229 			scmd->device->hostdata;
230 		atomic_inc(&mr_device_priv_data->sdev_priv_busy);
231 	}
232 }
233 
234 static inline void
235 megasas_sdev_busy_dec(struct megasas_instance *instance,
236 		      struct scsi_cmnd *scmd)
237 {
238 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
239 		struct MR_PRIV_DEVICE *mr_device_priv_data =
240 			scmd->device->hostdata;
241 		atomic_dec(&mr_device_priv_data->sdev_priv_busy);
242 	}
243 }
244 
245 static inline int
246 megasas_sdev_busy_read(struct megasas_instance *instance,
247 		       struct scsi_cmnd *scmd)
248 {
249 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
250 		struct MR_PRIV_DEVICE *mr_device_priv_data =
251 			scmd->device->hostdata;
252 		return atomic_read(&mr_device_priv_data->sdev_priv_busy);
253 	}
254 	return 0;
255 }
256 
257 /**
258  * megasas_get_cmd_fusion -	Get a command from the free pool
259  * @instance:		Adapter soft state
260  * @blk_tag:		Command tag
261  *
262  * Returns a blk_tag indexed mpt frame
263  */
264 inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
265 						  *instance, u32 blk_tag)
266 {
267 	struct fusion_context *fusion;
268 
269 	fusion = instance->ctrl_context;
270 	return fusion->cmd_list[blk_tag];
271 }
272 
273 /**
274  * megasas_return_cmd_fusion -	Return a cmd to free command pool
275  * @instance:		Adapter soft state
276  * @cmd:		Command packet to be returned to free command pool
277  */
278 inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
279 	struct megasas_cmd_fusion *cmd)
280 {
281 	cmd->scmd = NULL;
282 	memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
283 	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
284 	cmd->cmd_completed = false;
285 }
286 
287 /**
288  * megasas_write_64bit_req_desc -	PCI writes 64bit request descriptor
289  * @instance:				Adapter soft state
290  * @req_desc:				64bit Request descriptor
291  */
292 static void
293 megasas_write_64bit_req_desc(struct megasas_instance *instance,
294 		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
295 {
296 #if defined(writeq) && defined(CONFIG_64BIT)
297 	u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
298 		le32_to_cpu(req_desc->u.low));
299 	writeq(req_data, &instance->reg_set->inbound_low_queue_port);
300 #else
301 	unsigned long flags;
302 	spin_lock_irqsave(&instance->hba_lock, flags);
303 	writel(le32_to_cpu(req_desc->u.low),
304 		&instance->reg_set->inbound_low_queue_port);
305 	writel(le32_to_cpu(req_desc->u.high),
306 		&instance->reg_set->inbound_high_queue_port);
307 	spin_unlock_irqrestore(&instance->hba_lock, flags);
308 #endif
309 }
310 
311 /**
312  * megasas_fire_cmd_fusion -	Sends command to the FW
313  * @instance:			Adapter soft state
314  * @req_desc:			32bit or 64bit Request descriptor
315  *
316  * Perform PCI Write. AERO SERIES supports 32 bit Descriptor.
317  * Prior to AERO_SERIES support 64 bit Descriptor.
318  */
319 static void
320 megasas_fire_cmd_fusion(struct megasas_instance *instance,
321 		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
322 {
323 	if (instance->atomic_desc_support)
324 		writel(le32_to_cpu(req_desc->u.low),
325 			&instance->reg_set->inbound_single_queue_port);
326 	else
327 		megasas_write_64bit_req_desc(instance, req_desc);
328 }
329 
330 /**
331  * megasas_fusion_update_can_queue -	Do all Adapter Queue depth related calculations here
332  * @instance:		Adapter soft state
333  * @fw_boot_context:	Whether this function called during probe or after OCR
334  *
335  * This function is only for fusion controllers.
336  * Update host can queue, if firmware downgrade max supported firmware commands.
337  * Firmware upgrade case will be skipped because underlying firmware has
338  * more resource than exposed to the OS.
339  *
340  */
341 static void
342 megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
343 {
344 	u16 cur_max_fw_cmds = 0;
345 	u16 ldio_threshold = 0;
346 
347 	/* ventura FW does not fill outbound_scratch_pad_2 with queue depth */
348 	if (instance->adapter_type < VENTURA_SERIES)
349 		cur_max_fw_cmds =
350 		megasas_readl(instance,
351 			      &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF;
352 
353 	if (dual_qdepth_disable || !cur_max_fw_cmds)
354 		cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF;
355 	else
356 		ldio_threshold =
357 			(instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
358 
359 	dev_info(&instance->pdev->dev,
360 		 "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",
361 		 cur_max_fw_cmds, ldio_threshold);
362 
363 	if (fw_boot_context == OCR_CONTEXT) {
364 		cur_max_fw_cmds = cur_max_fw_cmds - 1;
365 		if (cur_max_fw_cmds < instance->max_fw_cmds) {
366 			instance->cur_can_queue =
367 				cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
368 						MEGASAS_FUSION_IOCTL_CMDS);
369 			instance->host->can_queue = instance->cur_can_queue;
370 			instance->ldio_threshold = ldio_threshold;
371 		}
372 	} else {
373 		instance->max_fw_cmds = cur_max_fw_cmds;
374 		instance->ldio_threshold = ldio_threshold;
375 
376 		if (reset_devices)
377 			instance->max_fw_cmds = min(instance->max_fw_cmds,
378 						(u16)MEGASAS_KDUMP_QUEUE_DEPTH);
379 		/*
380 		* Reduce the max supported cmds by 1. This is to ensure that the
381 		* reply_q_sz (1 more than the max cmd that driver may send)
382 		* does not exceed max cmds that the FW can support
383 		*/
384 		instance->max_fw_cmds = instance->max_fw_cmds-1;
385 	}
386 }
387 
388 static inline void
389 megasas_get_msix_index(struct megasas_instance *instance,
390 		       struct scsi_cmnd *scmd,
391 		       struct megasas_cmd_fusion *cmd,
392 		       u8 data_arms)
393 {
394 	if (instance->perf_mode == MR_BALANCED_PERF_MODE &&
395 	    (megasas_sdev_busy_read(instance, scmd) >
396 	     (data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) {
397 		cmd->request_desc->SCSIIO.MSIxIndex =
398 			mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) /
399 					MR_HIGH_IOPS_BATCH_COUNT), instance->low_latency_index_start);
400 	} else if (instance->msix_load_balance) {
401 		cmd->request_desc->SCSIIO.MSIxIndex =
402 			(mega_mod64(atomic64_add_return(1, &instance->total_io_count),
403 				instance->msix_vectors));
404 	} else if (instance->host->nr_hw_queues > 1) {
405 		u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
406 
407 		cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(tag) +
408 			instance->low_latency_index_start;
409 	} else {
410 		cmd->request_desc->SCSIIO.MSIxIndex =
411 			instance->reply_map[raw_smp_processor_id()];
412 	}
413 }
414 
415 /**
416  * megasas_free_cmds_fusion -	Free all the cmds in the free cmd pool
417  * @instance:		Adapter soft state
418  */
419 void
420 megasas_free_cmds_fusion(struct megasas_instance *instance)
421 {
422 	int i;
423 	struct fusion_context *fusion = instance->ctrl_context;
424 	struct megasas_cmd_fusion *cmd;
425 
426 	if (fusion->sense)
427 		dma_pool_free(fusion->sense_dma_pool, fusion->sense,
428 			      fusion->sense_phys_addr);
429 
430 	/* SG */
431 	if (fusion->cmd_list) {
432 		for (i = 0; i < instance->max_mpt_cmds; i++) {
433 			cmd = fusion->cmd_list[i];
434 			if (cmd) {
435 				if (cmd->sg_frame)
436 					dma_pool_free(fusion->sg_dma_pool,
437 						      cmd->sg_frame,
438 						      cmd->sg_frame_phys_addr);
439 			}
440 			kfree(cmd);
441 		}
442 		kfree(fusion->cmd_list);
443 	}
444 
445 	if (fusion->sg_dma_pool) {
446 		dma_pool_destroy(fusion->sg_dma_pool);
447 		fusion->sg_dma_pool = NULL;
448 	}
449 	if (fusion->sense_dma_pool) {
450 		dma_pool_destroy(fusion->sense_dma_pool);
451 		fusion->sense_dma_pool = NULL;
452 	}
453 
454 
455 	/* Reply Frame, Desc*/
456 	if (instance->is_rdpq)
457 		megasas_free_rdpq_fusion(instance);
458 	else
459 		megasas_free_reply_fusion(instance);
460 
461 	/* Request Frame, Desc*/
462 	if (fusion->req_frames_desc)
463 		dma_free_coherent(&instance->pdev->dev,
464 			fusion->request_alloc_sz, fusion->req_frames_desc,
465 			fusion->req_frames_desc_phys);
466 	if (fusion->io_request_frames)
467 		dma_pool_free(fusion->io_request_frames_pool,
468 			fusion->io_request_frames,
469 			fusion->io_request_frames_phys);
470 	if (fusion->io_request_frames_pool) {
471 		dma_pool_destroy(fusion->io_request_frames_pool);
472 		fusion->io_request_frames_pool = NULL;
473 	}
474 }
475 
476 /**
477  * megasas_create_sg_sense_fusion -	Creates DMA pool for cmd frames
478  * @instance:			Adapter soft state
479  *
480  */
481 static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
482 {
483 	int i;
484 	u16 max_cmd;
485 	struct fusion_context *fusion;
486 	struct megasas_cmd_fusion *cmd;
487 	int sense_sz;
488 	u32 offset;
489 
490 	fusion = instance->ctrl_context;
491 	max_cmd = instance->max_fw_cmds;
492 	sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;
493 
494 	fusion->sg_dma_pool =
495 			dma_pool_create("mr_sg", &instance->pdev->dev,
496 				instance->max_chain_frame_sz,
497 				MR_DEFAULT_NVME_PAGE_SIZE, 0);
498 	/* SCSI_SENSE_BUFFERSIZE  = 96 bytes */
499 	fusion->sense_dma_pool =
500 			dma_pool_create("mr_sense", &instance->pdev->dev,
501 				sense_sz, 64, 0);
502 
503 	if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
504 		dev_err(&instance->pdev->dev,
505 			"Failed from %s %d\n",  __func__, __LINE__);
506 		return -ENOMEM;
507 	}
508 
509 	fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
510 				       GFP_KERNEL, &fusion->sense_phys_addr);
511 	if (!fusion->sense) {
512 		dev_err(&instance->pdev->dev,
513 			"failed from %s %d\n",  __func__, __LINE__);
514 		return -ENOMEM;
515 	}
516 
517 	/* sense buffer, request frame and reply desc pool requires to be in
518 	 * same 4 gb region. Below function will check this.
519 	 * In case of failure, new pci pool will be created with updated
520 	 * alignment.
521 	 * Older allocation and pool will be destroyed.
522 	 * Alignment will be used such a way that next allocation if success,
523 	 * will always meet same 4gb region requirement.
524 	 * Actual requirement is not alignment, but we need start and end of
525 	 * DMA address must have same upper 32 bit address.
526 	 */
527 
528 	if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr,
529 					   sense_sz)) {
530 		dma_pool_free(fusion->sense_dma_pool, fusion->sense,
531 			      fusion->sense_phys_addr);
532 		fusion->sense = NULL;
533 		dma_pool_destroy(fusion->sense_dma_pool);
534 
535 		fusion->sense_dma_pool =
536 			dma_pool_create("mr_sense_align", &instance->pdev->dev,
537 					sense_sz, roundup_pow_of_two(sense_sz),
538 					0);
539 		if (!fusion->sense_dma_pool) {
540 			dev_err(&instance->pdev->dev,
541 				"Failed from %s %d\n",  __func__, __LINE__);
542 			return -ENOMEM;
543 		}
544 		fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
545 					       GFP_KERNEL,
546 					       &fusion->sense_phys_addr);
547 		if (!fusion->sense) {
548 			dev_err(&instance->pdev->dev,
549 				"failed from %s %d\n",  __func__, __LINE__);
550 			return -ENOMEM;
551 		}
552 	}
553 
554 	/*
555 	 * Allocate and attach a frame to each of the commands in cmd_list
556 	 */
557 	for (i = 0; i < max_cmd; i++) {
558 		cmd = fusion->cmd_list[i];
559 		cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,
560 					GFP_KERNEL, &cmd->sg_frame_phys_addr);
561 
562 		offset = SCSI_SENSE_BUFFERSIZE * i;
563 		cmd->sense = (u8 *)fusion->sense + offset;
564 		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
565 
566 		if (!cmd->sg_frame) {
567 			dev_err(&instance->pdev->dev,
568 				"Failed from %s %d\n",  __func__, __LINE__);
569 			return -ENOMEM;
570 		}
571 	}
572 
573 	/* create sense buffer for the raid 1/10 fp */
574 	for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
575 		cmd = fusion->cmd_list[i];
576 		offset = SCSI_SENSE_BUFFERSIZE * i;
577 		cmd->sense = (u8 *)fusion->sense + offset;
578 		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
579 
580 	}
581 
582 	return 0;
583 }
584 
585 static int
586 megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
587 {
588 	u32 max_mpt_cmd, i, j;
589 	struct fusion_context *fusion;
590 
591 	fusion = instance->ctrl_context;
592 
593 	max_mpt_cmd = instance->max_mpt_cmds;
594 
595 	/*
596 	 * fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.
597 	 * Allocate the dynamic array first and then allocate individual
598 	 * commands.
599 	 */
600 	fusion->cmd_list =
601 		kcalloc(max_mpt_cmd, sizeof(struct megasas_cmd_fusion *),
602 			GFP_KERNEL);
603 	if (!fusion->cmd_list) {
604 		dev_err(&instance->pdev->dev,
605 			"Failed from %s %d\n",  __func__, __LINE__);
606 		return -ENOMEM;
607 	}
608 
609 	for (i = 0; i < max_mpt_cmd; i++) {
610 		fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),
611 					      GFP_KERNEL);
612 		if (!fusion->cmd_list[i]) {
613 			for (j = 0; j < i; j++)
614 				kfree(fusion->cmd_list[j]);
615 			kfree(fusion->cmd_list);
616 			dev_err(&instance->pdev->dev,
617 				"Failed from %s %d\n",  __func__, __LINE__);
618 			return -ENOMEM;
619 		}
620 	}
621 
622 	return 0;
623 }
624 
625 static int
626 megasas_alloc_request_fusion(struct megasas_instance *instance)
627 {
628 	struct fusion_context *fusion;
629 
630 	fusion = instance->ctrl_context;
631 
632 retry_alloc:
633 	fusion->io_request_frames_pool =
634 			dma_pool_create("mr_ioreq", &instance->pdev->dev,
635 				fusion->io_frames_alloc_sz, 16, 0);
636 
637 	if (!fusion->io_request_frames_pool) {
638 		dev_err(&instance->pdev->dev,
639 			"Failed from %s %d\n",  __func__, __LINE__);
640 		return -ENOMEM;
641 	}
642 
643 	fusion->io_request_frames =
644 			dma_pool_alloc(fusion->io_request_frames_pool,
645 				GFP_KERNEL | __GFP_NOWARN,
646 				&fusion->io_request_frames_phys);
647 	if (!fusion->io_request_frames) {
648 		if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {
649 			instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;
650 			dma_pool_destroy(fusion->io_request_frames_pool);
651 			megasas_configure_queue_sizes(instance);
652 			goto retry_alloc;
653 		} else {
654 			dev_err(&instance->pdev->dev,
655 				"Failed from %s %d\n",  __func__, __LINE__);
656 			return -ENOMEM;
657 		}
658 	}
659 
660 	if (!megasas_check_same_4gb_region(instance,
661 					   fusion->io_request_frames_phys,
662 					   fusion->io_frames_alloc_sz)) {
663 		dma_pool_free(fusion->io_request_frames_pool,
664 			      fusion->io_request_frames,
665 			      fusion->io_request_frames_phys);
666 		fusion->io_request_frames = NULL;
667 		dma_pool_destroy(fusion->io_request_frames_pool);
668 
669 		fusion->io_request_frames_pool =
670 			dma_pool_create("mr_ioreq_align",
671 					&instance->pdev->dev,
672 					fusion->io_frames_alloc_sz,
673 					roundup_pow_of_two(fusion->io_frames_alloc_sz),
674 					0);
675 
676 		if (!fusion->io_request_frames_pool) {
677 			dev_err(&instance->pdev->dev,
678 				"Failed from %s %d\n",  __func__, __LINE__);
679 			return -ENOMEM;
680 		}
681 
682 		fusion->io_request_frames =
683 			dma_pool_alloc(fusion->io_request_frames_pool,
684 				       GFP_KERNEL | __GFP_NOWARN,
685 				       &fusion->io_request_frames_phys);
686 
687 		if (!fusion->io_request_frames) {
688 			dev_err(&instance->pdev->dev,
689 				"Failed from %s %d\n",  __func__, __LINE__);
690 			return -ENOMEM;
691 		}
692 	}
693 
694 	fusion->req_frames_desc =
695 		dma_alloc_coherent(&instance->pdev->dev,
696 				   fusion->request_alloc_sz,
697 				   &fusion->req_frames_desc_phys, GFP_KERNEL);
698 	if (!fusion->req_frames_desc) {
699 		dev_err(&instance->pdev->dev,
700 			"Failed from %s %d\n",  __func__, __LINE__);
701 		return -ENOMEM;
702 	}
703 
704 	return 0;
705 }
706 
707 static int
708 megasas_alloc_reply_fusion(struct megasas_instance *instance)
709 {
710 	int i, count;
711 	struct fusion_context *fusion;
712 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
713 	fusion = instance->ctrl_context;
714 
715 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
716 	count += instance->iopoll_q_count;
717 
718 	fusion->reply_frames_desc_pool =
719 			dma_pool_create("mr_reply", &instance->pdev->dev,
720 				fusion->reply_alloc_sz * count, 16, 0);
721 
722 	if (!fusion->reply_frames_desc_pool) {
723 		dev_err(&instance->pdev->dev,
724 			"Failed from %s %d\n",  __func__, __LINE__);
725 		return -ENOMEM;
726 	}
727 
728 	fusion->reply_frames_desc[0] =
729 		dma_pool_alloc(fusion->reply_frames_desc_pool,
730 			GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
731 	if (!fusion->reply_frames_desc[0]) {
732 		dev_err(&instance->pdev->dev,
733 			"Failed from %s %d\n",  __func__, __LINE__);
734 		return -ENOMEM;
735 	}
736 
737 	if (!megasas_check_same_4gb_region(instance,
738 					   fusion->reply_frames_desc_phys[0],
739 					   (fusion->reply_alloc_sz * count))) {
740 		dma_pool_free(fusion->reply_frames_desc_pool,
741 			      fusion->reply_frames_desc[0],
742 			      fusion->reply_frames_desc_phys[0]);
743 		fusion->reply_frames_desc[0] = NULL;
744 		dma_pool_destroy(fusion->reply_frames_desc_pool);
745 
746 		fusion->reply_frames_desc_pool =
747 			dma_pool_create("mr_reply_align",
748 					&instance->pdev->dev,
749 					fusion->reply_alloc_sz * count,
750 					roundup_pow_of_two(fusion->reply_alloc_sz * count),
751 					0);
752 
753 		if (!fusion->reply_frames_desc_pool) {
754 			dev_err(&instance->pdev->dev,
755 				"Failed from %s %d\n",  __func__, __LINE__);
756 			return -ENOMEM;
757 		}
758 
759 		fusion->reply_frames_desc[0] =
760 			dma_pool_alloc(fusion->reply_frames_desc_pool,
761 				       GFP_KERNEL,
762 				       &fusion->reply_frames_desc_phys[0]);
763 
764 		if (!fusion->reply_frames_desc[0]) {
765 			dev_err(&instance->pdev->dev,
766 				"Failed from %s %d\n",  __func__, __LINE__);
767 			return -ENOMEM;
768 		}
769 	}
770 
771 	reply_desc = fusion->reply_frames_desc[0];
772 	for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
773 		reply_desc->Words = cpu_to_le64(ULLONG_MAX);
774 
775 	/* This is not a rdpq mode, but driver still populate
776 	 * reply_frame_desc array to use same msix index in ISR path.
777 	 */
778 	for (i = 0; i < (count - 1); i++)
779 		fusion->reply_frames_desc[i + 1] =
780 			fusion->reply_frames_desc[i] +
781 			(fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
782 
783 	return 0;
784 }
785 
786 static int
787 megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
788 {
789 	int i, j, k, msix_count;
790 	struct fusion_context *fusion;
791 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
792 	union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];
793 	dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];
794 	u8 dma_alloc_count, abs_index;
795 	u32 chunk_size, array_size, offset;
796 
797 	fusion = instance->ctrl_context;
798 	chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
799 	array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
800 		     MAX_MSIX_QUEUES_FUSION;
801 
802 	fusion->rdpq_virt = dma_alloc_coherent(&instance->pdev->dev,
803 					       array_size, &fusion->rdpq_phys,
804 					       GFP_KERNEL);
805 	if (!fusion->rdpq_virt) {
806 		dev_err(&instance->pdev->dev,
807 			"Failed from %s %d\n",  __func__, __LINE__);
808 		return -ENOMEM;
809 	}
810 
811 	msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
812 	msix_count += instance->iopoll_q_count;
813 
814 	fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
815 							 &instance->pdev->dev,
816 							 chunk_size, 16, 0);
817 	fusion->reply_frames_desc_pool_align =
818 				dma_pool_create("mr_rdpq_align",
819 						&instance->pdev->dev,
820 						chunk_size,
821 						roundup_pow_of_two(chunk_size),
822 						0);
823 
824 	if (!fusion->reply_frames_desc_pool ||
825 	    !fusion->reply_frames_desc_pool_align) {
826 		dev_err(&instance->pdev->dev,
827 			"Failed from %s %d\n",  __func__, __LINE__);
828 		return -ENOMEM;
829 	}
830 
831 /*
832  * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
833  * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be
834  * within 4GB boundary and also reply queues in a set must have same
835  * upper 32-bits in their memory address. so here driver is allocating the
836  * DMA'able memory for reply queues according. Driver uses limitation of
837  * VENTURA_SERIES to manage INVADER_SERIES as well.
838  */
839 	dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);
840 
841 	for (i = 0; i < dma_alloc_count; i++) {
842 		rdpq_chunk_virt[i] =
843 			dma_pool_alloc(fusion->reply_frames_desc_pool,
844 				       GFP_KERNEL, &rdpq_chunk_phys[i]);
845 		if (!rdpq_chunk_virt[i]) {
846 			dev_err(&instance->pdev->dev,
847 				"Failed from %s %d\n",  __func__, __LINE__);
848 			return -ENOMEM;
849 		}
850 		/* reply desc pool requires to be in same 4 gb region.
851 		 * Below function will check this.
852 		 * In case of failure, new pci pool will be created with updated
853 		 * alignment.
854 		 * For RDPQ buffers, driver always allocate two separate pci pool.
855 		 * Alignment will be used such a way that next allocation if
856 		 * success, will always meet same 4gb region requirement.
857 		 * rdpq_tracker keep track of each buffer's physical,
858 		 * virtual address and pci pool descriptor. It will help driver
859 		 * while freeing the resources.
860 		 *
861 		 */
862 		if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i],
863 						   chunk_size)) {
864 			dma_pool_free(fusion->reply_frames_desc_pool,
865 				      rdpq_chunk_virt[i],
866 				      rdpq_chunk_phys[i]);
867 
868 			rdpq_chunk_virt[i] =
869 				dma_pool_alloc(fusion->reply_frames_desc_pool_align,
870 					       GFP_KERNEL, &rdpq_chunk_phys[i]);
871 			if (!rdpq_chunk_virt[i]) {
872 				dev_err(&instance->pdev->dev,
873 					"Failed from %s %d\n",
874 					__func__, __LINE__);
875 				return -ENOMEM;
876 			}
877 			fusion->rdpq_tracker[i].dma_pool_ptr =
878 					fusion->reply_frames_desc_pool_align;
879 		} else {
880 			fusion->rdpq_tracker[i].dma_pool_ptr =
881 					fusion->reply_frames_desc_pool;
882 		}
883 
884 		fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];
885 		fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];
886 	}
887 
888 	for (k = 0; k < dma_alloc_count; k++) {
889 		for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {
890 			abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;
891 
892 			if (abs_index == msix_count)
893 				break;
894 			offset = fusion->reply_alloc_sz * i;
895 			fusion->rdpq_virt[abs_index].RDPQBaseAddress =
896 					cpu_to_le64(rdpq_chunk_phys[k] + offset);
897 			fusion->reply_frames_desc_phys[abs_index] =
898 					rdpq_chunk_phys[k] + offset;
899 			fusion->reply_frames_desc[abs_index] =
900 					(union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);
901 
902 			reply_desc = fusion->reply_frames_desc[abs_index];
903 			for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
904 				reply_desc->Words = ULLONG_MAX;
905 		}
906 	}
907 
908 	return 0;
909 }
910 
911 static void
912 megasas_free_rdpq_fusion(struct megasas_instance *instance) {
913 
914 	int i;
915 	struct fusion_context *fusion;
916 
917 	fusion = instance->ctrl_context;
918 
919 	for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {
920 		if (fusion->rdpq_tracker[i].pool_entry_virt)
921 			dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr,
922 				      fusion->rdpq_tracker[i].pool_entry_virt,
923 				      fusion->rdpq_tracker[i].pool_entry_phys);
924 
925 	}
926 
927 	dma_pool_destroy(fusion->reply_frames_desc_pool);
928 	dma_pool_destroy(fusion->reply_frames_desc_pool_align);
929 
930 	if (fusion->rdpq_virt)
931 		dma_free_coherent(&instance->pdev->dev,
932 			sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
933 			fusion->rdpq_virt, fusion->rdpq_phys);
934 }
935 
936 static void
937 megasas_free_reply_fusion(struct megasas_instance *instance) {
938 
939 	struct fusion_context *fusion;
940 
941 	fusion = instance->ctrl_context;
942 
943 	if (fusion->reply_frames_desc[0])
944 		dma_pool_free(fusion->reply_frames_desc_pool,
945 			fusion->reply_frames_desc[0],
946 			fusion->reply_frames_desc_phys[0]);
947 
948 	dma_pool_destroy(fusion->reply_frames_desc_pool);
949 
950 }
951 
952 
953 /**
954  * megasas_alloc_cmds_fusion -	Allocates the command packets
955  * @instance:		Adapter soft state
956  *
957  *
958  * Each frame has a 32-bit field called context. This context is used to get
959  * back the megasas_cmd_fusion from the frame when a frame gets completed
960  * In this driver, the 32 bit values are the indices into an array cmd_list.
961  * This array is used only to look up the megasas_cmd_fusion given the context.
962  * The free commands themselves are maintained in a linked list called cmd_pool.
963  *
964  * cmds are formed in the io_request and sg_frame members of the
965  * megasas_cmd_fusion. The context field is used to get a request descriptor
966  * and is used as SMID of the cmd.
967  * SMID value range is from 1 to max_fw_cmds.
968  */
969 static int
970 megasas_alloc_cmds_fusion(struct megasas_instance *instance)
971 {
972 	int i;
973 	struct fusion_context *fusion;
974 	struct megasas_cmd_fusion *cmd;
975 	u32 offset;
976 	dma_addr_t io_req_base_phys;
977 	u8 *io_req_base;
978 
979 
980 	fusion = instance->ctrl_context;
981 
982 	if (megasas_alloc_request_fusion(instance))
983 		goto fail_exit;
984 
985 	if (instance->is_rdpq) {
986 		if (megasas_alloc_rdpq_fusion(instance))
987 			goto fail_exit;
988 	} else
989 		if (megasas_alloc_reply_fusion(instance))
990 			goto fail_exit;
991 
992 	if (megasas_alloc_cmdlist_fusion(instance))
993 		goto fail_exit;
994 
995 	/* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */
996 	io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
997 	io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
998 
999 	/*
1000 	 * Add all the commands to command pool (fusion->cmd_pool)
1001 	 */
1002 
1003 	/* SMID 0 is reserved. Set SMID/index from 1 */
1004 	for (i = 0; i < instance->max_mpt_cmds; i++) {
1005 		cmd = fusion->cmd_list[i];
1006 		offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
1007 		memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
1008 		cmd->index = i + 1;
1009 		cmd->scmd = NULL;
1010 		cmd->sync_cmd_idx =
1011 		(i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
1012 				(i - instance->max_scsi_cmds) :
1013 				(u32)ULONG_MAX; /* Set to Invalid */
1014 		cmd->instance = instance;
1015 		cmd->io_request =
1016 			(struct MPI2_RAID_SCSI_IO_REQUEST *)
1017 		  (io_req_base + offset);
1018 		memset(cmd->io_request, 0,
1019 		       sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
1020 		cmd->io_request_phys_addr = io_req_base_phys + offset;
1021 		cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1022 	}
1023 
1024 	if (megasas_create_sg_sense_fusion(instance))
1025 		goto fail_exit;
1026 
1027 	return 0;
1028 
1029 fail_exit:
1030 	megasas_free_cmds_fusion(instance);
1031 	return -ENOMEM;
1032 }
1033 
1034 /**
1035  * wait_and_poll -	Issues a polling command
1036  * @instance:			Adapter soft state
1037  * @cmd:			Command packet to be issued
1038  * @seconds:			Maximum poll time
1039  *
1040  * For polling, MFI requires the cmd_status to be set to 0xFF before posting.
1041  */
1042 int
1043 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
1044 	int seconds)
1045 {
1046 	int i;
1047 	struct megasas_header *frame_hdr = &cmd->frame->hdr;
1048 	u32 status_reg;
1049 
1050 	u32 msecs = seconds * 1000;
1051 
1052 	/*
1053 	 * Wait for cmd_status to change
1054 	 */
1055 	for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
1056 		rmb();
1057 		msleep(20);
1058 		if (!(i % 5000)) {
1059 			status_reg = instance->instancet->read_fw_status_reg(instance)
1060 					& MFI_STATE_MASK;
1061 			if (status_reg == MFI_STATE_FAULT)
1062 				break;
1063 		}
1064 	}
1065 
1066 	if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
1067 		return DCMD_TIMEOUT;
1068 	else if (frame_hdr->cmd_status == MFI_STAT_OK)
1069 		return DCMD_SUCCESS;
1070 	else
1071 		return DCMD_FAILED;
1072 }
1073 
1074 /**
1075  * megasas_ioc_init_fusion -	Initializes the FW
1076  * @instance:		Adapter soft state
1077  *
1078  * Issues the IOC Init cmd
1079  */
1080 int
1081 megasas_ioc_init_fusion(struct megasas_instance *instance)
1082 {
1083 	struct megasas_init_frame *init_frame;
1084 	struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
1085 	dma_addr_t	ioc_init_handle;
1086 	struct megasas_cmd *cmd;
1087 	u8 ret, cur_rdpq_mode;
1088 	struct fusion_context *fusion;
1089 	union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
1090 	int i;
1091 	struct megasas_header *frame_hdr;
1092 	const char *sys_info;
1093 	MFI_CAPABILITIES *drv_ops;
1094 	u32 scratch_pad_1;
1095 	ktime_t time;
1096 	bool cur_fw_64bit_dma_capable;
1097 	bool cur_intr_coalescing;
1098 
1099 	fusion = instance->ctrl_context;
1100 
1101 	ioc_init_handle = fusion->ioc_init_request_phys;
1102 	IOCInitMessage = fusion->ioc_init_request;
1103 
1104 	cmd = fusion->ioc_init_cmd;
1105 
1106 	scratch_pad_1 = megasas_readl
1107 		(instance, &instance->reg_set->outbound_scratch_pad_1);
1108 
1109 	cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
1110 
1111 	if (instance->adapter_type == INVADER_SERIES) {
1112 		cur_fw_64bit_dma_capable =
1113 			(scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;
1114 
1115 		if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {
1116 			dev_err(&instance->pdev->dev, "Driver was operating on 64bit "
1117 				"DMA mask, but upcoming FW does not support 64bit DMA mask\n");
1118 			megaraid_sas_kill_hba(instance);
1119 			ret = 1;
1120 			goto fail_fw_init;
1121 		}
1122 	}
1123 
1124 	if (instance->is_rdpq && !cur_rdpq_mode) {
1125 		dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
1126 			" from RDPQ mode to non RDPQ mode\n");
1127 		ret = 1;
1128 		goto fail_fw_init;
1129 	}
1130 
1131 	cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ?
1132 							true : false;
1133 
1134 	if ((instance->low_latency_index_start ==
1135 		MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)
1136 		instance->perf_mode = MR_BALANCED_PERF_MODE;
1137 
1138 	dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n",
1139 		MEGASAS_PERF_MODE_2STR(instance->perf_mode),
1140 		instance->low_latency_index_start);
1141 
1142 	instance->fw_sync_cache_support = (scratch_pad_1 &
1143 		MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
1144 	dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
1145 		 instance->fw_sync_cache_support ? "Yes" : "No");
1146 
1147 	memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
1148 
1149 	IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
1150 	IOCInitMessage->WhoInit	= MPI2_WHOINIT_HOST_DRIVER;
1151 	IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
1152 	IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
1153 	IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
1154 
1155 	IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
1156 	IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
1157 			cpu_to_le64(fusion->rdpq_phys) :
1158 			cpu_to_le64(fusion->reply_frames_desc_phys[0]);
1159 	IOCInitMessage->MsgFlags = instance->is_rdpq ?
1160 			MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
1161 	IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
1162 	IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));
1163 	IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count;
1164 	IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
1165 
1166 	time = ktime_get_real();
1167 	/* Convert to milliseconds as per FW requirement */
1168 	IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));
1169 
1170 	init_frame = (struct megasas_init_frame *)cmd->frame;
1171 	memset(init_frame, 0, IOC_INIT_FRAME_SIZE);
1172 
1173 	frame_hdr = &cmd->frame->hdr;
1174 	frame_hdr->cmd_status = 0xFF;
1175 	frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
1176 
1177 	init_frame->cmd	= MFI_CMD_INIT;
1178 	init_frame->cmd_status = 0xFF;
1179 
1180 	drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
1181 
1182 	/* driver support Extended MSIX */
1183 	if (instance->adapter_type >= INVADER_SERIES)
1184 		drv_ops->mfi_capabilities.support_additional_msix = 1;
1185 	/* driver supports HA / Remote LUN over Fast Path interface */
1186 	drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
1187 
1188 	drv_ops->mfi_capabilities.support_max_255lds = 1;
1189 	drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
1190 	drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
1191 
1192 	if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
1193 		drv_ops->mfi_capabilities.support_ext_io_size = 1;
1194 
1195 	drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
1196 	if (!dual_qdepth_disable)
1197 		drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
1198 
1199 	drv_ops->mfi_capabilities.support_qd_throttling = 1;
1200 	drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
1201 	drv_ops->mfi_capabilities.support_nvme_passthru = 1;
1202 	drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1;
1203 
1204 	if (reset_devices)
1205 		drv_ops->mfi_capabilities.support_memdump = 1;
1206 
1207 	if (instance->consistent_mask_64bit)
1208 		drv_ops->mfi_capabilities.support_64bit_mode = 1;
1209 
1210 	/* Convert capability to LE32 */
1211 	cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
1212 
1213 	sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);
1214 	if (instance->system_info_buf && sys_info) {
1215 		memcpy(instance->system_info_buf->systemId, sys_info,
1216 			strlen(sys_info) > 64 ? 64 : strlen(sys_info));
1217 		instance->system_info_buf->systemIdLength =
1218 			strlen(sys_info) > 64 ? 64 : strlen(sys_info);
1219 		init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));
1220 		init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));
1221 	}
1222 
1223 	init_frame->queue_info_new_phys_addr_hi =
1224 		cpu_to_le32(upper_32_bits(ioc_init_handle));
1225 	init_frame->queue_info_new_phys_addr_lo =
1226 		cpu_to_le32(lower_32_bits(ioc_init_handle));
1227 	init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
1228 
1229 	/*
1230 	 * Each bit in replyqueue_mask represents one group of MSI-x vectors
1231 	 * (each group has 8 vectors)
1232 	 */
1233 	switch (instance->perf_mode) {
1234 	case MR_BALANCED_PERF_MODE:
1235 		init_frame->replyqueue_mask =
1236 		       cpu_to_le16(~(~0 << instance->low_latency_index_start/8));
1237 		break;
1238 	case MR_IOPS_PERF_MODE:
1239 		init_frame->replyqueue_mask =
1240 		       cpu_to_le16(~(~0 << instance->msix_vectors/8));
1241 		break;
1242 	}
1243 
1244 
1245 	req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
1246 	req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
1247 	req_desc.MFAIo.RequestFlags =
1248 		(MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
1249 		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1250 
1251 	/*
1252 	 * disable the intr before firing the init frame
1253 	 */
1254 	instance->instancet->disable_intr(instance);
1255 
1256 	for (i = 0; i < (10 * 1000); i += 20) {
1257 		if (megasas_readl(instance, &instance->reg_set->doorbell) & 1)
1258 			msleep(20);
1259 		else
1260 			break;
1261 	}
1262 
1263 	/* For AERO also, IOC_INIT requires 64 bit descriptor write */
1264 	megasas_write_64bit_req_desc(instance, &req_desc);
1265 
1266 	wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS);
1267 
1268 	frame_hdr = &cmd->frame->hdr;
1269 	if (frame_hdr->cmd_status != 0) {
1270 		ret = 1;
1271 		goto fail_fw_init;
1272 	}
1273 
1274 	if (instance->adapter_type >= AERO_SERIES) {
1275 		scratch_pad_1 = megasas_readl
1276 			(instance, &instance->reg_set->outbound_scratch_pad_1);
1277 
1278 		instance->atomic_desc_support =
1279 			(scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0;
1280 
1281 		dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n",
1282 			instance->atomic_desc_support ? "Yes" : "No");
1283 	}
1284 
1285 	return 0;
1286 
1287 fail_fw_init:
1288 	dev_err(&instance->pdev->dev,
1289 		"Init cmd return status FAILED for SCSI host %d\n",
1290 		instance->host->host_no);
1291 
1292 	return ret;
1293 }
1294 
1295 /**
1296  * megasas_sync_pd_seq_num -	JBOD SEQ MAP
1297  * @instance:		Adapter soft state
1298  * @pend:		set to 1, if it is pended jbod map.
1299  *
1300  * Issue Jbod map to the firmware. If it is pended command,
1301  * issue command and return. If it is first instance of jbod map
1302  * issue and receive command.
1303  */
1304 int
1305 megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
1306 	int ret = 0;
1307 	size_t pd_seq_map_sz;
1308 	struct megasas_cmd *cmd;
1309 	struct megasas_dcmd_frame *dcmd;
1310 	struct fusion_context *fusion = instance->ctrl_context;
1311 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1312 	dma_addr_t pd_seq_h;
1313 
1314 	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
1315 	pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
1316 	pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES);
1317 
1318 	cmd = megasas_get_cmd(instance);
1319 	if (!cmd) {
1320 		dev_err(&instance->pdev->dev,
1321 			"Could not get mfi cmd. Fail from %s %d\n",
1322 			__func__, __LINE__);
1323 		return -ENOMEM;
1324 	}
1325 
1326 	dcmd = &cmd->frame->dcmd;
1327 
1328 	memset(pd_sync, 0, pd_seq_map_sz);
1329 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1330 
1331 	if (pend) {
1332 		dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1333 		dcmd->flags = MFI_FRAME_DIR_WRITE;
1334 		instance->jbod_seq_cmd = cmd;
1335 	} else {
1336 		dcmd->flags = MFI_FRAME_DIR_READ;
1337 	}
1338 
1339 	dcmd->cmd = MFI_CMD_DCMD;
1340 	dcmd->cmd_status = 0xFF;
1341 	dcmd->sge_count = 1;
1342 	dcmd->timeout = 0;
1343 	dcmd->pad_0 = 0;
1344 	dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
1345 	dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
1346 
1347 	megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz);
1348 
1349 	if (pend) {
1350 		instance->instancet->issue_dcmd(instance, cmd);
1351 		return 0;
1352 	}
1353 
1354 	/* Below code is only for non pended DCMD */
1355 	if (!instance->mask_interrupts)
1356 		ret = megasas_issue_blocked_cmd(instance, cmd,
1357 			MFI_IO_TIMEOUT_SECS);
1358 	else
1359 		ret = megasas_issue_polled(instance, cmd);
1360 
1361 	if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
1362 		dev_warn(&instance->pdev->dev,
1363 			"driver supports max %d JBOD, but FW reports %d\n",
1364 			MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
1365 		ret = -EINVAL;
1366 	}
1367 
1368 	if (ret == DCMD_TIMEOUT)
1369 		dev_warn(&instance->pdev->dev,
1370 			 "%s DCMD timed out, continue without JBOD sequence map\n",
1371 			 __func__);
1372 
1373 	if (ret == DCMD_SUCCESS)
1374 		instance->pd_seq_map_id++;
1375 
1376 	megasas_return_cmd(instance, cmd);
1377 	return ret;
1378 }
1379 
1380 /*
1381  * megasas_get_ld_map_info -	Returns FW's ld_map structure
1382  * @instance:				Adapter soft state
1383  * @pend:				Pend the command or not
1384  * Issues an internal command (DCMD) to get the FW's controller PD
1385  * list structure.  This information is mainly used to find out SYSTEM
1386  * supported by the FW.
1387  * dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO
1388  * dcmd.mbox.b[0]	- number of LDs being sync'd
1389  * dcmd.mbox.b[1]	- 0 - complete command immediately.
1390  *			- 1 - pend till config change
1391  * dcmd.mbox.b[2]	- 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP
1392  *			- 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and
1393  *				uses extended struct MR_FW_RAID_MAP_EXT
1394  */
1395 static int
1396 megasas_get_ld_map_info(struct megasas_instance *instance)
1397 {
1398 	int ret = 0;
1399 	struct megasas_cmd *cmd;
1400 	struct megasas_dcmd_frame *dcmd;
1401 	void *ci;
1402 	dma_addr_t ci_h = 0;
1403 	u32 size_map_info;
1404 	struct fusion_context *fusion;
1405 
1406 	cmd = megasas_get_cmd(instance);
1407 
1408 	if (!cmd) {
1409 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1410 		return -ENOMEM;
1411 	}
1412 
1413 	fusion = instance->ctrl_context;
1414 
1415 	if (!fusion) {
1416 		megasas_return_cmd(instance, cmd);
1417 		return -ENXIO;
1418 	}
1419 
1420 	dcmd = &cmd->frame->dcmd;
1421 
1422 	size_map_info = fusion->current_map_sz;
1423 
1424 	ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1425 	ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1426 
1427 	if (!ci) {
1428 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1429 		megasas_return_cmd(instance, cmd);
1430 		return -ENOMEM;
1431 	}
1432 
1433 	memset(ci, 0, fusion->max_map_sz);
1434 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1435 	dcmd->cmd = MFI_CMD_DCMD;
1436 	dcmd->cmd_status = 0xFF;
1437 	dcmd->sge_count = 1;
1438 	dcmd->flags = MFI_FRAME_DIR_READ;
1439 	dcmd->timeout = 0;
1440 	dcmd->pad_0 = 0;
1441 	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1442 	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1443 
1444 	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1445 
1446 	if (!instance->mask_interrupts)
1447 		ret = megasas_issue_blocked_cmd(instance, cmd,
1448 			MFI_IO_TIMEOUT_SECS);
1449 	else
1450 		ret = megasas_issue_polled(instance, cmd);
1451 
1452 	if (ret == DCMD_TIMEOUT)
1453 		dev_warn(&instance->pdev->dev,
1454 			 "%s DCMD timed out, RAID map is disabled\n",
1455 			 __func__);
1456 
1457 	megasas_return_cmd(instance, cmd);
1458 
1459 	return ret;
1460 }
1461 
1462 u8
1463 megasas_get_map_info(struct megasas_instance *instance)
1464 {
1465 	struct fusion_context *fusion = instance->ctrl_context;
1466 
1467 	fusion->fast_path_io = 0;
1468 	if (!megasas_get_ld_map_info(instance)) {
1469 		if (MR_ValidateMapInfo(instance, instance->map_id)) {
1470 			fusion->fast_path_io = 1;
1471 			return 0;
1472 		}
1473 	}
1474 	return 1;
1475 }
1476 
1477 /*
1478  * megasas_sync_map_info -	Returns FW's ld_map structure
1479  * @instance:				Adapter soft state
1480  *
1481  * Issues an internal command (DCMD) to get the FW's controller PD
1482  * list structure.  This information is mainly used to find out SYSTEM
1483  * supported by the FW.
1484  */
1485 int
1486 megasas_sync_map_info(struct megasas_instance *instance)
1487 {
1488 	int i;
1489 	struct megasas_cmd *cmd;
1490 	struct megasas_dcmd_frame *dcmd;
1491 	u16 num_lds;
1492 	struct fusion_context *fusion;
1493 	struct MR_LD_TARGET_SYNC *ci = NULL;
1494 	struct MR_DRV_RAID_MAP_ALL *map;
1495 	struct MR_LD_RAID  *raid;
1496 	struct MR_LD_TARGET_SYNC *ld_sync;
1497 	dma_addr_t ci_h = 0;
1498 	u32 size_map_info;
1499 
1500 	cmd = megasas_get_cmd(instance);
1501 
1502 	if (!cmd) {
1503 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1504 		return -ENOMEM;
1505 	}
1506 
1507 	fusion = instance->ctrl_context;
1508 
1509 	if (!fusion) {
1510 		megasas_return_cmd(instance, cmd);
1511 		return 1;
1512 	}
1513 
1514 	map = fusion->ld_drv_map[instance->map_id & 1];
1515 
1516 	num_lds = le16_to_cpu(map->raidMap.ldCount);
1517 
1518 	dcmd = &cmd->frame->dcmd;
1519 
1520 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1521 
1522 	ci = (struct MR_LD_TARGET_SYNC *)
1523 	  fusion->ld_map[(instance->map_id - 1) & 1];
1524 	memset(ci, 0, fusion->max_map_sz);
1525 
1526 	ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1527 
1528 	ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1529 
1530 	for (i = 0; i < num_lds; i++, ld_sync++) {
1531 		raid = MR_LdRaidGet(i, map);
1532 		ld_sync->targetId = MR_GetLDTgtId(i, map);
1533 		ld_sync->seqNum = raid->seqNum;
1534 	}
1535 
1536 	size_map_info = fusion->current_map_sz;
1537 
1538 	dcmd->cmd = MFI_CMD_DCMD;
1539 	dcmd->cmd_status = 0xFF;
1540 	dcmd->sge_count = 1;
1541 	dcmd->flags = MFI_FRAME_DIR_WRITE;
1542 	dcmd->timeout = 0;
1543 	dcmd->pad_0 = 0;
1544 	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1545 	dcmd->mbox.b[0] = num_lds;
1546 	dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1547 	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1548 
1549 	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1550 
1551 	instance->map_update_cmd = cmd;
1552 
1553 	instance->instancet->issue_dcmd(instance, cmd);
1554 
1555 	return 0;
1556 }
1557 
1558 /*
1559  * meagasas_display_intel_branding - Display branding string
1560  * @instance: per adapter object
1561  *
1562  * Return nothing.
1563  */
1564 static void
1565 megasas_display_intel_branding(struct megasas_instance *instance)
1566 {
1567 	if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1568 		return;
1569 
1570 	switch (instance->pdev->device) {
1571 	case PCI_DEVICE_ID_LSI_INVADER:
1572 		switch (instance->pdev->subsystem_device) {
1573 		case MEGARAID_INTEL_RS3DC080_SSDID:
1574 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1575 				instance->host->host_no,
1576 				MEGARAID_INTEL_RS3DC080_BRANDING);
1577 			break;
1578 		case MEGARAID_INTEL_RS3DC040_SSDID:
1579 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1580 				instance->host->host_no,
1581 				MEGARAID_INTEL_RS3DC040_BRANDING);
1582 			break;
1583 		case MEGARAID_INTEL_RS3SC008_SSDID:
1584 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1585 				instance->host->host_no,
1586 				MEGARAID_INTEL_RS3SC008_BRANDING);
1587 			break;
1588 		case MEGARAID_INTEL_RS3MC044_SSDID:
1589 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1590 				instance->host->host_no,
1591 				MEGARAID_INTEL_RS3MC044_BRANDING);
1592 			break;
1593 		default:
1594 			break;
1595 		}
1596 		break;
1597 	case PCI_DEVICE_ID_LSI_FURY:
1598 		switch (instance->pdev->subsystem_device) {
1599 		case MEGARAID_INTEL_RS3WC080_SSDID:
1600 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1601 				instance->host->host_no,
1602 				MEGARAID_INTEL_RS3WC080_BRANDING);
1603 			break;
1604 		case MEGARAID_INTEL_RS3WC040_SSDID:
1605 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1606 				instance->host->host_no,
1607 				MEGARAID_INTEL_RS3WC040_BRANDING);
1608 			break;
1609 		default:
1610 			break;
1611 		}
1612 		break;
1613 	case PCI_DEVICE_ID_LSI_CUTLASS_52:
1614 	case PCI_DEVICE_ID_LSI_CUTLASS_53:
1615 		switch (instance->pdev->subsystem_device) {
1616 		case MEGARAID_INTEL_RMS3BC160_SSDID:
1617 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1618 				instance->host->host_no,
1619 				MEGARAID_INTEL_RMS3BC160_BRANDING);
1620 			break;
1621 		default:
1622 			break;
1623 		}
1624 		break;
1625 	default:
1626 		break;
1627 	}
1628 }
1629 
1630 /**
1631  * megasas_allocate_raid_maps -	Allocate memory for RAID maps
1632  * @instance:				Adapter soft state
1633  *
1634  * return:				if success: return 0
1635  *					failed:  return -ENOMEM
1636  */
1637 static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
1638 {
1639 	struct fusion_context *fusion;
1640 	int i = 0;
1641 
1642 	fusion = instance->ctrl_context;
1643 
1644 	fusion->drv_map_pages = get_order(fusion->drv_map_sz);
1645 
1646 	for (i = 0; i < 2; i++) {
1647 		fusion->ld_map[i] = NULL;
1648 
1649 		fusion->ld_drv_map[i] = (void *)
1650 			__get_free_pages(__GFP_ZERO | GFP_KERNEL,
1651 					 fusion->drv_map_pages);
1652 
1653 		if (!fusion->ld_drv_map[i]) {
1654 			fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);
1655 
1656 			if (!fusion->ld_drv_map[i]) {
1657 				dev_err(&instance->pdev->dev,
1658 					"Could not allocate memory for local map"
1659 					" size requested: %d\n",
1660 					fusion->drv_map_sz);
1661 				goto ld_drv_map_alloc_fail;
1662 			}
1663 		}
1664 	}
1665 
1666 	for (i = 0; i < 2; i++) {
1667 		fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
1668 						       fusion->max_map_sz,
1669 						       &fusion->ld_map_phys[i],
1670 						       GFP_KERNEL);
1671 		if (!fusion->ld_map[i]) {
1672 			dev_err(&instance->pdev->dev,
1673 				"Could not allocate memory for map info %s:%d\n",
1674 				__func__, __LINE__);
1675 			goto ld_map_alloc_fail;
1676 		}
1677 	}
1678 
1679 	return 0;
1680 
1681 ld_map_alloc_fail:
1682 	for (i = 0; i < 2; i++) {
1683 		if (fusion->ld_map[i])
1684 			dma_free_coherent(&instance->pdev->dev,
1685 					  fusion->max_map_sz,
1686 					  fusion->ld_map[i],
1687 					  fusion->ld_map_phys[i]);
1688 	}
1689 
1690 ld_drv_map_alloc_fail:
1691 	for (i = 0; i < 2; i++) {
1692 		if (fusion->ld_drv_map[i]) {
1693 			if (is_vmalloc_addr(fusion->ld_drv_map[i]))
1694 				vfree(fusion->ld_drv_map[i]);
1695 			else
1696 				free_pages((ulong)fusion->ld_drv_map[i],
1697 					   fusion->drv_map_pages);
1698 		}
1699 	}
1700 
1701 	return -ENOMEM;
1702 }
1703 
1704 /**
1705  * megasas_configure_queue_sizes -	Calculate size of request desc queue,
1706  *					reply desc queue,
1707  *					IO request frame queue, set can_queue.
1708  * @instance:				Adapter soft state
1709  * @return:				void
1710  */
1711 static inline
1712 void megasas_configure_queue_sizes(struct megasas_instance *instance)
1713 {
1714 	struct fusion_context *fusion;
1715 	u16 max_cmd;
1716 
1717 	fusion = instance->ctrl_context;
1718 	max_cmd = instance->max_fw_cmds;
1719 
1720 	if (instance->adapter_type >= VENTURA_SERIES)
1721 		instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;
1722 	else
1723 		instance->max_mpt_cmds = instance->max_fw_cmds;
1724 
1725 	instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds;
1726 	instance->cur_can_queue = instance->max_scsi_cmds;
1727 	instance->host->can_queue = instance->cur_can_queue;
1728 
1729 	fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;
1730 
1731 	fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *
1732 					  instance->max_mpt_cmds;
1733 	fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *
1734 					(fusion->reply_q_depth);
1735 	fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1736 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1737 		 * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */
1738 }
1739 
1740 static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)
1741 {
1742 	struct fusion_context *fusion;
1743 	struct megasas_cmd *cmd;
1744 
1745 	fusion = instance->ctrl_context;
1746 
1747 	cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL);
1748 
1749 	if (!cmd) {
1750 		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1751 			__func__, __LINE__);
1752 		return -ENOMEM;
1753 	}
1754 
1755 	cmd->frame = dma_alloc_coherent(&instance->pdev->dev,
1756 					IOC_INIT_FRAME_SIZE,
1757 					&cmd->frame_phys_addr, GFP_KERNEL);
1758 
1759 	if (!cmd->frame) {
1760 		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1761 			__func__, __LINE__);
1762 		kfree(cmd);
1763 		return -ENOMEM;
1764 	}
1765 
1766 	fusion->ioc_init_cmd = cmd;
1767 	return 0;
1768 }
1769 
1770 /**
1771  * megasas_free_ioc_init_cmd -	Free IOC INIT command frame
1772  * @instance:		Adapter soft state
1773  */
1774 static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)
1775 {
1776 	struct fusion_context *fusion;
1777 
1778 	fusion = instance->ctrl_context;
1779 
1780 	if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)
1781 		dma_free_coherent(&instance->pdev->dev,
1782 				  IOC_INIT_FRAME_SIZE,
1783 				  fusion->ioc_init_cmd->frame,
1784 				  fusion->ioc_init_cmd->frame_phys_addr);
1785 
1786 	kfree(fusion->ioc_init_cmd);
1787 }
1788 
1789 /**
1790  * megasas_init_adapter_fusion -	Initializes the FW
1791  * @instance:		Adapter soft state
1792  *
1793  * This is the main function for initializing firmware.
1794  */
1795 static u32
1796 megasas_init_adapter_fusion(struct megasas_instance *instance)
1797 {
1798 	struct fusion_context *fusion;
1799 	u32 scratch_pad_1;
1800 	int i = 0, count;
1801 	u32 status_reg;
1802 
1803 	fusion = instance->ctrl_context;
1804 
1805 	megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);
1806 
1807 	/*
1808 	 * Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames
1809 	 */
1810 	instance->max_mfi_cmds =
1811 		MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1812 
1813 	megasas_configure_queue_sizes(instance);
1814 
1815 	scratch_pad_1 = megasas_readl(instance,
1816 				      &instance->reg_set->outbound_scratch_pad_1);
1817 	/* If scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,
1818 	 * Firmware support extended IO chain frame which is 4 times more than
1819 	 * legacy Firmware.
1820 	 * Legacy Firmware - Frame size is (8 * 128) = 1K
1821 	 * 1M IO Firmware  - Frame size is (8 * 128 * 4)  = 4K
1822 	 */
1823 	if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1824 		instance->max_chain_frame_sz =
1825 			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1826 			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1827 	else
1828 		instance->max_chain_frame_sz =
1829 			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1830 			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1831 
1832 	if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1833 		dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1834 			instance->max_chain_frame_sz,
1835 			MEGASAS_CHAIN_FRAME_SZ_MIN);
1836 		instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1837 	}
1838 
1839 	fusion->max_sge_in_main_msg =
1840 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1841 			- offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1842 
1843 	fusion->max_sge_in_chain =
1844 		instance->max_chain_frame_sz
1845 			/ sizeof(union MPI2_SGE_IO_UNION);
1846 
1847 	instance->max_num_sge =
1848 		rounddown_pow_of_two(fusion->max_sge_in_main_msg
1849 			+ fusion->max_sge_in_chain - 2);
1850 
1851 	/* Used for pass thru MFI frame (DCMD) */
1852 	fusion->chain_offset_mfi_pthru =
1853 		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1854 
1855 	fusion->chain_offset_io_request =
1856 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1857 		 sizeof(union MPI2_SGE_IO_UNION))/16;
1858 
1859 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1860 	count += instance->iopoll_q_count;
1861 
1862 	for (i = 0 ; i < count; i++)
1863 		fusion->last_reply_idx[i] = 0;
1864 
1865 	/*
1866 	 * For fusion adapters, 3 commands for IOCTL and 8 commands
1867 	 * for driver's internal DCMDs.
1868 	 */
1869 	instance->max_scsi_cmds = instance->max_fw_cmds -
1870 				(MEGASAS_FUSION_INTERNAL_CMDS +
1871 				MEGASAS_FUSION_IOCTL_CMDS);
1872 	sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1873 
1874 	for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++)
1875 		atomic_set(&fusion->busy_mq_poll[i], 0);
1876 
1877 	if (megasas_alloc_ioc_init_frame(instance))
1878 		return 1;
1879 
1880 	/*
1881 	 * Allocate memory for descriptors
1882 	 * Create a pool of commands
1883 	 */
1884 	if (megasas_alloc_cmds(instance))
1885 		goto fail_alloc_mfi_cmds;
1886 	if (megasas_alloc_cmds_fusion(instance))
1887 		goto fail_alloc_cmds;
1888 
1889 	if (megasas_ioc_init_fusion(instance)) {
1890 		status_reg = instance->instancet->read_fw_status_reg(instance);
1891 		if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) &&
1892 		    (status_reg & MFI_RESET_ADAPTER)) {
1893 			/* Do a chip reset and then retry IOC INIT once */
1894 			if (megasas_adp_reset_wait_for_ready
1895 				(instance, true, 0) == FAILED)
1896 				goto fail_ioc_init;
1897 
1898 			if (megasas_ioc_init_fusion(instance))
1899 				goto fail_ioc_init;
1900 		} else {
1901 			goto fail_ioc_init;
1902 		}
1903 	}
1904 
1905 	megasas_display_intel_branding(instance);
1906 	if (megasas_get_ctrl_info(instance)) {
1907 		dev_err(&instance->pdev->dev,
1908 			"Could not get controller info. Fail from %s %d\n",
1909 			__func__, __LINE__);
1910 		goto fail_ioc_init;
1911 	}
1912 
1913 	instance->flag_ieee = 1;
1914 	instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;
1915 	instance->threshold_reply_count = instance->max_fw_cmds / 4;
1916 	fusion->fast_path_io = 0;
1917 
1918 	if (megasas_allocate_raid_maps(instance))
1919 		goto fail_ioc_init;
1920 
1921 	if (!megasas_get_map_info(instance))
1922 		megasas_sync_map_info(instance);
1923 
1924 	return 0;
1925 
1926 fail_ioc_init:
1927 	megasas_free_cmds_fusion(instance);
1928 fail_alloc_cmds:
1929 	megasas_free_cmds(instance);
1930 fail_alloc_mfi_cmds:
1931 	megasas_free_ioc_init_cmd(instance);
1932 	return 1;
1933 }
1934 
1935 /**
1936  * megasas_fault_detect_work	-	Worker function of
1937  *					FW fault handling workqueue.
1938  * @work:	FW fault work struct
1939  */
1940 static void
1941 megasas_fault_detect_work(struct work_struct *work)
1942 {
1943 	struct megasas_instance *instance =
1944 		container_of(work, struct megasas_instance,
1945 			     fw_fault_work.work);
1946 	u32 fw_state, dma_state, status;
1947 
1948 	/* Check the fw state */
1949 	fw_state = instance->instancet->read_fw_status_reg(instance) &
1950 			MFI_STATE_MASK;
1951 
1952 	if (fw_state == MFI_STATE_FAULT) {
1953 		dma_state = instance->instancet->read_fw_status_reg(instance) &
1954 				MFI_STATE_DMADONE;
1955 		/* Start collecting crash, if DMA bit is done */
1956 		if (instance->crash_dump_drv_support &&
1957 		    instance->crash_dump_app_support && dma_state) {
1958 			megasas_fusion_crash_dump(instance);
1959 		} else {
1960 			if (instance->unload == 0) {
1961 				status = megasas_reset_fusion(instance->host, 0);
1962 				if (status != SUCCESS) {
1963 					dev_err(&instance->pdev->dev,
1964 						"Failed from %s %d, do not re-arm timer\n",
1965 						__func__, __LINE__);
1966 					return;
1967 				}
1968 			}
1969 		}
1970 	}
1971 
1972 	if (instance->fw_fault_work_q)
1973 		queue_delayed_work(instance->fw_fault_work_q,
1974 			&instance->fw_fault_work,
1975 			msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1976 }
1977 
1978 int
1979 megasas_fusion_start_watchdog(struct megasas_instance *instance)
1980 {
1981 	/* Check if the Fault WQ is already started */
1982 	if (instance->fw_fault_work_q)
1983 		return SUCCESS;
1984 
1985 	INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work);
1986 
1987 	snprintf(instance->fault_handler_work_q_name,
1988 		 sizeof(instance->fault_handler_work_q_name),
1989 		 "poll_megasas%d_status", instance->host->host_no);
1990 
1991 	instance->fw_fault_work_q =
1992 		create_singlethread_workqueue(instance->fault_handler_work_q_name);
1993 	if (!instance->fw_fault_work_q) {
1994 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
1995 			__func__, __LINE__);
1996 		return FAILED;
1997 	}
1998 
1999 	queue_delayed_work(instance->fw_fault_work_q,
2000 			   &instance->fw_fault_work,
2001 			   msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
2002 
2003 	return SUCCESS;
2004 }
2005 
2006 void
2007 megasas_fusion_stop_watchdog(struct megasas_instance *instance)
2008 {
2009 	struct workqueue_struct *wq;
2010 
2011 	if (instance->fw_fault_work_q) {
2012 		wq = instance->fw_fault_work_q;
2013 		instance->fw_fault_work_q = NULL;
2014 		if (!cancel_delayed_work_sync(&instance->fw_fault_work))
2015 			flush_workqueue(wq);
2016 		destroy_workqueue(wq);
2017 	}
2018 }
2019 
2020 /**
2021  * map_cmd_status -	Maps FW cmd status to OS cmd status
2022  * @fusion:		fusion context
2023  * @scmd:		Pointer to cmd
2024  * @status:		status of cmd returned by FW
2025  * @ext_status:		ext status of cmd returned by FW
2026  * @data_length:	command data length
2027  * @sense:		command sense data
2028  */
2029 static void
2030 map_cmd_status(struct fusion_context *fusion,
2031 		struct scsi_cmnd *scmd, u8 status, u8 ext_status,
2032 		u32 data_length, u8 *sense)
2033 {
2034 	u8 cmd_type;
2035 	int resid;
2036 
2037 	cmd_type = megasas_cmd_type(scmd);
2038 	switch (status) {
2039 
2040 	case MFI_STAT_OK:
2041 		scmd->result = DID_OK << 16;
2042 		break;
2043 
2044 	case MFI_STAT_SCSI_IO_FAILED:
2045 	case MFI_STAT_LD_INIT_IN_PROGRESS:
2046 		scmd->result = (DID_ERROR << 16) | ext_status;
2047 		break;
2048 
2049 	case MFI_STAT_SCSI_DONE_WITH_ERROR:
2050 
2051 		scmd->result = (DID_OK << 16) | ext_status;
2052 		if (ext_status == SAM_STAT_CHECK_CONDITION) {
2053 			memcpy(scmd->sense_buffer, sense,
2054 			       SCSI_SENSE_BUFFERSIZE);
2055 		}
2056 
2057 		/*
2058 		 * If the  IO request is partially completed, then MR FW will
2059 		 * update "io_request->DataLength" field with actual number of
2060 		 * bytes transferred.Driver will set residual bytes count in
2061 		 * SCSI command structure.
2062 		 */
2063 		resid = (scsi_bufflen(scmd) - data_length);
2064 		scsi_set_resid(scmd, resid);
2065 
2066 		if (resid &&
2067 			((cmd_type == READ_WRITE_LDIO) ||
2068 			(cmd_type == READ_WRITE_SYSPDIO)))
2069 			scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
2070 				" requested/completed 0x%x/0x%x\n",
2071 				status, scsi_bufflen(scmd), data_length);
2072 		break;
2073 
2074 	case MFI_STAT_LD_OFFLINE:
2075 	case MFI_STAT_DEVICE_NOT_FOUND:
2076 		scmd->result = DID_BAD_TARGET << 16;
2077 		break;
2078 	case MFI_STAT_CONFIG_SEQ_MISMATCH:
2079 		scmd->result = DID_IMM_RETRY << 16;
2080 		break;
2081 	default:
2082 		scmd->result = DID_ERROR << 16;
2083 		break;
2084 	}
2085 }
2086 
2087 /**
2088  * megasas_is_prp_possible -
2089  * Checks if native NVMe PRPs can be built for the IO
2090  *
2091  * @instance:		Adapter soft state
2092  * @scmd:		SCSI command from the mid-layer
2093  * @sge_count:		scatter gather element count.
2094  *
2095  * Returns:		true: PRPs can be built
2096  *			false: IEEE SGLs needs to be built
2097  */
2098 static bool
2099 megasas_is_prp_possible(struct megasas_instance *instance,
2100 			struct scsi_cmnd *scmd, int sge_count)
2101 {
2102 	u32 data_length = 0;
2103 	struct scatterlist *sg_scmd;
2104 	bool build_prp = false;
2105 	u32 mr_nvme_pg_size;
2106 
2107 	mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2108 				MR_DEFAULT_NVME_PAGE_SIZE);
2109 	data_length = scsi_bufflen(scmd);
2110 	sg_scmd = scsi_sglist(scmd);
2111 
2112 	/*
2113 	 * NVMe uses one PRP for each page (or part of a page)
2114 	 * look at the data length - if 4 pages or less then IEEE is OK
2115 	 * if  > 5 pages then we need to build a native SGL
2116 	 * if > 4 and <= 5 pages, then check physical address of 1st SG entry
2117 	 * if this first size in the page is >= the residual beyond 4 pages
2118 	 * then use IEEE, otherwise use native SGL
2119 	 */
2120 
2121 	if (data_length > (mr_nvme_pg_size * 5)) {
2122 		build_prp = true;
2123 	} else if ((data_length > (mr_nvme_pg_size * 4)) &&
2124 			(data_length <= (mr_nvme_pg_size * 5)))  {
2125 		/* check if 1st SG entry size is < residual beyond 4 pages */
2126 		if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
2127 			build_prp = true;
2128 	}
2129 
2130 	return build_prp;
2131 }
2132 
2133 /**
2134  * megasas_make_prp_nvme -
2135  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
2136  *
2137  * @instance:		Adapter soft state
2138  * @scmd:		SCSI command from the mid-layer
2139  * @sgl_ptr:		SGL to be filled in
2140  * @cmd:		Fusion command frame
2141  * @sge_count:		scatter gather element count.
2142  *
2143  * Returns:		true: PRPs are built
2144  *			false: IEEE SGLs needs to be built
2145  */
2146 static bool
2147 megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
2148 		      struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2149 		      struct megasas_cmd_fusion *cmd, int sge_count)
2150 {
2151 	int sge_len, offset, num_prp_in_chain = 0;
2152 	struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
2153 	u64 *ptr_sgl;
2154 	dma_addr_t ptr_sgl_phys;
2155 	u64 sge_addr;
2156 	u32 page_mask, page_mask_result;
2157 	struct scatterlist *sg_scmd;
2158 	u32 first_prp_len;
2159 	bool build_prp = false;
2160 	int data_len = scsi_bufflen(scmd);
2161 	u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2162 					MR_DEFAULT_NVME_PAGE_SIZE);
2163 
2164 	build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
2165 
2166 	if (!build_prp)
2167 		return false;
2168 
2169 	/*
2170 	 * Nvme has a very convoluted prp format.  One prp is required
2171 	 * for each page or partial page. Driver need to split up OS sg_list
2172 	 * entries if it is longer than one page or cross a page
2173 	 * boundary.  Driver also have to insert a PRP list pointer entry as
2174 	 * the last entry in each physical page of the PRP list.
2175 	 *
2176 	 * NOTE: The first PRP "entry" is actually placed in the first
2177 	 * SGL entry in the main message as IEEE 64 format.  The 2nd
2178 	 * entry in the main message is the chain element, and the rest
2179 	 * of the PRP entries are built in the contiguous pcie buffer.
2180 	 */
2181 	page_mask = mr_nvme_pg_size - 1;
2182 	ptr_sgl = (u64 *)cmd->sg_frame;
2183 	ptr_sgl_phys = cmd->sg_frame_phys_addr;
2184 	memset(ptr_sgl, 0, instance->max_chain_frame_sz);
2185 
2186 	/* Build chain frame element which holds all prps except first*/
2187 	main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
2188 	    ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
2189 
2190 	main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
2191 	main_chain_element->NextChainOffset = 0;
2192 	main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2193 					IEEE_SGE_FLAGS_SYSTEM_ADDR |
2194 					MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2195 
2196 	/* Build first prp, sge need not to be page aligned*/
2197 	ptr_first_sgl = sgl_ptr;
2198 	sg_scmd = scsi_sglist(scmd);
2199 	sge_addr = sg_dma_address(sg_scmd);
2200 	sge_len = sg_dma_len(sg_scmd);
2201 
2202 	offset = (u32)(sge_addr & page_mask);
2203 	first_prp_len = mr_nvme_pg_size - offset;
2204 
2205 	ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2206 	ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2207 
2208 	data_len -= first_prp_len;
2209 
2210 	if (sge_len > first_prp_len) {
2211 		sge_addr += first_prp_len;
2212 		sge_len -= first_prp_len;
2213 	} else if (sge_len == first_prp_len) {
2214 		sg_scmd = sg_next(sg_scmd);
2215 		sge_addr = sg_dma_address(sg_scmd);
2216 		sge_len = sg_dma_len(sg_scmd);
2217 	}
2218 
2219 	for (;;) {
2220 		offset = (u32)(sge_addr & page_mask);
2221 
2222 		/* Put PRP pointer due to page boundary*/
2223 		page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
2224 		if (unlikely(!page_mask_result)) {
2225 			scmd_printk(KERN_NOTICE,
2226 				    scmd, "page boundary ptr_sgl: 0x%p\n",
2227 				    ptr_sgl);
2228 			ptr_sgl_phys += 8;
2229 			*ptr_sgl = cpu_to_le64(ptr_sgl_phys);
2230 			ptr_sgl++;
2231 			num_prp_in_chain++;
2232 		}
2233 
2234 		*ptr_sgl = cpu_to_le64(sge_addr);
2235 		ptr_sgl++;
2236 		ptr_sgl_phys += 8;
2237 		num_prp_in_chain++;
2238 
2239 		sge_addr += mr_nvme_pg_size;
2240 		sge_len -= mr_nvme_pg_size;
2241 		data_len -= mr_nvme_pg_size;
2242 
2243 		if (data_len <= 0)
2244 			break;
2245 
2246 		if (sge_len > 0)
2247 			continue;
2248 
2249 		sg_scmd = sg_next(sg_scmd);
2250 		sge_addr = sg_dma_address(sg_scmd);
2251 		sge_len = sg_dma_len(sg_scmd);
2252 	}
2253 
2254 	main_chain_element->Length =
2255 			cpu_to_le32(num_prp_in_chain * sizeof(u64));
2256 
2257 	return build_prp;
2258 }
2259 
2260 /**
2261  * megasas_make_sgl_fusion -	Prepares 32-bit SGL
2262  * @instance:		Adapter soft state
2263  * @scp:		SCSI command from the mid-layer
2264  * @sgl_ptr:		SGL to be filled in
2265  * @cmd:		cmd we are working on
2266  * @sge_count:		sge count
2267  *
2268  */
2269 static void
2270 megasas_make_sgl_fusion(struct megasas_instance *instance,
2271 			struct scsi_cmnd *scp,
2272 			struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2273 			struct megasas_cmd_fusion *cmd, int sge_count)
2274 {
2275 	int i, sg_processed;
2276 	struct scatterlist *os_sgl;
2277 	struct fusion_context *fusion;
2278 
2279 	fusion = instance->ctrl_context;
2280 
2281 	if (instance->adapter_type >= INVADER_SERIES) {
2282 		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
2283 		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
2284 		sgl_ptr_end->Flags = 0;
2285 	}
2286 
2287 	scsi_for_each_sg(scp, os_sgl, sge_count, i) {
2288 		sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
2289 		sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
2290 		sgl_ptr->Flags = 0;
2291 		if (instance->adapter_type >= INVADER_SERIES)
2292 			if (i == sge_count - 1)
2293 				sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
2294 		sgl_ptr++;
2295 		sg_processed = i + 1;
2296 
2297 		if ((sg_processed ==  (fusion->max_sge_in_main_msg - 1)) &&
2298 		    (sge_count > fusion->max_sge_in_main_msg)) {
2299 
2300 			struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
2301 			if (instance->adapter_type >= INVADER_SERIES) {
2302 				if ((le16_to_cpu(cmd->io_request->IoFlags) &
2303 					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
2304 					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
2305 					cmd->io_request->ChainOffset =
2306 						fusion->
2307 						chain_offset_io_request;
2308 				else
2309 					cmd->io_request->ChainOffset = 0;
2310 			} else
2311 				cmd->io_request->ChainOffset =
2312 					fusion->chain_offset_io_request;
2313 
2314 			sg_chain = sgl_ptr;
2315 			/* Prepare chain element */
2316 			sg_chain->NextChainOffset = 0;
2317 			if (instance->adapter_type >= INVADER_SERIES)
2318 				sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
2319 			else
2320 				sg_chain->Flags =
2321 					(IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2322 					 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
2323 			sg_chain->Length =  cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
2324 			sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
2325 
2326 			sgl_ptr =
2327 			  (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
2328 			memset(sgl_ptr, 0, instance->max_chain_frame_sz);
2329 		}
2330 	}
2331 }
2332 
2333 /**
2334  * megasas_make_sgl -	Build Scatter Gather List(SGLs)
2335  * @scp:		SCSI command pointer
2336  * @instance:		Soft instance of controller
2337  * @cmd:		Fusion command pointer
2338  *
2339  * This function will build sgls based on device type.
2340  * For nvme drives, there is different way of building sgls in nvme native
2341  * format- PRPs(Physical Region Page).
2342  *
2343  * Returns the number of sg lists actually used, zero if the sg lists
2344  * is NULL, or -ENOMEM if the mapping failed
2345  */
2346 static
2347 int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
2348 		     struct megasas_cmd_fusion *cmd)
2349 {
2350 	int sge_count;
2351 	bool build_prp = false;
2352 	struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
2353 
2354 	sge_count = scsi_dma_map(scp);
2355 
2356 	if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
2357 		return sge_count;
2358 
2359 	sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
2360 	if ((le16_to_cpu(cmd->io_request->IoFlags) &
2361 	    MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
2362 	    (cmd->pd_interface == NVME_PD))
2363 		build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,
2364 						  cmd, sge_count);
2365 
2366 	if (!build_prp)
2367 		megasas_make_sgl_fusion(instance, scp, sgl_chain64,
2368 					cmd, sge_count);
2369 
2370 	return sge_count;
2371 }
2372 
2373 /**
2374  * megasas_set_pd_lba -	Sets PD LBA
2375  * @io_request:		IO request
2376  * @cdb_len:		cdb length
2377  * @io_info:		IO information
2378  * @scp:		SCSI command
2379  * @local_map_ptr:	Raid map
2380  * @ref_tag:		Primary reference tag
2381  *
2382  * Used to set the PD LBA in CDB for FP IOs
2383  */
2384 static void
2385 megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
2386 		   struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
2387 		   struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
2388 {
2389 	struct MR_LD_RAID *raid;
2390 	u16 ld;
2391 	u64 start_blk = io_info->pdBlock;
2392 	u8 *cdb = io_request->CDB.CDB32;
2393 	u32 num_blocks = io_info->numBlocks;
2394 	u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
2395 
2396 	/* Check if T10 PI (DIF) is enabled for this LD */
2397 	ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);
2398 	raid = MR_LdRaidGet(ld, local_map_ptr);
2399 	if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
2400 		memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2401 		cdb[0] =  MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
2402 		cdb[7] =  MEGASAS_SCSI_ADDL_CDB_LEN;
2403 
2404 		if (scp->sc_data_direction == DMA_FROM_DEVICE)
2405 			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
2406 		else
2407 			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
2408 		cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
2409 
2410 		/* LBA */
2411 		cdb[12] = (u8)((start_blk >> 56) & 0xff);
2412 		cdb[13] = (u8)((start_blk >> 48) & 0xff);
2413 		cdb[14] = (u8)((start_blk >> 40) & 0xff);
2414 		cdb[15] = (u8)((start_blk >> 32) & 0xff);
2415 		cdb[16] = (u8)((start_blk >> 24) & 0xff);
2416 		cdb[17] = (u8)((start_blk >> 16) & 0xff);
2417 		cdb[18] = (u8)((start_blk >> 8) & 0xff);
2418 		cdb[19] = (u8)(start_blk & 0xff);
2419 
2420 		/* Logical block reference tag */
2421 		io_request->CDB.EEDP32.PrimaryReferenceTag =
2422 			cpu_to_be32(ref_tag);
2423 		io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
2424 		io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */
2425 
2426 		/* Transfer length */
2427 		cdb[28] = (u8)((num_blocks >> 24) & 0xff);
2428 		cdb[29] = (u8)((num_blocks >> 16) & 0xff);
2429 		cdb[30] = (u8)((num_blocks >> 8) & 0xff);
2430 		cdb[31] = (u8)(num_blocks & 0xff);
2431 
2432 		/* set SCSI IO EEDPFlags */
2433 		if (scp->sc_data_direction == DMA_FROM_DEVICE) {
2434 			io_request->EEDPFlags = cpu_to_le16(
2435 				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG  |
2436 				MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2437 				MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
2438 				MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
2439 				MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
2440 				MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
2441 		} else {
2442 			io_request->EEDPFlags = cpu_to_le16(
2443 				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2444 				MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
2445 		}
2446 		io_request->Control |= cpu_to_le32((0x4 << 26));
2447 		io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
2448 	} else {
2449 		/* Some drives don't support 16/12 byte CDB's, convert to 10 */
2450 		if (((cdb_len == 12) || (cdb_len == 16)) &&
2451 		    (start_blk <= 0xffffffff)) {
2452 			if (cdb_len == 16) {
2453 				opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
2454 				flagvals = cdb[1];
2455 				groupnum = cdb[14];
2456 				control = cdb[15];
2457 			} else {
2458 				opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
2459 				flagvals = cdb[1];
2460 				groupnum = cdb[10];
2461 				control = cdb[11];
2462 			}
2463 
2464 			memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2465 
2466 			cdb[0] = opcode;
2467 			cdb[1] = flagvals;
2468 			cdb[6] = groupnum;
2469 			cdb[9] = control;
2470 
2471 			/* Transfer length */
2472 			cdb[8] = (u8)(num_blocks & 0xff);
2473 			cdb[7] = (u8)((num_blocks >> 8) & 0xff);
2474 
2475 			io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */
2476 			cdb_len = 10;
2477 		} else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
2478 			/* Convert to 16 byte CDB for large LBA's */
2479 			switch (cdb_len) {
2480 			case 6:
2481 				opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
2482 				control = cdb[5];
2483 				break;
2484 			case 10:
2485 				opcode =
2486 					cdb[0] == READ_10 ? READ_16 : WRITE_16;
2487 				flagvals = cdb[1];
2488 				groupnum = cdb[6];
2489 				control = cdb[9];
2490 				break;
2491 			case 12:
2492 				opcode =
2493 					cdb[0] == READ_12 ? READ_16 : WRITE_16;
2494 				flagvals = cdb[1];
2495 				groupnum = cdb[10];
2496 				control = cdb[11];
2497 				break;
2498 			}
2499 
2500 			memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2501 
2502 			cdb[0] = opcode;
2503 			cdb[1] = flagvals;
2504 			cdb[14] = groupnum;
2505 			cdb[15] = control;
2506 
2507 			/* Transfer length */
2508 			cdb[13] = (u8)(num_blocks & 0xff);
2509 			cdb[12] = (u8)((num_blocks >> 8) & 0xff);
2510 			cdb[11] = (u8)((num_blocks >> 16) & 0xff);
2511 			cdb[10] = (u8)((num_blocks >> 24) & 0xff);
2512 
2513 			io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */
2514 			cdb_len = 16;
2515 		}
2516 
2517 		/* Normal case, just load LBA here */
2518 		switch (cdb_len) {
2519 		case 6:
2520 		{
2521 			u8 val = cdb[1] & 0xE0;
2522 			cdb[3] = (u8)(start_blk & 0xff);
2523 			cdb[2] = (u8)((start_blk >> 8) & 0xff);
2524 			cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
2525 			break;
2526 		}
2527 		case 10:
2528 			cdb[5] = (u8)(start_blk & 0xff);
2529 			cdb[4] = (u8)((start_blk >> 8) & 0xff);
2530 			cdb[3] = (u8)((start_blk >> 16) & 0xff);
2531 			cdb[2] = (u8)((start_blk >> 24) & 0xff);
2532 			break;
2533 		case 12:
2534 			cdb[5]    = (u8)(start_blk & 0xff);
2535 			cdb[4]    = (u8)((start_blk >> 8) & 0xff);
2536 			cdb[3]    = (u8)((start_blk >> 16) & 0xff);
2537 			cdb[2]    = (u8)((start_blk >> 24) & 0xff);
2538 			break;
2539 		case 16:
2540 			cdb[9]    = (u8)(start_blk & 0xff);
2541 			cdb[8]    = (u8)((start_blk >> 8) & 0xff);
2542 			cdb[7]    = (u8)((start_blk >> 16) & 0xff);
2543 			cdb[6]    = (u8)((start_blk >> 24) & 0xff);
2544 			cdb[5]    = (u8)((start_blk >> 32) & 0xff);
2545 			cdb[4]    = (u8)((start_blk >> 40) & 0xff);
2546 			cdb[3]    = (u8)((start_blk >> 48) & 0xff);
2547 			cdb[2]    = (u8)((start_blk >> 56) & 0xff);
2548 			break;
2549 		}
2550 	}
2551 }
2552 
2553 /**
2554  * megasas_stream_detect -	stream detection on read and and write IOs
2555  * @instance:		Adapter soft state
2556  * @cmd:		    Command to be prepared
2557  * @io_info:		IO Request info
2558  *
2559  */
2560 
2561 /** stream detection on read and and write IOs */
2562 static void megasas_stream_detect(struct megasas_instance *instance,
2563 				  struct megasas_cmd_fusion *cmd,
2564 				  struct IO_REQUEST_INFO *io_info)
2565 {
2566 	struct fusion_context *fusion = instance->ctrl_context;
2567 	u32 device_id = io_info->ldTgtId;
2568 	struct LD_STREAM_DETECT *current_ld_sd
2569 		= fusion->stream_detect_by_ld[device_id];
2570 	u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;
2571 	u32 shifted_values, unshifted_values;
2572 	u32 index_value_mask, shifted_values_mask;
2573 	int i;
2574 	bool is_read_ahead = false;
2575 	struct STREAM_DETECT *current_sd;
2576 	/* find possible stream */
2577 	for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2578 		stream_num = (*track_stream >>
2579 			(i * BITS_PER_INDEX_STREAM)) &
2580 			STREAM_MASK;
2581 		current_sd = &current_ld_sd->stream_track[stream_num];
2582 		/* if we found a stream, update the raid
2583 		 *  context and also update the mruBitMap
2584 		 */
2585 		/*	boundary condition */
2586 		if ((current_sd->next_seq_lba) &&
2587 		    (io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2588 		    (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2589 		    (current_sd->is_read == io_info->isRead)) {
2590 
2591 			if ((io_info->ldStartBlock != current_sd->next_seq_lba)	&&
2592 			    ((!io_info->isRead) || (!is_read_ahead)))
2593 				/*
2594 				 * Once the API is available we need to change this.
2595 				 * At this point we are not allowing any gap
2596 				 */
2597 				continue;
2598 
2599 			SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2600 			current_sd->next_seq_lba =
2601 			io_info->ldStartBlock + io_info->numBlocks;
2602 			/*
2603 			 *	update the mruBitMap LRU
2604 			 */
2605 			shifted_values_mask =
2606 				(1 <<  i * BITS_PER_INDEX_STREAM) - 1;
2607 			shifted_values = ((*track_stream & shifted_values_mask)
2608 						<< BITS_PER_INDEX_STREAM);
2609 			index_value_mask =
2610 				STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2611 			unshifted_values =
2612 				*track_stream & ~(shifted_values_mask |
2613 				index_value_mask);
2614 			*track_stream =
2615 				unshifted_values | shifted_values | stream_num;
2616 			return;
2617 		}
2618 	}
2619 	/*
2620 	 * if we did not find any stream, create a new one
2621 	 * from the least recently used
2622 	 */
2623 	stream_num = (*track_stream >>
2624 		((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2625 		STREAM_MASK;
2626 	current_sd = &current_ld_sd->stream_track[stream_num];
2627 	current_sd->is_read = io_info->isRead;
2628 	current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2629 	*track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2630 	return;
2631 }
2632 
2633 /**
2634  * megasas_set_raidflag_cpu_affinity - This function sets the cpu
2635  * affinity (cpu of the controller) and raid_flags in the raid context
2636  * based on IO type.
2637  *
2638  * @fusion:		Fusion context
2639  * @praid_context:	IO RAID context
2640  * @raid:		LD raid map
2641  * @fp_possible:	Is fast path possible?
2642  * @is_read:		Is read IO?
2643  * @scsi_buff_len:	SCSI command buffer length
2644  *
2645  */
2646 static void
2647 megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion,
2648 				union RAID_CONTEXT_UNION *praid_context,
2649 				struct MR_LD_RAID *raid, bool fp_possible,
2650 				u8 is_read, u32 scsi_buff_len)
2651 {
2652 	u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2653 	struct RAID_CONTEXT_G35 *rctx_g35;
2654 
2655 	rctx_g35 = &praid_context->raid_context_g35;
2656 	if (fp_possible) {
2657 		if (is_read) {
2658 			if ((raid->cpuAffinity.pdRead.cpu0) &&
2659 			    (raid->cpuAffinity.pdRead.cpu1))
2660 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2661 			else if (raid->cpuAffinity.pdRead.cpu1)
2662 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2663 		} else {
2664 			if ((raid->cpuAffinity.pdWrite.cpu0) &&
2665 			    (raid->cpuAffinity.pdWrite.cpu1))
2666 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2667 			else if (raid->cpuAffinity.pdWrite.cpu1)
2668 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2669 			/* Fast path cache by pass capable R0/R1 VD */
2670 			if ((raid->level <= 1) &&
2671 			    (raid->capability.fp_cache_bypass_capable)) {
2672 				rctx_g35->routing_flags |=
2673 					(1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2674 				rctx_g35->raid_flags =
2675 					(MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2676 					<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2677 			}
2678 		}
2679 	} else {
2680 		if (is_read) {
2681 			if ((raid->cpuAffinity.ldRead.cpu0) &&
2682 			    (raid->cpuAffinity.ldRead.cpu1))
2683 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2684 			else if (raid->cpuAffinity.ldRead.cpu1)
2685 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2686 		} else {
2687 			if ((raid->cpuAffinity.ldWrite.cpu0) &&
2688 			    (raid->cpuAffinity.ldWrite.cpu1))
2689 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2690 			else if (raid->cpuAffinity.ldWrite.cpu1)
2691 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2692 
2693 			if (is_stream_detected(rctx_g35) &&
2694 			    ((raid->level == 5) || (raid->level == 6)) &&
2695 			    (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2696 			    (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2697 				cpu_sel = MR_RAID_CTX_CPUSEL_0;
2698 		}
2699 	}
2700 
2701 	rctx_g35->routing_flags |=
2702 		(cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2703 
2704 	/* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2705 	 * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.
2706 	 * IO Subtype is not bitmap.
2707 	 */
2708 	if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) &&
2709 			(scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) {
2710 		praid_context->raid_context_g35.raid_flags =
2711 			(MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2712 			<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2713 	}
2714 }
2715 
2716 /**
2717  * megasas_build_ldio_fusion -	Prepares IOs to devices
2718  * @instance:		Adapter soft state
2719  * @scp:		SCSI command
2720  * @cmd:		Command to be prepared
2721  *
2722  * Prepares the io_request and chain elements (sg_frame) for IO
2723  * The IO can be for PD (Fast Path) or LD
2724  */
2725 static void
2726 megasas_build_ldio_fusion(struct megasas_instance *instance,
2727 			  struct scsi_cmnd *scp,
2728 			  struct megasas_cmd_fusion *cmd)
2729 {
2730 	bool fp_possible;
2731 	u16 ld;
2732 	u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2733 	u32 scsi_buff_len;
2734 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2735 	struct IO_REQUEST_INFO io_info;
2736 	struct fusion_context *fusion;
2737 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2738 	u8 *raidLUN;
2739 	unsigned long spinlock_flags;
2740 	struct MR_LD_RAID *raid = NULL;
2741 	struct MR_PRIV_DEVICE *mrdev_priv;
2742 	struct RAID_CONTEXT *rctx;
2743 	struct RAID_CONTEXT_G35 *rctx_g35;
2744 
2745 	device_id = MEGASAS_DEV_INDEX(scp);
2746 
2747 	fusion = instance->ctrl_context;
2748 
2749 	io_request = cmd->io_request;
2750 	rctx = &io_request->RaidContext.raid_context;
2751 	rctx_g35 = &io_request->RaidContext.raid_context_g35;
2752 
2753 	rctx->virtual_disk_tgt_id = cpu_to_le16(device_id);
2754 	rctx->status = 0;
2755 	rctx->ex_status = 0;
2756 
2757 	start_lba_lo = 0;
2758 	start_lba_hi = 0;
2759 	fp_possible = false;
2760 
2761 	/*
2762 	 * 6-byte READ(0x08) or WRITE(0x0A) cdb
2763 	 */
2764 	if (scp->cmd_len == 6) {
2765 		datalength = (u32) scp->cmnd[4];
2766 		start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2767 			((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2768 
2769 		start_lba_lo &= 0x1FFFFF;
2770 	}
2771 
2772 	/*
2773 	 * 10-byte READ(0x28) or WRITE(0x2A) cdb
2774 	 */
2775 	else if (scp->cmd_len == 10) {
2776 		datalength = (u32) scp->cmnd[8] |
2777 			((u32) scp->cmnd[7] << 8);
2778 		start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2779 			((u32) scp->cmnd[3] << 16) |
2780 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2781 	}
2782 
2783 	/*
2784 	 * 12-byte READ(0xA8) or WRITE(0xAA) cdb
2785 	 */
2786 	else if (scp->cmd_len == 12) {
2787 		datalength = ((u32) scp->cmnd[6] << 24) |
2788 			((u32) scp->cmnd[7] << 16) |
2789 			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2790 		start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2791 			((u32) scp->cmnd[3] << 16) |
2792 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2793 	}
2794 
2795 	/*
2796 	 * 16-byte READ(0x88) or WRITE(0x8A) cdb
2797 	 */
2798 	else if (scp->cmd_len == 16) {
2799 		datalength = ((u32) scp->cmnd[10] << 24) |
2800 			((u32) scp->cmnd[11] << 16) |
2801 			((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2802 		start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2803 			((u32) scp->cmnd[7] << 16) |
2804 			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2805 
2806 		start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2807 			((u32) scp->cmnd[3] << 16) |
2808 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2809 	}
2810 
2811 	memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2812 	io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2813 	io_info.numBlocks = datalength;
2814 	io_info.ldTgtId = device_id;
2815 	io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2816 	scsi_buff_len = scsi_bufflen(scp);
2817 	io_request->DataLength = cpu_to_le32(scsi_buff_len);
2818 	io_info.data_arms = 1;
2819 
2820 	if (scp->sc_data_direction == DMA_FROM_DEVICE)
2821 		io_info.isRead = 1;
2822 
2823 	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2824 	ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2825 
2826 	if (ld < instance->fw_supported_vd_count)
2827 		raid = MR_LdRaidGet(ld, local_map_ptr);
2828 
2829 	if (!raid || (!fusion->fast_path_io)) {
2830 		rctx->reg_lock_flags  = 0;
2831 		fp_possible = false;
2832 	} else {
2833 		if (MR_BuildRaidContext(instance, &io_info, rctx,
2834 					local_map_ptr, &raidLUN))
2835 			fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2836 	}
2837 
2838 	megasas_get_msix_index(instance, scp, cmd, io_info.data_arms);
2839 
2840 	if (instance->adapter_type >= VENTURA_SERIES) {
2841 		/* FP for Optimal raid level 1.
2842 		 * All large RAID-1 writes (> 32 KiB, both WT and WB modes)
2843 		 * are built by the driver as LD I/Os.
2844 		 * All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os
2845 		 * (there is never a reason to process these as buffered writes)
2846 		 * All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os
2847 		 * with the SLD bit asserted.
2848 		 */
2849 		if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2850 			mrdev_priv = scp->device->hostdata;
2851 
2852 			if (atomic_inc_return(&instance->fw_outstanding) >
2853 				(instance->host->can_queue)) {
2854 				fp_possible = false;
2855 				atomic_dec(&instance->fw_outstanding);
2856 			} else if (fusion->pcie_bw_limitation &&
2857 				((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2858 				   (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0))) {
2859 				fp_possible = false;
2860 				atomic_dec(&instance->fw_outstanding);
2861 				if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2862 					atomic_set(&mrdev_priv->r1_ldio_hint,
2863 						   instance->r1_ldio_hint_default);
2864 			}
2865 		}
2866 
2867 		if (!fp_possible ||
2868 		    (io_info.isRead && io_info.ra_capable)) {
2869 			spin_lock_irqsave(&instance->stream_lock,
2870 					  spinlock_flags);
2871 			megasas_stream_detect(instance, cmd, &io_info);
2872 			spin_unlock_irqrestore(&instance->stream_lock,
2873 					       spinlock_flags);
2874 			/* In ventura if stream detected for a read and it is
2875 			 * read ahead capable make this IO as LDIO
2876 			 */
2877 			if (is_stream_detected(rctx_g35))
2878 				fp_possible = false;
2879 		}
2880 
2881 		/* If raid is NULL, set CPU affinity to default CPU0 */
2882 		if (raid)
2883 			megasas_set_raidflag_cpu_affinity(fusion, &io_request->RaidContext,
2884 				raid, fp_possible, io_info.isRead,
2885 				scsi_buff_len);
2886 		else
2887 			rctx_g35->routing_flags |=
2888 				(MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2889 	}
2890 
2891 	if (fp_possible) {
2892 		megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
2893 				   local_map_ptr, start_lba_lo);
2894 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2895 		cmd->request_desc->SCSIIO.RequestFlags =
2896 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2897 			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2898 		if (instance->adapter_type == INVADER_SERIES) {
2899 			rctx->type = MPI2_TYPE_CUDA;
2900 			rctx->nseg = 0x1;
2901 			io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2902 			rctx->reg_lock_flags |=
2903 			  (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2904 			   MR_RL_FLAGS_SEQ_NUM_ENABLE);
2905 		} else if (instance->adapter_type >= VENTURA_SERIES) {
2906 			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2907 			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2908 			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2909 			io_request->IoFlags |=
2910 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2911 		}
2912 		if (fusion->load_balance_info &&
2913 			(fusion->load_balance_info[device_id].loadBalanceFlag) &&
2914 			(io_info.isRead)) {
2915 			io_info.devHandle =
2916 				get_updated_dev_handle(instance,
2917 					&fusion->load_balance_info[device_id],
2918 					&io_info, local_map_ptr);
2919 			megasas_priv(scp)->status |= MEGASAS_LOAD_BALANCE_FLAG;
2920 			cmd->pd_r1_lb = io_info.pd_after_lb;
2921 			if (instance->adapter_type >= VENTURA_SERIES)
2922 				rctx_g35->span_arm = io_info.span_arm;
2923 			else
2924 				rctx->span_arm = io_info.span_arm;
2925 
2926 		} else
2927 			megasas_priv(scp)->status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2928 
2929 		if (instance->adapter_type >= VENTURA_SERIES)
2930 			cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2931 		else
2932 			cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2933 
2934 		if ((raidLUN[0] == 1) &&
2935 			(local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2936 			instance->dev_handle = !(instance->dev_handle);
2937 			io_info.devHandle =
2938 				local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2939 		}
2940 
2941 		cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2942 		io_request->DevHandle = io_info.devHandle;
2943 		cmd->pd_interface = io_info.pd_interface;
2944 		/* populate the LUN field */
2945 		memcpy(io_request->LUN, raidLUN, 8);
2946 	} else {
2947 		rctx->timeout_value =
2948 			cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2949 		cmd->request_desc->SCSIIO.RequestFlags =
2950 			(MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2951 			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2952 		if (instance->adapter_type == INVADER_SERIES) {
2953 			if (io_info.do_fp_rlbypass ||
2954 			(rctx->reg_lock_flags == REGION_TYPE_UNUSED))
2955 				cmd->request_desc->SCSIIO.RequestFlags =
2956 					(MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2957 					MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2958 			rctx->type = MPI2_TYPE_CUDA;
2959 			rctx->reg_lock_flags |=
2960 				(MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2961 					MR_RL_FLAGS_SEQ_NUM_ENABLE);
2962 			rctx->nseg = 0x1;
2963 		} else if (instance->adapter_type >= VENTURA_SERIES) {
2964 			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2965 			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2966 			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2967 		}
2968 		io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2969 		io_request->DevHandle = cpu_to_le16(device_id);
2970 
2971 	} /* Not FP */
2972 }
2973 
2974 /**
2975  * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
2976  * @instance:		Adapter soft state
2977  * @scmd:		SCSI command
2978  * @cmd:		Command to be prepared
2979  *
2980  * Prepares the io_request frame for non-rw io cmds for vd.
2981  */
2982 static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2983 			  struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2984 {
2985 	u32 device_id;
2986 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2987 	u16 ld;
2988 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2989 	struct fusion_context *fusion = instance->ctrl_context;
2990 	u8                          span, physArm;
2991 	__le16                      devHandle;
2992 	u32                         arRef, pd;
2993 	struct MR_LD_RAID                  *raid;
2994 	struct RAID_CONTEXT                *pRAID_Context;
2995 	u8 fp_possible = 1;
2996 
2997 	io_request = cmd->io_request;
2998 	device_id = MEGASAS_DEV_INDEX(scmd);
2999 	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
3000 	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3001 	/* get RAID_Context pointer */
3002 	pRAID_Context = &io_request->RaidContext.raid_context;
3003 	/* Check with FW team */
3004 	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3005 	pRAID_Context->reg_lock_row_lba    = 0;
3006 	pRAID_Context->reg_lock_length    = 0;
3007 
3008 	if (fusion->fast_path_io && (
3009 		device_id < instance->fw_supported_vd_count)) {
3010 
3011 		ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
3012 		if (ld >= instance->fw_supported_vd_count - 1)
3013 			fp_possible = 0;
3014 		else {
3015 			raid = MR_LdRaidGet(ld, local_map_ptr);
3016 			if (!(raid->capability.fpNonRWCapable))
3017 				fp_possible = 0;
3018 		}
3019 	} else
3020 		fp_possible = 0;
3021 
3022 	if (!fp_possible) {
3023 		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3024 		io_request->DevHandle = cpu_to_le16(device_id);
3025 		io_request->LUN[1] = scmd->device->lun;
3026 		pRAID_Context->timeout_value =
3027 			cpu_to_le16(scsi_cmd_to_rq(scmd)->timeout / HZ);
3028 		cmd->request_desc->SCSIIO.RequestFlags =
3029 			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3030 			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3031 	} else {
3032 
3033 		/* set RAID context values */
3034 		pRAID_Context->config_seq_num = raid->seqNum;
3035 		if (instance->adapter_type < VENTURA_SERIES)
3036 			pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
3037 		pRAID_Context->timeout_value =
3038 			cpu_to_le16(raid->fpIoTimeoutForLd);
3039 
3040 		/* get the DevHandle for the PD (since this is
3041 		   fpNonRWCapable, this is a single disk RAID0) */
3042 		span = physArm = 0;
3043 		arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);
3044 		pd = MR_ArPdGet(arRef, physArm, local_map_ptr);
3045 		devHandle = MR_PdDevHandleGet(pd, local_map_ptr);
3046 
3047 		/* build request descriptor */
3048 		cmd->request_desc->SCSIIO.RequestFlags =
3049 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3050 			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3051 		cmd->request_desc->SCSIIO.DevHandle = devHandle;
3052 
3053 		/* populate the LUN field */
3054 		memcpy(io_request->LUN, raid->LUN, 8);
3055 
3056 		/* build the raidScsiIO structure */
3057 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3058 		io_request->DevHandle = devHandle;
3059 	}
3060 }
3061 
3062 /**
3063  * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
3064  * @instance:		Adapter soft state
3065  * @scmd:		SCSI command
3066  * @cmd:		Command to be prepared
3067  * @fp_possible:	parameter to detect fast path or firmware path io.
3068  *
3069  * Prepares the io_request frame for rw/non-rw io cmds for syspds
3070  */
3071 static void
3072 megasas_build_syspd_fusion(struct megasas_instance *instance,
3073 	struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
3074 	bool fp_possible)
3075 {
3076 	u32 device_id;
3077 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
3078 	u16 pd_index = 0;
3079 	u16 os_timeout_value;
3080 	u16 timeout_limit;
3081 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
3082 	struct RAID_CONTEXT	*pRAID_Context;
3083 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
3084 	struct MR_PRIV_DEVICE *mr_device_priv_data;
3085 	struct fusion_context *fusion = instance->ctrl_context;
3086 	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
3087 
3088 	device_id = MEGASAS_DEV_INDEX(scmd);
3089 	pd_index = MEGASAS_PD_INDEX(scmd);
3090 	os_timeout_value = scsi_cmd_to_rq(scmd)->timeout / HZ;
3091 	mr_device_priv_data = scmd->device->hostdata;
3092 	cmd->pd_interface = mr_device_priv_data->interface_type;
3093 
3094 	io_request = cmd->io_request;
3095 	/* get RAID_Context pointer */
3096 	pRAID_Context = &io_request->RaidContext.raid_context;
3097 	pRAID_Context->reg_lock_flags = 0;
3098 	pRAID_Context->reg_lock_row_lba = 0;
3099 	pRAID_Context->reg_lock_length = 0;
3100 	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3101 	io_request->LUN[1] = scmd->device->lun;
3102 	pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
3103 		<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
3104 
3105 	/* If FW supports PD sequence number */
3106 	if (instance->support_seqnum_jbod_fp) {
3107 		if (instance->use_seqnum_jbod_fp &&
3108 			instance->pd_list[pd_index].driveType == TYPE_DISK) {
3109 
3110 			/* More than 256 PD/JBOD support for Ventura */
3111 			if (instance->support_morethan256jbod)
3112 				pRAID_Context->virtual_disk_tgt_id =
3113 					pd_sync->seq[pd_index].pd_target_id;
3114 			else
3115 				pRAID_Context->virtual_disk_tgt_id =
3116 					cpu_to_le16(device_id +
3117 					(MAX_PHYSICAL_DEVICES - 1));
3118 			pRAID_Context->config_seq_num =
3119 				pd_sync->seq[pd_index].seqNum;
3120 			io_request->DevHandle =
3121 				pd_sync->seq[pd_index].devHandle;
3122 			if (instance->adapter_type >= VENTURA_SERIES) {
3123 				io_request->RaidContext.raid_context_g35.routing_flags |=
3124 					(1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
3125 				io_request->RaidContext.raid_context_g35.nseg_type |=
3126 					(1 << RAID_CONTEXT_NSEG_SHIFT);
3127 				io_request->RaidContext.raid_context_g35.nseg_type |=
3128 					(MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
3129 			} else {
3130 				pRAID_Context->type = MPI2_TYPE_CUDA;
3131 				pRAID_Context->nseg = 0x1;
3132 				pRAID_Context->reg_lock_flags |=
3133 					(MR_RL_FLAGS_SEQ_NUM_ENABLE |
3134 					 MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
3135 			}
3136 		} else {
3137 			pRAID_Context->virtual_disk_tgt_id =
3138 				cpu_to_le16(device_id +
3139 				(MAX_PHYSICAL_DEVICES - 1));
3140 			pRAID_Context->config_seq_num = 0;
3141 			io_request->DevHandle = cpu_to_le16(0xFFFF);
3142 		}
3143 	} else {
3144 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3145 		pRAID_Context->config_seq_num = 0;
3146 
3147 		if (fusion->fast_path_io) {
3148 			local_map_ptr =
3149 				fusion->ld_drv_map[(instance->map_id & 1)];
3150 			io_request->DevHandle =
3151 				local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
3152 		} else {
3153 			io_request->DevHandle = cpu_to_le16(0xFFFF);
3154 		}
3155 	}
3156 
3157 	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
3158 
3159 	megasas_get_msix_index(instance, scmd, cmd, 1);
3160 
3161 	if (!fp_possible) {
3162 		/* system pd firmware path */
3163 		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3164 		cmd->request_desc->SCSIIO.RequestFlags =
3165 			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3166 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3167 		pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
3168 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3169 	} else {
3170 		if (os_timeout_value)
3171 			os_timeout_value++;
3172 
3173 		/* system pd Fast Path */
3174 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3175 		timeout_limit = (scmd->device->type == TYPE_DISK) ?
3176 				255 : 0xFFFF;
3177 		pRAID_Context->timeout_value =
3178 			cpu_to_le16((os_timeout_value > timeout_limit) ?
3179 			timeout_limit : os_timeout_value);
3180 		if (instance->adapter_type >= INVADER_SERIES)
3181 			io_request->IoFlags |=
3182 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
3183 
3184 		cmd->request_desc->SCSIIO.RequestFlags =
3185 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3186 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3187 	}
3188 }
3189 
3190 /**
3191  * megasas_build_io_fusion -	Prepares IOs to devices
3192  * @instance:		Adapter soft state
3193  * @scp:		SCSI command
3194  * @cmd:		Command to be prepared
3195  *
3196  * Invokes helper functions to prepare request frames
3197  * and sets flags appropriate for IO/Non-IO cmd
3198  */
3199 static int
3200 megasas_build_io_fusion(struct megasas_instance *instance,
3201 			struct scsi_cmnd *scp,
3202 			struct megasas_cmd_fusion *cmd)
3203 {
3204 	int sge_count;
3205 	u16 pd_index = 0;
3206 	u8 drive_type = 0;
3207 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
3208 	struct MR_PRIV_DEVICE *mr_device_priv_data;
3209 	mr_device_priv_data = scp->device->hostdata;
3210 
3211 	/* Zero out some fields so they don't get reused */
3212 	memset(io_request->LUN, 0x0, 8);
3213 	io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
3214 	io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
3215 	io_request->EEDPFlags = 0;
3216 	io_request->Control = 0;
3217 	io_request->EEDPBlockSize = 0;
3218 	io_request->ChainOffset = 0;
3219 	io_request->RaidContext.raid_context.raid_flags = 0;
3220 	io_request->RaidContext.raid_context.type = 0;
3221 	io_request->RaidContext.raid_context.nseg = 0;
3222 
3223 	memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
3224 	/*
3225 	 * Just the CDB length,rest of the Flags are zero
3226 	 * This will be modified for FP in build_ldio_fusion
3227 	 */
3228 	io_request->IoFlags = cpu_to_le16(scp->cmd_len);
3229 
3230 	switch (megasas_cmd_type(scp)) {
3231 	case READ_WRITE_LDIO:
3232 		megasas_build_ldio_fusion(instance, scp, cmd);
3233 		break;
3234 	case NON_READ_WRITE_LDIO:
3235 		megasas_build_ld_nonrw_fusion(instance, scp, cmd);
3236 		break;
3237 	case READ_WRITE_SYSPDIO:
3238 		megasas_build_syspd_fusion(instance, scp, cmd, true);
3239 		break;
3240 	case NON_READ_WRITE_SYSPDIO:
3241 		pd_index = MEGASAS_PD_INDEX(scp);
3242 		drive_type = instance->pd_list[pd_index].driveType;
3243 		if ((instance->secure_jbod_support ||
3244 		     mr_device_priv_data->is_tm_capable) ||
3245 		     (instance->adapter_type >= VENTURA_SERIES &&
3246 		     drive_type == TYPE_ENCLOSURE))
3247 			megasas_build_syspd_fusion(instance, scp, cmd, false);
3248 		else
3249 			megasas_build_syspd_fusion(instance, scp, cmd, true);
3250 		break;
3251 	default:
3252 		break;
3253 	}
3254 
3255 	/*
3256 	 * Construct SGL
3257 	 */
3258 
3259 	sge_count = megasas_make_sgl(instance, scp, cmd);
3260 
3261 	if (sge_count > instance->max_num_sge || (sge_count < 0)) {
3262 		dev_err(&instance->pdev->dev,
3263 			"%s %d sge_count (%d) is out of range. Range is:  0-%d\n",
3264 			__func__, __LINE__, sge_count, instance->max_num_sge);
3265 		return 1;
3266 	}
3267 
3268 	if (instance->adapter_type >= VENTURA_SERIES) {
3269 		set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);
3270 		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
3271 		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
3272 	} else {
3273 		/* numSGE store lower 8 bit of sge_count.
3274 		 * numSGEExt store higher 8 bit of sge_count
3275 		 */
3276 		io_request->RaidContext.raid_context.num_sge = sge_count;
3277 		io_request->RaidContext.raid_context.num_sge_ext =
3278 			(u8)(sge_count >> 8);
3279 	}
3280 
3281 	io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
3282 
3283 	if (scp->sc_data_direction == DMA_TO_DEVICE)
3284 		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
3285 	else if (scp->sc_data_direction == DMA_FROM_DEVICE)
3286 		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
3287 
3288 	io_request->SGLOffset0 =
3289 		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
3290 
3291 	io_request->SenseBufferLowAddress =
3292 		cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));
3293 	io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
3294 
3295 	cmd->scmd = scp;
3296 	megasas_priv(scp)->cmd_priv = cmd;
3297 
3298 	return 0;
3299 }
3300 
3301 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3302 megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
3303 {
3304 	u8 *p;
3305 	struct fusion_context *fusion;
3306 
3307 	fusion = instance->ctrl_context;
3308 	p = fusion->req_frames_desc +
3309 		sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
3310 
3311 	return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
3312 }
3313 
3314 
3315 /* megasas_prepate_secondRaid1_IO
3316  *  It prepares the raid 1 second IO
3317  */
3318 static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
3319 					   struct megasas_cmd_fusion *cmd,
3320 					   struct megasas_cmd_fusion *r1_cmd)
3321 {
3322 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
3323 	struct fusion_context *fusion;
3324 	fusion = instance->ctrl_context;
3325 	req_desc = cmd->request_desc;
3326 	/* copy the io request frame as well as 8 SGEs data for r1 command*/
3327 	memcpy(r1_cmd->io_request, cmd->io_request,
3328 	       (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
3329 	memcpy(r1_cmd->io_request->SGLs, cmd->io_request->SGLs,
3330 	       (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
3331 	/*sense buffer is different for r1 command*/
3332 	r1_cmd->io_request->SenseBufferLowAddress =
3333 			cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));
3334 	r1_cmd->scmd = cmd->scmd;
3335 	req_desc2 = megasas_get_request_descriptor(instance,
3336 						   (r1_cmd->index - 1));
3337 	req_desc2->Words = 0;
3338 	r1_cmd->request_desc = req_desc2;
3339 	req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
3340 	req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
3341 	r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
3342 	r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
3343 	r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
3344 	cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3345 			cpu_to_le16(r1_cmd->index);
3346 	r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3347 			cpu_to_le16(cmd->index);
3348 	/*MSIxIndex of both commands request descriptors should be same*/
3349 	r1_cmd->request_desc->SCSIIO.MSIxIndex =
3350 			cmd->request_desc->SCSIIO.MSIxIndex;
3351 	/*span arm is different for r1 cmd*/
3352 	r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
3353 			cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
3354 }
3355 
3356 /**
3357  * megasas_build_and_issue_cmd_fusion -Main routine for building and
3358  *                                     issuing non IOCTL cmd
3359  * @instance:			Adapter soft state
3360  * @scmd:			pointer to scsi cmd from OS
3361  */
3362 static u32
3363 megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
3364 				   struct scsi_cmnd *scmd)
3365 {
3366 	struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
3367 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3368 	u32 index;
3369 
3370 	if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&
3371 		instance->ldio_threshold &&
3372 		(atomic_inc_return(&instance->ldio_outstanding) >
3373 		instance->ldio_threshold)) {
3374 		atomic_dec(&instance->ldio_outstanding);
3375 		return SCSI_MLQUEUE_DEVICE_BUSY;
3376 	}
3377 
3378 	if (atomic_inc_return(&instance->fw_outstanding) >
3379 			instance->host->can_queue) {
3380 		atomic_dec(&instance->fw_outstanding);
3381 		return SCSI_MLQUEUE_HOST_BUSY;
3382 	}
3383 
3384 	cmd = megasas_get_cmd_fusion(instance, scsi_cmd_to_rq(scmd)->tag);
3385 
3386 	if (!cmd) {
3387 		atomic_dec(&instance->fw_outstanding);
3388 		return SCSI_MLQUEUE_HOST_BUSY;
3389 	}
3390 
3391 	index = cmd->index;
3392 
3393 	req_desc = megasas_get_request_descriptor(instance, index-1);
3394 
3395 	req_desc->Words = 0;
3396 	cmd->request_desc = req_desc;
3397 
3398 	if (megasas_build_io_fusion(instance, scmd, cmd)) {
3399 		megasas_return_cmd_fusion(instance, cmd);
3400 		dev_err(&instance->pdev->dev, "Error building command\n");
3401 		cmd->request_desc = NULL;
3402 		atomic_dec(&instance->fw_outstanding);
3403 		return SCSI_MLQUEUE_HOST_BUSY;
3404 	}
3405 
3406 	req_desc = cmd->request_desc;
3407 	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3408 
3409 	if (cmd->io_request->ChainOffset != 0 &&
3410 	    cmd->io_request->ChainOffset != 0xF)
3411 		dev_err(&instance->pdev->dev, "The chain offset value is not "
3412 		       "correct : %x\n", cmd->io_request->ChainOffset);
3413 	/*
3414 	 *	if it is raid 1/10 fp write capable.
3415 	 *	try to get second command from pool and construct it.
3416 	 *	From FW, it has confirmed that lba values of two PDs
3417 	 *	corresponds to single R1/10 LD are always same
3418 	 *
3419 	 */
3420 	/*	driver side count always should be less than max_fw_cmds
3421 	 *	to get new command
3422 	 */
3423 	if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
3424 		r1_cmd = megasas_get_cmd_fusion(instance,
3425 				scsi_cmd_to_rq(scmd)->tag + instance->max_fw_cmds);
3426 		megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
3427 	}
3428 
3429 
3430 	/*
3431 	 * Issue the command to the FW
3432 	 */
3433 
3434 	megasas_sdev_busy_inc(instance, scmd);
3435 	megasas_fire_cmd_fusion(instance, req_desc);
3436 
3437 	if (r1_cmd)
3438 		megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);
3439 
3440 
3441 	return 0;
3442 }
3443 
3444 /**
3445  * megasas_complete_r1_command -
3446  * completes R1 FP write commands which has valid peer smid
3447  * @instance:			Adapter soft state
3448  * @cmd:			MPT command frame
3449  *
3450  */
3451 static inline void
3452 megasas_complete_r1_command(struct megasas_instance *instance,
3453 			    struct megasas_cmd_fusion *cmd)
3454 {
3455 	u8 *sense, status, ex_status;
3456 	u32 data_length;
3457 	u16 peer_smid;
3458 	struct fusion_context *fusion;
3459 	struct megasas_cmd_fusion *r1_cmd = NULL;
3460 	struct scsi_cmnd *scmd_local = NULL;
3461 	struct RAID_CONTEXT_G35 *rctx_g35;
3462 
3463 	rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
3464 	fusion = instance->ctrl_context;
3465 	peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid);
3466 
3467 	r1_cmd = fusion->cmd_list[peer_smid - 1];
3468 	scmd_local = cmd->scmd;
3469 	status = rctx_g35->status;
3470 	ex_status = rctx_g35->ex_status;
3471 	data_length = cmd->io_request->DataLength;
3472 	sense = cmd->sense;
3473 
3474 	cmd->cmd_completed = true;
3475 
3476 	/* Check if peer command is completed or not*/
3477 	if (r1_cmd->cmd_completed) {
3478 		rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
3479 		if (rctx_g35->status != MFI_STAT_OK) {
3480 			status = rctx_g35->status;
3481 			ex_status = rctx_g35->ex_status;
3482 			data_length = r1_cmd->io_request->DataLength;
3483 			sense = r1_cmd->sense;
3484 		}
3485 
3486 		megasas_return_cmd_fusion(instance, r1_cmd);
3487 		map_cmd_status(fusion, scmd_local, status, ex_status,
3488 			       le32_to_cpu(data_length), sense);
3489 		if (instance->ldio_threshold &&
3490 		    megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
3491 			atomic_dec(&instance->ldio_outstanding);
3492 		megasas_priv(scmd_local)->cmd_priv = NULL;
3493 		megasas_return_cmd_fusion(instance, cmd);
3494 		scsi_dma_unmap(scmd_local);
3495 		megasas_sdev_busy_dec(instance, scmd_local);
3496 		scsi_done(scmd_local);
3497 	}
3498 }
3499 
3500 /**
3501  * access_irq_context:		Access to reply processing
3502  * @irq_context:		IRQ context
3503  *
3504  * Synchronize access to reply processing.
3505  *
3506  * Return:  true on success, false on failure.
3507  */
3508 static inline
3509 bool access_irq_context(struct megasas_irq_context  *irq_context)
3510 {
3511 	if (!irq_context)
3512 		return true;
3513 
3514 	if (atomic_add_unless(&irq_context->in_used, 1, 1))
3515 		return true;
3516 
3517 	return false;
3518 }
3519 
3520 /**
3521  * release_irq_context:		Release reply processing
3522  * @irq_context:		IRQ context
3523  *
3524  * Release access of reply processing.
3525  *
3526  * Return: Nothing.
3527  */
3528 static inline
3529 void release_irq_context(struct megasas_irq_context  *irq_context)
3530 {
3531 	if (irq_context)
3532 		atomic_dec(&irq_context->in_used);
3533 }
3534 
3535 /**
3536  * complete_cmd_fusion -	Completes command
3537  * @instance:			Adapter soft state
3538  * @MSIxIndex:			MSI number
3539  * @irq_context:		IRQ context
3540  *
3541  * Completes all commands that is in reply descriptor queue
3542  */
3543 static int
3544 complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
3545 		    struct megasas_irq_context *irq_context)
3546 {
3547 	union MPI2_REPLY_DESCRIPTORS_UNION *desc;
3548 	struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
3549 	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
3550 	struct fusion_context *fusion;
3551 	struct megasas_cmd *cmd_mfi;
3552 	struct megasas_cmd_fusion *cmd_fusion;
3553 	u16 smid, num_completed;
3554 	u8 reply_descript_type, *sense, status, extStatus;
3555 	u32 device_id, data_length;
3556 	union desc_value d_val;
3557 	struct LD_LOAD_BALANCE_INFO *lbinfo;
3558 	int threshold_reply_count = 0;
3559 	struct scsi_cmnd *scmd_local = NULL;
3560 	struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
3561 	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
3562 
3563 	fusion = instance->ctrl_context;
3564 
3565 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3566 		return IRQ_HANDLED;
3567 
3568 	if (!access_irq_context(irq_context))
3569 		return 0;
3570 
3571 	desc = fusion->reply_frames_desc[MSIxIndex] +
3572 				fusion->last_reply_idx[MSIxIndex];
3573 
3574 	reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3575 
3576 	d_val.word = desc->Words;
3577 
3578 	reply_descript_type = reply_desc->ReplyFlags &
3579 		MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3580 
3581 	if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED) {
3582 		release_irq_context(irq_context);
3583 		return IRQ_NONE;
3584 	}
3585 
3586 	num_completed = 0;
3587 
3588 	while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3589 	       d_val.u.high != cpu_to_le32(UINT_MAX)) {
3590 
3591 		smid = le16_to_cpu(reply_desc->SMID);
3592 		cmd_fusion = fusion->cmd_list[smid - 1];
3593 		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3594 						cmd_fusion->io_request;
3595 
3596 		scmd_local = cmd_fusion->scmd;
3597 		status = scsi_io_req->RaidContext.raid_context.status;
3598 		extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3599 		sense = cmd_fusion->sense;
3600 		data_length = scsi_io_req->DataLength;
3601 
3602 		switch (scsi_io_req->Function) {
3603 		case MPI2_FUNCTION_SCSI_TASK_MGMT:
3604 			mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3605 						cmd_fusion->io_request;
3606 			mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3607 						&mr_tm_req->TmRequest;
3608 			dev_dbg(&instance->pdev->dev, "TM completion:"
3609 				"type: 0x%x TaskMID: 0x%x\n",
3610 				mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3611 			complete(&cmd_fusion->done);
3612 			break;
3613 		case MPI2_FUNCTION_SCSI_IO_REQUEST:  /*Fast Path IO.*/
3614 			/* Update load balancing info */
3615 			if (fusion->load_balance_info &&
3616 			    (megasas_priv(cmd_fusion->scmd)->status &
3617 			    MEGASAS_LOAD_BALANCE_FLAG)) {
3618 				device_id = MEGASAS_DEV_INDEX(scmd_local);
3619 				lbinfo = &fusion->load_balance_info[device_id];
3620 				atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3621 				megasas_priv(cmd_fusion->scmd)->status &=
3622 					~MEGASAS_LOAD_BALANCE_FLAG;
3623 			}
3624 			fallthrough;	/* and complete IO */
3625 		case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */
3626 			atomic_dec(&instance->fw_outstanding);
3627 			if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3628 				map_cmd_status(fusion, scmd_local, status,
3629 					       extStatus, le32_to_cpu(data_length),
3630 					       sense);
3631 				if (instance->ldio_threshold &&
3632 				    (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))
3633 					atomic_dec(&instance->ldio_outstanding);
3634 				megasas_priv(scmd_local)->cmd_priv = NULL;
3635 				megasas_return_cmd_fusion(instance, cmd_fusion);
3636 				scsi_dma_unmap(scmd_local);
3637 				megasas_sdev_busy_dec(instance, scmd_local);
3638 				scsi_done(scmd_local);
3639 			} else	/* Optimal VD - R1 FP command completion. */
3640 				megasas_complete_r1_command(instance, cmd_fusion);
3641 			break;
3642 		case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */
3643 			cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3644 			/* Poll mode. Dummy free.
3645 			 * In case of Interrupt mode, caller has reverse check.
3646 			 */
3647 			if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3648 				cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3649 				megasas_return_cmd(instance, cmd_mfi);
3650 			} else
3651 				megasas_complete_cmd(instance, cmd_mfi, DID_OK);
3652 			break;
3653 		}
3654 
3655 		fusion->last_reply_idx[MSIxIndex]++;
3656 		if (fusion->last_reply_idx[MSIxIndex] >=
3657 		    fusion->reply_q_depth)
3658 			fusion->last_reply_idx[MSIxIndex] = 0;
3659 
3660 		desc->Words = cpu_to_le64(ULLONG_MAX);
3661 		num_completed++;
3662 		threshold_reply_count++;
3663 
3664 		/* Get the next reply descriptor */
3665 		if (!fusion->last_reply_idx[MSIxIndex])
3666 			desc = fusion->reply_frames_desc[MSIxIndex];
3667 		else
3668 			desc++;
3669 
3670 		reply_desc =
3671 		  (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3672 
3673 		d_val.word = desc->Words;
3674 
3675 		reply_descript_type = reply_desc->ReplyFlags &
3676 			MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3677 
3678 		if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3679 			break;
3680 		/*
3681 		 * Write to reply post host index register after completing threshold
3682 		 * number of reply counts and still there are more replies in reply queue
3683 		 * pending to be completed
3684 		 */
3685 		if (threshold_reply_count >= instance->threshold_reply_count) {
3686 			if (instance->msix_combined)
3687 				writel(((MSIxIndex & 0x7) << 24) |
3688 					fusion->last_reply_idx[MSIxIndex],
3689 					instance->reply_post_host_index_addr[MSIxIndex/8]);
3690 			else
3691 				writel((MSIxIndex << 24) |
3692 					fusion->last_reply_idx[MSIxIndex],
3693 					instance->reply_post_host_index_addr[0]);
3694 			threshold_reply_count = 0;
3695 			if (irq_context) {
3696 				if (!irq_context->irq_poll_scheduled) {
3697 					irq_context->irq_poll_scheduled = true;
3698 					irq_context->irq_line_enable = true;
3699 					irq_poll_sched(&irq_context->irqpoll);
3700 				}
3701 				release_irq_context(irq_context);
3702 				return num_completed;
3703 			}
3704 		}
3705 	}
3706 
3707 	if (num_completed) {
3708 		wmb();
3709 		if (instance->msix_combined)
3710 			writel(((MSIxIndex & 0x7) << 24) |
3711 				fusion->last_reply_idx[MSIxIndex],
3712 				instance->reply_post_host_index_addr[MSIxIndex/8]);
3713 		else
3714 			writel((MSIxIndex << 24) |
3715 				fusion->last_reply_idx[MSIxIndex],
3716 				instance->reply_post_host_index_addr[0]);
3717 		megasas_check_and_restore_queue_depth(instance);
3718 	}
3719 
3720 	release_irq_context(irq_context);
3721 
3722 	return num_completed;
3723 }
3724 
3725 int megasas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
3726 {
3727 
3728 	struct megasas_instance *instance;
3729 	int num_entries = 0;
3730 	struct fusion_context *fusion;
3731 
3732 	instance = (struct megasas_instance *)shost->hostdata;
3733 
3734 	fusion = instance->ctrl_context;
3735 
3736 	queue_num = queue_num + instance->low_latency_index_start;
3737 
3738 	if (!atomic_add_unless(&fusion->busy_mq_poll[queue_num], 1, 1))
3739 		return 0;
3740 
3741 	num_entries = complete_cmd_fusion(instance, queue_num, NULL);
3742 	atomic_dec(&fusion->busy_mq_poll[queue_num]);
3743 
3744 	return num_entries;
3745 }
3746 
3747 /**
3748  * megasas_enable_irq_poll() - enable irqpoll
3749  * @instance:			Adapter soft state
3750  */
3751 static void megasas_enable_irq_poll(struct megasas_instance *instance)
3752 {
3753 	u32 count, i;
3754 	struct megasas_irq_context *irq_ctx;
3755 
3756 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3757 
3758 	for (i = 0; i < count; i++) {
3759 		irq_ctx = &instance->irq_context[i];
3760 		irq_poll_enable(&irq_ctx->irqpoll);
3761 	}
3762 }
3763 
3764 /**
3765  * megasas_sync_irqs -	Synchronizes all IRQs owned by adapter
3766  * @instance_addr:			Adapter soft state address
3767  */
3768 static void megasas_sync_irqs(unsigned long instance_addr)
3769 {
3770 	u32 count, i;
3771 	struct megasas_instance *instance =
3772 		(struct megasas_instance *)instance_addr;
3773 	struct megasas_irq_context *irq_ctx;
3774 
3775 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3776 
3777 	for (i = 0; i < count; i++) {
3778 		synchronize_irq(pci_irq_vector(instance->pdev, i));
3779 		irq_ctx = &instance->irq_context[i];
3780 		irq_poll_disable(&irq_ctx->irqpoll);
3781 		if (irq_ctx->irq_poll_scheduled) {
3782 			irq_ctx->irq_poll_scheduled = false;
3783 			enable_irq(irq_ctx->os_irq);
3784 			complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3785 		}
3786 	}
3787 }
3788 
3789 /**
3790  * megasas_irqpoll() - process a queue for completed reply descriptors
3791  * @irqpoll:	IRQ poll structure associated with queue to poll.
3792  * @budget:	Threshold of reply descriptors to process per poll.
3793  *
3794  * Return: The number of entries processed.
3795  */
3796 
3797 int megasas_irqpoll(struct irq_poll *irqpoll, int budget)
3798 {
3799 	struct megasas_irq_context *irq_ctx;
3800 	struct megasas_instance *instance;
3801 	int num_entries;
3802 
3803 	irq_ctx = container_of(irqpoll, struct megasas_irq_context, irqpoll);
3804 	instance = irq_ctx->instance;
3805 
3806 	if (irq_ctx->irq_line_enable) {
3807 		disable_irq_nosync(irq_ctx->os_irq);
3808 		irq_ctx->irq_line_enable = false;
3809 	}
3810 
3811 	num_entries = complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3812 	if (num_entries < budget) {
3813 		irq_poll_complete(irqpoll);
3814 		irq_ctx->irq_poll_scheduled = false;
3815 		enable_irq(irq_ctx->os_irq);
3816 		complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3817 	}
3818 
3819 	return num_entries;
3820 }
3821 
3822 /**
3823  * megasas_complete_cmd_dpc_fusion -	Completes command
3824  * @instance_addr:			Adapter soft state address
3825  *
3826  * Tasklet to complete cmds
3827  */
3828 static void
3829 megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
3830 {
3831 	struct megasas_instance *instance =
3832 		(struct megasas_instance *)instance_addr;
3833 	struct megasas_irq_context *irq_ctx = NULL;
3834 	u32 count, MSIxIndex;
3835 
3836 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3837 
3838 	/* If we have already declared adapter dead, donot complete cmds */
3839 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3840 		return;
3841 
3842 	for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) {
3843 		irq_ctx = &instance->irq_context[MSIxIndex];
3844 		complete_cmd_fusion(instance, MSIxIndex, irq_ctx);
3845 	}
3846 }
3847 
3848 /**
3849  * megasas_isr_fusion - isr entry point
3850  * @irq:	IRQ number
3851  * @devp:	IRQ context
3852  */
3853 static irqreturn_t megasas_isr_fusion(int irq, void *devp)
3854 {
3855 	struct megasas_irq_context *irq_context = devp;
3856 	struct megasas_instance *instance = irq_context->instance;
3857 	u32 mfiStatus;
3858 
3859 	if (instance->mask_interrupts)
3860 		return IRQ_NONE;
3861 
3862 	if (irq_context->irq_poll_scheduled)
3863 		return IRQ_HANDLED;
3864 
3865 	if (!instance->msix_vectors) {
3866 		mfiStatus = instance->instancet->clear_intr(instance);
3867 		if (!mfiStatus)
3868 			return IRQ_NONE;
3869 	}
3870 
3871 	/* If we are resetting, bail */
3872 	if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {
3873 		instance->instancet->clear_intr(instance);
3874 		return IRQ_HANDLED;
3875 	}
3876 
3877 	return complete_cmd_fusion(instance, irq_context->MSIxIndex, irq_context)
3878 			? IRQ_HANDLED : IRQ_NONE;
3879 }
3880 
3881 /**
3882  * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
3883  * @instance:			Adapter soft state
3884  * @mfi_cmd:			megasas_cmd pointer
3885  *
3886  */
3887 static void
3888 build_mpt_mfi_pass_thru(struct megasas_instance *instance,
3889 			struct megasas_cmd *mfi_cmd)
3890 {
3891 	struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;
3892 	struct MPI2_RAID_SCSI_IO_REQUEST *io_req;
3893 	struct megasas_cmd_fusion *cmd;
3894 	struct fusion_context *fusion;
3895 	struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;
3896 
3897 	fusion = instance->ctrl_context;
3898 
3899 	cmd = megasas_get_cmd_fusion(instance,
3900 			instance->max_scsi_cmds + mfi_cmd->index);
3901 
3902 	/*  Save the smid. To be used for returning the cmd */
3903 	mfi_cmd->context.smid = cmd->index;
3904 
3905 	/*
3906 	 * For cmds where the flag is set, store the flag and check
3907 	 * on completion. For cmds with this flag, don't call
3908 	 * megasas_complete_cmd
3909 	 */
3910 
3911 	if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))
3912 		mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;
3913 
3914 	io_req = cmd->io_request;
3915 
3916 	if (instance->adapter_type >= INVADER_SERIES) {
3917 		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =
3918 			(struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;
3919 		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
3920 		sgl_ptr_end->Flags = 0;
3921 	}
3922 
3923 	mpi25_ieee_chain =
3924 	  (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;
3925 
3926 	io_req->Function    = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
3927 	io_req->SGLOffset0  = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,
3928 				       SGL) / 4;
3929 	io_req->ChainOffset = fusion->chain_offset_mfi_pthru;
3930 
3931 	mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);
3932 
3933 	mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
3934 		MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
3935 
3936 	mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);
3937 }
3938 
3939 /**
3940  * build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd
3941  * @instance:			Adapter soft state
3942  * @cmd:			mfi cmd to build
3943  *
3944  */
3945 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3946 build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
3947 {
3948 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;
3949 	u16 index;
3950 
3951 	build_mpt_mfi_pass_thru(instance, cmd);
3952 	index = cmd->context.smid;
3953 
3954 	req_desc = megasas_get_request_descriptor(instance, index - 1);
3955 
3956 	req_desc->Words = 0;
3957 	req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3958 					 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3959 
3960 	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3961 
3962 	return req_desc;
3963 }
3964 
3965 /**
3966  * megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd
3967  * @instance:			Adapter soft state
3968  * @cmd:			mfi cmd pointer
3969  *
3970  */
3971 static void
3972 megasas_issue_dcmd_fusion(struct megasas_instance *instance,
3973 			  struct megasas_cmd *cmd)
3974 {
3975 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3976 
3977 	req_desc = build_mpt_cmd(instance, cmd);
3978 
3979 	megasas_fire_cmd_fusion(instance, req_desc);
3980 	return;
3981 }
3982 
3983 /**
3984  * megasas_release_fusion -	Reverses the FW initialization
3985  * @instance:			Adapter soft state
3986  */
3987 void
3988 megasas_release_fusion(struct megasas_instance *instance)
3989 {
3990 	megasas_free_ioc_init_cmd(instance);
3991 	megasas_free_cmds(instance);
3992 	megasas_free_cmds_fusion(instance);
3993 
3994 	iounmap(instance->reg_set);
3995 
3996 	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
3997 }
3998 
3999 /**
4000  * megasas_read_fw_status_reg_fusion - returns the current FW status value
4001  * @instance:			Adapter soft state
4002  */
4003 static u32
4004 megasas_read_fw_status_reg_fusion(struct megasas_instance *instance)
4005 {
4006 	return megasas_readl(instance, &instance->reg_set->outbound_scratch_pad_0);
4007 }
4008 
4009 /**
4010  * megasas_alloc_host_crash_buffer -	Host buffers for Crash dump collection from Firmware
4011  * @instance:				Controller's soft instance
4012  * @return:			        Number of allocated host crash buffers
4013  */
4014 static void
4015 megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
4016 {
4017 	unsigned int i;
4018 
4019 	for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
4020 		instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);
4021 		if (!instance->crash_buf[i]) {
4022 			dev_info(&instance->pdev->dev, "Firmware crash dump "
4023 				"memory allocation failed at index %d\n", i);
4024 			break;
4025 		}
4026 	}
4027 	instance->drv_buf_alloc = i;
4028 }
4029 
4030 /**
4031  * megasas_free_host_crash_buffer -	Host buffers for Crash dump collection from Firmware
4032  * @instance:				Controller's soft instance
4033  */
4034 void
4035 megasas_free_host_crash_buffer(struct megasas_instance *instance)
4036 {
4037 	unsigned int i;
4038 	for (i = 0; i < instance->drv_buf_alloc; i++) {
4039 		vfree(instance->crash_buf[i]);
4040 	}
4041 	instance->drv_buf_index = 0;
4042 	instance->drv_buf_alloc = 0;
4043 	instance->fw_crash_state = UNAVAILABLE;
4044 	instance->fw_crash_buffer_size = 0;
4045 }
4046 
4047 /**
4048  * megasas_adp_reset_fusion -	For controller reset
4049  * @instance:				Controller's soft instance
4050  * @regs:				MFI register set
4051  */
4052 static int
4053 megasas_adp_reset_fusion(struct megasas_instance *instance,
4054 			 struct megasas_register_set __iomem *regs)
4055 {
4056 	u32 host_diag, abs_state, retry;
4057 
4058 	/* Now try to reset the chip */
4059 	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4060 	writel(MPI2_WRSEQ_1ST_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4061 	writel(MPI2_WRSEQ_2ND_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4062 	writel(MPI2_WRSEQ_3RD_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4063 	writel(MPI2_WRSEQ_4TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4064 	writel(MPI2_WRSEQ_5TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4065 	writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4066 
4067 	/* Check that the diag write enable (DRWE) bit is on */
4068 	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);
4069 	retry = 0;
4070 	while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {
4071 		msleep(100);
4072 		host_diag = megasas_readl(instance,
4073 					  &instance->reg_set->fusion_host_diag);
4074 		if (retry++ == 100) {
4075 			dev_warn(&instance->pdev->dev,
4076 				"Host diag unlock failed from %s %d\n",
4077 				__func__, __LINE__);
4078 			break;
4079 		}
4080 	}
4081 	if (!(host_diag & HOST_DIAG_WRITE_ENABLE))
4082 		return -1;
4083 
4084 	/* Send chip reset command */
4085 	writel(host_diag | HOST_DIAG_RESET_ADAPTER,
4086 		&instance->reg_set->fusion_host_diag);
4087 	msleep(3000);
4088 
4089 	/* Make sure reset adapter bit is cleared */
4090 	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);
4091 	retry = 0;
4092 	while (host_diag & HOST_DIAG_RESET_ADAPTER) {
4093 		msleep(100);
4094 		host_diag = megasas_readl(instance,
4095 					  &instance->reg_set->fusion_host_diag);
4096 		if (retry++ == 1000) {
4097 			dev_warn(&instance->pdev->dev,
4098 				"Diag reset adapter never cleared %s %d\n",
4099 				__func__, __LINE__);
4100 			break;
4101 		}
4102 	}
4103 	if (host_diag & HOST_DIAG_RESET_ADAPTER)
4104 		return -1;
4105 
4106 	abs_state = instance->instancet->read_fw_status_reg(instance)
4107 			& MFI_STATE_MASK;
4108 	retry = 0;
4109 
4110 	while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {
4111 		msleep(100);
4112 		abs_state = instance->instancet->
4113 			read_fw_status_reg(instance) & MFI_STATE_MASK;
4114 	}
4115 	if (abs_state <= MFI_STATE_FW_INIT) {
4116 		dev_warn(&instance->pdev->dev,
4117 			"fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",
4118 			abs_state, __func__, __LINE__);
4119 		return -1;
4120 	}
4121 
4122 	return 0;
4123 }
4124 
4125 /**
4126  * megasas_check_reset_fusion -	For controller reset check
4127  * @instance:				Controller's soft instance
4128  * @regs:				MFI register set
4129  */
4130 static int
4131 megasas_check_reset_fusion(struct megasas_instance *instance,
4132 			   struct megasas_register_set __iomem *regs)
4133 {
4134 	return 0;
4135 }
4136 
4137 /**
4138  * megasas_trigger_snap_dump -	Trigger snap dump in FW
4139  * @instance:			Soft instance of adapter
4140  */
4141 static inline void megasas_trigger_snap_dump(struct megasas_instance *instance)
4142 {
4143 	int j;
4144 	u32 fw_state, abs_state;
4145 
4146 	if (!instance->disableOnlineCtrlReset) {
4147 		dev_info(&instance->pdev->dev, "Trigger snap dump\n");
4148 		writel(MFI_ADP_TRIGGER_SNAP_DUMP,
4149 		       &instance->reg_set->doorbell);
4150 		readl(&instance->reg_set->doorbell);
4151 	}
4152 
4153 	for (j = 0; j < instance->snapdump_wait_time; j++) {
4154 		abs_state = instance->instancet->read_fw_status_reg(instance);
4155 		fw_state = abs_state & MFI_STATE_MASK;
4156 		if (fw_state == MFI_STATE_FAULT) {
4157 			dev_printk(KERN_ERR, &instance->pdev->dev,
4158 				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4159 				   abs_state & MFI_STATE_FAULT_CODE,
4160 				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4161 			return;
4162 		}
4163 		msleep(1000);
4164 	}
4165 }
4166 
4167 /* This function waits for outstanding commands on fusion to complete */
4168 static int
4169 megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
4170 				    int reason, int *convert)
4171 {
4172 	int i, outstanding, retval = 0, hb_seconds_missed = 0;
4173 	u32 fw_state, abs_state;
4174 	u32 waittime_for_io_completion;
4175 
4176 	waittime_for_io_completion =
4177 		min_t(u32, resetwaittime,
4178 			(resetwaittime - instance->snapdump_wait_time));
4179 
4180 	if (reason == MFI_IO_TIMEOUT_OCR) {
4181 		dev_info(&instance->pdev->dev,
4182 			"MFI command is timed out\n");
4183 		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4184 		if (instance->snapdump_wait_time)
4185 			megasas_trigger_snap_dump(instance);
4186 		retval = 1;
4187 		goto out;
4188 	}
4189 
4190 	for (i = 0; i < waittime_for_io_completion; i++) {
4191 		/* Check if firmware is in fault state */
4192 		abs_state = instance->instancet->read_fw_status_reg(instance);
4193 		fw_state = abs_state & MFI_STATE_MASK;
4194 		if (fw_state == MFI_STATE_FAULT) {
4195 			dev_printk(KERN_ERR, &instance->pdev->dev,
4196 				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4197 				   abs_state & MFI_STATE_FAULT_CODE,
4198 				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4199 			megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4200 			if (instance->requestorId && reason) {
4201 				dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"
4202 				" state while polling during"
4203 				" I/O timeout handling for %d\n",
4204 				instance->host->host_no);
4205 				*convert = 1;
4206 			}
4207 
4208 			retval = 1;
4209 			goto out;
4210 		}
4211 
4212 
4213 		/* If SR-IOV VF mode & heartbeat timeout, don't wait */
4214 		if (instance->requestorId && !reason) {
4215 			retval = 1;
4216 			goto out;
4217 		}
4218 
4219 		/* If SR-IOV VF mode & I/O timeout, check for HB timeout */
4220 		if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {
4221 			if (instance->hb_host_mem->HB.fwCounter !=
4222 			    instance->hb_host_mem->HB.driverCounter) {
4223 				instance->hb_host_mem->HB.driverCounter =
4224 					instance->hb_host_mem->HB.fwCounter;
4225 				hb_seconds_missed = 0;
4226 			} else {
4227 				hb_seconds_missed++;
4228 				if (hb_seconds_missed ==
4229 				    (MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {
4230 					dev_warn(&instance->pdev->dev, "SR-IOV:"
4231 					       " Heartbeat never completed "
4232 					       " while polling during I/O "
4233 					       " timeout handling for "
4234 					       "scsi%d.\n",
4235 					       instance->host->host_no);
4236 					       *convert = 1;
4237 					       retval = 1;
4238 					       goto out;
4239 				}
4240 			}
4241 		}
4242 
4243 		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4244 		outstanding = atomic_read(&instance->fw_outstanding);
4245 		if (!outstanding)
4246 			goto out;
4247 
4248 		if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {
4249 			dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
4250 			       "commands to complete for scsi%d\n", i,
4251 			       outstanding, instance->host->host_no);
4252 		}
4253 		msleep(1000);
4254 	}
4255 
4256 	if (instance->snapdump_wait_time) {
4257 		megasas_trigger_snap_dump(instance);
4258 		retval = 1;
4259 		goto out;
4260 	}
4261 
4262 	if (atomic_read(&instance->fw_outstanding)) {
4263 		dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
4264 		       "will reset adapter scsi%d.\n",
4265 		       instance->host->host_no);
4266 		*convert = 1;
4267 		retval = 1;
4268 	}
4269 
4270 out:
4271 	return retval;
4272 }
4273 
4274 void  megasas_reset_reply_desc(struct megasas_instance *instance)
4275 {
4276 	int i, j, count;
4277 	struct fusion_context *fusion;
4278 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
4279 
4280 	fusion = instance->ctrl_context;
4281 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
4282 	count += instance->iopoll_q_count;
4283 
4284 	for (i = 0 ; i < count ; i++) {
4285 		fusion->last_reply_idx[i] = 0;
4286 		reply_desc = fusion->reply_frames_desc[i];
4287 		for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)
4288 			reply_desc->Words = cpu_to_le64(ULLONG_MAX);
4289 	}
4290 }
4291 
4292 /*
4293  * megasas_refire_mgmt_cmd :	Re-fire management commands
4294  * @instance:				Controller's soft instance
4295 */
4296 static void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
4297 			     bool return_ioctl)
4298 {
4299 	int j;
4300 	struct megasas_cmd_fusion *cmd_fusion;
4301 	struct fusion_context *fusion;
4302 	struct megasas_cmd *cmd_mfi;
4303 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4304 	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
4305 	u16 smid;
4306 	bool refire_cmd = false;
4307 	u8 result;
4308 	u32 opcode = 0;
4309 
4310 	fusion = instance->ctrl_context;
4311 
4312 	/* Re-fire management commands.
4313 	 * Do not traverse complet MPT frame pool. Start from max_scsi_cmds.
4314 	 */
4315 	for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {
4316 		cmd_fusion = fusion->cmd_list[j];
4317 		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4318 		smid = le16_to_cpu(cmd_mfi->context.smid);
4319 		result = REFIRE_CMD;
4320 
4321 		if (!smid)
4322 			continue;
4323 
4324 		req_desc = megasas_get_request_descriptor(instance, smid - 1);
4325 
4326 		switch (cmd_mfi->frame->hdr.cmd) {
4327 		case MFI_CMD_DCMD:
4328 			opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode);
4329 			 /* Do not refire shutdown command */
4330 			if (opcode == MR_DCMD_CTRL_SHUTDOWN) {
4331 				cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;
4332 				result = COMPLETE_CMD;
4333 				break;
4334 			}
4335 
4336 			refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) &&
4337 				      (opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&
4338 				      !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);
4339 
4340 			if (!refire_cmd)
4341 				result = RETURN_CMD;
4342 
4343 			break;
4344 		case MFI_CMD_NVME:
4345 			if (!instance->support_nvme_passthru) {
4346 				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4347 				result = COMPLETE_CMD;
4348 			}
4349 
4350 			break;
4351 		case MFI_CMD_TOOLBOX:
4352 			if (!instance->support_pci_lane_margining) {
4353 				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4354 				result = COMPLETE_CMD;
4355 			}
4356 
4357 			break;
4358 		default:
4359 			break;
4360 		}
4361 
4362 		if (return_ioctl && cmd_mfi->sync_cmd &&
4363 		    cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT) {
4364 			dev_err(&instance->pdev->dev,
4365 				"return -EBUSY from %s %d cmd 0x%x opcode 0x%x\n",
4366 				__func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4367 				le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4368 			cmd_mfi->cmd_status_drv = DCMD_BUSY;
4369 			result = COMPLETE_CMD;
4370 		}
4371 
4372 		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
4373 				cmd_fusion->io_request;
4374 		if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT)
4375 			result = RETURN_CMD;
4376 
4377 		switch (result) {
4378 		case REFIRE_CMD:
4379 			megasas_fire_cmd_fusion(instance, req_desc);
4380 			break;
4381 		case RETURN_CMD:
4382 			megasas_return_cmd(instance, cmd_mfi);
4383 			break;
4384 		case COMPLETE_CMD:
4385 			megasas_complete_cmd(instance, cmd_mfi, DID_OK);
4386 			break;
4387 		}
4388 	}
4389 }
4390 
4391 /*
4392  * megasas_return_polled_cmds: Return polled mode commands back to the pool
4393  *			       before initiating an OCR.
4394  * @instance:                  Controller's soft instance
4395  */
4396 static void
4397 megasas_return_polled_cmds(struct megasas_instance *instance)
4398 {
4399 	int i;
4400 	struct megasas_cmd_fusion *cmd_fusion;
4401 	struct fusion_context *fusion;
4402 	struct megasas_cmd *cmd_mfi;
4403 
4404 	fusion = instance->ctrl_context;
4405 
4406 	for (i = instance->max_scsi_cmds; i < instance->max_fw_cmds; i++) {
4407 		cmd_fusion = fusion->cmd_list[i];
4408 		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4409 
4410 		if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
4411 			if (megasas_dbg_lvl & OCR_DEBUG)
4412 				dev_info(&instance->pdev->dev,
4413 					 "%s %d return cmd 0x%x opcode 0x%x\n",
4414 					 __func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4415 					 le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4416 			cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
4417 			megasas_return_cmd(instance, cmd_mfi);
4418 		}
4419 	}
4420 }
4421 
4422 /*
4423  * megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device
4424  * @instance: per adapter struct
4425  * @channel: the channel assigned by the OS
4426  * @id: the id assigned by the OS
4427  *
4428  * Returns SUCCESS if no IOs pending to SCSI device, else return FAILED
4429  */
4430 
4431 static int megasas_track_scsiio(struct megasas_instance *instance,
4432 		int id, int channel)
4433 {
4434 	int i, found = 0;
4435 	struct megasas_cmd_fusion *cmd_fusion;
4436 	struct fusion_context *fusion;
4437 	fusion = instance->ctrl_context;
4438 
4439 	for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4440 		cmd_fusion = fusion->cmd_list[i];
4441 		if (cmd_fusion->scmd &&
4442 			(cmd_fusion->scmd->device->id == id &&
4443 			cmd_fusion->scmd->device->channel == channel)) {
4444 			dev_info(&instance->pdev->dev,
4445 				"SCSI commands pending to target"
4446 				"channel %d id %d \tSMID: 0x%x\n",
4447 				channel, id, cmd_fusion->index);
4448 			scsi_print_command(cmd_fusion->scmd);
4449 			found = 1;
4450 			break;
4451 		}
4452 	}
4453 
4454 	return found ? FAILED : SUCCESS;
4455 }
4456 
4457 /**
4458  * megasas_tm_response_code - translation of device response code
4459  * @instance:	Controller's soft instance
4460  * @mpi_reply:	MPI reply returned by firmware
4461  *
4462  * Return nothing.
4463  */
4464 static void
4465 megasas_tm_response_code(struct megasas_instance *instance,
4466 		struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
4467 {
4468 	char *desc;
4469 
4470 	switch (mpi_reply->ResponseCode) {
4471 	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
4472 		desc = "task management request completed";
4473 		break;
4474 	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
4475 		desc = "invalid frame";
4476 		break;
4477 	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
4478 		desc = "task management request not supported";
4479 		break;
4480 	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
4481 		desc = "task management request failed";
4482 		break;
4483 	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
4484 		desc = "task management request succeeded";
4485 		break;
4486 	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
4487 		desc = "invalid lun";
4488 		break;
4489 	case 0xA:
4490 		desc = "overlapped tag attempted";
4491 		break;
4492 	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
4493 		desc = "task queued, however not sent to target";
4494 		break;
4495 	default:
4496 		desc = "unknown";
4497 		break;
4498 	}
4499 	dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",
4500 		mpi_reply->ResponseCode, desc);
4501 	dev_dbg(&instance->pdev->dev,
4502 		"TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"
4503 		" 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
4504 		mpi_reply->TerminationCount, mpi_reply->DevHandle,
4505 		mpi_reply->Function, mpi_reply->TaskType,
4506 		mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
4507 }
4508 
4509 /**
4510  * megasas_issue_tm - main routine for sending tm requests
4511  * @instance: per adapter struct
4512  * @device_handle: device handle
4513  * @channel: the channel assigned by the OS
4514  * @id: the id assigned by the OS
4515  * @smid_task: smid assigned to the task
4516  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
4517  * @mr_device_priv_data: private data
4518  * Context: user
4519  *
4520  * MegaRaid use MPT interface for Task Magement request.
4521  * A generic API for sending task management requests to firmware.
4522  *
4523  * Return SUCCESS or FAILED.
4524  */
4525 static int
4526 megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
4527 	uint channel, uint id, u16 smid_task, u8 type,
4528 	struct MR_PRIV_DEVICE *mr_device_priv_data)
4529 {
4530 	struct MR_TASK_MANAGE_REQUEST *mr_request;
4531 	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;
4532 	unsigned long timeleft;
4533 	struct megasas_cmd_fusion *cmd_fusion;
4534 	struct megasas_cmd *cmd_mfi;
4535 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4536 	struct fusion_context *fusion = NULL;
4537 	struct megasas_cmd_fusion *scsi_lookup;
4538 	int rc;
4539 	int timeout = MEGASAS_DEFAULT_TM_TIMEOUT;
4540 	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
4541 
4542 	fusion = instance->ctrl_context;
4543 
4544 	cmd_mfi = megasas_get_cmd(instance);
4545 
4546 	if (!cmd_mfi) {
4547 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4548 			__func__, __LINE__);
4549 		return -ENOMEM;
4550 	}
4551 
4552 	cmd_fusion = megasas_get_cmd_fusion(instance,
4553 			instance->max_scsi_cmds + cmd_mfi->index);
4554 
4555 	/*  Save the smid. To be used for returning the cmd */
4556 	cmd_mfi->context.smid = cmd_fusion->index;
4557 
4558 	req_desc = megasas_get_request_descriptor(instance,
4559 			(cmd_fusion->index - 1));
4560 
4561 	cmd_fusion->request_desc = req_desc;
4562 	req_desc->Words = 0;
4563 
4564 	mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
4565 	memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
4566 	mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
4567 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4568 	mpi_request->DevHandle = cpu_to_le16(device_handle);
4569 	mpi_request->TaskType = type;
4570 	mpi_request->TaskMID = cpu_to_le16(smid_task);
4571 	mpi_request->LUN[1] = 0;
4572 
4573 
4574 	req_desc = cmd_fusion->request_desc;
4575 	req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);
4576 	req_desc->HighPriority.RequestFlags =
4577 		(MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
4578 		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
4579 	req_desc->HighPriority.MSIxIndex =  0;
4580 	req_desc->HighPriority.LMID = 0;
4581 	req_desc->HighPriority.Reserved1 = 0;
4582 
4583 	if (channel < MEGASAS_MAX_PD_CHANNELS)
4584 		mr_request->tmReqFlags.isTMForPD = 1;
4585 	else
4586 		mr_request->tmReqFlags.isTMForLD = 1;
4587 
4588 	init_completion(&cmd_fusion->done);
4589 	megasas_fire_cmd_fusion(instance, req_desc);
4590 
4591 	switch (type) {
4592 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4593 		timeout = mr_device_priv_data->task_abort_tmo;
4594 		break;
4595 	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4596 		timeout = mr_device_priv_data->target_reset_tmo;
4597 		break;
4598 	}
4599 
4600 	timeleft = wait_for_completion_timeout(&cmd_fusion->done, timeout * HZ);
4601 
4602 	if (!timeleft) {
4603 		dev_err(&instance->pdev->dev,
4604 			"task mgmt type 0x%x timed out\n", type);
4605 		mutex_unlock(&instance->reset_mutex);
4606 		rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);
4607 		mutex_lock(&instance->reset_mutex);
4608 		return rc;
4609 	}
4610 
4611 	mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;
4612 	megasas_tm_response_code(instance, mpi_reply);
4613 
4614 	megasas_return_cmd(instance, cmd_mfi);
4615 	rc = SUCCESS;
4616 	switch (type) {
4617 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4618 		scsi_lookup = fusion->cmd_list[smid_task - 1];
4619 
4620 		if (scsi_lookup->scmd == NULL)
4621 			break;
4622 		else {
4623 			instance->instancet->disable_intr(instance);
4624 			megasas_sync_irqs((unsigned long)instance);
4625 			instance->instancet->enable_intr(instance);
4626 			megasas_enable_irq_poll(instance);
4627 			if (scsi_lookup->scmd == NULL)
4628 				break;
4629 		}
4630 		rc = FAILED;
4631 		break;
4632 
4633 	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4634 		if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
4635 			break;
4636 		instance->instancet->disable_intr(instance);
4637 		megasas_sync_irqs((unsigned long)instance);
4638 		rc = megasas_track_scsiio(instance, id, channel);
4639 		instance->instancet->enable_intr(instance);
4640 		megasas_enable_irq_poll(instance);
4641 
4642 		break;
4643 	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
4644 	case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
4645 		break;
4646 	default:
4647 		rc = FAILED;
4648 		break;
4649 	}
4650 
4651 	return rc;
4652 
4653 }
4654 
4655 /*
4656  * megasas_fusion_smid_lookup : Look for fusion command corresponding to SCSI
4657  * @instance: per adapter struct
4658  *
4659  * Return Non Zero index, if SMID found in outstanding commands
4660  */
4661 static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)
4662 {
4663 	int i, ret = 0;
4664 	struct megasas_instance *instance;
4665 	struct megasas_cmd_fusion *cmd_fusion;
4666 	struct fusion_context *fusion;
4667 
4668 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4669 
4670 	fusion = instance->ctrl_context;
4671 
4672 	for (i = 0; i < instance->max_scsi_cmds; i++) {
4673 		cmd_fusion = fusion->cmd_list[i];
4674 		if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {
4675 			scmd_printk(KERN_NOTICE, scmd, "Abort request is for"
4676 				" SMID: %d\n", cmd_fusion->index);
4677 			ret = cmd_fusion->index;
4678 			break;
4679 		}
4680 	}
4681 
4682 	return ret;
4683 }
4684 
4685 /*
4686 * megasas_get_tm_devhandle - Get devhandle for TM request
4687 * @sdev-		     OS provided scsi device
4688 *
4689 * Returns-		     devhandle/targetID of SCSI device
4690 */
4691 static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)
4692 {
4693 	u16 pd_index = 0;
4694 	u32 device_id;
4695 	struct megasas_instance *instance;
4696 	struct fusion_context *fusion;
4697 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
4698 	u16 devhandle = (u16)ULONG_MAX;
4699 
4700 	instance = (struct megasas_instance *)sdev->host->hostdata;
4701 	fusion = instance->ctrl_context;
4702 
4703 	if (!MEGASAS_IS_LOGICAL(sdev)) {
4704 		if (instance->use_seqnum_jbod_fp) {
4705 			pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)
4706 				    + sdev->id;
4707 			pd_sync = (void *)fusion->pd_seq_sync
4708 					[(instance->pd_seq_map_id - 1) & 1];
4709 			devhandle = pd_sync->seq[pd_index].devHandle;
4710 		} else
4711 			sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"
4712 				" without JBOD MAP support from %s %d\n", __func__, __LINE__);
4713 	} else {
4714 		device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)
4715 				+ sdev->id;
4716 		devhandle = device_id;
4717 	}
4718 
4719 	return devhandle;
4720 }
4721 
4722 /*
4723  * megasas_task_abort_fusion : SCSI task abort function for fusion adapters
4724  * @scmd : pointer to scsi command object
4725  *
4726  * Return SUCCESS, if command aborted else FAILED
4727  */
4728 
4729 int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
4730 {
4731 	struct megasas_instance *instance;
4732 	u16 smid, devhandle;
4733 	int ret;
4734 	struct MR_PRIV_DEVICE *mr_device_priv_data;
4735 	mr_device_priv_data = scmd->device->hostdata;
4736 
4737 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4738 
4739 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4740 		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4741 		"SCSI host:%d\n", instance->host->host_no);
4742 		ret = FAILED;
4743 		return ret;
4744 	}
4745 
4746 	if (!mr_device_priv_data) {
4747 		sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4748 			"scmd(%p)\n", scmd);
4749 		scmd->result = DID_NO_CONNECT << 16;
4750 		ret = SUCCESS;
4751 		goto out;
4752 	}
4753 
4754 	if (!mr_device_priv_data->is_tm_capable) {
4755 		ret = FAILED;
4756 		goto out;
4757 	}
4758 
4759 	mutex_lock(&instance->reset_mutex);
4760 
4761 	smid = megasas_fusion_smid_lookup(scmd);
4762 
4763 	if (!smid) {
4764 		ret = SUCCESS;
4765 		scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
4766 			" issued is not found in outstanding commands\n");
4767 		mutex_unlock(&instance->reset_mutex);
4768 		goto out;
4769 	}
4770 
4771 	devhandle = megasas_get_tm_devhandle(scmd->device);
4772 
4773 	if (devhandle == (u16)ULONG_MAX) {
4774 		ret = FAILED;
4775 		sdev_printk(KERN_INFO, scmd->device,
4776 			"task abort issued for invalid devhandle\n");
4777 		mutex_unlock(&instance->reset_mutex);
4778 		goto out;
4779 	}
4780 	sdev_printk(KERN_INFO, scmd->device,
4781 		"attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n",
4782 		scmd, devhandle);
4783 
4784 	mr_device_priv_data->tm_busy = true;
4785 	ret = megasas_issue_tm(instance, devhandle,
4786 			scmd->device->channel, scmd->device->id, smid,
4787 			MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
4788 			mr_device_priv_data);
4789 	mr_device_priv_data->tm_busy = false;
4790 
4791 	mutex_unlock(&instance->reset_mutex);
4792 	scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n",
4793 			((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4794 out:
4795 	scsi_print_command(scmd);
4796 	if (megasas_dbg_lvl & TM_DEBUG)
4797 		megasas_dump_fusion_io(scmd);
4798 
4799 	return ret;
4800 }
4801 
4802 /*
4803  * megasas_reset_target_fusion : target reset function for fusion adapters
4804  * scmd: SCSI command pointer
4805  *
4806  * Returns SUCCESS if all commands associated with target aborted else FAILED
4807  */
4808 
4809 int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
4810 {
4811 
4812 	struct megasas_instance *instance;
4813 	int ret = FAILED;
4814 	u16 devhandle;
4815 	struct MR_PRIV_DEVICE *mr_device_priv_data;
4816 	mr_device_priv_data = scmd->device->hostdata;
4817 
4818 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4819 
4820 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4821 		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4822 		"SCSI host:%d\n", instance->host->host_no);
4823 		ret = FAILED;
4824 		return ret;
4825 	}
4826 
4827 	if (!mr_device_priv_data) {
4828 		sdev_printk(KERN_INFO, scmd->device,
4829 			    "device been deleted! scmd: (0x%p)\n", scmd);
4830 		scmd->result = DID_NO_CONNECT << 16;
4831 		ret = SUCCESS;
4832 		goto out;
4833 	}
4834 
4835 	if (!mr_device_priv_data->is_tm_capable) {
4836 		ret = FAILED;
4837 		goto out;
4838 	}
4839 
4840 	mutex_lock(&instance->reset_mutex);
4841 	devhandle = megasas_get_tm_devhandle(scmd->device);
4842 
4843 	if (devhandle == (u16)ULONG_MAX) {
4844 		ret = FAILED;
4845 		sdev_printk(KERN_INFO, scmd->device,
4846 			"target reset issued for invalid devhandle\n");
4847 		mutex_unlock(&instance->reset_mutex);
4848 		goto out;
4849 	}
4850 
4851 	sdev_printk(KERN_INFO, scmd->device,
4852 		"attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n",
4853 		scmd, devhandle);
4854 	mr_device_priv_data->tm_busy = true;
4855 	ret = megasas_issue_tm(instance, devhandle,
4856 			scmd->device->channel, scmd->device->id, 0,
4857 			MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET,
4858 			mr_device_priv_data);
4859 	mr_device_priv_data->tm_busy = false;
4860 	mutex_unlock(&instance->reset_mutex);
4861 	scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n",
4862 		(ret == SUCCESS) ? "SUCCESS" : "FAILED");
4863 
4864 out:
4865 	return ret;
4866 }
4867 
4868 /*SRIOV get other instance in cluster if any*/
4869 static struct
4870 megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)
4871 {
4872 	int i;
4873 
4874 	for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {
4875 		if (megasas_mgmt_info.instance[i] &&
4876 			(megasas_mgmt_info.instance[i] != instance) &&
4877 			 megasas_mgmt_info.instance[i]->requestorId &&
4878 			 megasas_mgmt_info.instance[i]->peerIsPresent &&
4879 			(memcmp((megasas_mgmt_info.instance[i]->clusterId),
4880 			instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))
4881 			return megasas_mgmt_info.instance[i];
4882 	}
4883 	return NULL;
4884 }
4885 
4886 /* Check for a second path that is currently UP */
4887 int megasas_check_mpio_paths(struct megasas_instance *instance,
4888 	struct scsi_cmnd *scmd)
4889 {
4890 	struct megasas_instance *peer_instance = NULL;
4891 	int retval = (DID_REQUEUE << 16);
4892 
4893 	if (instance->peerIsPresent) {
4894 		peer_instance = megasas_get_peer_instance(instance);
4895 		if ((peer_instance) &&
4896 			(atomic_read(&peer_instance->adprecovery) ==
4897 			MEGASAS_HBA_OPERATIONAL))
4898 			retval = (DID_NO_CONNECT << 16);
4899 	}
4900 	return retval;
4901 }
4902 
4903 /* Core fusion reset function */
4904 int megasas_reset_fusion(struct Scsi_Host *shost, int reason)
4905 {
4906 	int retval = SUCCESS, i, j, convert = 0;
4907 	struct megasas_instance *instance;
4908 	struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;
4909 	struct fusion_context *fusion;
4910 	u32 abs_state, status_reg, reset_adapter, fpio_count = 0;
4911 	u32 io_timeout_in_crash_mode = 0;
4912 	struct scsi_cmnd *scmd_local = NULL;
4913 	struct scsi_device *sdev;
4914 	int ret_target_prop = DCMD_FAILED;
4915 	bool is_target_prop = false;
4916 	bool do_adp_reset = true;
4917 	int max_reset_tries = MEGASAS_FUSION_MAX_RESET_TRIES;
4918 
4919 	instance = (struct megasas_instance *)shost->hostdata;
4920 	fusion = instance->ctrl_context;
4921 
4922 	mutex_lock(&instance->reset_mutex);
4923 
4924 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
4925 		dev_warn(&instance->pdev->dev, "Hardware critical error, "
4926 		       "returning FAILED for scsi%d.\n",
4927 			instance->host->host_no);
4928 		mutex_unlock(&instance->reset_mutex);
4929 		return FAILED;
4930 	}
4931 	status_reg = instance->instancet->read_fw_status_reg(instance);
4932 	abs_state = status_reg & MFI_STATE_MASK;
4933 
4934 	/* IO timeout detected, forcibly put FW in FAULT state */
4935 	if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&
4936 		instance->crash_dump_app_support && reason) {
4937 		dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "
4938 			"forcibly FAULT Firmware\n");
4939 		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4940 		status_reg = megasas_readl(instance, &instance->reg_set->doorbell);
4941 		writel(status_reg | MFI_STATE_FORCE_OCR,
4942 			&instance->reg_set->doorbell);
4943 		readl(&instance->reg_set->doorbell);
4944 		mutex_unlock(&instance->reset_mutex);
4945 		do {
4946 			ssleep(3);
4947 			io_timeout_in_crash_mode++;
4948 			dev_dbg(&instance->pdev->dev, "waiting for [%d] "
4949 				"seconds for crash dump collection and OCR "
4950 				"to be done\n", (io_timeout_in_crash_mode * 3));
4951 		} while ((atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&
4952 			(io_timeout_in_crash_mode < 80));
4953 
4954 		if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {
4955 			dev_info(&instance->pdev->dev, "OCR done for IO "
4956 				"timeout case\n");
4957 			retval = SUCCESS;
4958 		} else {
4959 			dev_info(&instance->pdev->dev, "Controller is not "
4960 				"operational after 240 seconds wait for IO "
4961 				"timeout case in FW crash dump mode\n do "
4962 				"OCR/kill adapter\n");
4963 			retval = megasas_reset_fusion(shost, 0);
4964 		}
4965 		return retval;
4966 	}
4967 
4968 	if (instance->requestorId && !instance->skip_heartbeat_timer_del)
4969 		del_timer_sync(&instance->sriov_heartbeat_timer);
4970 	set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4971 	set_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);
4972 	atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING);
4973 	instance->instancet->disable_intr(instance);
4974 	megasas_sync_irqs((unsigned long)instance);
4975 
4976 	/* First try waiting for commands to complete */
4977 	if (megasas_wait_for_outstanding_fusion(instance, reason,
4978 						&convert)) {
4979 		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4980 		dev_warn(&instance->pdev->dev, "resetting fusion "
4981 		       "adapter scsi%d.\n", instance->host->host_no);
4982 		if (convert)
4983 			reason = 0;
4984 
4985 		if (megasas_dbg_lvl & OCR_DEBUG)
4986 			dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");
4987 
4988 		/* Now return commands back to the OS */
4989 		for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4990 			cmd_fusion = fusion->cmd_list[i];
4991 			/*check for extra commands issued by driver*/
4992 			if (instance->adapter_type >= VENTURA_SERIES) {
4993 				r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];
4994 				megasas_return_cmd_fusion(instance, r1_cmd);
4995 			}
4996 			scmd_local = cmd_fusion->scmd;
4997 			if (cmd_fusion->scmd) {
4998 				if (megasas_dbg_lvl & OCR_DEBUG) {
4999 					sdev_printk(KERN_INFO,
5000 						cmd_fusion->scmd->device, "SMID: 0x%x\n",
5001 						cmd_fusion->index);
5002 					megasas_dump_fusion_io(cmd_fusion->scmd);
5003 				}
5004 
5005 				if (cmd_fusion->io_request->Function ==
5006 					MPI2_FUNCTION_SCSI_IO_REQUEST)
5007 					fpio_count++;
5008 
5009 				scmd_local->result =
5010 					megasas_check_mpio_paths(instance,
5011 							scmd_local);
5012 				if (instance->ldio_threshold &&
5013 					megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
5014 					atomic_dec(&instance->ldio_outstanding);
5015 				megasas_return_cmd_fusion(instance, cmd_fusion);
5016 				scsi_dma_unmap(scmd_local);
5017 				scsi_done(scmd_local);
5018 			}
5019 		}
5020 
5021 		dev_info(&instance->pdev->dev, "Outstanding fastpath IOs: %d\n",
5022 			fpio_count);
5023 
5024 		atomic_set(&instance->fw_outstanding, 0);
5025 
5026 		status_reg = instance->instancet->read_fw_status_reg(instance);
5027 		abs_state = status_reg & MFI_STATE_MASK;
5028 		reset_adapter = status_reg & MFI_RESET_ADAPTER;
5029 		if (instance->disableOnlineCtrlReset ||
5030 		    (abs_state == MFI_STATE_FAULT && !reset_adapter)) {
5031 			/* Reset not supported, kill adapter */
5032 			dev_warn(&instance->pdev->dev, "Reset not supported"
5033 			       ", killing adapter scsi%d.\n",
5034 				instance->host->host_no);
5035 			goto kill_hba;
5036 		}
5037 
5038 		/* Let SR-IOV VF & PF sync up if there was a HB failure */
5039 		if (instance->requestorId && !reason) {
5040 			msleep(MEGASAS_OCR_SETTLE_TIME_VF);
5041 			do_adp_reset = false;
5042 			max_reset_tries = MEGASAS_SRIOV_MAX_RESET_TRIES_VF;
5043 		}
5044 
5045 		/* Now try to reset the chip */
5046 		for (i = 0; i < max_reset_tries; i++) {
5047 			/*
5048 			 * Do adp reset and wait for
5049 			 * controller to transition to ready
5050 			 */
5051 			if (megasas_adp_reset_wait_for_ready(instance,
5052 				do_adp_reset, 1) == FAILED)
5053 				continue;
5054 
5055 			/* Wait for FW to become ready */
5056 			if (megasas_transition_to_ready(instance, 1)) {
5057 				dev_warn(&instance->pdev->dev,
5058 					"Failed to transition controller to ready for "
5059 					"scsi%d.\n", instance->host->host_no);
5060 				continue;
5061 			}
5062 			megasas_reset_reply_desc(instance);
5063 			megasas_fusion_update_can_queue(instance, OCR_CONTEXT);
5064 
5065 			if (megasas_ioc_init_fusion(instance)) {
5066 				continue;
5067 			}
5068 
5069 			if (megasas_get_ctrl_info(instance)) {
5070 				dev_info(&instance->pdev->dev,
5071 					"Failed from %s %d\n",
5072 					__func__, __LINE__);
5073 				goto kill_hba;
5074 			}
5075 
5076 			megasas_refire_mgmt_cmd(instance,
5077 						(i == (MEGASAS_FUSION_MAX_RESET_TRIES - 1)
5078 							? 1 : 0));
5079 
5080 			/* Reset load balance info */
5081 			if (fusion->load_balance_info)
5082 				memset(fusion->load_balance_info, 0,
5083 				       (sizeof(struct LD_LOAD_BALANCE_INFO) *
5084 				       MAX_LOGICAL_DRIVES_EXT));
5085 
5086 			if (!megasas_get_map_info(instance)) {
5087 				megasas_sync_map_info(instance);
5088 			} else {
5089 				/*
5090 				 * Return pending polled mode cmds before
5091 				 * retrying OCR
5092 				 */
5093 				megasas_return_polled_cmds(instance);
5094 				continue;
5095 			}
5096 
5097 			megasas_setup_jbod_map(instance);
5098 
5099 			/* reset stream detection array */
5100 			if (instance->adapter_type >= VENTURA_SERIES) {
5101 				for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {
5102 					memset(fusion->stream_detect_by_ld[j],
5103 					       0, sizeof(struct LD_STREAM_DETECT));
5104 					fusion->stream_detect_by_ld[j]->mru_bit_map
5105 						= MR_STREAM_BITMAP;
5106 				}
5107 			}
5108 
5109 			clear_bit(MEGASAS_FUSION_IN_RESET,
5110 				  &instance->reset_flags);
5111 			instance->instancet->enable_intr(instance);
5112 			megasas_enable_irq_poll(instance);
5113 			shost_for_each_device(sdev, shost) {
5114 				if ((instance->tgt_prop) &&
5115 				    (instance->nvme_page_size))
5116 					ret_target_prop = megasas_get_target_prop(instance, sdev);
5117 
5118 				is_target_prop = (ret_target_prop == DCMD_SUCCESS) ? true : false;
5119 				megasas_set_dynamic_target_properties(sdev, is_target_prop);
5120 			}
5121 
5122 			status_reg = instance->instancet->read_fw_status_reg
5123 					(instance);
5124 			abs_state = status_reg & MFI_STATE_MASK;
5125 			if (abs_state != MFI_STATE_OPERATIONAL) {
5126 				dev_info(&instance->pdev->dev,
5127 					 "Adapter is not OPERATIONAL, state 0x%x for scsi:%d\n",
5128 					 abs_state, instance->host->host_no);
5129 				goto out;
5130 			}
5131 			atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
5132 
5133 			dev_info(&instance->pdev->dev,
5134 				 "Adapter is OPERATIONAL for scsi:%d\n",
5135 				 instance->host->host_no);
5136 
5137 			/* Restart SR-IOV heartbeat */
5138 			if (instance->requestorId) {
5139 				if (!megasas_sriov_start_heartbeat(instance, 0))
5140 					megasas_start_timer(instance);
5141 				else
5142 					instance->skip_heartbeat_timer_del = 1;
5143 			}
5144 
5145 			if (instance->crash_dump_drv_support &&
5146 				instance->crash_dump_app_support)
5147 				megasas_set_crash_dump_params(instance,
5148 					MR_CRASH_BUF_TURN_ON);
5149 			else
5150 				megasas_set_crash_dump_params(instance,
5151 					MR_CRASH_BUF_TURN_OFF);
5152 
5153 			if (instance->snapdump_wait_time) {
5154 				megasas_get_snapdump_properties(instance);
5155 				dev_info(&instance->pdev->dev,
5156 					 "Snap dump wait time\t: %d\n",
5157 					 instance->snapdump_wait_time);
5158 			}
5159 
5160 			retval = SUCCESS;
5161 
5162 			/* Adapter reset completed successfully */
5163 			dev_warn(&instance->pdev->dev,
5164 				 "Reset successful for scsi%d.\n",
5165 				 instance->host->host_no);
5166 
5167 			goto out;
5168 		}
5169 		/* Reset failed, kill the adapter */
5170 		dev_warn(&instance->pdev->dev, "Reset failed, killing "
5171 		       "adapter scsi%d.\n", instance->host->host_no);
5172 		goto kill_hba;
5173 	} else {
5174 		/* For VF: Restart HB timer if we didn't OCR */
5175 		if (instance->requestorId) {
5176 			megasas_start_timer(instance);
5177 		}
5178 		clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
5179 		instance->instancet->enable_intr(instance);
5180 		megasas_enable_irq_poll(instance);
5181 		atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
5182 		goto out;
5183 	}
5184 kill_hba:
5185 	megaraid_sas_kill_hba(instance);
5186 	megasas_enable_irq_poll(instance);
5187 	instance->skip_heartbeat_timer_del = 1;
5188 	retval = FAILED;
5189 out:
5190 	clear_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);
5191 	mutex_unlock(&instance->reset_mutex);
5192 	return retval;
5193 }
5194 
5195 /* Fusion Crash dump collection */
5196 static void  megasas_fusion_crash_dump(struct megasas_instance *instance)
5197 {
5198 	u32 status_reg;
5199 	u8 partial_copy = 0;
5200 	int wait = 0;
5201 
5202 
5203 	status_reg = instance->instancet->read_fw_status_reg(instance);
5204 
5205 	/*
5206 	 * Allocate host crash buffers to copy data from 1 MB DMA crash buffer
5207 	 * to host crash buffers
5208 	 */
5209 	if (instance->drv_buf_index == 0) {
5210 		/* Buffer is already allocated for old Crash dump.
5211 		 * Do OCR and do not wait for crash dump collection
5212 		 */
5213 		if (instance->drv_buf_alloc) {
5214 			dev_info(&instance->pdev->dev, "earlier crash dump is "
5215 				"not yet copied by application, ignoring this "
5216 				"crash dump and initiating OCR\n");
5217 			status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5218 			writel(status_reg,
5219 				&instance->reg_set->outbound_scratch_pad_0);
5220 			readl(&instance->reg_set->outbound_scratch_pad_0);
5221 			return;
5222 		}
5223 		megasas_alloc_host_crash_buffer(instance);
5224 		dev_info(&instance->pdev->dev, "Number of host crash buffers "
5225 			"allocated: %d\n", instance->drv_buf_alloc);
5226 	}
5227 
5228 	while (!(status_reg & MFI_STATE_CRASH_DUMP_DONE) &&
5229 	       (wait < MEGASAS_WATCHDOG_WAIT_COUNT)) {
5230 		if (!(status_reg & MFI_STATE_DMADONE)) {
5231 			/*
5232 			 * Next crash dump buffer is not yet DMA'd by FW
5233 			 * Check after 10ms. Wait for 1 second for FW to
5234 			 * post the next buffer. If not bail out.
5235 			 */
5236 			wait++;
5237 			msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5238 			status_reg = instance->instancet->read_fw_status_reg(
5239 					instance);
5240 			continue;
5241 		}
5242 
5243 		wait = 0;
5244 		if (instance->drv_buf_index >= instance->drv_buf_alloc) {
5245 			dev_info(&instance->pdev->dev,
5246 				 "Driver is done copying the buffer: %d\n",
5247 				 instance->drv_buf_alloc);
5248 			status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5249 			partial_copy = 1;
5250 			break;
5251 		} else {
5252 			memcpy(instance->crash_buf[instance->drv_buf_index],
5253 			       instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);
5254 			instance->drv_buf_index++;
5255 			status_reg &= ~MFI_STATE_DMADONE;
5256 		}
5257 
5258 		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);
5259 		readl(&instance->reg_set->outbound_scratch_pad_0);
5260 
5261 		msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5262 		status_reg = instance->instancet->read_fw_status_reg(instance);
5263 	}
5264 
5265 	if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {
5266 		dev_info(&instance->pdev->dev, "Crash Dump is available,number "
5267 			"of copied buffers: %d\n", instance->drv_buf_index);
5268 		instance->fw_crash_buffer_size =  instance->drv_buf_index;
5269 		instance->fw_crash_state = AVAILABLE;
5270 		instance->drv_buf_index = 0;
5271 		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);
5272 		readl(&instance->reg_set->outbound_scratch_pad_0);
5273 		if (!partial_copy)
5274 			megasas_reset_fusion(instance->host, 0);
5275 	}
5276 }
5277 
5278 
5279 /* Fusion OCR work queue */
5280 void megasas_fusion_ocr_wq(struct work_struct *work)
5281 {
5282 	struct megasas_instance *instance =
5283 		container_of(work, struct megasas_instance, work_init);
5284 
5285 	megasas_reset_fusion(instance->host, 0);
5286 }
5287 
5288 /* Allocate fusion context */
5289 int
5290 megasas_alloc_fusion_context(struct megasas_instance *instance)
5291 {
5292 	struct fusion_context *fusion;
5293 
5294 	instance->ctrl_context = kzalloc(sizeof(struct fusion_context),
5295 					 GFP_KERNEL);
5296 	if (!instance->ctrl_context) {
5297 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5298 			__func__, __LINE__);
5299 		return -ENOMEM;
5300 	}
5301 
5302 	fusion = instance->ctrl_context;
5303 
5304 	fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5305 					      sizeof(LD_SPAN_INFO));
5306 	fusion->log_to_span =
5307 		(PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5308 						fusion->log_to_span_pages);
5309 	if (!fusion->log_to_span) {
5310 		fusion->log_to_span =
5311 			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5312 					   sizeof(LD_SPAN_INFO)));
5313 		if (!fusion->log_to_span) {
5314 			dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5315 				__func__, __LINE__);
5316 			return -ENOMEM;
5317 		}
5318 	}
5319 
5320 	fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5321 		sizeof(struct LD_LOAD_BALANCE_INFO));
5322 	fusion->load_balance_info =
5323 		(struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5324 		fusion->load_balance_info_pages);
5325 	if (!fusion->load_balance_info) {
5326 		fusion->load_balance_info =
5327 			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5328 					   sizeof(struct LD_LOAD_BALANCE_INFO)));
5329 		if (!fusion->load_balance_info)
5330 			dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "
5331 				"continuing without Load Balance support\n");
5332 	}
5333 
5334 	return 0;
5335 }
5336 
5337 void
5338 megasas_free_fusion_context(struct megasas_instance *instance)
5339 {
5340 	struct fusion_context *fusion = instance->ctrl_context;
5341 
5342 	if (fusion) {
5343 		if (fusion->load_balance_info) {
5344 			if (is_vmalloc_addr(fusion->load_balance_info))
5345 				vfree(fusion->load_balance_info);
5346 			else
5347 				free_pages((ulong)fusion->load_balance_info,
5348 					fusion->load_balance_info_pages);
5349 		}
5350 
5351 		if (fusion->log_to_span) {
5352 			if (is_vmalloc_addr(fusion->log_to_span))
5353 				vfree(fusion->log_to_span);
5354 			else
5355 				free_pages((ulong)fusion->log_to_span,
5356 					   fusion->log_to_span_pages);
5357 		}
5358 
5359 		kfree(fusion);
5360 	}
5361 }
5362 
5363 struct megasas_instance_template megasas_instance_template_fusion = {
5364 	.enable_intr = megasas_enable_intr_fusion,
5365 	.disable_intr = megasas_disable_intr_fusion,
5366 	.clear_intr = megasas_clear_intr_fusion,
5367 	.read_fw_status_reg = megasas_read_fw_status_reg_fusion,
5368 	.adp_reset = megasas_adp_reset_fusion,
5369 	.check_reset = megasas_check_reset_fusion,
5370 	.service_isr = megasas_isr_fusion,
5371 	.tasklet = megasas_complete_cmd_dpc_fusion,
5372 	.init_adapter = megasas_init_adapter_fusion,
5373 	.build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,
5374 	.issue_dcmd = megasas_issue_dcmd_fusion,
5375 };
5376