1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Linux MegaRAID driver for SAS based RAID controllers
4  *
5  *  Copyright (c) 2009-2013  LSI Corporation
6  *  Copyright (c) 2013-2016  Avago Technologies
7  *  Copyright (c) 2016-2018  Broadcom Inc.
8  *
9  *  FILE: megaraid_sas_fusion.c
10  *
11  *  Authors: Broadcom Inc.
12  *           Sumant Patro
13  *           Adam Radford
14  *           Kashyap Desai <kashyap.desai@broadcom.com>
15  *           Sumit Saxena <sumit.saxena@broadcom.com>
16  *
17  *  Send feedback to: megaraidlinux.pdl@broadcom.com
18  */
19 
20 #include <linux/kernel.h>
21 #include <linux/types.h>
22 #include <linux/pci.h>
23 #include <linux/list.h>
24 #include <linux/moduleparam.h>
25 #include <linux/module.h>
26 #include <linux/spinlock.h>
27 #include <linux/interrupt.h>
28 #include <linux/delay.h>
29 #include <linux/uio.h>
30 #include <linux/uaccess.h>
31 #include <linux/fs.h>
32 #include <linux/compat.h>
33 #include <linux/blkdev.h>
34 #include <linux/mutex.h>
35 #include <linux/poll.h>
36 #include <linux/vmalloc.h>
37 #include <linux/workqueue.h>
38 #include <linux/irq_poll.h>
39 
40 #include <scsi/scsi.h>
41 #include <scsi/scsi_cmnd.h>
42 #include <scsi/scsi_device.h>
43 #include <scsi/scsi_host.h>
44 #include <scsi/scsi_dbg.h>
45 #include <linux/dmi.h>
46 
47 #include "megaraid_sas_fusion.h"
48 #include "megaraid_sas.h"
49 
50 
51 extern void
52 megasas_complete_cmd(struct megasas_instance *instance,
53 		     struct megasas_cmd *cmd, u8 alt_status);
54 int
55 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
56 	      int seconds);
57 
58 int
59 megasas_clear_intr_fusion(struct megasas_instance *instance);
60 
61 int megasas_transition_to_ready(struct megasas_instance *instance, int ocr);
62 
63 extern u32 megasas_dbg_lvl;
64 int megasas_sriov_start_heartbeat(struct megasas_instance *instance,
65 				  int initial);
66 extern struct megasas_mgmt_info megasas_mgmt_info;
67 extern unsigned int resetwaittime;
68 extern unsigned int dual_qdepth_disable;
69 static void megasas_free_rdpq_fusion(struct megasas_instance *instance);
70 static void megasas_free_reply_fusion(struct megasas_instance *instance);
71 static inline
72 void megasas_configure_queue_sizes(struct megasas_instance *instance);
73 static void megasas_fusion_crash_dump(struct megasas_instance *instance);
74 
75 /**
76  * megasas_adp_reset_wait_for_ready -	initiate chip reset and wait for
77  *					controller to come to ready state
78  * @instance:				adapter's soft state
79  * @do_adp_reset:			If true, do a chip reset
80  * @ocr_context:			If called from OCR context this will
81  *					be set to 1, else 0
82  *
83  * This function initates a chip reset followed by a wait for controller to
84  * transition to ready state.
85  * During this, driver will block all access to PCI config space from userspace
86  */
87 int
88 megasas_adp_reset_wait_for_ready(struct megasas_instance *instance,
89 				 bool do_adp_reset,
90 				 int ocr_context)
91 {
92 	int ret = FAILED;
93 
94 	/*
95 	 * Block access to PCI config space from userspace
96 	 * when diag reset is initiated from driver
97 	 */
98 	if (megasas_dbg_lvl & OCR_DEBUG)
99 		dev_info(&instance->pdev->dev,
100 			 "Block access to PCI config space %s %d\n",
101 			 __func__, __LINE__);
102 
103 	pci_cfg_access_lock(instance->pdev);
104 
105 	if (do_adp_reset) {
106 		if (instance->instancet->adp_reset
107 			(instance, instance->reg_set))
108 			goto out;
109 	}
110 
111 	/* Wait for FW to become ready */
112 	if (megasas_transition_to_ready(instance, ocr_context)) {
113 		dev_warn(&instance->pdev->dev,
114 			 "Failed to transition controller to ready for scsi%d.\n",
115 			 instance->host->host_no);
116 		goto out;
117 	}
118 
119 	ret = SUCCESS;
120 out:
121 	if (megasas_dbg_lvl & OCR_DEBUG)
122 		dev_info(&instance->pdev->dev,
123 			 "Unlock access to PCI config space %s %d\n",
124 			 __func__, __LINE__);
125 
126 	pci_cfg_access_unlock(instance->pdev);
127 
128 	return ret;
129 }
130 
131 /**
132  * megasas_check_same_4gb_region -	check if allocation
133  *					crosses same 4GB boundary or not
134  * @instance:				adapter's soft instance
135  * @start_addr:				start address of DMA allocation
136  * @size:				size of allocation in bytes
137  * @return:				true : allocation does not cross same
138  *					4GB boundary
139  *					false: allocation crosses same
140  *					4GB boundary
141  */
142 static inline bool megasas_check_same_4gb_region
143 	(struct megasas_instance *instance, dma_addr_t start_addr, size_t size)
144 {
145 	dma_addr_t end_addr;
146 
147 	end_addr = start_addr + size;
148 
149 	if (upper_32_bits(start_addr) != upper_32_bits(end_addr)) {
150 		dev_err(&instance->pdev->dev,
151 			"Failed to get same 4GB boundary: start_addr: 0x%llx end_addr: 0x%llx\n",
152 			(unsigned long long)start_addr,
153 			(unsigned long long)end_addr);
154 		return false;
155 	}
156 
157 	return true;
158 }
159 
160 /**
161  * megasas_enable_intr_fusion -	Enables interrupts
162  * @instance:	adapter's soft instance
163  */
164 static void
165 megasas_enable_intr_fusion(struct megasas_instance *instance)
166 {
167 	struct megasas_register_set __iomem *regs;
168 	regs = instance->reg_set;
169 
170 	instance->mask_interrupts = 0;
171 	/* For Thunderbolt/Invader also clear intr on enable */
172 	writel(~0, &regs->outbound_intr_status);
173 	readl(&regs->outbound_intr_status);
174 
175 	writel(~MFI_FUSION_ENABLE_INTERRUPT_MASK, &(regs)->outbound_intr_mask);
176 
177 	/* Dummy readl to force pci flush */
178 	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
179 		 __func__, readl(&regs->outbound_intr_mask));
180 }
181 
182 /**
183  * megasas_disable_intr_fusion - Disables interrupt
184  * @instance:	adapter's soft instance
185  */
186 static void
187 megasas_disable_intr_fusion(struct megasas_instance *instance)
188 {
189 	u32 mask = 0xFFFFFFFF;
190 	struct megasas_register_set __iomem *regs;
191 	regs = instance->reg_set;
192 	instance->mask_interrupts = 1;
193 
194 	writel(mask, &regs->outbound_intr_mask);
195 	/* Dummy readl to force pci flush */
196 	dev_info(&instance->pdev->dev, "%s is called outbound_intr_mask:0x%08x\n",
197 		 __func__, readl(&regs->outbound_intr_mask));
198 }
199 
200 int
201 megasas_clear_intr_fusion(struct megasas_instance *instance)
202 {
203 	u32 status;
204 	struct megasas_register_set __iomem *regs;
205 	regs = instance->reg_set;
206 	/*
207 	 * Check if it is our interrupt
208 	 */
209 	status = megasas_readl(instance,
210 			       &regs->outbound_intr_status);
211 
212 	if (status & 1) {
213 		writel(status, &regs->outbound_intr_status);
214 		readl(&regs->outbound_intr_status);
215 		return 1;
216 	}
217 	if (!(status & MFI_FUSION_ENABLE_INTERRUPT_MASK))
218 		return 0;
219 
220 	return 1;
221 }
222 
223 static inline void
224 megasas_sdev_busy_inc(struct megasas_instance *instance,
225 		      struct scsi_cmnd *scmd)
226 {
227 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
228 		struct MR_PRIV_DEVICE *mr_device_priv_data =
229 			scmd->device->hostdata;
230 		atomic_inc(&mr_device_priv_data->sdev_priv_busy);
231 	}
232 }
233 
234 static inline void
235 megasas_sdev_busy_dec(struct megasas_instance *instance,
236 		      struct scsi_cmnd *scmd)
237 {
238 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
239 		struct MR_PRIV_DEVICE *mr_device_priv_data =
240 			scmd->device->hostdata;
241 		atomic_dec(&mr_device_priv_data->sdev_priv_busy);
242 	}
243 }
244 
245 static inline int
246 megasas_sdev_busy_read(struct megasas_instance *instance,
247 		       struct scsi_cmnd *scmd)
248 {
249 	if (instance->perf_mode == MR_BALANCED_PERF_MODE) {
250 		struct MR_PRIV_DEVICE *mr_device_priv_data =
251 			scmd->device->hostdata;
252 		return atomic_read(&mr_device_priv_data->sdev_priv_busy);
253 	}
254 	return 0;
255 }
256 
257 /**
258  * megasas_get_cmd_fusion -	Get a command from the free pool
259  * @instance:		Adapter soft state
260  * @blk_tag:		Command tag
261  *
262  * Returns a blk_tag indexed mpt frame
263  */
264 inline struct megasas_cmd_fusion *megasas_get_cmd_fusion(struct megasas_instance
265 						  *instance, u32 blk_tag)
266 {
267 	struct fusion_context *fusion;
268 
269 	fusion = instance->ctrl_context;
270 	return fusion->cmd_list[blk_tag];
271 }
272 
273 /**
274  * megasas_return_cmd_fusion -	Return a cmd to free command pool
275  * @instance:		Adapter soft state
276  * @cmd:		Command packet to be returned to free command pool
277  */
278 inline void megasas_return_cmd_fusion(struct megasas_instance *instance,
279 	struct megasas_cmd_fusion *cmd)
280 {
281 	cmd->scmd = NULL;
282 	memset(cmd->io_request, 0, MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE);
283 	cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
284 	cmd->cmd_completed = false;
285 }
286 
287 /**
288  * megasas_write_64bit_req_desc -	PCI writes 64bit request descriptor
289  * @instance:				Adapter soft state
290  * @req_desc:				64bit Request descriptor
291  */
292 static void
293 megasas_write_64bit_req_desc(struct megasas_instance *instance,
294 		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
295 {
296 #if defined(writeq) && defined(CONFIG_64BIT)
297 	u64 req_data = (((u64)le32_to_cpu(req_desc->u.high) << 32) |
298 		le32_to_cpu(req_desc->u.low));
299 	writeq(req_data, &instance->reg_set->inbound_low_queue_port);
300 #else
301 	unsigned long flags;
302 	spin_lock_irqsave(&instance->hba_lock, flags);
303 	writel(le32_to_cpu(req_desc->u.low),
304 		&instance->reg_set->inbound_low_queue_port);
305 	writel(le32_to_cpu(req_desc->u.high),
306 		&instance->reg_set->inbound_high_queue_port);
307 	spin_unlock_irqrestore(&instance->hba_lock, flags);
308 #endif
309 }
310 
311 /**
312  * megasas_fire_cmd_fusion -	Sends command to the FW
313  * @instance:			Adapter soft state
314  * @req_desc:			32bit or 64bit Request descriptor
315  *
316  * Perform PCI Write. AERO SERIES supports 32 bit Descriptor.
317  * Prior to AERO_SERIES support 64 bit Descriptor.
318  */
319 static void
320 megasas_fire_cmd_fusion(struct megasas_instance *instance,
321 		union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc)
322 {
323 	if (instance->atomic_desc_support)
324 		writel(le32_to_cpu(req_desc->u.low),
325 			&instance->reg_set->inbound_single_queue_port);
326 	else
327 		megasas_write_64bit_req_desc(instance, req_desc);
328 }
329 
330 /**
331  * megasas_fusion_update_can_queue -	Do all Adapter Queue depth related calculations here
332  * @instance:		Adapter soft state
333  * @fw_boot_context:	Whether this function called during probe or after OCR
334  *
335  * This function is only for fusion controllers.
336  * Update host can queue, if firmware downgrade max supported firmware commands.
337  * Firmware upgrade case will be skiped because underlying firmware has
338  * more resource than exposed to the OS.
339  *
340  */
341 static void
342 megasas_fusion_update_can_queue(struct megasas_instance *instance, int fw_boot_context)
343 {
344 	u16 cur_max_fw_cmds = 0;
345 	u16 ldio_threshold = 0;
346 
347 	/* ventura FW does not fill outbound_scratch_pad_2 with queue depth */
348 	if (instance->adapter_type < VENTURA_SERIES)
349 		cur_max_fw_cmds =
350 		megasas_readl(instance,
351 			      &instance->reg_set->outbound_scratch_pad_2) & 0x00FFFF;
352 
353 	if (dual_qdepth_disable || !cur_max_fw_cmds)
354 		cur_max_fw_cmds = instance->instancet->read_fw_status_reg(instance) & 0x00FFFF;
355 	else
356 		ldio_threshold =
357 			(instance->instancet->read_fw_status_reg(instance) & 0x00FFFF) - MEGASAS_FUSION_IOCTL_CMDS;
358 
359 	dev_info(&instance->pdev->dev,
360 		 "Current firmware supports maximum commands: %d\t LDIO threshold: %d\n",
361 		 cur_max_fw_cmds, ldio_threshold);
362 
363 	if (fw_boot_context == OCR_CONTEXT) {
364 		cur_max_fw_cmds = cur_max_fw_cmds - 1;
365 		if (cur_max_fw_cmds < instance->max_fw_cmds) {
366 			instance->cur_can_queue =
367 				cur_max_fw_cmds - (MEGASAS_FUSION_INTERNAL_CMDS +
368 						MEGASAS_FUSION_IOCTL_CMDS);
369 			instance->host->can_queue = instance->cur_can_queue;
370 			instance->ldio_threshold = ldio_threshold;
371 		}
372 	} else {
373 		instance->max_fw_cmds = cur_max_fw_cmds;
374 		instance->ldio_threshold = ldio_threshold;
375 
376 		if (reset_devices)
377 			instance->max_fw_cmds = min(instance->max_fw_cmds,
378 						(u16)MEGASAS_KDUMP_QUEUE_DEPTH);
379 		/*
380 		* Reduce the max supported cmds by 1. This is to ensure that the
381 		* reply_q_sz (1 more than the max cmd that driver may send)
382 		* does not exceed max cmds that the FW can support
383 		*/
384 		instance->max_fw_cmds = instance->max_fw_cmds-1;
385 	}
386 }
387 
388 static inline void
389 megasas_get_msix_index(struct megasas_instance *instance,
390 		       struct scsi_cmnd *scmd,
391 		       struct megasas_cmd_fusion *cmd,
392 		       u8 data_arms)
393 {
394 	if (instance->perf_mode == MR_BALANCED_PERF_MODE &&
395 	    (megasas_sdev_busy_read(instance, scmd) >
396 	     (data_arms * MR_DEVICE_HIGH_IOPS_DEPTH))) {
397 		cmd->request_desc->SCSIIO.MSIxIndex =
398 			mega_mod64((atomic64_add_return(1, &instance->high_iops_outstanding) /
399 					MR_HIGH_IOPS_BATCH_COUNT), instance->low_latency_index_start);
400 	} else if (instance->msix_load_balance) {
401 		cmd->request_desc->SCSIIO.MSIxIndex =
402 			(mega_mod64(atomic64_add_return(1, &instance->total_io_count),
403 				instance->msix_vectors));
404 	} else if (instance->host->nr_hw_queues > 1) {
405 		u32 tag = blk_mq_unique_tag(scmd->request);
406 
407 		cmd->request_desc->SCSIIO.MSIxIndex = blk_mq_unique_tag_to_hwq(tag) +
408 			instance->low_latency_index_start;
409 	} else {
410 		cmd->request_desc->SCSIIO.MSIxIndex =
411 			instance->reply_map[raw_smp_processor_id()];
412 	}
413 }
414 
415 /**
416  * megasas_free_cmds_fusion -	Free all the cmds in the free cmd pool
417  * @instance:		Adapter soft state
418  */
419 void
420 megasas_free_cmds_fusion(struct megasas_instance *instance)
421 {
422 	int i;
423 	struct fusion_context *fusion = instance->ctrl_context;
424 	struct megasas_cmd_fusion *cmd;
425 
426 	if (fusion->sense)
427 		dma_pool_free(fusion->sense_dma_pool, fusion->sense,
428 			      fusion->sense_phys_addr);
429 
430 	/* SG */
431 	if (fusion->cmd_list) {
432 		for (i = 0; i < instance->max_mpt_cmds; i++) {
433 			cmd = fusion->cmd_list[i];
434 			if (cmd) {
435 				if (cmd->sg_frame)
436 					dma_pool_free(fusion->sg_dma_pool,
437 						      cmd->sg_frame,
438 						      cmd->sg_frame_phys_addr);
439 			}
440 			kfree(cmd);
441 		}
442 		kfree(fusion->cmd_list);
443 	}
444 
445 	if (fusion->sg_dma_pool) {
446 		dma_pool_destroy(fusion->sg_dma_pool);
447 		fusion->sg_dma_pool = NULL;
448 	}
449 	if (fusion->sense_dma_pool) {
450 		dma_pool_destroy(fusion->sense_dma_pool);
451 		fusion->sense_dma_pool = NULL;
452 	}
453 
454 
455 	/* Reply Frame, Desc*/
456 	if (instance->is_rdpq)
457 		megasas_free_rdpq_fusion(instance);
458 	else
459 		megasas_free_reply_fusion(instance);
460 
461 	/* Request Frame, Desc*/
462 	if (fusion->req_frames_desc)
463 		dma_free_coherent(&instance->pdev->dev,
464 			fusion->request_alloc_sz, fusion->req_frames_desc,
465 			fusion->req_frames_desc_phys);
466 	if (fusion->io_request_frames)
467 		dma_pool_free(fusion->io_request_frames_pool,
468 			fusion->io_request_frames,
469 			fusion->io_request_frames_phys);
470 	if (fusion->io_request_frames_pool) {
471 		dma_pool_destroy(fusion->io_request_frames_pool);
472 		fusion->io_request_frames_pool = NULL;
473 	}
474 }
475 
476 /**
477  * megasas_create_sg_sense_fusion -	Creates DMA pool for cmd frames
478  * @instance:			Adapter soft state
479  *
480  */
481 static int megasas_create_sg_sense_fusion(struct megasas_instance *instance)
482 {
483 	int i;
484 	u16 max_cmd;
485 	struct fusion_context *fusion;
486 	struct megasas_cmd_fusion *cmd;
487 	int sense_sz;
488 	u32 offset;
489 
490 	fusion = instance->ctrl_context;
491 	max_cmd = instance->max_fw_cmds;
492 	sense_sz = instance->max_mpt_cmds * SCSI_SENSE_BUFFERSIZE;
493 
494 	fusion->sg_dma_pool =
495 			dma_pool_create("mr_sg", &instance->pdev->dev,
496 				instance->max_chain_frame_sz,
497 				MR_DEFAULT_NVME_PAGE_SIZE, 0);
498 	/* SCSI_SENSE_BUFFERSIZE  = 96 bytes */
499 	fusion->sense_dma_pool =
500 			dma_pool_create("mr_sense", &instance->pdev->dev,
501 				sense_sz, 64, 0);
502 
503 	if (!fusion->sense_dma_pool || !fusion->sg_dma_pool) {
504 		dev_err(&instance->pdev->dev,
505 			"Failed from %s %d\n",  __func__, __LINE__);
506 		return -ENOMEM;
507 	}
508 
509 	fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
510 				       GFP_KERNEL, &fusion->sense_phys_addr);
511 	if (!fusion->sense) {
512 		dev_err(&instance->pdev->dev,
513 			"failed from %s %d\n",  __func__, __LINE__);
514 		return -ENOMEM;
515 	}
516 
517 	/* sense buffer, request frame and reply desc pool requires to be in
518 	 * same 4 gb region. Below function will check this.
519 	 * In case of failure, new pci pool will be created with updated
520 	 * alignment.
521 	 * Older allocation and pool will be destroyed.
522 	 * Alignment will be used such a way that next allocation if success,
523 	 * will always meet same 4gb region requirement.
524 	 * Actual requirement is not alignment, but we need start and end of
525 	 * DMA address must have same upper 32 bit address.
526 	 */
527 
528 	if (!megasas_check_same_4gb_region(instance, fusion->sense_phys_addr,
529 					   sense_sz)) {
530 		dma_pool_free(fusion->sense_dma_pool, fusion->sense,
531 			      fusion->sense_phys_addr);
532 		fusion->sense = NULL;
533 		dma_pool_destroy(fusion->sense_dma_pool);
534 
535 		fusion->sense_dma_pool =
536 			dma_pool_create("mr_sense_align", &instance->pdev->dev,
537 					sense_sz, roundup_pow_of_two(sense_sz),
538 					0);
539 		if (!fusion->sense_dma_pool) {
540 			dev_err(&instance->pdev->dev,
541 				"Failed from %s %d\n",  __func__, __LINE__);
542 			return -ENOMEM;
543 		}
544 		fusion->sense = dma_pool_alloc(fusion->sense_dma_pool,
545 					       GFP_KERNEL,
546 					       &fusion->sense_phys_addr);
547 		if (!fusion->sense) {
548 			dev_err(&instance->pdev->dev,
549 				"failed from %s %d\n",  __func__, __LINE__);
550 			return -ENOMEM;
551 		}
552 	}
553 
554 	/*
555 	 * Allocate and attach a frame to each of the commands in cmd_list
556 	 */
557 	for (i = 0; i < max_cmd; i++) {
558 		cmd = fusion->cmd_list[i];
559 		cmd->sg_frame = dma_pool_alloc(fusion->sg_dma_pool,
560 					GFP_KERNEL, &cmd->sg_frame_phys_addr);
561 
562 		offset = SCSI_SENSE_BUFFERSIZE * i;
563 		cmd->sense = (u8 *)fusion->sense + offset;
564 		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
565 
566 		if (!cmd->sg_frame) {
567 			dev_err(&instance->pdev->dev,
568 				"Failed from %s %d\n",  __func__, __LINE__);
569 			return -ENOMEM;
570 		}
571 	}
572 
573 	/* create sense buffer for the raid 1/10 fp */
574 	for (i = max_cmd; i < instance->max_mpt_cmds; i++) {
575 		cmd = fusion->cmd_list[i];
576 		offset = SCSI_SENSE_BUFFERSIZE * i;
577 		cmd->sense = (u8 *)fusion->sense + offset;
578 		cmd->sense_phys_addr = fusion->sense_phys_addr + offset;
579 
580 	}
581 
582 	return 0;
583 }
584 
585 static int
586 megasas_alloc_cmdlist_fusion(struct megasas_instance *instance)
587 {
588 	u32 max_mpt_cmd, i, j;
589 	struct fusion_context *fusion;
590 
591 	fusion = instance->ctrl_context;
592 
593 	max_mpt_cmd = instance->max_mpt_cmds;
594 
595 	/*
596 	 * fusion->cmd_list is an array of struct megasas_cmd_fusion pointers.
597 	 * Allocate the dynamic array first and then allocate individual
598 	 * commands.
599 	 */
600 	fusion->cmd_list =
601 		kcalloc(max_mpt_cmd, sizeof(struct megasas_cmd_fusion *),
602 			GFP_KERNEL);
603 	if (!fusion->cmd_list) {
604 		dev_err(&instance->pdev->dev,
605 			"Failed from %s %d\n",  __func__, __LINE__);
606 		return -ENOMEM;
607 	}
608 
609 	for (i = 0; i < max_mpt_cmd; i++) {
610 		fusion->cmd_list[i] = kzalloc(sizeof(struct megasas_cmd_fusion),
611 					      GFP_KERNEL);
612 		if (!fusion->cmd_list[i]) {
613 			for (j = 0; j < i; j++)
614 				kfree(fusion->cmd_list[j]);
615 			kfree(fusion->cmd_list);
616 			dev_err(&instance->pdev->dev,
617 				"Failed from %s %d\n",  __func__, __LINE__);
618 			return -ENOMEM;
619 		}
620 	}
621 
622 	return 0;
623 }
624 
625 static int
626 megasas_alloc_request_fusion(struct megasas_instance *instance)
627 {
628 	struct fusion_context *fusion;
629 
630 	fusion = instance->ctrl_context;
631 
632 retry_alloc:
633 	fusion->io_request_frames_pool =
634 			dma_pool_create("mr_ioreq", &instance->pdev->dev,
635 				fusion->io_frames_alloc_sz, 16, 0);
636 
637 	if (!fusion->io_request_frames_pool) {
638 		dev_err(&instance->pdev->dev,
639 			"Failed from %s %d\n",  __func__, __LINE__);
640 		return -ENOMEM;
641 	}
642 
643 	fusion->io_request_frames =
644 			dma_pool_alloc(fusion->io_request_frames_pool,
645 				GFP_KERNEL | __GFP_NOWARN,
646 				&fusion->io_request_frames_phys);
647 	if (!fusion->io_request_frames) {
648 		if (instance->max_fw_cmds >= (MEGASAS_REDUCE_QD_COUNT * 2)) {
649 			instance->max_fw_cmds -= MEGASAS_REDUCE_QD_COUNT;
650 			dma_pool_destroy(fusion->io_request_frames_pool);
651 			megasas_configure_queue_sizes(instance);
652 			goto retry_alloc;
653 		} else {
654 			dev_err(&instance->pdev->dev,
655 				"Failed from %s %d\n",  __func__, __LINE__);
656 			return -ENOMEM;
657 		}
658 	}
659 
660 	if (!megasas_check_same_4gb_region(instance,
661 					   fusion->io_request_frames_phys,
662 					   fusion->io_frames_alloc_sz)) {
663 		dma_pool_free(fusion->io_request_frames_pool,
664 			      fusion->io_request_frames,
665 			      fusion->io_request_frames_phys);
666 		fusion->io_request_frames = NULL;
667 		dma_pool_destroy(fusion->io_request_frames_pool);
668 
669 		fusion->io_request_frames_pool =
670 			dma_pool_create("mr_ioreq_align",
671 					&instance->pdev->dev,
672 					fusion->io_frames_alloc_sz,
673 					roundup_pow_of_two(fusion->io_frames_alloc_sz),
674 					0);
675 
676 		if (!fusion->io_request_frames_pool) {
677 			dev_err(&instance->pdev->dev,
678 				"Failed from %s %d\n",  __func__, __LINE__);
679 			return -ENOMEM;
680 		}
681 
682 		fusion->io_request_frames =
683 			dma_pool_alloc(fusion->io_request_frames_pool,
684 				       GFP_KERNEL | __GFP_NOWARN,
685 				       &fusion->io_request_frames_phys);
686 
687 		if (!fusion->io_request_frames) {
688 			dev_err(&instance->pdev->dev,
689 				"Failed from %s %d\n",  __func__, __LINE__);
690 			return -ENOMEM;
691 		}
692 	}
693 
694 	fusion->req_frames_desc =
695 		dma_alloc_coherent(&instance->pdev->dev,
696 				   fusion->request_alloc_sz,
697 				   &fusion->req_frames_desc_phys, GFP_KERNEL);
698 	if (!fusion->req_frames_desc) {
699 		dev_err(&instance->pdev->dev,
700 			"Failed from %s %d\n",  __func__, __LINE__);
701 		return -ENOMEM;
702 	}
703 
704 	return 0;
705 }
706 
707 static int
708 megasas_alloc_reply_fusion(struct megasas_instance *instance)
709 {
710 	int i, count;
711 	struct fusion_context *fusion;
712 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
713 	fusion = instance->ctrl_context;
714 
715 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
716 	count += instance->iopoll_q_count;
717 
718 	fusion->reply_frames_desc_pool =
719 			dma_pool_create("mr_reply", &instance->pdev->dev,
720 				fusion->reply_alloc_sz * count, 16, 0);
721 
722 	if (!fusion->reply_frames_desc_pool) {
723 		dev_err(&instance->pdev->dev,
724 			"Failed from %s %d\n",  __func__, __LINE__);
725 		return -ENOMEM;
726 	}
727 
728 	fusion->reply_frames_desc[0] =
729 		dma_pool_alloc(fusion->reply_frames_desc_pool,
730 			GFP_KERNEL, &fusion->reply_frames_desc_phys[0]);
731 	if (!fusion->reply_frames_desc[0]) {
732 		dev_err(&instance->pdev->dev,
733 			"Failed from %s %d\n",  __func__, __LINE__);
734 		return -ENOMEM;
735 	}
736 
737 	if (!megasas_check_same_4gb_region(instance,
738 					   fusion->reply_frames_desc_phys[0],
739 					   (fusion->reply_alloc_sz * count))) {
740 		dma_pool_free(fusion->reply_frames_desc_pool,
741 			      fusion->reply_frames_desc[0],
742 			      fusion->reply_frames_desc_phys[0]);
743 		fusion->reply_frames_desc[0] = NULL;
744 		dma_pool_destroy(fusion->reply_frames_desc_pool);
745 
746 		fusion->reply_frames_desc_pool =
747 			dma_pool_create("mr_reply_align",
748 					&instance->pdev->dev,
749 					fusion->reply_alloc_sz * count,
750 					roundup_pow_of_two(fusion->reply_alloc_sz * count),
751 					0);
752 
753 		if (!fusion->reply_frames_desc_pool) {
754 			dev_err(&instance->pdev->dev,
755 				"Failed from %s %d\n",  __func__, __LINE__);
756 			return -ENOMEM;
757 		}
758 
759 		fusion->reply_frames_desc[0] =
760 			dma_pool_alloc(fusion->reply_frames_desc_pool,
761 				       GFP_KERNEL,
762 				       &fusion->reply_frames_desc_phys[0]);
763 
764 		if (!fusion->reply_frames_desc[0]) {
765 			dev_err(&instance->pdev->dev,
766 				"Failed from %s %d\n",  __func__, __LINE__);
767 			return -ENOMEM;
768 		}
769 	}
770 
771 	reply_desc = fusion->reply_frames_desc[0];
772 	for (i = 0; i < fusion->reply_q_depth * count; i++, reply_desc++)
773 		reply_desc->Words = cpu_to_le64(ULLONG_MAX);
774 
775 	/* This is not a rdpq mode, but driver still populate
776 	 * reply_frame_desc array to use same msix index in ISR path.
777 	 */
778 	for (i = 0; i < (count - 1); i++)
779 		fusion->reply_frames_desc[i + 1] =
780 			fusion->reply_frames_desc[i] +
781 			(fusion->reply_alloc_sz)/sizeof(union MPI2_REPLY_DESCRIPTORS_UNION);
782 
783 	return 0;
784 }
785 
786 static int
787 megasas_alloc_rdpq_fusion(struct megasas_instance *instance)
788 {
789 	int i, j, k, msix_count;
790 	struct fusion_context *fusion;
791 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
792 	union MPI2_REPLY_DESCRIPTORS_UNION *rdpq_chunk_virt[RDPQ_MAX_CHUNK_COUNT];
793 	dma_addr_t rdpq_chunk_phys[RDPQ_MAX_CHUNK_COUNT];
794 	u8 dma_alloc_count, abs_index;
795 	u32 chunk_size, array_size, offset;
796 
797 	fusion = instance->ctrl_context;
798 	chunk_size = fusion->reply_alloc_sz * RDPQ_MAX_INDEX_IN_ONE_CHUNK;
799 	array_size = sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) *
800 		     MAX_MSIX_QUEUES_FUSION;
801 
802 	fusion->rdpq_virt = dma_alloc_coherent(&instance->pdev->dev,
803 					       array_size, &fusion->rdpq_phys,
804 					       GFP_KERNEL);
805 	if (!fusion->rdpq_virt) {
806 		dev_err(&instance->pdev->dev,
807 			"Failed from %s %d\n",  __func__, __LINE__);
808 		return -ENOMEM;
809 	}
810 
811 	msix_count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
812 	msix_count += instance->iopoll_q_count;
813 
814 	fusion->reply_frames_desc_pool = dma_pool_create("mr_rdpq",
815 							 &instance->pdev->dev,
816 							 chunk_size, 16, 0);
817 	fusion->reply_frames_desc_pool_align =
818 				dma_pool_create("mr_rdpq_align",
819 						&instance->pdev->dev,
820 						chunk_size,
821 						roundup_pow_of_two(chunk_size),
822 						0);
823 
824 	if (!fusion->reply_frames_desc_pool ||
825 	    !fusion->reply_frames_desc_pool_align) {
826 		dev_err(&instance->pdev->dev,
827 			"Failed from %s %d\n",  __func__, __LINE__);
828 		return -ENOMEM;
829 	}
830 
831 /*
832  * For INVADER_SERIES each set of 8 reply queues(0-7, 8-15, ..) and
833  * VENTURA_SERIES each set of 16 reply queues(0-15, 16-31, ..) should be
834  * within 4GB boundary and also reply queues in a set must have same
835  * upper 32-bits in their memory address. so here driver is allocating the
836  * DMA'able memory for reply queues according. Driver uses limitation of
837  * VENTURA_SERIES to manage INVADER_SERIES as well.
838  */
839 	dma_alloc_count = DIV_ROUND_UP(msix_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK);
840 
841 	for (i = 0; i < dma_alloc_count; i++) {
842 		rdpq_chunk_virt[i] =
843 			dma_pool_alloc(fusion->reply_frames_desc_pool,
844 				       GFP_KERNEL, &rdpq_chunk_phys[i]);
845 		if (!rdpq_chunk_virt[i]) {
846 			dev_err(&instance->pdev->dev,
847 				"Failed from %s %d\n",  __func__, __LINE__);
848 			return -ENOMEM;
849 		}
850 		/* reply desc pool requires to be in same 4 gb region.
851 		 * Below function will check this.
852 		 * In case of failure, new pci pool will be created with updated
853 		 * alignment.
854 		 * For RDPQ buffers, driver always allocate two separate pci pool.
855 		 * Alignment will be used such a way that next allocation if
856 		 * success, will always meet same 4gb region requirement.
857 		 * rdpq_tracker keep track of each buffer's physical,
858 		 * virtual address and pci pool descriptor. It will help driver
859 		 * while freeing the resources.
860 		 *
861 		 */
862 		if (!megasas_check_same_4gb_region(instance, rdpq_chunk_phys[i],
863 						   chunk_size)) {
864 			dma_pool_free(fusion->reply_frames_desc_pool,
865 				      rdpq_chunk_virt[i],
866 				      rdpq_chunk_phys[i]);
867 
868 			rdpq_chunk_virt[i] =
869 				dma_pool_alloc(fusion->reply_frames_desc_pool_align,
870 					       GFP_KERNEL, &rdpq_chunk_phys[i]);
871 			if (!rdpq_chunk_virt[i]) {
872 				dev_err(&instance->pdev->dev,
873 					"Failed from %s %d\n",
874 					__func__, __LINE__);
875 				return -ENOMEM;
876 			}
877 			fusion->rdpq_tracker[i].dma_pool_ptr =
878 					fusion->reply_frames_desc_pool_align;
879 		} else {
880 			fusion->rdpq_tracker[i].dma_pool_ptr =
881 					fusion->reply_frames_desc_pool;
882 		}
883 
884 		fusion->rdpq_tracker[i].pool_entry_phys = rdpq_chunk_phys[i];
885 		fusion->rdpq_tracker[i].pool_entry_virt = rdpq_chunk_virt[i];
886 	}
887 
888 	for (k = 0; k < dma_alloc_count; k++) {
889 		for (i = 0; i < RDPQ_MAX_INDEX_IN_ONE_CHUNK; i++) {
890 			abs_index = (k * RDPQ_MAX_INDEX_IN_ONE_CHUNK) + i;
891 
892 			if (abs_index == msix_count)
893 				break;
894 			offset = fusion->reply_alloc_sz * i;
895 			fusion->rdpq_virt[abs_index].RDPQBaseAddress =
896 					cpu_to_le64(rdpq_chunk_phys[k] + offset);
897 			fusion->reply_frames_desc_phys[abs_index] =
898 					rdpq_chunk_phys[k] + offset;
899 			fusion->reply_frames_desc[abs_index] =
900 					(union MPI2_REPLY_DESCRIPTORS_UNION *)((u8 *)rdpq_chunk_virt[k] + offset);
901 
902 			reply_desc = fusion->reply_frames_desc[abs_index];
903 			for (j = 0; j < fusion->reply_q_depth; j++, reply_desc++)
904 				reply_desc->Words = ULLONG_MAX;
905 		}
906 	}
907 
908 	return 0;
909 }
910 
911 static void
912 megasas_free_rdpq_fusion(struct megasas_instance *instance) {
913 
914 	int i;
915 	struct fusion_context *fusion;
916 
917 	fusion = instance->ctrl_context;
918 
919 	for (i = 0; i < RDPQ_MAX_CHUNK_COUNT; i++) {
920 		if (fusion->rdpq_tracker[i].pool_entry_virt)
921 			dma_pool_free(fusion->rdpq_tracker[i].dma_pool_ptr,
922 				      fusion->rdpq_tracker[i].pool_entry_virt,
923 				      fusion->rdpq_tracker[i].pool_entry_phys);
924 
925 	}
926 
927 	dma_pool_destroy(fusion->reply_frames_desc_pool);
928 	dma_pool_destroy(fusion->reply_frames_desc_pool_align);
929 
930 	if (fusion->rdpq_virt)
931 		dma_free_coherent(&instance->pdev->dev,
932 			sizeof(struct MPI2_IOC_INIT_RDPQ_ARRAY_ENTRY) * MAX_MSIX_QUEUES_FUSION,
933 			fusion->rdpq_virt, fusion->rdpq_phys);
934 }
935 
936 static void
937 megasas_free_reply_fusion(struct megasas_instance *instance) {
938 
939 	struct fusion_context *fusion;
940 
941 	fusion = instance->ctrl_context;
942 
943 	if (fusion->reply_frames_desc[0])
944 		dma_pool_free(fusion->reply_frames_desc_pool,
945 			fusion->reply_frames_desc[0],
946 			fusion->reply_frames_desc_phys[0]);
947 
948 	dma_pool_destroy(fusion->reply_frames_desc_pool);
949 
950 }
951 
952 
953 /**
954  * megasas_alloc_cmds_fusion -	Allocates the command packets
955  * @instance:		Adapter soft state
956  *
957  *
958  * Each frame has a 32-bit field called context. This context is used to get
959  * back the megasas_cmd_fusion from the frame when a frame gets completed
960  * In this driver, the 32 bit values are the indices into an array cmd_list.
961  * This array is used only to look up the megasas_cmd_fusion given the context.
962  * The free commands themselves are maintained in a linked list called cmd_pool.
963  *
964  * cmds are formed in the io_request and sg_frame members of the
965  * megasas_cmd_fusion. The context field is used to get a request descriptor
966  * and is used as SMID of the cmd.
967  * SMID value range is from 1 to max_fw_cmds.
968  */
969 static int
970 megasas_alloc_cmds_fusion(struct megasas_instance *instance)
971 {
972 	int i;
973 	struct fusion_context *fusion;
974 	struct megasas_cmd_fusion *cmd;
975 	u32 offset;
976 	dma_addr_t io_req_base_phys;
977 	u8 *io_req_base;
978 
979 
980 	fusion = instance->ctrl_context;
981 
982 	if (megasas_alloc_request_fusion(instance))
983 		goto fail_exit;
984 
985 	if (instance->is_rdpq) {
986 		if (megasas_alloc_rdpq_fusion(instance))
987 			goto fail_exit;
988 	} else
989 		if (megasas_alloc_reply_fusion(instance))
990 			goto fail_exit;
991 
992 	if (megasas_alloc_cmdlist_fusion(instance))
993 		goto fail_exit;
994 
995 	/* The first 256 bytes (SMID 0) is not used. Don't add to the cmd list */
996 	io_req_base = fusion->io_request_frames + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
997 	io_req_base_phys = fusion->io_request_frames_phys + MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE;
998 
999 	/*
1000 	 * Add all the commands to command pool (fusion->cmd_pool)
1001 	 */
1002 
1003 	/* SMID 0 is reserved. Set SMID/index from 1 */
1004 	for (i = 0; i < instance->max_mpt_cmds; i++) {
1005 		cmd = fusion->cmd_list[i];
1006 		offset = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE * i;
1007 		memset(cmd, 0, sizeof(struct megasas_cmd_fusion));
1008 		cmd->index = i + 1;
1009 		cmd->scmd = NULL;
1010 		cmd->sync_cmd_idx =
1011 		(i >= instance->max_scsi_cmds && i < instance->max_fw_cmds) ?
1012 				(i - instance->max_scsi_cmds) :
1013 				(u32)ULONG_MAX; /* Set to Invalid */
1014 		cmd->instance = instance;
1015 		cmd->io_request =
1016 			(struct MPI2_RAID_SCSI_IO_REQUEST *)
1017 		  (io_req_base + offset);
1018 		memset(cmd->io_request, 0,
1019 		       sizeof(struct MPI2_RAID_SCSI_IO_REQUEST));
1020 		cmd->io_request_phys_addr = io_req_base_phys + offset;
1021 		cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
1022 	}
1023 
1024 	if (megasas_create_sg_sense_fusion(instance))
1025 		goto fail_exit;
1026 
1027 	return 0;
1028 
1029 fail_exit:
1030 	megasas_free_cmds_fusion(instance);
1031 	return -ENOMEM;
1032 }
1033 
1034 /**
1035  * wait_and_poll -	Issues a polling command
1036  * @instance:			Adapter soft state
1037  * @cmd:			Command packet to be issued
1038  * @seconds:			Maximum poll time
1039  *
1040  * For polling, MFI requires the cmd_status to be set to 0xFF before posting.
1041  */
1042 int
1043 wait_and_poll(struct megasas_instance *instance, struct megasas_cmd *cmd,
1044 	int seconds)
1045 {
1046 	int i;
1047 	struct megasas_header *frame_hdr = &cmd->frame->hdr;
1048 	u32 status_reg;
1049 
1050 	u32 msecs = seconds * 1000;
1051 
1052 	/*
1053 	 * Wait for cmd_status to change
1054 	 */
1055 	for (i = 0; (i < msecs) && (frame_hdr->cmd_status == 0xff); i += 20) {
1056 		rmb();
1057 		msleep(20);
1058 		if (!(i % 5000)) {
1059 			status_reg = instance->instancet->read_fw_status_reg(instance)
1060 					& MFI_STATE_MASK;
1061 			if (status_reg == MFI_STATE_FAULT)
1062 				break;
1063 		}
1064 	}
1065 
1066 	if (frame_hdr->cmd_status == MFI_STAT_INVALID_STATUS)
1067 		return DCMD_TIMEOUT;
1068 	else if (frame_hdr->cmd_status == MFI_STAT_OK)
1069 		return DCMD_SUCCESS;
1070 	else
1071 		return DCMD_FAILED;
1072 }
1073 
1074 /**
1075  * megasas_ioc_init_fusion -	Initializes the FW
1076  * @instance:		Adapter soft state
1077  *
1078  * Issues the IOC Init cmd
1079  */
1080 int
1081 megasas_ioc_init_fusion(struct megasas_instance *instance)
1082 {
1083 	struct megasas_init_frame *init_frame;
1084 	struct MPI2_IOC_INIT_REQUEST *IOCInitMessage = NULL;
1085 	dma_addr_t	ioc_init_handle;
1086 	struct megasas_cmd *cmd;
1087 	u8 ret, cur_rdpq_mode;
1088 	struct fusion_context *fusion;
1089 	union MEGASAS_REQUEST_DESCRIPTOR_UNION req_desc;
1090 	int i;
1091 	struct megasas_header *frame_hdr;
1092 	const char *sys_info;
1093 	MFI_CAPABILITIES *drv_ops;
1094 	u32 scratch_pad_1;
1095 	ktime_t time;
1096 	bool cur_fw_64bit_dma_capable;
1097 	bool cur_intr_coalescing;
1098 
1099 	fusion = instance->ctrl_context;
1100 
1101 	ioc_init_handle = fusion->ioc_init_request_phys;
1102 	IOCInitMessage = fusion->ioc_init_request;
1103 
1104 	cmd = fusion->ioc_init_cmd;
1105 
1106 	scratch_pad_1 = megasas_readl
1107 		(instance, &instance->reg_set->outbound_scratch_pad_1);
1108 
1109 	cur_rdpq_mode = (scratch_pad_1 & MR_RDPQ_MODE_OFFSET) ? 1 : 0;
1110 
1111 	if (instance->adapter_type == INVADER_SERIES) {
1112 		cur_fw_64bit_dma_capable =
1113 			(scratch_pad_1 & MR_CAN_HANDLE_64_BIT_DMA_OFFSET) ? true : false;
1114 
1115 		if (instance->consistent_mask_64bit && !cur_fw_64bit_dma_capable) {
1116 			dev_err(&instance->pdev->dev, "Driver was operating on 64bit "
1117 				"DMA mask, but upcoming FW does not support 64bit DMA mask\n");
1118 			megaraid_sas_kill_hba(instance);
1119 			ret = 1;
1120 			goto fail_fw_init;
1121 		}
1122 	}
1123 
1124 	if (instance->is_rdpq && !cur_rdpq_mode) {
1125 		dev_err(&instance->pdev->dev, "Firmware downgrade *NOT SUPPORTED*"
1126 			" from RDPQ mode to non RDPQ mode\n");
1127 		ret = 1;
1128 		goto fail_fw_init;
1129 	}
1130 
1131 	cur_intr_coalescing = (scratch_pad_1 & MR_INTR_COALESCING_SUPPORT_OFFSET) ?
1132 							true : false;
1133 
1134 	if ((instance->low_latency_index_start ==
1135 		MR_HIGH_IOPS_QUEUE_COUNT) && cur_intr_coalescing)
1136 		instance->perf_mode = MR_BALANCED_PERF_MODE;
1137 
1138 	dev_info(&instance->pdev->dev, "Performance mode :%s (latency index = %d)\n",
1139 		MEGASAS_PERF_MODE_2STR(instance->perf_mode),
1140 		instance->low_latency_index_start);
1141 
1142 	instance->fw_sync_cache_support = (scratch_pad_1 &
1143 		MR_CAN_HANDLE_SYNC_CACHE_OFFSET) ? 1 : 0;
1144 	dev_info(&instance->pdev->dev, "FW supports sync cache\t: %s\n",
1145 		 instance->fw_sync_cache_support ? "Yes" : "No");
1146 
1147 	memset(IOCInitMessage, 0, sizeof(struct MPI2_IOC_INIT_REQUEST));
1148 
1149 	IOCInitMessage->Function = MPI2_FUNCTION_IOC_INIT;
1150 	IOCInitMessage->WhoInit	= MPI2_WHOINIT_HOST_DRIVER;
1151 	IOCInitMessage->MsgVersion = cpu_to_le16(MPI2_VERSION);
1152 	IOCInitMessage->HeaderVersion = cpu_to_le16(MPI2_HEADER_VERSION);
1153 	IOCInitMessage->SystemRequestFrameSize = cpu_to_le16(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE / 4);
1154 
1155 	IOCInitMessage->ReplyDescriptorPostQueueDepth = cpu_to_le16(fusion->reply_q_depth);
1156 	IOCInitMessage->ReplyDescriptorPostQueueAddress = instance->is_rdpq ?
1157 			cpu_to_le64(fusion->rdpq_phys) :
1158 			cpu_to_le64(fusion->reply_frames_desc_phys[0]);
1159 	IOCInitMessage->MsgFlags = instance->is_rdpq ?
1160 			MPI2_IOCINIT_MSGFLAG_RDPQ_ARRAY_MODE : 0;
1161 	IOCInitMessage->SystemRequestFrameBaseAddress = cpu_to_le64(fusion->io_request_frames_phys);
1162 	IOCInitMessage->SenseBufferAddressHigh = cpu_to_le32(upper_32_bits(fusion->sense_phys_addr));
1163 	IOCInitMessage->HostMSIxVectors = instance->msix_vectors + instance->iopoll_q_count;
1164 	IOCInitMessage->HostPageSize = MR_DEFAULT_NVME_PAGE_SHIFT;
1165 
1166 	time = ktime_get_real();
1167 	/* Convert to milliseconds as per FW requirement */
1168 	IOCInitMessage->TimeStamp = cpu_to_le64(ktime_to_ms(time));
1169 
1170 	init_frame = (struct megasas_init_frame *)cmd->frame;
1171 	memset(init_frame, 0, IOC_INIT_FRAME_SIZE);
1172 
1173 	frame_hdr = &cmd->frame->hdr;
1174 	frame_hdr->cmd_status = 0xFF;
1175 	frame_hdr->flags |= cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE);
1176 
1177 	init_frame->cmd	= MFI_CMD_INIT;
1178 	init_frame->cmd_status = 0xFF;
1179 
1180 	drv_ops = (MFI_CAPABILITIES *) &(init_frame->driver_operations);
1181 
1182 	/* driver support Extended MSIX */
1183 	if (instance->adapter_type >= INVADER_SERIES)
1184 		drv_ops->mfi_capabilities.support_additional_msix = 1;
1185 	/* driver supports HA / Remote LUN over Fast Path interface */
1186 	drv_ops->mfi_capabilities.support_fp_remote_lun = 1;
1187 
1188 	drv_ops->mfi_capabilities.support_max_255lds = 1;
1189 	drv_ops->mfi_capabilities.support_ndrive_r1_lb = 1;
1190 	drv_ops->mfi_capabilities.security_protocol_cmds_fw = 1;
1191 
1192 	if (instance->max_chain_frame_sz > MEGASAS_CHAIN_FRAME_SZ_MIN)
1193 		drv_ops->mfi_capabilities.support_ext_io_size = 1;
1194 
1195 	drv_ops->mfi_capabilities.support_fp_rlbypass = 1;
1196 	if (!dual_qdepth_disable)
1197 		drv_ops->mfi_capabilities.support_ext_queue_depth = 1;
1198 
1199 	drv_ops->mfi_capabilities.support_qd_throttling = 1;
1200 	drv_ops->mfi_capabilities.support_pd_map_target_id = 1;
1201 	drv_ops->mfi_capabilities.support_nvme_passthru = 1;
1202 	drv_ops->mfi_capabilities.support_fw_exposed_dev_list = 1;
1203 
1204 	if (instance->consistent_mask_64bit)
1205 		drv_ops->mfi_capabilities.support_64bit_mode = 1;
1206 
1207 	/* Convert capability to LE32 */
1208 	cpu_to_le32s((u32 *)&init_frame->driver_operations.mfi_capabilities);
1209 
1210 	sys_info = dmi_get_system_info(DMI_PRODUCT_UUID);
1211 	if (instance->system_info_buf && sys_info) {
1212 		memcpy(instance->system_info_buf->systemId, sys_info,
1213 			strlen(sys_info) > 64 ? 64 : strlen(sys_info));
1214 		instance->system_info_buf->systemIdLength =
1215 			strlen(sys_info) > 64 ? 64 : strlen(sys_info);
1216 		init_frame->system_info_lo = cpu_to_le32(lower_32_bits(instance->system_info_h));
1217 		init_frame->system_info_hi = cpu_to_le32(upper_32_bits(instance->system_info_h));
1218 	}
1219 
1220 	init_frame->queue_info_new_phys_addr_hi =
1221 		cpu_to_le32(upper_32_bits(ioc_init_handle));
1222 	init_frame->queue_info_new_phys_addr_lo =
1223 		cpu_to_le32(lower_32_bits(ioc_init_handle));
1224 	init_frame->data_xfer_len = cpu_to_le32(sizeof(struct MPI2_IOC_INIT_REQUEST));
1225 
1226 	/*
1227 	 * Each bit in replyqueue_mask represents one group of MSI-x vectors
1228 	 * (each group has 8 vectors)
1229 	 */
1230 	switch (instance->perf_mode) {
1231 	case MR_BALANCED_PERF_MODE:
1232 		init_frame->replyqueue_mask =
1233 		       cpu_to_le16(~(~0 << instance->low_latency_index_start/8));
1234 		break;
1235 	case MR_IOPS_PERF_MODE:
1236 		init_frame->replyqueue_mask =
1237 		       cpu_to_le16(~(~0 << instance->msix_vectors/8));
1238 		break;
1239 	}
1240 
1241 
1242 	req_desc.u.low = cpu_to_le32(lower_32_bits(cmd->frame_phys_addr));
1243 	req_desc.u.high = cpu_to_le32(upper_32_bits(cmd->frame_phys_addr));
1244 	req_desc.MFAIo.RequestFlags =
1245 		(MEGASAS_REQ_DESCRIPT_FLAGS_MFA <<
1246 		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
1247 
1248 	/*
1249 	 * disable the intr before firing the init frame
1250 	 */
1251 	instance->instancet->disable_intr(instance);
1252 
1253 	for (i = 0; i < (10 * 1000); i += 20) {
1254 		if (megasas_readl(instance, &instance->reg_set->doorbell) & 1)
1255 			msleep(20);
1256 		else
1257 			break;
1258 	}
1259 
1260 	/* For AERO also, IOC_INIT requires 64 bit descriptor write */
1261 	megasas_write_64bit_req_desc(instance, &req_desc);
1262 
1263 	wait_and_poll(instance, cmd, MFI_IO_TIMEOUT_SECS);
1264 
1265 	frame_hdr = &cmd->frame->hdr;
1266 	if (frame_hdr->cmd_status != 0) {
1267 		ret = 1;
1268 		goto fail_fw_init;
1269 	}
1270 
1271 	if (instance->adapter_type >= AERO_SERIES) {
1272 		scratch_pad_1 = megasas_readl
1273 			(instance, &instance->reg_set->outbound_scratch_pad_1);
1274 
1275 		instance->atomic_desc_support =
1276 			(scratch_pad_1 & MR_ATOMIC_DESCRIPTOR_SUPPORT_OFFSET) ? 1 : 0;
1277 
1278 		dev_info(&instance->pdev->dev, "FW supports atomic descriptor\t: %s\n",
1279 			instance->atomic_desc_support ? "Yes" : "No");
1280 	}
1281 
1282 	return 0;
1283 
1284 fail_fw_init:
1285 	dev_err(&instance->pdev->dev,
1286 		"Init cmd return status FAILED for SCSI host %d\n",
1287 		instance->host->host_no);
1288 
1289 	return ret;
1290 }
1291 
1292 /**
1293  * megasas_sync_pd_seq_num -	JBOD SEQ MAP
1294  * @instance:		Adapter soft state
1295  * @pend:		set to 1, if it is pended jbod map.
1296  *
1297  * Issue Jbod map to the firmware. If it is pended command,
1298  * issue command and return. If it is first instance of jbod map
1299  * issue and receive command.
1300  */
1301 int
1302 megasas_sync_pd_seq_num(struct megasas_instance *instance, bool pend) {
1303 	int ret = 0;
1304 	size_t pd_seq_map_sz;
1305 	struct megasas_cmd *cmd;
1306 	struct megasas_dcmd_frame *dcmd;
1307 	struct fusion_context *fusion = instance->ctrl_context;
1308 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
1309 	dma_addr_t pd_seq_h;
1310 
1311 	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id & 1)];
1312 	pd_seq_h = fusion->pd_seq_phys[(instance->pd_seq_map_id & 1)];
1313 	pd_seq_map_sz = struct_size(pd_sync, seq, MAX_PHYSICAL_DEVICES - 1);
1314 
1315 	cmd = megasas_get_cmd(instance);
1316 	if (!cmd) {
1317 		dev_err(&instance->pdev->dev,
1318 			"Could not get mfi cmd. Fail from %s %d\n",
1319 			__func__, __LINE__);
1320 		return -ENOMEM;
1321 	}
1322 
1323 	dcmd = &cmd->frame->dcmd;
1324 
1325 	memset(pd_sync, 0, pd_seq_map_sz);
1326 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1327 
1328 	if (pend) {
1329 		dcmd->mbox.b[0] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1330 		dcmd->flags = MFI_FRAME_DIR_WRITE;
1331 		instance->jbod_seq_cmd = cmd;
1332 	} else {
1333 		dcmd->flags = MFI_FRAME_DIR_READ;
1334 	}
1335 
1336 	dcmd->cmd = MFI_CMD_DCMD;
1337 	dcmd->cmd_status = 0xFF;
1338 	dcmd->sge_count = 1;
1339 	dcmd->timeout = 0;
1340 	dcmd->pad_0 = 0;
1341 	dcmd->data_xfer_len = cpu_to_le32(pd_seq_map_sz);
1342 	dcmd->opcode = cpu_to_le32(MR_DCMD_SYSTEM_PD_MAP_GET_INFO);
1343 
1344 	megasas_set_dma_settings(instance, dcmd, pd_seq_h, pd_seq_map_sz);
1345 
1346 	if (pend) {
1347 		instance->instancet->issue_dcmd(instance, cmd);
1348 		return 0;
1349 	}
1350 
1351 	/* Below code is only for non pended DCMD */
1352 	if (!instance->mask_interrupts)
1353 		ret = megasas_issue_blocked_cmd(instance, cmd,
1354 			MFI_IO_TIMEOUT_SECS);
1355 	else
1356 		ret = megasas_issue_polled(instance, cmd);
1357 
1358 	if (le32_to_cpu(pd_sync->count) > MAX_PHYSICAL_DEVICES) {
1359 		dev_warn(&instance->pdev->dev,
1360 			"driver supports max %d JBOD, but FW reports %d\n",
1361 			MAX_PHYSICAL_DEVICES, le32_to_cpu(pd_sync->count));
1362 		ret = -EINVAL;
1363 	}
1364 
1365 	if (ret == DCMD_TIMEOUT)
1366 		dev_warn(&instance->pdev->dev,
1367 			 "%s DCMD timed out, continue without JBOD sequence map\n",
1368 			 __func__);
1369 
1370 	if (ret == DCMD_SUCCESS)
1371 		instance->pd_seq_map_id++;
1372 
1373 	megasas_return_cmd(instance, cmd);
1374 	return ret;
1375 }
1376 
1377 /*
1378  * megasas_get_ld_map_info -	Returns FW's ld_map structure
1379  * @instance:				Adapter soft state
1380  * @pend:				Pend the command or not
1381  * Issues an internal command (DCMD) to get the FW's controller PD
1382  * list structure.  This information is mainly used to find out SYSTEM
1383  * supported by the FW.
1384  * dcmd.mbox value setting for MR_DCMD_LD_MAP_GET_INFO
1385  * dcmd.mbox.b[0]	- number of LDs being sync'd
1386  * dcmd.mbox.b[1]	- 0 - complete command immediately.
1387  *			- 1 - pend till config change
1388  * dcmd.mbox.b[2]	- 0 - supports max 64 lds and uses legacy MR_FW_RAID_MAP
1389  *			- 1 - supports max MAX_LOGICAL_DRIVES_EXT lds and
1390  *				uses extended struct MR_FW_RAID_MAP_EXT
1391  */
1392 static int
1393 megasas_get_ld_map_info(struct megasas_instance *instance)
1394 {
1395 	int ret = 0;
1396 	struct megasas_cmd *cmd;
1397 	struct megasas_dcmd_frame *dcmd;
1398 	void *ci;
1399 	dma_addr_t ci_h = 0;
1400 	u32 size_map_info;
1401 	struct fusion_context *fusion;
1402 
1403 	cmd = megasas_get_cmd(instance);
1404 
1405 	if (!cmd) {
1406 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for map info\n");
1407 		return -ENOMEM;
1408 	}
1409 
1410 	fusion = instance->ctrl_context;
1411 
1412 	if (!fusion) {
1413 		megasas_return_cmd(instance, cmd);
1414 		return -ENXIO;
1415 	}
1416 
1417 	dcmd = &cmd->frame->dcmd;
1418 
1419 	size_map_info = fusion->current_map_sz;
1420 
1421 	ci = (void *) fusion->ld_map[(instance->map_id & 1)];
1422 	ci_h = fusion->ld_map_phys[(instance->map_id & 1)];
1423 
1424 	if (!ci) {
1425 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to alloc mem for ld_map_info\n");
1426 		megasas_return_cmd(instance, cmd);
1427 		return -ENOMEM;
1428 	}
1429 
1430 	memset(ci, 0, fusion->max_map_sz);
1431 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1432 	dcmd->cmd = MFI_CMD_DCMD;
1433 	dcmd->cmd_status = 0xFF;
1434 	dcmd->sge_count = 1;
1435 	dcmd->flags = MFI_FRAME_DIR_READ;
1436 	dcmd->timeout = 0;
1437 	dcmd->pad_0 = 0;
1438 	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1439 	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1440 
1441 	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1442 
1443 	if (!instance->mask_interrupts)
1444 		ret = megasas_issue_blocked_cmd(instance, cmd,
1445 			MFI_IO_TIMEOUT_SECS);
1446 	else
1447 		ret = megasas_issue_polled(instance, cmd);
1448 
1449 	if (ret == DCMD_TIMEOUT)
1450 		dev_warn(&instance->pdev->dev,
1451 			 "%s DCMD timed out, RAID map is disabled\n",
1452 			 __func__);
1453 
1454 	megasas_return_cmd(instance, cmd);
1455 
1456 	return ret;
1457 }
1458 
1459 u8
1460 megasas_get_map_info(struct megasas_instance *instance)
1461 {
1462 	struct fusion_context *fusion = instance->ctrl_context;
1463 
1464 	fusion->fast_path_io = 0;
1465 	if (!megasas_get_ld_map_info(instance)) {
1466 		if (MR_ValidateMapInfo(instance, instance->map_id)) {
1467 			fusion->fast_path_io = 1;
1468 			return 0;
1469 		}
1470 	}
1471 	return 1;
1472 }
1473 
1474 /*
1475  * megasas_sync_map_info -	Returns FW's ld_map structure
1476  * @instance:				Adapter soft state
1477  *
1478  * Issues an internal command (DCMD) to get the FW's controller PD
1479  * list structure.  This information is mainly used to find out SYSTEM
1480  * supported by the FW.
1481  */
1482 int
1483 megasas_sync_map_info(struct megasas_instance *instance)
1484 {
1485 	int i;
1486 	struct megasas_cmd *cmd;
1487 	struct megasas_dcmd_frame *dcmd;
1488 	u16 num_lds;
1489 	struct fusion_context *fusion;
1490 	struct MR_LD_TARGET_SYNC *ci = NULL;
1491 	struct MR_DRV_RAID_MAP_ALL *map;
1492 	struct MR_LD_RAID  *raid;
1493 	struct MR_LD_TARGET_SYNC *ld_sync;
1494 	dma_addr_t ci_h = 0;
1495 	u32 size_map_info;
1496 
1497 	cmd = megasas_get_cmd(instance);
1498 
1499 	if (!cmd) {
1500 		dev_printk(KERN_DEBUG, &instance->pdev->dev, "Failed to get cmd for sync info\n");
1501 		return -ENOMEM;
1502 	}
1503 
1504 	fusion = instance->ctrl_context;
1505 
1506 	if (!fusion) {
1507 		megasas_return_cmd(instance, cmd);
1508 		return 1;
1509 	}
1510 
1511 	map = fusion->ld_drv_map[instance->map_id & 1];
1512 
1513 	num_lds = le16_to_cpu(map->raidMap.ldCount);
1514 
1515 	dcmd = &cmd->frame->dcmd;
1516 
1517 	memset(dcmd->mbox.b, 0, MFI_MBOX_SIZE);
1518 
1519 	ci = (struct MR_LD_TARGET_SYNC *)
1520 	  fusion->ld_map[(instance->map_id - 1) & 1];
1521 	memset(ci, 0, fusion->max_map_sz);
1522 
1523 	ci_h = fusion->ld_map_phys[(instance->map_id - 1) & 1];
1524 
1525 	ld_sync = (struct MR_LD_TARGET_SYNC *)ci;
1526 
1527 	for (i = 0; i < num_lds; i++, ld_sync++) {
1528 		raid = MR_LdRaidGet(i, map);
1529 		ld_sync->targetId = MR_GetLDTgtId(i, map);
1530 		ld_sync->seqNum = raid->seqNum;
1531 	}
1532 
1533 	size_map_info = fusion->current_map_sz;
1534 
1535 	dcmd->cmd = MFI_CMD_DCMD;
1536 	dcmd->cmd_status = 0xFF;
1537 	dcmd->sge_count = 1;
1538 	dcmd->flags = MFI_FRAME_DIR_WRITE;
1539 	dcmd->timeout = 0;
1540 	dcmd->pad_0 = 0;
1541 	dcmd->data_xfer_len = cpu_to_le32(size_map_info);
1542 	dcmd->mbox.b[0] = num_lds;
1543 	dcmd->mbox.b[1] = MEGASAS_DCMD_MBOX_PEND_FLAG;
1544 	dcmd->opcode = cpu_to_le32(MR_DCMD_LD_MAP_GET_INFO);
1545 
1546 	megasas_set_dma_settings(instance, dcmd, ci_h, size_map_info);
1547 
1548 	instance->map_update_cmd = cmd;
1549 
1550 	instance->instancet->issue_dcmd(instance, cmd);
1551 
1552 	return 0;
1553 }
1554 
1555 /*
1556  * meagasas_display_intel_branding - Display branding string
1557  * @instance: per adapter object
1558  *
1559  * Return nothing.
1560  */
1561 static void
1562 megasas_display_intel_branding(struct megasas_instance *instance)
1563 {
1564 	if (instance->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)
1565 		return;
1566 
1567 	switch (instance->pdev->device) {
1568 	case PCI_DEVICE_ID_LSI_INVADER:
1569 		switch (instance->pdev->subsystem_device) {
1570 		case MEGARAID_INTEL_RS3DC080_SSDID:
1571 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1572 				instance->host->host_no,
1573 				MEGARAID_INTEL_RS3DC080_BRANDING);
1574 			break;
1575 		case MEGARAID_INTEL_RS3DC040_SSDID:
1576 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1577 				instance->host->host_no,
1578 				MEGARAID_INTEL_RS3DC040_BRANDING);
1579 			break;
1580 		case MEGARAID_INTEL_RS3SC008_SSDID:
1581 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1582 				instance->host->host_no,
1583 				MEGARAID_INTEL_RS3SC008_BRANDING);
1584 			break;
1585 		case MEGARAID_INTEL_RS3MC044_SSDID:
1586 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1587 				instance->host->host_no,
1588 				MEGARAID_INTEL_RS3MC044_BRANDING);
1589 			break;
1590 		default:
1591 			break;
1592 		}
1593 		break;
1594 	case PCI_DEVICE_ID_LSI_FURY:
1595 		switch (instance->pdev->subsystem_device) {
1596 		case MEGARAID_INTEL_RS3WC080_SSDID:
1597 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1598 				instance->host->host_no,
1599 				MEGARAID_INTEL_RS3WC080_BRANDING);
1600 			break;
1601 		case MEGARAID_INTEL_RS3WC040_SSDID:
1602 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1603 				instance->host->host_no,
1604 				MEGARAID_INTEL_RS3WC040_BRANDING);
1605 			break;
1606 		default:
1607 			break;
1608 		}
1609 		break;
1610 	case PCI_DEVICE_ID_LSI_CUTLASS_52:
1611 	case PCI_DEVICE_ID_LSI_CUTLASS_53:
1612 		switch (instance->pdev->subsystem_device) {
1613 		case MEGARAID_INTEL_RMS3BC160_SSDID:
1614 			dev_info(&instance->pdev->dev, "scsi host %d: %s\n",
1615 				instance->host->host_no,
1616 				MEGARAID_INTEL_RMS3BC160_BRANDING);
1617 			break;
1618 		default:
1619 			break;
1620 		}
1621 		break;
1622 	default:
1623 		break;
1624 	}
1625 }
1626 
1627 /**
1628  * megasas_allocate_raid_maps -	Allocate memory for RAID maps
1629  * @instance:				Adapter soft state
1630  *
1631  * return:				if success: return 0
1632  *					failed:  return -ENOMEM
1633  */
1634 static inline int megasas_allocate_raid_maps(struct megasas_instance *instance)
1635 {
1636 	struct fusion_context *fusion;
1637 	int i = 0;
1638 
1639 	fusion = instance->ctrl_context;
1640 
1641 	fusion->drv_map_pages = get_order(fusion->drv_map_sz);
1642 
1643 	for (i = 0; i < 2; i++) {
1644 		fusion->ld_map[i] = NULL;
1645 
1646 		fusion->ld_drv_map[i] = (void *)
1647 			__get_free_pages(__GFP_ZERO | GFP_KERNEL,
1648 					 fusion->drv_map_pages);
1649 
1650 		if (!fusion->ld_drv_map[i]) {
1651 			fusion->ld_drv_map[i] = vzalloc(fusion->drv_map_sz);
1652 
1653 			if (!fusion->ld_drv_map[i]) {
1654 				dev_err(&instance->pdev->dev,
1655 					"Could not allocate memory for local map"
1656 					" size requested: %d\n",
1657 					fusion->drv_map_sz);
1658 				goto ld_drv_map_alloc_fail;
1659 			}
1660 		}
1661 	}
1662 
1663 	for (i = 0; i < 2; i++) {
1664 		fusion->ld_map[i] = dma_alloc_coherent(&instance->pdev->dev,
1665 						       fusion->max_map_sz,
1666 						       &fusion->ld_map_phys[i],
1667 						       GFP_KERNEL);
1668 		if (!fusion->ld_map[i]) {
1669 			dev_err(&instance->pdev->dev,
1670 				"Could not allocate memory for map info %s:%d\n",
1671 				__func__, __LINE__);
1672 			goto ld_map_alloc_fail;
1673 		}
1674 	}
1675 
1676 	return 0;
1677 
1678 ld_map_alloc_fail:
1679 	for (i = 0; i < 2; i++) {
1680 		if (fusion->ld_map[i])
1681 			dma_free_coherent(&instance->pdev->dev,
1682 					  fusion->max_map_sz,
1683 					  fusion->ld_map[i],
1684 					  fusion->ld_map_phys[i]);
1685 	}
1686 
1687 ld_drv_map_alloc_fail:
1688 	for (i = 0; i < 2; i++) {
1689 		if (fusion->ld_drv_map[i]) {
1690 			if (is_vmalloc_addr(fusion->ld_drv_map[i]))
1691 				vfree(fusion->ld_drv_map[i]);
1692 			else
1693 				free_pages((ulong)fusion->ld_drv_map[i],
1694 					   fusion->drv_map_pages);
1695 		}
1696 	}
1697 
1698 	return -ENOMEM;
1699 }
1700 
1701 /**
1702  * megasas_configure_queue_sizes -	Calculate size of request desc queue,
1703  *					reply desc queue,
1704  *					IO request frame queue, set can_queue.
1705  * @instance:				Adapter soft state
1706  * @return:				void
1707  */
1708 static inline
1709 void megasas_configure_queue_sizes(struct megasas_instance *instance)
1710 {
1711 	struct fusion_context *fusion;
1712 	u16 max_cmd;
1713 
1714 	fusion = instance->ctrl_context;
1715 	max_cmd = instance->max_fw_cmds;
1716 
1717 	if (instance->adapter_type >= VENTURA_SERIES)
1718 		instance->max_mpt_cmds = instance->max_fw_cmds * RAID_1_PEER_CMDS;
1719 	else
1720 		instance->max_mpt_cmds = instance->max_fw_cmds;
1721 
1722 	instance->max_scsi_cmds = instance->max_fw_cmds - instance->max_mfi_cmds;
1723 	instance->cur_can_queue = instance->max_scsi_cmds;
1724 	instance->host->can_queue = instance->cur_can_queue;
1725 
1726 	fusion->reply_q_depth = 2 * ((max_cmd + 1 + 15) / 16) * 16;
1727 
1728 	fusion->request_alloc_sz = sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) *
1729 					  instance->max_mpt_cmds;
1730 	fusion->reply_alloc_sz = sizeof(union MPI2_REPLY_DESCRIPTORS_UNION) *
1731 					(fusion->reply_q_depth);
1732 	fusion->io_frames_alloc_sz = MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE +
1733 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1734 		 * (instance->max_mpt_cmds + 1)); /* Extra 1 for SMID 0 */
1735 }
1736 
1737 static int megasas_alloc_ioc_init_frame(struct megasas_instance *instance)
1738 {
1739 	struct fusion_context *fusion;
1740 	struct megasas_cmd *cmd;
1741 
1742 	fusion = instance->ctrl_context;
1743 
1744 	cmd = kzalloc(sizeof(struct megasas_cmd), GFP_KERNEL);
1745 
1746 	if (!cmd) {
1747 		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1748 			__func__, __LINE__);
1749 		return -ENOMEM;
1750 	}
1751 
1752 	cmd->frame = dma_alloc_coherent(&instance->pdev->dev,
1753 					IOC_INIT_FRAME_SIZE,
1754 					&cmd->frame_phys_addr, GFP_KERNEL);
1755 
1756 	if (!cmd->frame) {
1757 		dev_err(&instance->pdev->dev, "Failed from func: %s line: %d\n",
1758 			__func__, __LINE__);
1759 		kfree(cmd);
1760 		return -ENOMEM;
1761 	}
1762 
1763 	fusion->ioc_init_cmd = cmd;
1764 	return 0;
1765 }
1766 
1767 /**
1768  * megasas_free_ioc_init_cmd -	Free IOC INIT command frame
1769  * @instance:		Adapter soft state
1770  */
1771 static inline void megasas_free_ioc_init_cmd(struct megasas_instance *instance)
1772 {
1773 	struct fusion_context *fusion;
1774 
1775 	fusion = instance->ctrl_context;
1776 
1777 	if (fusion->ioc_init_cmd && fusion->ioc_init_cmd->frame)
1778 		dma_free_coherent(&instance->pdev->dev,
1779 				  IOC_INIT_FRAME_SIZE,
1780 				  fusion->ioc_init_cmd->frame,
1781 				  fusion->ioc_init_cmd->frame_phys_addr);
1782 
1783 	kfree(fusion->ioc_init_cmd);
1784 }
1785 
1786 /**
1787  * megasas_init_adapter_fusion -	Initializes the FW
1788  * @instance:		Adapter soft state
1789  *
1790  * This is the main function for initializing firmware.
1791  */
1792 static u32
1793 megasas_init_adapter_fusion(struct megasas_instance *instance)
1794 {
1795 	struct fusion_context *fusion;
1796 	u32 scratch_pad_1;
1797 	int i = 0, count;
1798 	u32 status_reg;
1799 
1800 	fusion = instance->ctrl_context;
1801 
1802 	megasas_fusion_update_can_queue(instance, PROBE_CONTEXT);
1803 
1804 	/*
1805 	 * Only Driver's internal DCMDs and IOCTL DCMDs needs to have MFI frames
1806 	 */
1807 	instance->max_mfi_cmds =
1808 		MEGASAS_FUSION_INTERNAL_CMDS + MEGASAS_FUSION_IOCTL_CMDS;
1809 
1810 	megasas_configure_queue_sizes(instance);
1811 
1812 	scratch_pad_1 = megasas_readl(instance,
1813 				      &instance->reg_set->outbound_scratch_pad_1);
1814 	/* If scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK is set,
1815 	 * Firmware support extended IO chain frame which is 4 times more than
1816 	 * legacy Firmware.
1817 	 * Legacy Firmware - Frame size is (8 * 128) = 1K
1818 	 * 1M IO Firmware  - Frame size is (8 * 128 * 4)  = 4K
1819 	 */
1820 	if (scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_UNITS_MASK)
1821 		instance->max_chain_frame_sz =
1822 			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1823 			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_1MB_IO;
1824 	else
1825 		instance->max_chain_frame_sz =
1826 			((scratch_pad_1 & MEGASAS_MAX_CHAIN_SIZE_MASK) >>
1827 			MEGASAS_MAX_CHAIN_SHIFT) * MEGASAS_256K_IO;
1828 
1829 	if (instance->max_chain_frame_sz < MEGASAS_CHAIN_FRAME_SZ_MIN) {
1830 		dev_warn(&instance->pdev->dev, "frame size %d invalid, fall back to legacy max frame size %d\n",
1831 			instance->max_chain_frame_sz,
1832 			MEGASAS_CHAIN_FRAME_SZ_MIN);
1833 		instance->max_chain_frame_sz = MEGASAS_CHAIN_FRAME_SZ_MIN;
1834 	}
1835 
1836 	fusion->max_sge_in_main_msg =
1837 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE
1838 			- offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL))/16;
1839 
1840 	fusion->max_sge_in_chain =
1841 		instance->max_chain_frame_sz
1842 			/ sizeof(union MPI2_SGE_IO_UNION);
1843 
1844 	instance->max_num_sge =
1845 		rounddown_pow_of_two(fusion->max_sge_in_main_msg
1846 			+ fusion->max_sge_in_chain - 2);
1847 
1848 	/* Used for pass thru MFI frame (DCMD) */
1849 	fusion->chain_offset_mfi_pthru =
1850 		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL)/16;
1851 
1852 	fusion->chain_offset_io_request =
1853 		(MEGA_MPI2_RAID_DEFAULT_IO_FRAME_SIZE -
1854 		 sizeof(union MPI2_SGE_IO_UNION))/16;
1855 
1856 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
1857 	count += instance->iopoll_q_count;
1858 
1859 	for (i = 0 ; i < count; i++)
1860 		fusion->last_reply_idx[i] = 0;
1861 
1862 	/*
1863 	 * For fusion adapters, 3 commands for IOCTL and 8 commands
1864 	 * for driver's internal DCMDs.
1865 	 */
1866 	instance->max_scsi_cmds = instance->max_fw_cmds -
1867 				(MEGASAS_FUSION_INTERNAL_CMDS +
1868 				MEGASAS_FUSION_IOCTL_CMDS);
1869 	sema_init(&instance->ioctl_sem, MEGASAS_FUSION_IOCTL_CMDS);
1870 
1871 	for (i = 0; i < MAX_MSIX_QUEUES_FUSION; i++)
1872 		atomic_set(&fusion->busy_mq_poll[i], 0);
1873 
1874 	if (megasas_alloc_ioc_init_frame(instance))
1875 		return 1;
1876 
1877 	/*
1878 	 * Allocate memory for descriptors
1879 	 * Create a pool of commands
1880 	 */
1881 	if (megasas_alloc_cmds(instance))
1882 		goto fail_alloc_mfi_cmds;
1883 	if (megasas_alloc_cmds_fusion(instance))
1884 		goto fail_alloc_cmds;
1885 
1886 	if (megasas_ioc_init_fusion(instance)) {
1887 		status_reg = instance->instancet->read_fw_status_reg(instance);
1888 		if (((status_reg & MFI_STATE_MASK) == MFI_STATE_FAULT) &&
1889 		    (status_reg & MFI_RESET_ADAPTER)) {
1890 			/* Do a chip reset and then retry IOC INIT once */
1891 			if (megasas_adp_reset_wait_for_ready
1892 				(instance, true, 0) == FAILED)
1893 				goto fail_ioc_init;
1894 
1895 			if (megasas_ioc_init_fusion(instance))
1896 				goto fail_ioc_init;
1897 		} else {
1898 			goto fail_ioc_init;
1899 		}
1900 	}
1901 
1902 	megasas_display_intel_branding(instance);
1903 	if (megasas_get_ctrl_info(instance)) {
1904 		dev_err(&instance->pdev->dev,
1905 			"Could not get controller info. Fail from %s %d\n",
1906 			__func__, __LINE__);
1907 		goto fail_ioc_init;
1908 	}
1909 
1910 	instance->flag_ieee = 1;
1911 	instance->r1_ldio_hint_default =  MR_R1_LDIO_PIGGYBACK_DEFAULT;
1912 	instance->threshold_reply_count = instance->max_fw_cmds / 4;
1913 	fusion->fast_path_io = 0;
1914 
1915 	if (megasas_allocate_raid_maps(instance))
1916 		goto fail_ioc_init;
1917 
1918 	if (!megasas_get_map_info(instance))
1919 		megasas_sync_map_info(instance);
1920 
1921 	return 0;
1922 
1923 fail_ioc_init:
1924 	megasas_free_cmds_fusion(instance);
1925 fail_alloc_cmds:
1926 	megasas_free_cmds(instance);
1927 fail_alloc_mfi_cmds:
1928 	megasas_free_ioc_init_cmd(instance);
1929 	return 1;
1930 }
1931 
1932 /**
1933  * megasas_fault_detect_work	-	Worker function of
1934  *					FW fault handling workqueue.
1935  * @work:	FW fault work struct
1936  */
1937 static void
1938 megasas_fault_detect_work(struct work_struct *work)
1939 {
1940 	struct megasas_instance *instance =
1941 		container_of(work, struct megasas_instance,
1942 			     fw_fault_work.work);
1943 	u32 fw_state, dma_state, status;
1944 
1945 	/* Check the fw state */
1946 	fw_state = instance->instancet->read_fw_status_reg(instance) &
1947 			MFI_STATE_MASK;
1948 
1949 	if (fw_state == MFI_STATE_FAULT) {
1950 		dma_state = instance->instancet->read_fw_status_reg(instance) &
1951 				MFI_STATE_DMADONE;
1952 		/* Start collecting crash, if DMA bit is done */
1953 		if (instance->crash_dump_drv_support &&
1954 		    instance->crash_dump_app_support && dma_state) {
1955 			megasas_fusion_crash_dump(instance);
1956 		} else {
1957 			if (instance->unload == 0) {
1958 				status = megasas_reset_fusion(instance->host, 0);
1959 				if (status != SUCCESS) {
1960 					dev_err(&instance->pdev->dev,
1961 						"Failed from %s %d, do not re-arm timer\n",
1962 						__func__, __LINE__);
1963 					return;
1964 				}
1965 			}
1966 		}
1967 	}
1968 
1969 	if (instance->fw_fault_work_q)
1970 		queue_delayed_work(instance->fw_fault_work_q,
1971 			&instance->fw_fault_work,
1972 			msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1973 }
1974 
1975 int
1976 megasas_fusion_start_watchdog(struct megasas_instance *instance)
1977 {
1978 	/* Check if the Fault WQ is already started */
1979 	if (instance->fw_fault_work_q)
1980 		return SUCCESS;
1981 
1982 	INIT_DELAYED_WORK(&instance->fw_fault_work, megasas_fault_detect_work);
1983 
1984 	snprintf(instance->fault_handler_work_q_name,
1985 		 sizeof(instance->fault_handler_work_q_name),
1986 		 "poll_megasas%d_status", instance->host->host_no);
1987 
1988 	instance->fw_fault_work_q =
1989 		create_singlethread_workqueue(instance->fault_handler_work_q_name);
1990 	if (!instance->fw_fault_work_q) {
1991 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
1992 			__func__, __LINE__);
1993 		return FAILED;
1994 	}
1995 
1996 	queue_delayed_work(instance->fw_fault_work_q,
1997 			   &instance->fw_fault_work,
1998 			   msecs_to_jiffies(MEGASAS_WATCHDOG_THREAD_INTERVAL));
1999 
2000 	return SUCCESS;
2001 }
2002 
2003 void
2004 megasas_fusion_stop_watchdog(struct megasas_instance *instance)
2005 {
2006 	struct workqueue_struct *wq;
2007 
2008 	if (instance->fw_fault_work_q) {
2009 		wq = instance->fw_fault_work_q;
2010 		instance->fw_fault_work_q = NULL;
2011 		if (!cancel_delayed_work_sync(&instance->fw_fault_work))
2012 			flush_workqueue(wq);
2013 		destroy_workqueue(wq);
2014 	}
2015 }
2016 
2017 /**
2018  * map_cmd_status -	Maps FW cmd status to OS cmd status
2019  * @fusion:		fusion context
2020  * @scmd:		Pointer to cmd
2021  * @status:		status of cmd returned by FW
2022  * @ext_status:		ext status of cmd returned by FW
2023  * @data_length:	command data length
2024  * @sense:		command sense data
2025  */
2026 static void
2027 map_cmd_status(struct fusion_context *fusion,
2028 		struct scsi_cmnd *scmd, u8 status, u8 ext_status,
2029 		u32 data_length, u8 *sense)
2030 {
2031 	u8 cmd_type;
2032 	int resid;
2033 
2034 	cmd_type = megasas_cmd_type(scmd);
2035 	switch (status) {
2036 
2037 	case MFI_STAT_OK:
2038 		scmd->result = DID_OK << 16;
2039 		break;
2040 
2041 	case MFI_STAT_SCSI_IO_FAILED:
2042 	case MFI_STAT_LD_INIT_IN_PROGRESS:
2043 		scmd->result = (DID_ERROR << 16) | ext_status;
2044 		break;
2045 
2046 	case MFI_STAT_SCSI_DONE_WITH_ERROR:
2047 
2048 		scmd->result = (DID_OK << 16) | ext_status;
2049 		if (ext_status == SAM_STAT_CHECK_CONDITION) {
2050 			memset(scmd->sense_buffer, 0,
2051 			       SCSI_SENSE_BUFFERSIZE);
2052 			memcpy(scmd->sense_buffer, sense,
2053 			       SCSI_SENSE_BUFFERSIZE);
2054 		}
2055 
2056 		/*
2057 		 * If the  IO request is partially completed, then MR FW will
2058 		 * update "io_request->DataLength" field with actual number of
2059 		 * bytes transferred.Driver will set residual bytes count in
2060 		 * SCSI command structure.
2061 		 */
2062 		resid = (scsi_bufflen(scmd) - data_length);
2063 		scsi_set_resid(scmd, resid);
2064 
2065 		if (resid &&
2066 			((cmd_type == READ_WRITE_LDIO) ||
2067 			(cmd_type == READ_WRITE_SYSPDIO)))
2068 			scmd_printk(KERN_INFO, scmd, "BRCM Debug mfi stat 0x%x, data len"
2069 				" requested/completed 0x%x/0x%x\n",
2070 				status, scsi_bufflen(scmd), data_length);
2071 		break;
2072 
2073 	case MFI_STAT_LD_OFFLINE:
2074 	case MFI_STAT_DEVICE_NOT_FOUND:
2075 		scmd->result = DID_BAD_TARGET << 16;
2076 		break;
2077 	case MFI_STAT_CONFIG_SEQ_MISMATCH:
2078 		scmd->result = DID_IMM_RETRY << 16;
2079 		break;
2080 	default:
2081 		scmd->result = DID_ERROR << 16;
2082 		break;
2083 	}
2084 }
2085 
2086 /**
2087  * megasas_is_prp_possible -
2088  * Checks if native NVMe PRPs can be built for the IO
2089  *
2090  * @instance:		Adapter soft state
2091  * @scmd:		SCSI command from the mid-layer
2092  * @sge_count:		scatter gather element count.
2093  *
2094  * Returns:		true: PRPs can be built
2095  *			false: IEEE SGLs needs to be built
2096  */
2097 static bool
2098 megasas_is_prp_possible(struct megasas_instance *instance,
2099 			struct scsi_cmnd *scmd, int sge_count)
2100 {
2101 	u32 data_length = 0;
2102 	struct scatterlist *sg_scmd;
2103 	bool build_prp = false;
2104 	u32 mr_nvme_pg_size;
2105 
2106 	mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2107 				MR_DEFAULT_NVME_PAGE_SIZE);
2108 	data_length = scsi_bufflen(scmd);
2109 	sg_scmd = scsi_sglist(scmd);
2110 
2111 	/*
2112 	 * NVMe uses one PRP for each page (or part of a page)
2113 	 * look at the data length - if 4 pages or less then IEEE is OK
2114 	 * if  > 5 pages then we need to build a native SGL
2115 	 * if > 4 and <= 5 pages, then check physical address of 1st SG entry
2116 	 * if this first size in the page is >= the residual beyond 4 pages
2117 	 * then use IEEE, otherwise use native SGL
2118 	 */
2119 
2120 	if (data_length > (mr_nvme_pg_size * 5)) {
2121 		build_prp = true;
2122 	} else if ((data_length > (mr_nvme_pg_size * 4)) &&
2123 			(data_length <= (mr_nvme_pg_size * 5)))  {
2124 		/* check if 1st SG entry size is < residual beyond 4 pages */
2125 		if (sg_dma_len(sg_scmd) < (data_length - (mr_nvme_pg_size * 4)))
2126 			build_prp = true;
2127 	}
2128 
2129 	return build_prp;
2130 }
2131 
2132 /**
2133  * megasas_make_prp_nvme -
2134  * Prepare PRPs(Physical Region Page)- SGLs specific to NVMe drives only
2135  *
2136  * @instance:		Adapter soft state
2137  * @scmd:		SCSI command from the mid-layer
2138  * @sgl_ptr:		SGL to be filled in
2139  * @cmd:		Fusion command frame
2140  * @sge_count:		scatter gather element count.
2141  *
2142  * Returns:		true: PRPs are built
2143  *			false: IEEE SGLs needs to be built
2144  */
2145 static bool
2146 megasas_make_prp_nvme(struct megasas_instance *instance, struct scsi_cmnd *scmd,
2147 		      struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2148 		      struct megasas_cmd_fusion *cmd, int sge_count)
2149 {
2150 	int sge_len, offset, num_prp_in_chain = 0;
2151 	struct MPI25_IEEE_SGE_CHAIN64 *main_chain_element, *ptr_first_sgl;
2152 	u64 *ptr_sgl;
2153 	dma_addr_t ptr_sgl_phys;
2154 	u64 sge_addr;
2155 	u32 page_mask, page_mask_result;
2156 	struct scatterlist *sg_scmd;
2157 	u32 first_prp_len;
2158 	bool build_prp = false;
2159 	int data_len = scsi_bufflen(scmd);
2160 	u32 mr_nvme_pg_size = max_t(u32, instance->nvme_page_size,
2161 					MR_DEFAULT_NVME_PAGE_SIZE);
2162 
2163 	build_prp = megasas_is_prp_possible(instance, scmd, sge_count);
2164 
2165 	if (!build_prp)
2166 		return false;
2167 
2168 	/*
2169 	 * Nvme has a very convoluted prp format.  One prp is required
2170 	 * for each page or partial page. Driver need to split up OS sg_list
2171 	 * entries if it is longer than one page or cross a page
2172 	 * boundary.  Driver also have to insert a PRP list pointer entry as
2173 	 * the last entry in each physical page of the PRP list.
2174 	 *
2175 	 * NOTE: The first PRP "entry" is actually placed in the first
2176 	 * SGL entry in the main message as IEEE 64 format.  The 2nd
2177 	 * entry in the main message is the chain element, and the rest
2178 	 * of the PRP entries are built in the contiguous pcie buffer.
2179 	 */
2180 	page_mask = mr_nvme_pg_size - 1;
2181 	ptr_sgl = (u64 *)cmd->sg_frame;
2182 	ptr_sgl_phys = cmd->sg_frame_phys_addr;
2183 	memset(ptr_sgl, 0, instance->max_chain_frame_sz);
2184 
2185 	/* Build chain frame element which holds all prps except first*/
2186 	main_chain_element = (struct MPI25_IEEE_SGE_CHAIN64 *)
2187 	    ((u8 *)sgl_ptr + sizeof(struct MPI25_IEEE_SGE_CHAIN64));
2188 
2189 	main_chain_element->Address = cpu_to_le64(ptr_sgl_phys);
2190 	main_chain_element->NextChainOffset = 0;
2191 	main_chain_element->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2192 					IEEE_SGE_FLAGS_SYSTEM_ADDR |
2193 					MPI26_IEEE_SGE_FLAGS_NSF_NVME_PRP;
2194 
2195 	/* Build first prp, sge need not to be page aligned*/
2196 	ptr_first_sgl = sgl_ptr;
2197 	sg_scmd = scsi_sglist(scmd);
2198 	sge_addr = sg_dma_address(sg_scmd);
2199 	sge_len = sg_dma_len(sg_scmd);
2200 
2201 	offset = (u32)(sge_addr & page_mask);
2202 	first_prp_len = mr_nvme_pg_size - offset;
2203 
2204 	ptr_first_sgl->Address = cpu_to_le64(sge_addr);
2205 	ptr_first_sgl->Length = cpu_to_le32(first_prp_len);
2206 
2207 	data_len -= first_prp_len;
2208 
2209 	if (sge_len > first_prp_len) {
2210 		sge_addr += first_prp_len;
2211 		sge_len -= first_prp_len;
2212 	} else if (sge_len == first_prp_len) {
2213 		sg_scmd = sg_next(sg_scmd);
2214 		sge_addr = sg_dma_address(sg_scmd);
2215 		sge_len = sg_dma_len(sg_scmd);
2216 	}
2217 
2218 	for (;;) {
2219 		offset = (u32)(sge_addr & page_mask);
2220 
2221 		/* Put PRP pointer due to page boundary*/
2222 		page_mask_result = (uintptr_t)(ptr_sgl + 1) & page_mask;
2223 		if (unlikely(!page_mask_result)) {
2224 			scmd_printk(KERN_NOTICE,
2225 				    scmd, "page boundary ptr_sgl: 0x%p\n",
2226 				    ptr_sgl);
2227 			ptr_sgl_phys += 8;
2228 			*ptr_sgl = cpu_to_le64(ptr_sgl_phys);
2229 			ptr_sgl++;
2230 			num_prp_in_chain++;
2231 		}
2232 
2233 		*ptr_sgl = cpu_to_le64(sge_addr);
2234 		ptr_sgl++;
2235 		ptr_sgl_phys += 8;
2236 		num_prp_in_chain++;
2237 
2238 		sge_addr += mr_nvme_pg_size;
2239 		sge_len -= mr_nvme_pg_size;
2240 		data_len -= mr_nvme_pg_size;
2241 
2242 		if (data_len <= 0)
2243 			break;
2244 
2245 		if (sge_len > 0)
2246 			continue;
2247 
2248 		sg_scmd = sg_next(sg_scmd);
2249 		sge_addr = sg_dma_address(sg_scmd);
2250 		sge_len = sg_dma_len(sg_scmd);
2251 	}
2252 
2253 	main_chain_element->Length =
2254 			cpu_to_le32(num_prp_in_chain * sizeof(u64));
2255 
2256 	return build_prp;
2257 }
2258 
2259 /**
2260  * megasas_make_sgl_fusion -	Prepares 32-bit SGL
2261  * @instance:		Adapter soft state
2262  * @scp:		SCSI command from the mid-layer
2263  * @sgl_ptr:		SGL to be filled in
2264  * @cmd:		cmd we are working on
2265  * @sge_count:		sge count
2266  *
2267  */
2268 static void
2269 megasas_make_sgl_fusion(struct megasas_instance *instance,
2270 			struct scsi_cmnd *scp,
2271 			struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr,
2272 			struct megasas_cmd_fusion *cmd, int sge_count)
2273 {
2274 	int i, sg_processed;
2275 	struct scatterlist *os_sgl;
2276 	struct fusion_context *fusion;
2277 
2278 	fusion = instance->ctrl_context;
2279 
2280 	if (instance->adapter_type >= INVADER_SERIES) {
2281 		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end = sgl_ptr;
2282 		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
2283 		sgl_ptr_end->Flags = 0;
2284 	}
2285 
2286 	scsi_for_each_sg(scp, os_sgl, sge_count, i) {
2287 		sgl_ptr->Length = cpu_to_le32(sg_dma_len(os_sgl));
2288 		sgl_ptr->Address = cpu_to_le64(sg_dma_address(os_sgl));
2289 		sgl_ptr->Flags = 0;
2290 		if (instance->adapter_type >= INVADER_SERIES)
2291 			if (i == sge_count - 1)
2292 				sgl_ptr->Flags = IEEE_SGE_FLAGS_END_OF_LIST;
2293 		sgl_ptr++;
2294 		sg_processed = i + 1;
2295 
2296 		if ((sg_processed ==  (fusion->max_sge_in_main_msg - 1)) &&
2297 		    (sge_count > fusion->max_sge_in_main_msg)) {
2298 
2299 			struct MPI25_IEEE_SGE_CHAIN64 *sg_chain;
2300 			if (instance->adapter_type >= INVADER_SERIES) {
2301 				if ((le16_to_cpu(cmd->io_request->IoFlags) &
2302 					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) !=
2303 					MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH)
2304 					cmd->io_request->ChainOffset =
2305 						fusion->
2306 						chain_offset_io_request;
2307 				else
2308 					cmd->io_request->ChainOffset = 0;
2309 			} else
2310 				cmd->io_request->ChainOffset =
2311 					fusion->chain_offset_io_request;
2312 
2313 			sg_chain = sgl_ptr;
2314 			/* Prepare chain element */
2315 			sg_chain->NextChainOffset = 0;
2316 			if (instance->adapter_type >= INVADER_SERIES)
2317 				sg_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT;
2318 			else
2319 				sg_chain->Flags =
2320 					(IEEE_SGE_FLAGS_CHAIN_ELEMENT |
2321 					 MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR);
2322 			sg_chain->Length =  cpu_to_le32((sizeof(union MPI2_SGE_IO_UNION) * (sge_count - sg_processed)));
2323 			sg_chain->Address = cpu_to_le64(cmd->sg_frame_phys_addr);
2324 
2325 			sgl_ptr =
2326 			  (struct MPI25_IEEE_SGE_CHAIN64 *)cmd->sg_frame;
2327 			memset(sgl_ptr, 0, instance->max_chain_frame_sz);
2328 		}
2329 	}
2330 }
2331 
2332 /**
2333  * megasas_make_sgl -	Build Scatter Gather List(SGLs)
2334  * @scp:		SCSI command pointer
2335  * @instance:		Soft instance of controller
2336  * @cmd:		Fusion command pointer
2337  *
2338  * This function will build sgls based on device type.
2339  * For nvme drives, there is different way of building sgls in nvme native
2340  * format- PRPs(Physical Region Page).
2341  *
2342  * Returns the number of sg lists actually used, zero if the sg lists
2343  * is NULL, or -ENOMEM if the mapping failed
2344  */
2345 static
2346 int megasas_make_sgl(struct megasas_instance *instance, struct scsi_cmnd *scp,
2347 		     struct megasas_cmd_fusion *cmd)
2348 {
2349 	int sge_count;
2350 	bool build_prp = false;
2351 	struct MPI25_IEEE_SGE_CHAIN64 *sgl_chain64;
2352 
2353 	sge_count = scsi_dma_map(scp);
2354 
2355 	if ((sge_count > instance->max_num_sge) || (sge_count <= 0))
2356 		return sge_count;
2357 
2358 	sgl_chain64 = (struct MPI25_IEEE_SGE_CHAIN64 *)&cmd->io_request->SGL;
2359 	if ((le16_to_cpu(cmd->io_request->IoFlags) &
2360 	    MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH) &&
2361 	    (cmd->pd_interface == NVME_PD))
2362 		build_prp = megasas_make_prp_nvme(instance, scp, sgl_chain64,
2363 						  cmd, sge_count);
2364 
2365 	if (!build_prp)
2366 		megasas_make_sgl_fusion(instance, scp, sgl_chain64,
2367 					cmd, sge_count);
2368 
2369 	return sge_count;
2370 }
2371 
2372 /**
2373  * megasas_set_pd_lba -	Sets PD LBA
2374  * @io_request:		IO request
2375  * @cdb_len:		cdb length
2376  * @io_info:		IO information
2377  * @scp:		SCSI command
2378  * @local_map_ptr:	Raid map
2379  * @ref_tag:		Primary reference tag
2380  *
2381  * Used to set the PD LBA in CDB for FP IOs
2382  */
2383 static void
2384 megasas_set_pd_lba(struct MPI2_RAID_SCSI_IO_REQUEST *io_request, u8 cdb_len,
2385 		   struct IO_REQUEST_INFO *io_info, struct scsi_cmnd *scp,
2386 		   struct MR_DRV_RAID_MAP_ALL *local_map_ptr, u32 ref_tag)
2387 {
2388 	struct MR_LD_RAID *raid;
2389 	u16 ld;
2390 	u64 start_blk = io_info->pdBlock;
2391 	u8 *cdb = io_request->CDB.CDB32;
2392 	u32 num_blocks = io_info->numBlocks;
2393 	u8 opcode = 0, flagvals = 0, groupnum = 0, control = 0;
2394 
2395 	/* Check if T10 PI (DIF) is enabled for this LD */
2396 	ld = MR_TargetIdToLdGet(io_info->ldTgtId, local_map_ptr);
2397 	raid = MR_LdRaidGet(ld, local_map_ptr);
2398 	if (raid->capability.ldPiMode == MR_PROT_INFO_TYPE_CONTROLLER) {
2399 		memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2400 		cdb[0] =  MEGASAS_SCSI_VARIABLE_LENGTH_CMD;
2401 		cdb[7] =  MEGASAS_SCSI_ADDL_CDB_LEN;
2402 
2403 		if (scp->sc_data_direction == DMA_FROM_DEVICE)
2404 			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_READ32;
2405 		else
2406 			cdb[9] = MEGASAS_SCSI_SERVICE_ACTION_WRITE32;
2407 		cdb[10] = MEGASAS_RD_WR_PROTECT_CHECK_ALL;
2408 
2409 		/* LBA */
2410 		cdb[12] = (u8)((start_blk >> 56) & 0xff);
2411 		cdb[13] = (u8)((start_blk >> 48) & 0xff);
2412 		cdb[14] = (u8)((start_blk >> 40) & 0xff);
2413 		cdb[15] = (u8)((start_blk >> 32) & 0xff);
2414 		cdb[16] = (u8)((start_blk >> 24) & 0xff);
2415 		cdb[17] = (u8)((start_blk >> 16) & 0xff);
2416 		cdb[18] = (u8)((start_blk >> 8) & 0xff);
2417 		cdb[19] = (u8)(start_blk & 0xff);
2418 
2419 		/* Logical block reference tag */
2420 		io_request->CDB.EEDP32.PrimaryReferenceTag =
2421 			cpu_to_be32(ref_tag);
2422 		io_request->CDB.EEDP32.PrimaryApplicationTagMask = cpu_to_be16(0xffff);
2423 		io_request->IoFlags = cpu_to_le16(32); /* Specify 32-byte cdb */
2424 
2425 		/* Transfer length */
2426 		cdb[28] = (u8)((num_blocks >> 24) & 0xff);
2427 		cdb[29] = (u8)((num_blocks >> 16) & 0xff);
2428 		cdb[30] = (u8)((num_blocks >> 8) & 0xff);
2429 		cdb[31] = (u8)(num_blocks & 0xff);
2430 
2431 		/* set SCSI IO EEDPFlags */
2432 		if (scp->sc_data_direction == DMA_FROM_DEVICE) {
2433 			io_request->EEDPFlags = cpu_to_le16(
2434 				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG  |
2435 				MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
2436 				MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP |
2437 				MPI2_SCSIIO_EEDPFLAGS_CHECK_APPTAG |
2438 				MPI25_SCSIIO_EEDPFLAGS_DO_NOT_DISABLE_MODE |
2439 				MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
2440 		} else {
2441 			io_request->EEDPFlags = cpu_to_le16(
2442 				MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
2443 				MPI2_SCSIIO_EEDPFLAGS_INSERT_OP);
2444 		}
2445 		io_request->Control |= cpu_to_le32((0x4 << 26));
2446 		io_request->EEDPBlockSize = cpu_to_le32(scp->device->sector_size);
2447 	} else {
2448 		/* Some drives don't support 16/12 byte CDB's, convert to 10 */
2449 		if (((cdb_len == 12) || (cdb_len == 16)) &&
2450 		    (start_blk <= 0xffffffff)) {
2451 			if (cdb_len == 16) {
2452 				opcode = cdb[0] == READ_16 ? READ_10 : WRITE_10;
2453 				flagvals = cdb[1];
2454 				groupnum = cdb[14];
2455 				control = cdb[15];
2456 			} else {
2457 				opcode = cdb[0] == READ_12 ? READ_10 : WRITE_10;
2458 				flagvals = cdb[1];
2459 				groupnum = cdb[10];
2460 				control = cdb[11];
2461 			}
2462 
2463 			memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2464 
2465 			cdb[0] = opcode;
2466 			cdb[1] = flagvals;
2467 			cdb[6] = groupnum;
2468 			cdb[9] = control;
2469 
2470 			/* Transfer length */
2471 			cdb[8] = (u8)(num_blocks & 0xff);
2472 			cdb[7] = (u8)((num_blocks >> 8) & 0xff);
2473 
2474 			io_request->IoFlags = cpu_to_le16(10); /* Specify 10-byte cdb */
2475 			cdb_len = 10;
2476 		} else if ((cdb_len < 16) && (start_blk > 0xffffffff)) {
2477 			/* Convert to 16 byte CDB for large LBA's */
2478 			switch (cdb_len) {
2479 			case 6:
2480 				opcode = cdb[0] == READ_6 ? READ_16 : WRITE_16;
2481 				control = cdb[5];
2482 				break;
2483 			case 10:
2484 				opcode =
2485 					cdb[0] == READ_10 ? READ_16 : WRITE_16;
2486 				flagvals = cdb[1];
2487 				groupnum = cdb[6];
2488 				control = cdb[9];
2489 				break;
2490 			case 12:
2491 				opcode =
2492 					cdb[0] == READ_12 ? READ_16 : WRITE_16;
2493 				flagvals = cdb[1];
2494 				groupnum = cdb[10];
2495 				control = cdb[11];
2496 				break;
2497 			}
2498 
2499 			memset(cdb, 0, sizeof(io_request->CDB.CDB32));
2500 
2501 			cdb[0] = opcode;
2502 			cdb[1] = flagvals;
2503 			cdb[14] = groupnum;
2504 			cdb[15] = control;
2505 
2506 			/* Transfer length */
2507 			cdb[13] = (u8)(num_blocks & 0xff);
2508 			cdb[12] = (u8)((num_blocks >> 8) & 0xff);
2509 			cdb[11] = (u8)((num_blocks >> 16) & 0xff);
2510 			cdb[10] = (u8)((num_blocks >> 24) & 0xff);
2511 
2512 			io_request->IoFlags = cpu_to_le16(16); /* Specify 16-byte cdb */
2513 			cdb_len = 16;
2514 		}
2515 
2516 		/* Normal case, just load LBA here */
2517 		switch (cdb_len) {
2518 		case 6:
2519 		{
2520 			u8 val = cdb[1] & 0xE0;
2521 			cdb[3] = (u8)(start_blk & 0xff);
2522 			cdb[2] = (u8)((start_blk >> 8) & 0xff);
2523 			cdb[1] = val | ((u8)(start_blk >> 16) & 0x1f);
2524 			break;
2525 		}
2526 		case 10:
2527 			cdb[5] = (u8)(start_blk & 0xff);
2528 			cdb[4] = (u8)((start_blk >> 8) & 0xff);
2529 			cdb[3] = (u8)((start_blk >> 16) & 0xff);
2530 			cdb[2] = (u8)((start_blk >> 24) & 0xff);
2531 			break;
2532 		case 12:
2533 			cdb[5]    = (u8)(start_blk & 0xff);
2534 			cdb[4]    = (u8)((start_blk >> 8) & 0xff);
2535 			cdb[3]    = (u8)((start_blk >> 16) & 0xff);
2536 			cdb[2]    = (u8)((start_blk >> 24) & 0xff);
2537 			break;
2538 		case 16:
2539 			cdb[9]    = (u8)(start_blk & 0xff);
2540 			cdb[8]    = (u8)((start_blk >> 8) & 0xff);
2541 			cdb[7]    = (u8)((start_blk >> 16) & 0xff);
2542 			cdb[6]    = (u8)((start_blk >> 24) & 0xff);
2543 			cdb[5]    = (u8)((start_blk >> 32) & 0xff);
2544 			cdb[4]    = (u8)((start_blk >> 40) & 0xff);
2545 			cdb[3]    = (u8)((start_blk >> 48) & 0xff);
2546 			cdb[2]    = (u8)((start_blk >> 56) & 0xff);
2547 			break;
2548 		}
2549 	}
2550 }
2551 
2552 /**
2553  * megasas_stream_detect -	stream detection on read and and write IOs
2554  * @instance:		Adapter soft state
2555  * @cmd:		    Command to be prepared
2556  * @io_info:		IO Request info
2557  *
2558  */
2559 
2560 /** stream detection on read and and write IOs */
2561 static void megasas_stream_detect(struct megasas_instance *instance,
2562 				  struct megasas_cmd_fusion *cmd,
2563 				  struct IO_REQUEST_INFO *io_info)
2564 {
2565 	struct fusion_context *fusion = instance->ctrl_context;
2566 	u32 device_id = io_info->ldTgtId;
2567 	struct LD_STREAM_DETECT *current_ld_sd
2568 		= fusion->stream_detect_by_ld[device_id];
2569 	u32 *track_stream = &current_ld_sd->mru_bit_map, stream_num;
2570 	u32 shifted_values, unshifted_values;
2571 	u32 index_value_mask, shifted_values_mask;
2572 	int i;
2573 	bool is_read_ahead = false;
2574 	struct STREAM_DETECT *current_sd;
2575 	/* find possible stream */
2576 	for (i = 0; i < MAX_STREAMS_TRACKED; ++i) {
2577 		stream_num = (*track_stream >>
2578 			(i * BITS_PER_INDEX_STREAM)) &
2579 			STREAM_MASK;
2580 		current_sd = &current_ld_sd->stream_track[stream_num];
2581 		/* if we found a stream, update the raid
2582 		 *  context and also update the mruBitMap
2583 		 */
2584 		/*	boundary condition */
2585 		if ((current_sd->next_seq_lba) &&
2586 		    (io_info->ldStartBlock >= current_sd->next_seq_lba) &&
2587 		    (io_info->ldStartBlock <= (current_sd->next_seq_lba + 32)) &&
2588 		    (current_sd->is_read == io_info->isRead)) {
2589 
2590 			if ((io_info->ldStartBlock != current_sd->next_seq_lba)	&&
2591 			    ((!io_info->isRead) || (!is_read_ahead)))
2592 				/*
2593 				 * Once the API availible we need to change this.
2594 				 * At this point we are not allowing any gap
2595 				 */
2596 				continue;
2597 
2598 			SET_STREAM_DETECTED(cmd->io_request->RaidContext.raid_context_g35);
2599 			current_sd->next_seq_lba =
2600 			io_info->ldStartBlock + io_info->numBlocks;
2601 			/*
2602 			 *	update the mruBitMap LRU
2603 			 */
2604 			shifted_values_mask =
2605 				(1 <<  i * BITS_PER_INDEX_STREAM) - 1;
2606 			shifted_values = ((*track_stream & shifted_values_mask)
2607 						<< BITS_PER_INDEX_STREAM);
2608 			index_value_mask =
2609 				STREAM_MASK << i * BITS_PER_INDEX_STREAM;
2610 			unshifted_values =
2611 				*track_stream & ~(shifted_values_mask |
2612 				index_value_mask);
2613 			*track_stream =
2614 				unshifted_values | shifted_values | stream_num;
2615 			return;
2616 		}
2617 	}
2618 	/*
2619 	 * if we did not find any stream, create a new one
2620 	 * from the least recently used
2621 	 */
2622 	stream_num = (*track_stream >>
2623 		((MAX_STREAMS_TRACKED - 1) * BITS_PER_INDEX_STREAM)) &
2624 		STREAM_MASK;
2625 	current_sd = &current_ld_sd->stream_track[stream_num];
2626 	current_sd->is_read = io_info->isRead;
2627 	current_sd->next_seq_lba = io_info->ldStartBlock + io_info->numBlocks;
2628 	*track_stream = (((*track_stream & ZERO_LAST_STREAM) << 4) | stream_num);
2629 	return;
2630 }
2631 
2632 /**
2633  * megasas_set_raidflag_cpu_affinity - This function sets the cpu
2634  * affinity (cpu of the controller) and raid_flags in the raid context
2635  * based on IO type.
2636  *
2637  * @fusion:		Fusion context
2638  * @praid_context:	IO RAID context
2639  * @raid:		LD raid map
2640  * @fp_possible:	Is fast path possible?
2641  * @is_read:		Is read IO?
2642  * @scsi_buff_len:	SCSI command buffer length
2643  *
2644  */
2645 static void
2646 megasas_set_raidflag_cpu_affinity(struct fusion_context *fusion,
2647 				union RAID_CONTEXT_UNION *praid_context,
2648 				struct MR_LD_RAID *raid, bool fp_possible,
2649 				u8 is_read, u32 scsi_buff_len)
2650 {
2651 	u8 cpu_sel = MR_RAID_CTX_CPUSEL_0;
2652 	struct RAID_CONTEXT_G35 *rctx_g35;
2653 
2654 	rctx_g35 = &praid_context->raid_context_g35;
2655 	if (fp_possible) {
2656 		if (is_read) {
2657 			if ((raid->cpuAffinity.pdRead.cpu0) &&
2658 			    (raid->cpuAffinity.pdRead.cpu1))
2659 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2660 			else if (raid->cpuAffinity.pdRead.cpu1)
2661 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2662 		} else {
2663 			if ((raid->cpuAffinity.pdWrite.cpu0) &&
2664 			    (raid->cpuAffinity.pdWrite.cpu1))
2665 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2666 			else if (raid->cpuAffinity.pdWrite.cpu1)
2667 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2668 			/* Fast path cache by pass capable R0/R1 VD */
2669 			if ((raid->level <= 1) &&
2670 			    (raid->capability.fp_cache_bypass_capable)) {
2671 				rctx_g35->routing_flags |=
2672 					(1 << MR_RAID_CTX_ROUTINGFLAGS_SLD_SHIFT);
2673 				rctx_g35->raid_flags =
2674 					(MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS
2675 					<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2676 			}
2677 		}
2678 	} else {
2679 		if (is_read) {
2680 			if ((raid->cpuAffinity.ldRead.cpu0) &&
2681 			    (raid->cpuAffinity.ldRead.cpu1))
2682 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2683 			else if (raid->cpuAffinity.ldRead.cpu1)
2684 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2685 		} else {
2686 			if ((raid->cpuAffinity.ldWrite.cpu0) &&
2687 			    (raid->cpuAffinity.ldWrite.cpu1))
2688 				cpu_sel = MR_RAID_CTX_CPUSEL_FCFS;
2689 			else if (raid->cpuAffinity.ldWrite.cpu1)
2690 				cpu_sel = MR_RAID_CTX_CPUSEL_1;
2691 
2692 			if (is_stream_detected(rctx_g35) &&
2693 			    ((raid->level == 5) || (raid->level == 6)) &&
2694 			    (raid->writeMode == MR_RL_WRITE_THROUGH_MODE) &&
2695 			    (cpu_sel == MR_RAID_CTX_CPUSEL_FCFS))
2696 				cpu_sel = MR_RAID_CTX_CPUSEL_0;
2697 		}
2698 	}
2699 
2700 	rctx_g35->routing_flags |=
2701 		(cpu_sel << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2702 
2703 	/* Always give priority to MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2704 	 * vs MR_RAID_FLAGS_IO_SUB_TYPE_CACHE_BYPASS.
2705 	 * IO Subtype is not bitmap.
2706 	 */
2707 	if ((fusion->pcie_bw_limitation) && (raid->level == 1) && (!is_read) &&
2708 			(scsi_buff_len > MR_LARGE_IO_MIN_SIZE)) {
2709 		praid_context->raid_context_g35.raid_flags =
2710 			(MR_RAID_FLAGS_IO_SUB_TYPE_LDIO_BW_LIMIT
2711 			<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT);
2712 	}
2713 }
2714 
2715 /**
2716  * megasas_build_ldio_fusion -	Prepares IOs to devices
2717  * @instance:		Adapter soft state
2718  * @scp:		SCSI command
2719  * @cmd:		Command to be prepared
2720  *
2721  * Prepares the io_request and chain elements (sg_frame) for IO
2722  * The IO can be for PD (Fast Path) or LD
2723  */
2724 static void
2725 megasas_build_ldio_fusion(struct megasas_instance *instance,
2726 			  struct scsi_cmnd *scp,
2727 			  struct megasas_cmd_fusion *cmd)
2728 {
2729 	bool fp_possible;
2730 	u16 ld;
2731 	u32 start_lba_lo, start_lba_hi, device_id, datalength = 0;
2732 	u32 scsi_buff_len;
2733 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2734 	struct IO_REQUEST_INFO io_info;
2735 	struct fusion_context *fusion;
2736 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2737 	u8 *raidLUN;
2738 	unsigned long spinlock_flags;
2739 	struct MR_LD_RAID *raid = NULL;
2740 	struct MR_PRIV_DEVICE *mrdev_priv;
2741 	struct RAID_CONTEXT *rctx;
2742 	struct RAID_CONTEXT_G35 *rctx_g35;
2743 
2744 	device_id = MEGASAS_DEV_INDEX(scp);
2745 
2746 	fusion = instance->ctrl_context;
2747 
2748 	io_request = cmd->io_request;
2749 	rctx = &io_request->RaidContext.raid_context;
2750 	rctx_g35 = &io_request->RaidContext.raid_context_g35;
2751 
2752 	rctx->virtual_disk_tgt_id = cpu_to_le16(device_id);
2753 	rctx->status = 0;
2754 	rctx->ex_status = 0;
2755 
2756 	start_lba_lo = 0;
2757 	start_lba_hi = 0;
2758 	fp_possible = false;
2759 
2760 	/*
2761 	 * 6-byte READ(0x08) or WRITE(0x0A) cdb
2762 	 */
2763 	if (scp->cmd_len == 6) {
2764 		datalength = (u32) scp->cmnd[4];
2765 		start_lba_lo = ((u32) scp->cmnd[1] << 16) |
2766 			((u32) scp->cmnd[2] << 8) | (u32) scp->cmnd[3];
2767 
2768 		start_lba_lo &= 0x1FFFFF;
2769 	}
2770 
2771 	/*
2772 	 * 10-byte READ(0x28) or WRITE(0x2A) cdb
2773 	 */
2774 	else if (scp->cmd_len == 10) {
2775 		datalength = (u32) scp->cmnd[8] |
2776 			((u32) scp->cmnd[7] << 8);
2777 		start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2778 			((u32) scp->cmnd[3] << 16) |
2779 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2780 	}
2781 
2782 	/*
2783 	 * 12-byte READ(0xA8) or WRITE(0xAA) cdb
2784 	 */
2785 	else if (scp->cmd_len == 12) {
2786 		datalength = ((u32) scp->cmnd[6] << 24) |
2787 			((u32) scp->cmnd[7] << 16) |
2788 			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2789 		start_lba_lo = ((u32) scp->cmnd[2] << 24) |
2790 			((u32) scp->cmnd[3] << 16) |
2791 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2792 	}
2793 
2794 	/*
2795 	 * 16-byte READ(0x88) or WRITE(0x8A) cdb
2796 	 */
2797 	else if (scp->cmd_len == 16) {
2798 		datalength = ((u32) scp->cmnd[10] << 24) |
2799 			((u32) scp->cmnd[11] << 16) |
2800 			((u32) scp->cmnd[12] << 8) | (u32) scp->cmnd[13];
2801 		start_lba_lo = ((u32) scp->cmnd[6] << 24) |
2802 			((u32) scp->cmnd[7] << 16) |
2803 			((u32) scp->cmnd[8] << 8) | (u32) scp->cmnd[9];
2804 
2805 		start_lba_hi = ((u32) scp->cmnd[2] << 24) |
2806 			((u32) scp->cmnd[3] << 16) |
2807 			((u32) scp->cmnd[4] << 8) | (u32) scp->cmnd[5];
2808 	}
2809 
2810 	memset(&io_info, 0, sizeof(struct IO_REQUEST_INFO));
2811 	io_info.ldStartBlock = ((u64)start_lba_hi << 32) | start_lba_lo;
2812 	io_info.numBlocks = datalength;
2813 	io_info.ldTgtId = device_id;
2814 	io_info.r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2815 	scsi_buff_len = scsi_bufflen(scp);
2816 	io_request->DataLength = cpu_to_le32(scsi_buff_len);
2817 	io_info.data_arms = 1;
2818 
2819 	if (scp->sc_data_direction == DMA_FROM_DEVICE)
2820 		io_info.isRead = 1;
2821 
2822 	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2823 	ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
2824 
2825 	if (ld < instance->fw_supported_vd_count)
2826 		raid = MR_LdRaidGet(ld, local_map_ptr);
2827 
2828 	if (!raid || (!fusion->fast_path_io)) {
2829 		rctx->reg_lock_flags  = 0;
2830 		fp_possible = false;
2831 	} else {
2832 		if (MR_BuildRaidContext(instance, &io_info, rctx,
2833 					local_map_ptr, &raidLUN))
2834 			fp_possible = (io_info.fpOkForIo > 0) ? true : false;
2835 	}
2836 
2837 	megasas_get_msix_index(instance, scp, cmd, io_info.data_arms);
2838 
2839 	if (instance->adapter_type >= VENTURA_SERIES) {
2840 		/* FP for Optimal raid level 1.
2841 		 * All large RAID-1 writes (> 32 KiB, both WT and WB modes)
2842 		 * are built by the driver as LD I/Os.
2843 		 * All small RAID-1 WT writes (<= 32 KiB) are built as FP I/Os
2844 		 * (there is never a reason to process these as buffered writes)
2845 		 * All small RAID-1 WB writes (<= 32 KiB) are built as FP I/Os
2846 		 * with the SLD bit asserted.
2847 		 */
2848 		if (io_info.r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
2849 			mrdev_priv = scp->device->hostdata;
2850 
2851 			if (atomic_inc_return(&instance->fw_outstanding) >
2852 				(instance->host->can_queue)) {
2853 				fp_possible = false;
2854 				atomic_dec(&instance->fw_outstanding);
2855 			} else if (fusion->pcie_bw_limitation &&
2856 				((scsi_buff_len > MR_LARGE_IO_MIN_SIZE) ||
2857 				   (atomic_dec_if_positive(&mrdev_priv->r1_ldio_hint) > 0))) {
2858 				fp_possible = false;
2859 				atomic_dec(&instance->fw_outstanding);
2860 				if (scsi_buff_len > MR_LARGE_IO_MIN_SIZE)
2861 					atomic_set(&mrdev_priv->r1_ldio_hint,
2862 						   instance->r1_ldio_hint_default);
2863 			}
2864 		}
2865 
2866 		if (!fp_possible ||
2867 		    (io_info.isRead && io_info.ra_capable)) {
2868 			spin_lock_irqsave(&instance->stream_lock,
2869 					  spinlock_flags);
2870 			megasas_stream_detect(instance, cmd, &io_info);
2871 			spin_unlock_irqrestore(&instance->stream_lock,
2872 					       spinlock_flags);
2873 			/* In ventura if stream detected for a read and it is
2874 			 * read ahead capable make this IO as LDIO
2875 			 */
2876 			if (is_stream_detected(rctx_g35))
2877 				fp_possible = false;
2878 		}
2879 
2880 		/* If raid is NULL, set CPU affinity to default CPU0 */
2881 		if (raid)
2882 			megasas_set_raidflag_cpu_affinity(fusion, &io_request->RaidContext,
2883 				raid, fp_possible, io_info.isRead,
2884 				scsi_buff_len);
2885 		else
2886 			rctx_g35->routing_flags |=
2887 				(MR_RAID_CTX_CPUSEL_0 << MR_RAID_CTX_ROUTINGFLAGS_CPUSEL_SHIFT);
2888 	}
2889 
2890 	if (fp_possible) {
2891 		megasas_set_pd_lba(io_request, scp->cmd_len, &io_info, scp,
2892 				   local_map_ptr, start_lba_lo);
2893 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
2894 		cmd->request_desc->SCSIIO.RequestFlags =
2895 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO
2896 			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2897 		if (instance->adapter_type == INVADER_SERIES) {
2898 			rctx->type = MPI2_TYPE_CUDA;
2899 			rctx->nseg = 0x1;
2900 			io_request->IoFlags |= cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2901 			rctx->reg_lock_flags |=
2902 			  (MR_RL_FLAGS_GRANT_DESTINATION_CUDA |
2903 			   MR_RL_FLAGS_SEQ_NUM_ENABLE);
2904 		} else if (instance->adapter_type >= VENTURA_SERIES) {
2905 			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2906 			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2907 			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2908 			io_request->IoFlags |=
2909 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
2910 		}
2911 		if (fusion->load_balance_info &&
2912 			(fusion->load_balance_info[device_id].loadBalanceFlag) &&
2913 			(io_info.isRead)) {
2914 			io_info.devHandle =
2915 				get_updated_dev_handle(instance,
2916 					&fusion->load_balance_info[device_id],
2917 					&io_info, local_map_ptr);
2918 			scp->SCp.Status |= MEGASAS_LOAD_BALANCE_FLAG;
2919 			cmd->pd_r1_lb = io_info.pd_after_lb;
2920 			if (instance->adapter_type >= VENTURA_SERIES)
2921 				rctx_g35->span_arm = io_info.span_arm;
2922 			else
2923 				rctx->span_arm = io_info.span_arm;
2924 
2925 		} else
2926 			scp->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
2927 
2928 		if (instance->adapter_type >= VENTURA_SERIES)
2929 			cmd->r1_alt_dev_handle = io_info.r1_alt_dev_handle;
2930 		else
2931 			cmd->r1_alt_dev_handle = MR_DEVHANDLE_INVALID;
2932 
2933 		if ((raidLUN[0] == 1) &&
2934 			(local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].validHandles > 1)) {
2935 			instance->dev_handle = !(instance->dev_handle);
2936 			io_info.devHandle =
2937 				local_map_ptr->raidMap.devHndlInfo[io_info.pd_after_lb].devHandle[instance->dev_handle];
2938 		}
2939 
2940 		cmd->request_desc->SCSIIO.DevHandle = io_info.devHandle;
2941 		io_request->DevHandle = io_info.devHandle;
2942 		cmd->pd_interface = io_info.pd_interface;
2943 		/* populate the LUN field */
2944 		memcpy(io_request->LUN, raidLUN, 8);
2945 	} else {
2946 		rctx->timeout_value =
2947 			cpu_to_le16(local_map_ptr->raidMap.fpPdIoTimeoutSec);
2948 		cmd->request_desc->SCSIIO.RequestFlags =
2949 			(MEGASAS_REQ_DESCRIPT_FLAGS_LD_IO
2950 			 << MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2951 		if (instance->adapter_type == INVADER_SERIES) {
2952 			if (io_info.do_fp_rlbypass ||
2953 			(rctx->reg_lock_flags == REGION_TYPE_UNUSED))
2954 				cmd->request_desc->SCSIIO.RequestFlags =
2955 					(MEGASAS_REQ_DESCRIPT_FLAGS_NO_LOCK <<
2956 					MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
2957 			rctx->type = MPI2_TYPE_CUDA;
2958 			rctx->reg_lock_flags |=
2959 				(MR_RL_FLAGS_GRANT_DESTINATION_CPU0 |
2960 					MR_RL_FLAGS_SEQ_NUM_ENABLE);
2961 			rctx->nseg = 0x1;
2962 		} else if (instance->adapter_type >= VENTURA_SERIES) {
2963 			rctx_g35->routing_flags |= (1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
2964 			rctx_g35->nseg_type |= (1 << RAID_CONTEXT_NSEG_SHIFT);
2965 			rctx_g35->nseg_type |= (MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
2966 		}
2967 		io_request->Function = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
2968 		io_request->DevHandle = cpu_to_le16(device_id);
2969 
2970 	} /* Not FP */
2971 }
2972 
2973 /**
2974  * megasas_build_ld_nonrw_fusion - prepares non rw ios for virtual disk
2975  * @instance:		Adapter soft state
2976  * @scmd:		SCSI command
2977  * @cmd:		Command to be prepared
2978  *
2979  * Prepares the io_request frame for non-rw io cmds for vd.
2980  */
2981 static void megasas_build_ld_nonrw_fusion(struct megasas_instance *instance,
2982 			  struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd)
2983 {
2984 	u32 device_id;
2985 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
2986 	u16 ld;
2987 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
2988 	struct fusion_context *fusion = instance->ctrl_context;
2989 	u8                          span, physArm;
2990 	__le16                      devHandle;
2991 	u32                         arRef, pd;
2992 	struct MR_LD_RAID                  *raid;
2993 	struct RAID_CONTEXT                *pRAID_Context;
2994 	u8 fp_possible = 1;
2995 
2996 	io_request = cmd->io_request;
2997 	device_id = MEGASAS_DEV_INDEX(scmd);
2998 	local_map_ptr = fusion->ld_drv_map[(instance->map_id & 1)];
2999 	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3000 	/* get RAID_Context pointer */
3001 	pRAID_Context = &io_request->RaidContext.raid_context;
3002 	/* Check with FW team */
3003 	pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3004 	pRAID_Context->reg_lock_row_lba    = 0;
3005 	pRAID_Context->reg_lock_length    = 0;
3006 
3007 	if (fusion->fast_path_io && (
3008 		device_id < instance->fw_supported_vd_count)) {
3009 
3010 		ld = MR_TargetIdToLdGet(device_id, local_map_ptr);
3011 		if (ld >= instance->fw_supported_vd_count - 1)
3012 			fp_possible = 0;
3013 		else {
3014 			raid = MR_LdRaidGet(ld, local_map_ptr);
3015 			if (!(raid->capability.fpNonRWCapable))
3016 				fp_possible = 0;
3017 		}
3018 	} else
3019 		fp_possible = 0;
3020 
3021 	if (!fp_possible) {
3022 		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3023 		io_request->DevHandle = cpu_to_le16(device_id);
3024 		io_request->LUN[1] = scmd->device->lun;
3025 		pRAID_Context->timeout_value =
3026 			cpu_to_le16 (scmd->request->timeout / HZ);
3027 		cmd->request_desc->SCSIIO.RequestFlags =
3028 			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3029 			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3030 	} else {
3031 
3032 		/* set RAID context values */
3033 		pRAID_Context->config_seq_num = raid->seqNum;
3034 		if (instance->adapter_type < VENTURA_SERIES)
3035 			pRAID_Context->reg_lock_flags = REGION_TYPE_SHARED_READ;
3036 		pRAID_Context->timeout_value =
3037 			cpu_to_le16(raid->fpIoTimeoutForLd);
3038 
3039 		/* get the DevHandle for the PD (since this is
3040 		   fpNonRWCapable, this is a single disk RAID0) */
3041 		span = physArm = 0;
3042 		arRef = MR_LdSpanArrayGet(ld, span, local_map_ptr);
3043 		pd = MR_ArPdGet(arRef, physArm, local_map_ptr);
3044 		devHandle = MR_PdDevHandleGet(pd, local_map_ptr);
3045 
3046 		/* build request descriptor */
3047 		cmd->request_desc->SCSIIO.RequestFlags =
3048 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3049 			MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3050 		cmd->request_desc->SCSIIO.DevHandle = devHandle;
3051 
3052 		/* populate the LUN field */
3053 		memcpy(io_request->LUN, raid->LUN, 8);
3054 
3055 		/* build the raidScsiIO structure */
3056 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3057 		io_request->DevHandle = devHandle;
3058 	}
3059 }
3060 
3061 /**
3062  * megasas_build_syspd_fusion - prepares rw/non-rw ios for syspd
3063  * @instance:		Adapter soft state
3064  * @scmd:		SCSI command
3065  * @cmd:		Command to be prepared
3066  * @fp_possible:	parameter to detect fast path or firmware path io.
3067  *
3068  * Prepares the io_request frame for rw/non-rw io cmds for syspds
3069  */
3070 static void
3071 megasas_build_syspd_fusion(struct megasas_instance *instance,
3072 	struct scsi_cmnd *scmd, struct megasas_cmd_fusion *cmd,
3073 	bool fp_possible)
3074 {
3075 	u32 device_id;
3076 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request;
3077 	u16 pd_index = 0;
3078 	u16 os_timeout_value;
3079 	u16 timeout_limit;
3080 	struct MR_DRV_RAID_MAP_ALL *local_map_ptr;
3081 	struct RAID_CONTEXT	*pRAID_Context;
3082 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
3083 	struct MR_PRIV_DEVICE *mr_device_priv_data;
3084 	struct fusion_context *fusion = instance->ctrl_context;
3085 	pd_sync = (void *)fusion->pd_seq_sync[(instance->pd_seq_map_id - 1) & 1];
3086 
3087 	device_id = MEGASAS_DEV_INDEX(scmd);
3088 	pd_index = MEGASAS_PD_INDEX(scmd);
3089 	os_timeout_value = scmd->request->timeout / HZ;
3090 	mr_device_priv_data = scmd->device->hostdata;
3091 	cmd->pd_interface = mr_device_priv_data->interface_type;
3092 
3093 	io_request = cmd->io_request;
3094 	/* get RAID_Context pointer */
3095 	pRAID_Context = &io_request->RaidContext.raid_context;
3096 	pRAID_Context->reg_lock_flags = 0;
3097 	pRAID_Context->reg_lock_row_lba = 0;
3098 	pRAID_Context->reg_lock_length = 0;
3099 	io_request->DataLength = cpu_to_le32(scsi_bufflen(scmd));
3100 	io_request->LUN[1] = scmd->device->lun;
3101 	pRAID_Context->raid_flags = MR_RAID_FLAGS_IO_SUB_TYPE_SYSTEM_PD
3102 		<< MR_RAID_CTX_RAID_FLAGS_IO_SUB_TYPE_SHIFT;
3103 
3104 	/* If FW supports PD sequence number */
3105 	if (instance->support_seqnum_jbod_fp) {
3106 		if (instance->use_seqnum_jbod_fp &&
3107 			instance->pd_list[pd_index].driveType == TYPE_DISK) {
3108 
3109 			/* More than 256 PD/JBOD support for Ventura */
3110 			if (instance->support_morethan256jbod)
3111 				pRAID_Context->virtual_disk_tgt_id =
3112 					pd_sync->seq[pd_index].pd_target_id;
3113 			else
3114 				pRAID_Context->virtual_disk_tgt_id =
3115 					cpu_to_le16(device_id +
3116 					(MAX_PHYSICAL_DEVICES - 1));
3117 			pRAID_Context->config_seq_num =
3118 				pd_sync->seq[pd_index].seqNum;
3119 			io_request->DevHandle =
3120 				pd_sync->seq[pd_index].devHandle;
3121 			if (instance->adapter_type >= VENTURA_SERIES) {
3122 				io_request->RaidContext.raid_context_g35.routing_flags |=
3123 					(1 << MR_RAID_CTX_ROUTINGFLAGS_SQN_SHIFT);
3124 				io_request->RaidContext.raid_context_g35.nseg_type |=
3125 					(1 << RAID_CONTEXT_NSEG_SHIFT);
3126 				io_request->RaidContext.raid_context_g35.nseg_type |=
3127 					(MPI2_TYPE_CUDA << RAID_CONTEXT_TYPE_SHIFT);
3128 			} else {
3129 				pRAID_Context->type = MPI2_TYPE_CUDA;
3130 				pRAID_Context->nseg = 0x1;
3131 				pRAID_Context->reg_lock_flags |=
3132 					(MR_RL_FLAGS_SEQ_NUM_ENABLE |
3133 					 MR_RL_FLAGS_GRANT_DESTINATION_CUDA);
3134 			}
3135 		} else {
3136 			pRAID_Context->virtual_disk_tgt_id =
3137 				cpu_to_le16(device_id +
3138 				(MAX_PHYSICAL_DEVICES - 1));
3139 			pRAID_Context->config_seq_num = 0;
3140 			io_request->DevHandle = cpu_to_le16(0xFFFF);
3141 		}
3142 	} else {
3143 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3144 		pRAID_Context->config_seq_num = 0;
3145 
3146 		if (fusion->fast_path_io) {
3147 			local_map_ptr =
3148 				fusion->ld_drv_map[(instance->map_id & 1)];
3149 			io_request->DevHandle =
3150 				local_map_ptr->raidMap.devHndlInfo[device_id].curDevHdl;
3151 		} else {
3152 			io_request->DevHandle = cpu_to_le16(0xFFFF);
3153 		}
3154 	}
3155 
3156 	cmd->request_desc->SCSIIO.DevHandle = io_request->DevHandle;
3157 
3158 	megasas_get_msix_index(instance, scmd, cmd, 1);
3159 
3160 	if (!fp_possible) {
3161 		/* system pd firmware path */
3162 		io_request->Function  = MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST;
3163 		cmd->request_desc->SCSIIO.RequestFlags =
3164 			(MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3165 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3166 		pRAID_Context->timeout_value = cpu_to_le16(os_timeout_value);
3167 		pRAID_Context->virtual_disk_tgt_id = cpu_to_le16(device_id);
3168 	} else {
3169 		if (os_timeout_value)
3170 			os_timeout_value++;
3171 
3172 		/* system pd Fast Path */
3173 		io_request->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
3174 		timeout_limit = (scmd->device->type == TYPE_DISK) ?
3175 				255 : 0xFFFF;
3176 		pRAID_Context->timeout_value =
3177 			cpu_to_le16((os_timeout_value > timeout_limit) ?
3178 			timeout_limit : os_timeout_value);
3179 		if (instance->adapter_type >= INVADER_SERIES)
3180 			io_request->IoFlags |=
3181 				cpu_to_le16(MPI25_SAS_DEVICE0_FLAGS_ENABLED_FAST_PATH);
3182 
3183 		cmd->request_desc->SCSIIO.RequestFlags =
3184 			(MPI2_REQ_DESCRIPT_FLAGS_FP_IO <<
3185 				MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3186 	}
3187 }
3188 
3189 /**
3190  * megasas_build_io_fusion -	Prepares IOs to devices
3191  * @instance:		Adapter soft state
3192  * @scp:		SCSI command
3193  * @cmd:		Command to be prepared
3194  *
3195  * Invokes helper functions to prepare request frames
3196  * and sets flags appropriate for IO/Non-IO cmd
3197  */
3198 static int
3199 megasas_build_io_fusion(struct megasas_instance *instance,
3200 			struct scsi_cmnd *scp,
3201 			struct megasas_cmd_fusion *cmd)
3202 {
3203 	int sge_count;
3204 	u8  cmd_type;
3205 	u16 pd_index = 0;
3206 	u8 drive_type = 0;
3207 	struct MPI2_RAID_SCSI_IO_REQUEST *io_request = cmd->io_request;
3208 	struct MR_PRIV_DEVICE *mr_device_priv_data;
3209 	mr_device_priv_data = scp->device->hostdata;
3210 
3211 	/* Zero out some fields so they don't get reused */
3212 	memset(io_request->LUN, 0x0, 8);
3213 	io_request->CDB.EEDP32.PrimaryReferenceTag = 0;
3214 	io_request->CDB.EEDP32.PrimaryApplicationTagMask = 0;
3215 	io_request->EEDPFlags = 0;
3216 	io_request->Control = 0;
3217 	io_request->EEDPBlockSize = 0;
3218 	io_request->ChainOffset = 0;
3219 	io_request->RaidContext.raid_context.raid_flags = 0;
3220 	io_request->RaidContext.raid_context.type = 0;
3221 	io_request->RaidContext.raid_context.nseg = 0;
3222 
3223 	memcpy(io_request->CDB.CDB32, scp->cmnd, scp->cmd_len);
3224 	/*
3225 	 * Just the CDB length,rest of the Flags are zero
3226 	 * This will be modified for FP in build_ldio_fusion
3227 	 */
3228 	io_request->IoFlags = cpu_to_le16(scp->cmd_len);
3229 
3230 	switch (cmd_type = megasas_cmd_type(scp)) {
3231 	case READ_WRITE_LDIO:
3232 		megasas_build_ldio_fusion(instance, scp, cmd);
3233 		break;
3234 	case NON_READ_WRITE_LDIO:
3235 		megasas_build_ld_nonrw_fusion(instance, scp, cmd);
3236 		break;
3237 	case READ_WRITE_SYSPDIO:
3238 		megasas_build_syspd_fusion(instance, scp, cmd, true);
3239 		break;
3240 	case NON_READ_WRITE_SYSPDIO:
3241 		pd_index = MEGASAS_PD_INDEX(scp);
3242 		drive_type = instance->pd_list[pd_index].driveType;
3243 		if ((instance->secure_jbod_support ||
3244 		     mr_device_priv_data->is_tm_capable) ||
3245 		     (instance->adapter_type >= VENTURA_SERIES &&
3246 		     drive_type == TYPE_ENCLOSURE))
3247 			megasas_build_syspd_fusion(instance, scp, cmd, false);
3248 		else
3249 			megasas_build_syspd_fusion(instance, scp, cmd, true);
3250 		break;
3251 	default:
3252 		break;
3253 	}
3254 
3255 	/*
3256 	 * Construct SGL
3257 	 */
3258 
3259 	sge_count = megasas_make_sgl(instance, scp, cmd);
3260 
3261 	if (sge_count > instance->max_num_sge || (sge_count < 0)) {
3262 		dev_err(&instance->pdev->dev,
3263 			"%s %d sge_count (%d) is out of range. Range is:  0-%d\n",
3264 			__func__, __LINE__, sge_count, instance->max_num_sge);
3265 		return 1;
3266 	}
3267 
3268 	if (instance->adapter_type >= VENTURA_SERIES) {
3269 		set_num_sge(&io_request->RaidContext.raid_context_g35, sge_count);
3270 		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.routing_flags);
3271 		cpu_to_le16s(&io_request->RaidContext.raid_context_g35.nseg_type);
3272 	} else {
3273 		/* numSGE store lower 8 bit of sge_count.
3274 		 * numSGEExt store higher 8 bit of sge_count
3275 		 */
3276 		io_request->RaidContext.raid_context.num_sge = sge_count;
3277 		io_request->RaidContext.raid_context.num_sge_ext =
3278 			(u8)(sge_count >> 8);
3279 	}
3280 
3281 	io_request->SGLFlags = cpu_to_le16(MPI2_SGE_FLAGS_64_BIT_ADDRESSING);
3282 
3283 	if (scp->sc_data_direction == DMA_TO_DEVICE)
3284 		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_WRITE);
3285 	else if (scp->sc_data_direction == DMA_FROM_DEVICE)
3286 		io_request->Control |= cpu_to_le32(MPI2_SCSIIO_CONTROL_READ);
3287 
3288 	io_request->SGLOffset0 =
3289 		offsetof(struct MPI2_RAID_SCSI_IO_REQUEST, SGL) / 4;
3290 
3291 	io_request->SenseBufferLowAddress =
3292 		cpu_to_le32(lower_32_bits(cmd->sense_phys_addr));
3293 	io_request->SenseBufferLength = SCSI_SENSE_BUFFERSIZE;
3294 
3295 	cmd->scmd = scp;
3296 	scp->SCp.ptr = (char *)cmd;
3297 
3298 	return 0;
3299 }
3300 
3301 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3302 megasas_get_request_descriptor(struct megasas_instance *instance, u16 index)
3303 {
3304 	u8 *p;
3305 	struct fusion_context *fusion;
3306 
3307 	fusion = instance->ctrl_context;
3308 	p = fusion->req_frames_desc +
3309 		sizeof(union MEGASAS_REQUEST_DESCRIPTOR_UNION) * index;
3310 
3311 	return (union MEGASAS_REQUEST_DESCRIPTOR_UNION *)p;
3312 }
3313 
3314 
3315 /* megasas_prepate_secondRaid1_IO
3316  *  It prepares the raid 1 second IO
3317  */
3318 static void megasas_prepare_secondRaid1_IO(struct megasas_instance *instance,
3319 					   struct megasas_cmd_fusion *cmd,
3320 					   struct megasas_cmd_fusion *r1_cmd)
3321 {
3322 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc, *req_desc2 = NULL;
3323 	struct fusion_context *fusion;
3324 	fusion = instance->ctrl_context;
3325 	req_desc = cmd->request_desc;
3326 	/* copy the io request frame as well as 8 SGEs data for r1 command*/
3327 	memcpy(r1_cmd->io_request, cmd->io_request,
3328 	       (sizeof(struct MPI2_RAID_SCSI_IO_REQUEST)));
3329 	memcpy(&r1_cmd->io_request->SGL, &cmd->io_request->SGL,
3330 	       (fusion->max_sge_in_main_msg * sizeof(union MPI2_SGE_IO_UNION)));
3331 	/*sense buffer is different for r1 command*/
3332 	r1_cmd->io_request->SenseBufferLowAddress =
3333 			cpu_to_le32(lower_32_bits(r1_cmd->sense_phys_addr));
3334 	r1_cmd->scmd = cmd->scmd;
3335 	req_desc2 = megasas_get_request_descriptor(instance,
3336 						   (r1_cmd->index - 1));
3337 	req_desc2->Words = 0;
3338 	r1_cmd->request_desc = req_desc2;
3339 	req_desc2->SCSIIO.SMID = cpu_to_le16(r1_cmd->index);
3340 	req_desc2->SCSIIO.RequestFlags = req_desc->SCSIIO.RequestFlags;
3341 	r1_cmd->request_desc->SCSIIO.DevHandle = cmd->r1_alt_dev_handle;
3342 	r1_cmd->io_request->DevHandle = cmd->r1_alt_dev_handle;
3343 	r1_cmd->r1_alt_dev_handle = cmd->io_request->DevHandle;
3344 	cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3345 			cpu_to_le16(r1_cmd->index);
3346 	r1_cmd->io_request->RaidContext.raid_context_g35.flow_specific.peer_smid =
3347 			cpu_to_le16(cmd->index);
3348 	/*MSIxIndex of both commands request descriptors should be same*/
3349 	r1_cmd->request_desc->SCSIIO.MSIxIndex =
3350 			cmd->request_desc->SCSIIO.MSIxIndex;
3351 	/*span arm is different for r1 cmd*/
3352 	r1_cmd->io_request->RaidContext.raid_context_g35.span_arm =
3353 			cmd->io_request->RaidContext.raid_context_g35.span_arm + 1;
3354 }
3355 
3356 /**
3357  * megasas_build_and_issue_cmd_fusion -Main routine for building and
3358  *                                     issuing non IOCTL cmd
3359  * @instance:			Adapter soft state
3360  * @scmd:			pointer to scsi cmd from OS
3361  */
3362 static u32
3363 megasas_build_and_issue_cmd_fusion(struct megasas_instance *instance,
3364 				   struct scsi_cmnd *scmd)
3365 {
3366 	struct megasas_cmd_fusion *cmd, *r1_cmd = NULL;
3367 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3368 	u32 index;
3369 
3370 	if ((megasas_cmd_type(scmd) == READ_WRITE_LDIO) &&
3371 		instance->ldio_threshold &&
3372 		(atomic_inc_return(&instance->ldio_outstanding) >
3373 		instance->ldio_threshold)) {
3374 		atomic_dec(&instance->ldio_outstanding);
3375 		return SCSI_MLQUEUE_DEVICE_BUSY;
3376 	}
3377 
3378 	if (atomic_inc_return(&instance->fw_outstanding) >
3379 			instance->host->can_queue) {
3380 		atomic_dec(&instance->fw_outstanding);
3381 		return SCSI_MLQUEUE_HOST_BUSY;
3382 	}
3383 
3384 	cmd = megasas_get_cmd_fusion(instance, scmd->request->tag);
3385 
3386 	if (!cmd) {
3387 		atomic_dec(&instance->fw_outstanding);
3388 		return SCSI_MLQUEUE_HOST_BUSY;
3389 	}
3390 
3391 	index = cmd->index;
3392 
3393 	req_desc = megasas_get_request_descriptor(instance, index-1);
3394 
3395 	req_desc->Words = 0;
3396 	cmd->request_desc = req_desc;
3397 
3398 	if (megasas_build_io_fusion(instance, scmd, cmd)) {
3399 		megasas_return_cmd_fusion(instance, cmd);
3400 		dev_err(&instance->pdev->dev, "Error building command\n");
3401 		cmd->request_desc = NULL;
3402 		atomic_dec(&instance->fw_outstanding);
3403 		return SCSI_MLQUEUE_HOST_BUSY;
3404 	}
3405 
3406 	req_desc = cmd->request_desc;
3407 	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3408 
3409 	if (cmd->io_request->ChainOffset != 0 &&
3410 	    cmd->io_request->ChainOffset != 0xF)
3411 		dev_err(&instance->pdev->dev, "The chain offset value is not "
3412 		       "correct : %x\n", cmd->io_request->ChainOffset);
3413 	/*
3414 	 *	if it is raid 1/10 fp write capable.
3415 	 *	try to get second command from pool and construct it.
3416 	 *	From FW, it has confirmed that lba values of two PDs
3417 	 *	corresponds to single R1/10 LD are always same
3418 	 *
3419 	 */
3420 	/*	driver side count always should be less than max_fw_cmds
3421 	 *	to get new command
3422 	 */
3423 	if (cmd->r1_alt_dev_handle != MR_DEVHANDLE_INVALID) {
3424 		r1_cmd = megasas_get_cmd_fusion(instance,
3425 				(scmd->request->tag + instance->max_fw_cmds));
3426 		megasas_prepare_secondRaid1_IO(instance, cmd, r1_cmd);
3427 	}
3428 
3429 
3430 	/*
3431 	 * Issue the command to the FW
3432 	 */
3433 
3434 	megasas_sdev_busy_inc(instance, scmd);
3435 	megasas_fire_cmd_fusion(instance, req_desc);
3436 
3437 	if (r1_cmd)
3438 		megasas_fire_cmd_fusion(instance, r1_cmd->request_desc);
3439 
3440 
3441 	return 0;
3442 }
3443 
3444 /**
3445  * megasas_complete_r1_command -
3446  * completes R1 FP write commands which has valid peer smid
3447  * @instance:			Adapter soft state
3448  * @cmd:			MPT command frame
3449  *
3450  */
3451 static inline void
3452 megasas_complete_r1_command(struct megasas_instance *instance,
3453 			    struct megasas_cmd_fusion *cmd)
3454 {
3455 	u8 *sense, status, ex_status;
3456 	u32 data_length;
3457 	u16 peer_smid;
3458 	struct fusion_context *fusion;
3459 	struct megasas_cmd_fusion *r1_cmd = NULL;
3460 	struct scsi_cmnd *scmd_local = NULL;
3461 	struct RAID_CONTEXT_G35 *rctx_g35;
3462 
3463 	rctx_g35 = &cmd->io_request->RaidContext.raid_context_g35;
3464 	fusion = instance->ctrl_context;
3465 	peer_smid = le16_to_cpu(rctx_g35->flow_specific.peer_smid);
3466 
3467 	r1_cmd = fusion->cmd_list[peer_smid - 1];
3468 	scmd_local = cmd->scmd;
3469 	status = rctx_g35->status;
3470 	ex_status = rctx_g35->ex_status;
3471 	data_length = cmd->io_request->DataLength;
3472 	sense = cmd->sense;
3473 
3474 	cmd->cmd_completed = true;
3475 
3476 	/* Check if peer command is completed or not*/
3477 	if (r1_cmd->cmd_completed) {
3478 		rctx_g35 = &r1_cmd->io_request->RaidContext.raid_context_g35;
3479 		if (rctx_g35->status != MFI_STAT_OK) {
3480 			status = rctx_g35->status;
3481 			ex_status = rctx_g35->ex_status;
3482 			data_length = r1_cmd->io_request->DataLength;
3483 			sense = r1_cmd->sense;
3484 		}
3485 
3486 		megasas_return_cmd_fusion(instance, r1_cmd);
3487 		map_cmd_status(fusion, scmd_local, status, ex_status,
3488 			       le32_to_cpu(data_length), sense);
3489 		if (instance->ldio_threshold &&
3490 		    megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
3491 			atomic_dec(&instance->ldio_outstanding);
3492 		scmd_local->SCp.ptr = NULL;
3493 		megasas_return_cmd_fusion(instance, cmd);
3494 		scsi_dma_unmap(scmd_local);
3495 		megasas_sdev_busy_dec(instance, scmd_local);
3496 		scmd_local->scsi_done(scmd_local);
3497 	}
3498 }
3499 
3500 /**
3501  * complete_cmd_fusion -	Completes command
3502  * @instance:			Adapter soft state
3503  * @MSIxIndex:			MSI number
3504  * @irq_context:		IRQ context
3505  *
3506  * Completes all commands that is in reply descriptor queue
3507  */
3508 static int
3509 complete_cmd_fusion(struct megasas_instance *instance, u32 MSIxIndex,
3510 		    struct megasas_irq_context *irq_context)
3511 {
3512 	union MPI2_REPLY_DESCRIPTORS_UNION *desc;
3513 	struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *reply_desc;
3514 	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
3515 	struct fusion_context *fusion;
3516 	struct megasas_cmd *cmd_mfi;
3517 	struct megasas_cmd_fusion *cmd_fusion;
3518 	u16 smid, num_completed;
3519 	u8 reply_descript_type, *sense, status, extStatus;
3520 	u32 device_id, data_length;
3521 	union desc_value d_val;
3522 	struct LD_LOAD_BALANCE_INFO *lbinfo;
3523 	int threshold_reply_count = 0;
3524 	struct scsi_cmnd *scmd_local = NULL;
3525 	struct MR_TASK_MANAGE_REQUEST *mr_tm_req;
3526 	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_tm_req;
3527 
3528 	fusion = instance->ctrl_context;
3529 
3530 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3531 		return IRQ_HANDLED;
3532 
3533 	desc = fusion->reply_frames_desc[MSIxIndex] +
3534 				fusion->last_reply_idx[MSIxIndex];
3535 
3536 	reply_desc = (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3537 
3538 	d_val.word = desc->Words;
3539 
3540 	reply_descript_type = reply_desc->ReplyFlags &
3541 		MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3542 
3543 	if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3544 		return IRQ_NONE;
3545 
3546 	if (irq_context && !atomic_add_unless(&irq_context->in_used, 1, 1))
3547 		return 0;
3548 
3549 	num_completed = 0;
3550 
3551 	while (d_val.u.low != cpu_to_le32(UINT_MAX) &&
3552 	       d_val.u.high != cpu_to_le32(UINT_MAX)) {
3553 
3554 		smid = le16_to_cpu(reply_desc->SMID);
3555 		cmd_fusion = fusion->cmd_list[smid - 1];
3556 		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
3557 						cmd_fusion->io_request;
3558 
3559 		scmd_local = cmd_fusion->scmd;
3560 		status = scsi_io_req->RaidContext.raid_context.status;
3561 		extStatus = scsi_io_req->RaidContext.raid_context.ex_status;
3562 		sense = cmd_fusion->sense;
3563 		data_length = scsi_io_req->DataLength;
3564 
3565 		switch (scsi_io_req->Function) {
3566 		case MPI2_FUNCTION_SCSI_TASK_MGMT:
3567 			mr_tm_req = (struct MR_TASK_MANAGE_REQUEST *)
3568 						cmd_fusion->io_request;
3569 			mpi_tm_req = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *)
3570 						&mr_tm_req->TmRequest;
3571 			dev_dbg(&instance->pdev->dev, "TM completion:"
3572 				"type: 0x%x TaskMID: 0x%x\n",
3573 				mpi_tm_req->TaskType, mpi_tm_req->TaskMID);
3574 			complete(&cmd_fusion->done);
3575 			break;
3576 		case MPI2_FUNCTION_SCSI_IO_REQUEST:  /*Fast Path IO.*/
3577 			/* Update load balancing info */
3578 			if (fusion->load_balance_info &&
3579 			    (cmd_fusion->scmd->SCp.Status &
3580 			    MEGASAS_LOAD_BALANCE_FLAG)) {
3581 				device_id = MEGASAS_DEV_INDEX(scmd_local);
3582 				lbinfo = &fusion->load_balance_info[device_id];
3583 				atomic_dec(&lbinfo->scsi_pending_cmds[cmd_fusion->pd_r1_lb]);
3584 				cmd_fusion->scmd->SCp.Status &= ~MEGASAS_LOAD_BALANCE_FLAG;
3585 			}
3586 			fallthrough;	/* and complete IO */
3587 		case MEGASAS_MPI2_FUNCTION_LD_IO_REQUEST: /* LD-IO Path */
3588 			atomic_dec(&instance->fw_outstanding);
3589 			if (cmd_fusion->r1_alt_dev_handle == MR_DEVHANDLE_INVALID) {
3590 				map_cmd_status(fusion, scmd_local, status,
3591 					       extStatus, le32_to_cpu(data_length),
3592 					       sense);
3593 				if (instance->ldio_threshold &&
3594 				    (megasas_cmd_type(scmd_local) == READ_WRITE_LDIO))
3595 					atomic_dec(&instance->ldio_outstanding);
3596 				scmd_local->SCp.ptr = NULL;
3597 				megasas_return_cmd_fusion(instance, cmd_fusion);
3598 				scsi_dma_unmap(scmd_local);
3599 				megasas_sdev_busy_dec(instance, scmd_local);
3600 				scmd_local->scsi_done(scmd_local);
3601 			} else	/* Optimal VD - R1 FP command completion. */
3602 				megasas_complete_r1_command(instance, cmd_fusion);
3603 			break;
3604 		case MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST: /*MFI command */
3605 			cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
3606 			/* Poll mode. Dummy free.
3607 			 * In case of Interrupt mode, caller has reverse check.
3608 			 */
3609 			if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
3610 				cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
3611 				megasas_return_cmd(instance, cmd_mfi);
3612 			} else
3613 				megasas_complete_cmd(instance, cmd_mfi, DID_OK);
3614 			break;
3615 		}
3616 
3617 		fusion->last_reply_idx[MSIxIndex]++;
3618 		if (fusion->last_reply_idx[MSIxIndex] >=
3619 		    fusion->reply_q_depth)
3620 			fusion->last_reply_idx[MSIxIndex] = 0;
3621 
3622 		desc->Words = cpu_to_le64(ULLONG_MAX);
3623 		num_completed++;
3624 		threshold_reply_count++;
3625 
3626 		/* Get the next reply descriptor */
3627 		if (!fusion->last_reply_idx[MSIxIndex])
3628 			desc = fusion->reply_frames_desc[MSIxIndex];
3629 		else
3630 			desc++;
3631 
3632 		reply_desc =
3633 		  (struct MPI2_SCSI_IO_SUCCESS_REPLY_DESCRIPTOR *)desc;
3634 
3635 		d_val.word = desc->Words;
3636 
3637 		reply_descript_type = reply_desc->ReplyFlags &
3638 			MPI2_RPY_DESCRIPT_FLAGS_TYPE_MASK;
3639 
3640 		if (reply_descript_type == MPI2_RPY_DESCRIPT_FLAGS_UNUSED)
3641 			break;
3642 		/*
3643 		 * Write to reply post host index register after completing threshold
3644 		 * number of reply counts and still there are more replies in reply queue
3645 		 * pending to be completed
3646 		 */
3647 		if (threshold_reply_count >= instance->threshold_reply_count) {
3648 			if (instance->msix_combined)
3649 				writel(((MSIxIndex & 0x7) << 24) |
3650 					fusion->last_reply_idx[MSIxIndex],
3651 					instance->reply_post_host_index_addr[MSIxIndex/8]);
3652 			else
3653 				writel((MSIxIndex << 24) |
3654 					fusion->last_reply_idx[MSIxIndex],
3655 					instance->reply_post_host_index_addr[0]);
3656 			threshold_reply_count = 0;
3657 			if (irq_context) {
3658 				if (!irq_context->irq_poll_scheduled) {
3659 					irq_context->irq_poll_scheduled = true;
3660 					irq_context->irq_line_enable = true;
3661 					irq_poll_sched(&irq_context->irqpoll);
3662 				}
3663 				atomic_dec(&irq_context->in_used);
3664 				return num_completed;
3665 			}
3666 		}
3667 	}
3668 
3669 	if (num_completed) {
3670 		wmb();
3671 		if (instance->msix_combined)
3672 			writel(((MSIxIndex & 0x7) << 24) |
3673 				fusion->last_reply_idx[MSIxIndex],
3674 				instance->reply_post_host_index_addr[MSIxIndex/8]);
3675 		else
3676 			writel((MSIxIndex << 24) |
3677 				fusion->last_reply_idx[MSIxIndex],
3678 				instance->reply_post_host_index_addr[0]);
3679 		megasas_check_and_restore_queue_depth(instance);
3680 	}
3681 
3682 	if (irq_context)
3683 		atomic_dec(&irq_context->in_used);
3684 
3685 	return num_completed;
3686 }
3687 
3688 int megasas_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
3689 {
3690 
3691 	struct megasas_instance *instance;
3692 	int num_entries = 0;
3693 	struct fusion_context *fusion;
3694 
3695 	instance = (struct megasas_instance *)shost->hostdata;
3696 
3697 	fusion = instance->ctrl_context;
3698 
3699 	queue_num = queue_num + instance->low_latency_index_start;
3700 
3701 	if (!atomic_add_unless(&fusion->busy_mq_poll[queue_num], 1, 1))
3702 		return 0;
3703 
3704 	num_entries = complete_cmd_fusion(instance, queue_num, NULL);
3705 	atomic_dec(&fusion->busy_mq_poll[queue_num]);
3706 
3707 	return num_entries;
3708 }
3709 
3710 /**
3711  * megasas_enable_irq_poll() - enable irqpoll
3712  * @instance:			Adapter soft state
3713  */
3714 static void megasas_enable_irq_poll(struct megasas_instance *instance)
3715 {
3716 	u32 count, i;
3717 	struct megasas_irq_context *irq_ctx;
3718 
3719 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3720 
3721 	for (i = 0; i < count; i++) {
3722 		irq_ctx = &instance->irq_context[i];
3723 		irq_poll_enable(&irq_ctx->irqpoll);
3724 	}
3725 }
3726 
3727 /**
3728  * megasas_sync_irqs -	Synchronizes all IRQs owned by adapter
3729  * @instance_addr:			Adapter soft state address
3730  */
3731 static void megasas_sync_irqs(unsigned long instance_addr)
3732 {
3733 	u32 count, i;
3734 	struct megasas_instance *instance =
3735 		(struct megasas_instance *)instance_addr;
3736 	struct megasas_irq_context *irq_ctx;
3737 
3738 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3739 
3740 	for (i = 0; i < count; i++) {
3741 		synchronize_irq(pci_irq_vector(instance->pdev, i));
3742 		irq_ctx = &instance->irq_context[i];
3743 		irq_poll_disable(&irq_ctx->irqpoll);
3744 		if (irq_ctx->irq_poll_scheduled) {
3745 			irq_ctx->irq_poll_scheduled = false;
3746 			enable_irq(irq_ctx->os_irq);
3747 			complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3748 		}
3749 	}
3750 }
3751 
3752 /**
3753  * megasas_irqpoll() - process a queue for completed reply descriptors
3754  * @irqpoll:	IRQ poll structure associated with queue to poll.
3755  * @budget:	Threshold of reply descriptors to process per poll.
3756  *
3757  * Return: The number of entries processed.
3758  */
3759 
3760 int megasas_irqpoll(struct irq_poll *irqpoll, int budget)
3761 {
3762 	struct megasas_irq_context *irq_ctx;
3763 	struct megasas_instance *instance;
3764 	int num_entries;
3765 
3766 	irq_ctx = container_of(irqpoll, struct megasas_irq_context, irqpoll);
3767 	instance = irq_ctx->instance;
3768 
3769 	if (irq_ctx->irq_line_enable) {
3770 		disable_irq_nosync(irq_ctx->os_irq);
3771 		irq_ctx->irq_line_enable = false;
3772 	}
3773 
3774 	num_entries = complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3775 	if (num_entries < budget) {
3776 		irq_poll_complete(irqpoll);
3777 		irq_ctx->irq_poll_scheduled = false;
3778 		enable_irq(irq_ctx->os_irq);
3779 		complete_cmd_fusion(instance, irq_ctx->MSIxIndex, irq_ctx);
3780 	}
3781 
3782 	return num_entries;
3783 }
3784 
3785 /**
3786  * megasas_complete_cmd_dpc_fusion -	Completes command
3787  * @instance_addr:			Adapter soft state address
3788  *
3789  * Tasklet to complete cmds
3790  */
3791 static void
3792 megasas_complete_cmd_dpc_fusion(unsigned long instance_addr)
3793 {
3794 	struct megasas_instance *instance =
3795 		(struct megasas_instance *)instance_addr;
3796 	struct megasas_irq_context *irq_ctx = NULL;
3797 	u32 count, MSIxIndex;
3798 
3799 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
3800 
3801 	/* If we have already declared adapter dead, donot complete cmds */
3802 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR)
3803 		return;
3804 
3805 	for (MSIxIndex = 0 ; MSIxIndex < count; MSIxIndex++) {
3806 		irq_ctx = &instance->irq_context[MSIxIndex];
3807 		complete_cmd_fusion(instance, MSIxIndex, irq_ctx);
3808 	}
3809 }
3810 
3811 /**
3812  * megasas_isr_fusion - isr entry point
3813  * @irq:	IRQ number
3814  * @devp:	IRQ context
3815  */
3816 static irqreturn_t megasas_isr_fusion(int irq, void *devp)
3817 {
3818 	struct megasas_irq_context *irq_context = devp;
3819 	struct megasas_instance *instance = irq_context->instance;
3820 	u32 mfiStatus;
3821 
3822 	if (instance->mask_interrupts)
3823 		return IRQ_NONE;
3824 
3825 	if (irq_context->irq_poll_scheduled)
3826 		return IRQ_HANDLED;
3827 
3828 	if (!instance->msix_vectors) {
3829 		mfiStatus = instance->instancet->clear_intr(instance);
3830 		if (!mfiStatus)
3831 			return IRQ_NONE;
3832 	}
3833 
3834 	/* If we are resetting, bail */
3835 	if (test_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags)) {
3836 		instance->instancet->clear_intr(instance);
3837 		return IRQ_HANDLED;
3838 	}
3839 
3840 	return complete_cmd_fusion(instance, irq_context->MSIxIndex, irq_context)
3841 			? IRQ_HANDLED : IRQ_NONE;
3842 }
3843 
3844 /**
3845  * build_mpt_mfi_pass_thru - builds a cmd fo MFI Pass thru
3846  * @instance:			Adapter soft state
3847  * @mfi_cmd:			megasas_cmd pointer
3848  *
3849  */
3850 static void
3851 build_mpt_mfi_pass_thru(struct megasas_instance *instance,
3852 			struct megasas_cmd *mfi_cmd)
3853 {
3854 	struct MPI25_IEEE_SGE_CHAIN64 *mpi25_ieee_chain;
3855 	struct MPI2_RAID_SCSI_IO_REQUEST *io_req;
3856 	struct megasas_cmd_fusion *cmd;
3857 	struct fusion_context *fusion;
3858 	struct megasas_header *frame_hdr = &mfi_cmd->frame->hdr;
3859 
3860 	fusion = instance->ctrl_context;
3861 
3862 	cmd = megasas_get_cmd_fusion(instance,
3863 			instance->max_scsi_cmds + mfi_cmd->index);
3864 
3865 	/*  Save the smid. To be used for returning the cmd */
3866 	mfi_cmd->context.smid = cmd->index;
3867 
3868 	/*
3869 	 * For cmds where the flag is set, store the flag and check
3870 	 * on completion. For cmds with this flag, don't call
3871 	 * megasas_complete_cmd
3872 	 */
3873 
3874 	if (frame_hdr->flags & cpu_to_le16(MFI_FRAME_DONT_POST_IN_REPLY_QUEUE))
3875 		mfi_cmd->flags |= DRV_DCMD_POLLED_MODE;
3876 
3877 	io_req = cmd->io_request;
3878 
3879 	if (instance->adapter_type >= INVADER_SERIES) {
3880 		struct MPI25_IEEE_SGE_CHAIN64 *sgl_ptr_end =
3881 			(struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL;
3882 		sgl_ptr_end += fusion->max_sge_in_main_msg - 1;
3883 		sgl_ptr_end->Flags = 0;
3884 	}
3885 
3886 	mpi25_ieee_chain =
3887 	  (struct MPI25_IEEE_SGE_CHAIN64 *)&io_req->SGL.IeeeChain;
3888 
3889 	io_req->Function    = MEGASAS_MPI2_FUNCTION_PASSTHRU_IO_REQUEST;
3890 	io_req->SGLOffset0  = offsetof(struct MPI2_RAID_SCSI_IO_REQUEST,
3891 				       SGL) / 4;
3892 	io_req->ChainOffset = fusion->chain_offset_mfi_pthru;
3893 
3894 	mpi25_ieee_chain->Address = cpu_to_le64(mfi_cmd->frame_phys_addr);
3895 
3896 	mpi25_ieee_chain->Flags = IEEE_SGE_FLAGS_CHAIN_ELEMENT |
3897 		MPI2_IEEE_SGE_FLAGS_IOCPLBNTA_ADDR;
3898 
3899 	mpi25_ieee_chain->Length = cpu_to_le32(instance->mfi_frame_size);
3900 }
3901 
3902 /**
3903  * build_mpt_cmd - Calls helper function to build a cmd MFI Pass thru cmd
3904  * @instance:			Adapter soft state
3905  * @cmd:			mfi cmd to build
3906  *
3907  */
3908 static union MEGASAS_REQUEST_DESCRIPTOR_UNION *
3909 build_mpt_cmd(struct megasas_instance *instance, struct megasas_cmd *cmd)
3910 {
3911 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc = NULL;
3912 	u16 index;
3913 
3914 	build_mpt_mfi_pass_thru(instance, cmd);
3915 	index = cmd->context.smid;
3916 
3917 	req_desc = megasas_get_request_descriptor(instance, index - 1);
3918 
3919 	req_desc->Words = 0;
3920 	req_desc->SCSIIO.RequestFlags = (MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO <<
3921 					 MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
3922 
3923 	req_desc->SCSIIO.SMID = cpu_to_le16(index);
3924 
3925 	return req_desc;
3926 }
3927 
3928 /**
3929  * megasas_issue_dcmd_fusion - Issues a MFI Pass thru cmd
3930  * @instance:			Adapter soft state
3931  * @cmd:			mfi cmd pointer
3932  *
3933  */
3934 static void
3935 megasas_issue_dcmd_fusion(struct megasas_instance *instance,
3936 			  struct megasas_cmd *cmd)
3937 {
3938 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
3939 
3940 	req_desc = build_mpt_cmd(instance, cmd);
3941 
3942 	megasas_fire_cmd_fusion(instance, req_desc);
3943 	return;
3944 }
3945 
3946 /**
3947  * megasas_release_fusion -	Reverses the FW initialization
3948  * @instance:			Adapter soft state
3949  */
3950 void
3951 megasas_release_fusion(struct megasas_instance *instance)
3952 {
3953 	megasas_free_ioc_init_cmd(instance);
3954 	megasas_free_cmds(instance);
3955 	megasas_free_cmds_fusion(instance);
3956 
3957 	iounmap(instance->reg_set);
3958 
3959 	pci_release_selected_regions(instance->pdev, 1<<instance->bar);
3960 }
3961 
3962 /**
3963  * megasas_read_fw_status_reg_fusion - returns the current FW status value
3964  * @instance:			Adapter soft state
3965  */
3966 static u32
3967 megasas_read_fw_status_reg_fusion(struct megasas_instance *instance)
3968 {
3969 	return megasas_readl(instance, &instance->reg_set->outbound_scratch_pad_0);
3970 }
3971 
3972 /**
3973  * megasas_alloc_host_crash_buffer -	Host buffers for Crash dump collection from Firmware
3974  * @instance:				Controller's soft instance
3975  * @return:			        Number of allocated host crash buffers
3976  */
3977 static void
3978 megasas_alloc_host_crash_buffer(struct megasas_instance *instance)
3979 {
3980 	unsigned int i;
3981 
3982 	for (i = 0; i < MAX_CRASH_DUMP_SIZE; i++) {
3983 		instance->crash_buf[i] = vzalloc(CRASH_DMA_BUF_SIZE);
3984 		if (!instance->crash_buf[i]) {
3985 			dev_info(&instance->pdev->dev, "Firmware crash dump "
3986 				"memory allocation failed at index %d\n", i);
3987 			break;
3988 		}
3989 	}
3990 	instance->drv_buf_alloc = i;
3991 }
3992 
3993 /**
3994  * megasas_free_host_crash_buffer -	Host buffers for Crash dump collection from Firmware
3995  * @instance:				Controller's soft instance
3996  */
3997 void
3998 megasas_free_host_crash_buffer(struct megasas_instance *instance)
3999 {
4000 	unsigned int i;
4001 	for (i = 0; i < instance->drv_buf_alloc; i++) {
4002 		vfree(instance->crash_buf[i]);
4003 	}
4004 	instance->drv_buf_index = 0;
4005 	instance->drv_buf_alloc = 0;
4006 	instance->fw_crash_state = UNAVAILABLE;
4007 	instance->fw_crash_buffer_size = 0;
4008 }
4009 
4010 /**
4011  * megasas_adp_reset_fusion -	For controller reset
4012  * @instance:				Controller's soft instance
4013  * @regs:				MFI register set
4014  */
4015 static int
4016 megasas_adp_reset_fusion(struct megasas_instance *instance,
4017 			 struct megasas_register_set __iomem *regs)
4018 {
4019 	u32 host_diag, abs_state, retry;
4020 
4021 	/* Now try to reset the chip */
4022 	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4023 	writel(MPI2_WRSEQ_1ST_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4024 	writel(MPI2_WRSEQ_2ND_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4025 	writel(MPI2_WRSEQ_3RD_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4026 	writel(MPI2_WRSEQ_4TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4027 	writel(MPI2_WRSEQ_5TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4028 	writel(MPI2_WRSEQ_6TH_KEY_VALUE, &instance->reg_set->fusion_seq_offset);
4029 
4030 	/* Check that the diag write enable (DRWE) bit is on */
4031 	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);
4032 	retry = 0;
4033 	while (!(host_diag & HOST_DIAG_WRITE_ENABLE)) {
4034 		msleep(100);
4035 		host_diag = megasas_readl(instance,
4036 					  &instance->reg_set->fusion_host_diag);
4037 		if (retry++ == 100) {
4038 			dev_warn(&instance->pdev->dev,
4039 				"Host diag unlock failed from %s %d\n",
4040 				__func__, __LINE__);
4041 			break;
4042 		}
4043 	}
4044 	if (!(host_diag & HOST_DIAG_WRITE_ENABLE))
4045 		return -1;
4046 
4047 	/* Send chip reset command */
4048 	writel(host_diag | HOST_DIAG_RESET_ADAPTER,
4049 		&instance->reg_set->fusion_host_diag);
4050 	msleep(3000);
4051 
4052 	/* Make sure reset adapter bit is cleared */
4053 	host_diag = megasas_readl(instance, &instance->reg_set->fusion_host_diag);
4054 	retry = 0;
4055 	while (host_diag & HOST_DIAG_RESET_ADAPTER) {
4056 		msleep(100);
4057 		host_diag = megasas_readl(instance,
4058 					  &instance->reg_set->fusion_host_diag);
4059 		if (retry++ == 1000) {
4060 			dev_warn(&instance->pdev->dev,
4061 				"Diag reset adapter never cleared %s %d\n",
4062 				__func__, __LINE__);
4063 			break;
4064 		}
4065 	}
4066 	if (host_diag & HOST_DIAG_RESET_ADAPTER)
4067 		return -1;
4068 
4069 	abs_state = instance->instancet->read_fw_status_reg(instance)
4070 			& MFI_STATE_MASK;
4071 	retry = 0;
4072 
4073 	while ((abs_state <= MFI_STATE_FW_INIT) && (retry++ < 1000)) {
4074 		msleep(100);
4075 		abs_state = instance->instancet->
4076 			read_fw_status_reg(instance) & MFI_STATE_MASK;
4077 	}
4078 	if (abs_state <= MFI_STATE_FW_INIT) {
4079 		dev_warn(&instance->pdev->dev,
4080 			"fw state < MFI_STATE_FW_INIT, state = 0x%x %s %d\n",
4081 			abs_state, __func__, __LINE__);
4082 		return -1;
4083 	}
4084 
4085 	return 0;
4086 }
4087 
4088 /**
4089  * megasas_check_reset_fusion -	For controller reset check
4090  * @instance:				Controller's soft instance
4091  * @regs:				MFI register set
4092  */
4093 static int
4094 megasas_check_reset_fusion(struct megasas_instance *instance,
4095 			   struct megasas_register_set __iomem *regs)
4096 {
4097 	return 0;
4098 }
4099 
4100 /**
4101  * megasas_trigger_snap_dump -	Trigger snap dump in FW
4102  * @instance:			Soft instance of adapter
4103  */
4104 static inline void megasas_trigger_snap_dump(struct megasas_instance *instance)
4105 {
4106 	int j;
4107 	u32 fw_state, abs_state;
4108 
4109 	if (!instance->disableOnlineCtrlReset) {
4110 		dev_info(&instance->pdev->dev, "Trigger snap dump\n");
4111 		writel(MFI_ADP_TRIGGER_SNAP_DUMP,
4112 		       &instance->reg_set->doorbell);
4113 		readl(&instance->reg_set->doorbell);
4114 	}
4115 
4116 	for (j = 0; j < instance->snapdump_wait_time; j++) {
4117 		abs_state = instance->instancet->read_fw_status_reg(instance);
4118 		fw_state = abs_state & MFI_STATE_MASK;
4119 		if (fw_state == MFI_STATE_FAULT) {
4120 			dev_printk(KERN_ERR, &instance->pdev->dev,
4121 				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4122 				   abs_state & MFI_STATE_FAULT_CODE,
4123 				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4124 			return;
4125 		}
4126 		msleep(1000);
4127 	}
4128 }
4129 
4130 /* This function waits for outstanding commands on fusion to complete */
4131 static int
4132 megasas_wait_for_outstanding_fusion(struct megasas_instance *instance,
4133 				    int reason, int *convert)
4134 {
4135 	int i, outstanding, retval = 0, hb_seconds_missed = 0;
4136 	u32 fw_state, abs_state;
4137 	u32 waittime_for_io_completion;
4138 
4139 	waittime_for_io_completion =
4140 		min_t(u32, resetwaittime,
4141 			(resetwaittime - instance->snapdump_wait_time));
4142 
4143 	if (reason == MFI_IO_TIMEOUT_OCR) {
4144 		dev_info(&instance->pdev->dev,
4145 			"MFI command is timed out\n");
4146 		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4147 		if (instance->snapdump_wait_time)
4148 			megasas_trigger_snap_dump(instance);
4149 		retval = 1;
4150 		goto out;
4151 	}
4152 
4153 	for (i = 0; i < waittime_for_io_completion; i++) {
4154 		/* Check if firmware is in fault state */
4155 		abs_state = instance->instancet->read_fw_status_reg(instance);
4156 		fw_state = abs_state & MFI_STATE_MASK;
4157 		if (fw_state == MFI_STATE_FAULT) {
4158 			dev_printk(KERN_ERR, &instance->pdev->dev,
4159 				   "FW in FAULT state Fault code:0x%x subcode:0x%x func:%s\n",
4160 				   abs_state & MFI_STATE_FAULT_CODE,
4161 				   abs_state & MFI_STATE_FAULT_SUBCODE, __func__);
4162 			megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4163 			if (instance->requestorId && reason) {
4164 				dev_warn(&instance->pdev->dev, "SR-IOV Found FW in FAULT"
4165 				" state while polling during"
4166 				" I/O timeout handling for %d\n",
4167 				instance->host->host_no);
4168 				*convert = 1;
4169 			}
4170 
4171 			retval = 1;
4172 			goto out;
4173 		}
4174 
4175 
4176 		/* If SR-IOV VF mode & heartbeat timeout, don't wait */
4177 		if (instance->requestorId && !reason) {
4178 			retval = 1;
4179 			goto out;
4180 		}
4181 
4182 		/* If SR-IOV VF mode & I/O timeout, check for HB timeout */
4183 		if (instance->requestorId && (reason == SCSIIO_TIMEOUT_OCR)) {
4184 			if (instance->hb_host_mem->HB.fwCounter !=
4185 			    instance->hb_host_mem->HB.driverCounter) {
4186 				instance->hb_host_mem->HB.driverCounter =
4187 					instance->hb_host_mem->HB.fwCounter;
4188 				hb_seconds_missed = 0;
4189 			} else {
4190 				hb_seconds_missed++;
4191 				if (hb_seconds_missed ==
4192 				    (MEGASAS_SRIOV_HEARTBEAT_INTERVAL_VF/HZ)) {
4193 					dev_warn(&instance->pdev->dev, "SR-IOV:"
4194 					       " Heartbeat never completed "
4195 					       " while polling during I/O "
4196 					       " timeout handling for "
4197 					       "scsi%d.\n",
4198 					       instance->host->host_no);
4199 					       *convert = 1;
4200 					       retval = 1;
4201 					       goto out;
4202 				}
4203 			}
4204 		}
4205 
4206 		megasas_complete_cmd_dpc_fusion((unsigned long)instance);
4207 		outstanding = atomic_read(&instance->fw_outstanding);
4208 		if (!outstanding)
4209 			goto out;
4210 
4211 		if (!(i % MEGASAS_RESET_NOTICE_INTERVAL)) {
4212 			dev_notice(&instance->pdev->dev, "[%2d]waiting for %d "
4213 			       "commands to complete for scsi%d\n", i,
4214 			       outstanding, instance->host->host_no);
4215 		}
4216 		msleep(1000);
4217 	}
4218 
4219 	if (instance->snapdump_wait_time) {
4220 		megasas_trigger_snap_dump(instance);
4221 		retval = 1;
4222 		goto out;
4223 	}
4224 
4225 	if (atomic_read(&instance->fw_outstanding)) {
4226 		dev_err(&instance->pdev->dev, "pending commands remain after waiting, "
4227 		       "will reset adapter scsi%d.\n",
4228 		       instance->host->host_no);
4229 		*convert = 1;
4230 		retval = 1;
4231 	}
4232 
4233 out:
4234 	return retval;
4235 }
4236 
4237 void  megasas_reset_reply_desc(struct megasas_instance *instance)
4238 {
4239 	int i, j, count;
4240 	struct fusion_context *fusion;
4241 	union MPI2_REPLY_DESCRIPTORS_UNION *reply_desc;
4242 
4243 	fusion = instance->ctrl_context;
4244 	count = instance->msix_vectors > 0 ? instance->msix_vectors : 1;
4245 	count += instance->iopoll_q_count;
4246 
4247 	for (i = 0 ; i < count ; i++) {
4248 		fusion->last_reply_idx[i] = 0;
4249 		reply_desc = fusion->reply_frames_desc[i];
4250 		for (j = 0 ; j < fusion->reply_q_depth; j++, reply_desc++)
4251 			reply_desc->Words = cpu_to_le64(ULLONG_MAX);
4252 	}
4253 }
4254 
4255 /*
4256  * megasas_refire_mgmt_cmd :	Re-fire management commands
4257  * @instance:				Controller's soft instance
4258 */
4259 static void megasas_refire_mgmt_cmd(struct megasas_instance *instance,
4260 			     bool return_ioctl)
4261 {
4262 	int j;
4263 	struct megasas_cmd_fusion *cmd_fusion;
4264 	struct fusion_context *fusion;
4265 	struct megasas_cmd *cmd_mfi;
4266 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4267 	struct MPI2_RAID_SCSI_IO_REQUEST *scsi_io_req;
4268 	u16 smid;
4269 	bool refire_cmd = false;
4270 	u8 result;
4271 	u32 opcode = 0;
4272 
4273 	fusion = instance->ctrl_context;
4274 
4275 	/* Re-fire management commands.
4276 	 * Do not traverse complet MPT frame pool. Start from max_scsi_cmds.
4277 	 */
4278 	for (j = instance->max_scsi_cmds ; j < instance->max_fw_cmds; j++) {
4279 		cmd_fusion = fusion->cmd_list[j];
4280 		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4281 		smid = le16_to_cpu(cmd_mfi->context.smid);
4282 		result = REFIRE_CMD;
4283 
4284 		if (!smid)
4285 			continue;
4286 
4287 		req_desc = megasas_get_request_descriptor(instance, smid - 1);
4288 
4289 		switch (cmd_mfi->frame->hdr.cmd) {
4290 		case MFI_CMD_DCMD:
4291 			opcode = le32_to_cpu(cmd_mfi->frame->dcmd.opcode);
4292 			 /* Do not refire shutdown command */
4293 			if (opcode == MR_DCMD_CTRL_SHUTDOWN) {
4294 				cmd_mfi->frame->dcmd.cmd_status = MFI_STAT_OK;
4295 				result = COMPLETE_CMD;
4296 				break;
4297 			}
4298 
4299 			refire_cmd = ((opcode != MR_DCMD_LD_MAP_GET_INFO)) &&
4300 				      (opcode != MR_DCMD_SYSTEM_PD_MAP_GET_INFO) &&
4301 				      !(cmd_mfi->flags & DRV_DCMD_SKIP_REFIRE);
4302 
4303 			if (!refire_cmd)
4304 				result = RETURN_CMD;
4305 
4306 			break;
4307 		case MFI_CMD_NVME:
4308 			if (!instance->support_nvme_passthru) {
4309 				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4310 				result = COMPLETE_CMD;
4311 			}
4312 
4313 			break;
4314 		case MFI_CMD_TOOLBOX:
4315 			if (!instance->support_pci_lane_margining) {
4316 				cmd_mfi->frame->hdr.cmd_status = MFI_STAT_INVALID_CMD;
4317 				result = COMPLETE_CMD;
4318 			}
4319 
4320 			break;
4321 		default:
4322 			break;
4323 		}
4324 
4325 		if (return_ioctl && cmd_mfi->sync_cmd &&
4326 		    cmd_mfi->frame->hdr.cmd != MFI_CMD_ABORT) {
4327 			dev_err(&instance->pdev->dev,
4328 				"return -EBUSY from %s %d cmd 0x%x opcode 0x%x\n",
4329 				__func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4330 				le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4331 			cmd_mfi->cmd_status_drv = DCMD_BUSY;
4332 			result = COMPLETE_CMD;
4333 		}
4334 
4335 		scsi_io_req = (struct MPI2_RAID_SCSI_IO_REQUEST *)
4336 				cmd_fusion->io_request;
4337 		if (scsi_io_req->Function == MPI2_FUNCTION_SCSI_TASK_MGMT)
4338 			result = RETURN_CMD;
4339 
4340 		switch (result) {
4341 		case REFIRE_CMD:
4342 			megasas_fire_cmd_fusion(instance, req_desc);
4343 			break;
4344 		case RETURN_CMD:
4345 			megasas_return_cmd(instance, cmd_mfi);
4346 			break;
4347 		case COMPLETE_CMD:
4348 			megasas_complete_cmd(instance, cmd_mfi, DID_OK);
4349 			break;
4350 		}
4351 	}
4352 }
4353 
4354 /*
4355  * megasas_return_polled_cmds: Return polled mode commands back to the pool
4356  *			       before initiating an OCR.
4357  * @instance:                  Controller's soft instance
4358  */
4359 static void
4360 megasas_return_polled_cmds(struct megasas_instance *instance)
4361 {
4362 	int i;
4363 	struct megasas_cmd_fusion *cmd_fusion;
4364 	struct fusion_context *fusion;
4365 	struct megasas_cmd *cmd_mfi;
4366 
4367 	fusion = instance->ctrl_context;
4368 
4369 	for (i = instance->max_scsi_cmds; i < instance->max_fw_cmds; i++) {
4370 		cmd_fusion = fusion->cmd_list[i];
4371 		cmd_mfi = instance->cmd_list[cmd_fusion->sync_cmd_idx];
4372 
4373 		if (cmd_mfi->flags & DRV_DCMD_POLLED_MODE) {
4374 			if (megasas_dbg_lvl & OCR_DEBUG)
4375 				dev_info(&instance->pdev->dev,
4376 					 "%s %d return cmd 0x%x opcode 0x%x\n",
4377 					 __func__, __LINE__, cmd_mfi->frame->hdr.cmd,
4378 					 le32_to_cpu(cmd_mfi->frame->dcmd.opcode));
4379 			cmd_mfi->flags &= ~DRV_DCMD_POLLED_MODE;
4380 			megasas_return_cmd(instance, cmd_mfi);
4381 		}
4382 	}
4383 }
4384 
4385 /*
4386  * megasas_track_scsiio : Track SCSI IOs outstanding to a SCSI device
4387  * @instance: per adapter struct
4388  * @channel: the channel assigned by the OS
4389  * @id: the id assigned by the OS
4390  *
4391  * Returns SUCCESS if no IOs pending to SCSI device, else return FAILED
4392  */
4393 
4394 static int megasas_track_scsiio(struct megasas_instance *instance,
4395 		int id, int channel)
4396 {
4397 	int i, found = 0;
4398 	struct megasas_cmd_fusion *cmd_fusion;
4399 	struct fusion_context *fusion;
4400 	fusion = instance->ctrl_context;
4401 
4402 	for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4403 		cmd_fusion = fusion->cmd_list[i];
4404 		if (cmd_fusion->scmd &&
4405 			(cmd_fusion->scmd->device->id == id &&
4406 			cmd_fusion->scmd->device->channel == channel)) {
4407 			dev_info(&instance->pdev->dev,
4408 				"SCSI commands pending to target"
4409 				"channel %d id %d \tSMID: 0x%x\n",
4410 				channel, id, cmd_fusion->index);
4411 			scsi_print_command(cmd_fusion->scmd);
4412 			found = 1;
4413 			break;
4414 		}
4415 	}
4416 
4417 	return found ? FAILED : SUCCESS;
4418 }
4419 
4420 /**
4421  * megasas_tm_response_code - translation of device response code
4422  * @instance:	Controller's soft instance
4423  * @mpi_reply:	MPI reply returned by firmware
4424  *
4425  * Return nothing.
4426  */
4427 static void
4428 megasas_tm_response_code(struct megasas_instance *instance,
4429 		struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply)
4430 {
4431 	char *desc;
4432 
4433 	switch (mpi_reply->ResponseCode) {
4434 	case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
4435 		desc = "task management request completed";
4436 		break;
4437 	case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
4438 		desc = "invalid frame";
4439 		break;
4440 	case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
4441 		desc = "task management request not supported";
4442 		break;
4443 	case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
4444 		desc = "task management request failed";
4445 		break;
4446 	case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
4447 		desc = "task management request succeeded";
4448 		break;
4449 	case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
4450 		desc = "invalid lun";
4451 		break;
4452 	case 0xA:
4453 		desc = "overlapped tag attempted";
4454 		break;
4455 	case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
4456 		desc = "task queued, however not sent to target";
4457 		break;
4458 	default:
4459 		desc = "unknown";
4460 		break;
4461 	}
4462 	dev_dbg(&instance->pdev->dev, "response_code(%01x): %s\n",
4463 		mpi_reply->ResponseCode, desc);
4464 	dev_dbg(&instance->pdev->dev,
4465 		"TerminationCount/DevHandle/Function/TaskType/IOCStat/IOCLoginfo"
4466 		" 0x%x/0x%x/0x%x/0x%x/0x%x/0x%x\n",
4467 		mpi_reply->TerminationCount, mpi_reply->DevHandle,
4468 		mpi_reply->Function, mpi_reply->TaskType,
4469 		mpi_reply->IOCStatus, mpi_reply->IOCLogInfo);
4470 }
4471 
4472 /**
4473  * megasas_issue_tm - main routine for sending tm requests
4474  * @instance: per adapter struct
4475  * @device_handle: device handle
4476  * @channel: the channel assigned by the OS
4477  * @id: the id assigned by the OS
4478  * @smid_task: smid assigned to the task
4479  * @type: MPI2_SCSITASKMGMT_TASKTYPE__XXX (defined in megaraid_sas_fusion.c)
4480  * @mr_device_priv_data: private data
4481  * Context: user
4482  *
4483  * MegaRaid use MPT interface for Task Magement request.
4484  * A generic API for sending task management requests to firmware.
4485  *
4486  * Return SUCCESS or FAILED.
4487  */
4488 static int
4489 megasas_issue_tm(struct megasas_instance *instance, u16 device_handle,
4490 	uint channel, uint id, u16 smid_task, u8 type,
4491 	struct MR_PRIV_DEVICE *mr_device_priv_data)
4492 {
4493 	struct MR_TASK_MANAGE_REQUEST *mr_request;
4494 	struct MPI2_SCSI_TASK_MANAGE_REQUEST *mpi_request;
4495 	unsigned long timeleft;
4496 	struct megasas_cmd_fusion *cmd_fusion;
4497 	struct megasas_cmd *cmd_mfi;
4498 	union MEGASAS_REQUEST_DESCRIPTOR_UNION *req_desc;
4499 	struct fusion_context *fusion = NULL;
4500 	struct megasas_cmd_fusion *scsi_lookup;
4501 	int rc;
4502 	int timeout = MEGASAS_DEFAULT_TM_TIMEOUT;
4503 	struct MPI2_SCSI_TASK_MANAGE_REPLY *mpi_reply;
4504 
4505 	fusion = instance->ctrl_context;
4506 
4507 	cmd_mfi = megasas_get_cmd(instance);
4508 
4509 	if (!cmd_mfi) {
4510 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
4511 			__func__, __LINE__);
4512 		return -ENOMEM;
4513 	}
4514 
4515 	cmd_fusion = megasas_get_cmd_fusion(instance,
4516 			instance->max_scsi_cmds + cmd_mfi->index);
4517 
4518 	/*  Save the smid. To be used for returning the cmd */
4519 	cmd_mfi->context.smid = cmd_fusion->index;
4520 
4521 	req_desc = megasas_get_request_descriptor(instance,
4522 			(cmd_fusion->index - 1));
4523 
4524 	cmd_fusion->request_desc = req_desc;
4525 	req_desc->Words = 0;
4526 
4527 	mr_request = (struct MR_TASK_MANAGE_REQUEST *) cmd_fusion->io_request;
4528 	memset(mr_request, 0, sizeof(struct MR_TASK_MANAGE_REQUEST));
4529 	mpi_request = (struct MPI2_SCSI_TASK_MANAGE_REQUEST *) &mr_request->TmRequest;
4530 	mpi_request->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
4531 	mpi_request->DevHandle = cpu_to_le16(device_handle);
4532 	mpi_request->TaskType = type;
4533 	mpi_request->TaskMID = cpu_to_le16(smid_task);
4534 	mpi_request->LUN[1] = 0;
4535 
4536 
4537 	req_desc = cmd_fusion->request_desc;
4538 	req_desc->HighPriority.SMID = cpu_to_le16(cmd_fusion->index);
4539 	req_desc->HighPriority.RequestFlags =
4540 		(MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY <<
4541 		MEGASAS_REQ_DESCRIPT_FLAGS_TYPE_SHIFT);
4542 	req_desc->HighPriority.MSIxIndex =  0;
4543 	req_desc->HighPriority.LMID = 0;
4544 	req_desc->HighPriority.Reserved1 = 0;
4545 
4546 	if (channel < MEGASAS_MAX_PD_CHANNELS)
4547 		mr_request->tmReqFlags.isTMForPD = 1;
4548 	else
4549 		mr_request->tmReqFlags.isTMForLD = 1;
4550 
4551 	init_completion(&cmd_fusion->done);
4552 	megasas_fire_cmd_fusion(instance, req_desc);
4553 
4554 	switch (type) {
4555 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4556 		timeout = mr_device_priv_data->task_abort_tmo;
4557 		break;
4558 	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4559 		timeout = mr_device_priv_data->target_reset_tmo;
4560 		break;
4561 	}
4562 
4563 	timeleft = wait_for_completion_timeout(&cmd_fusion->done, timeout * HZ);
4564 
4565 	if (!timeleft) {
4566 		dev_err(&instance->pdev->dev,
4567 			"task mgmt type 0x%x timed out\n", type);
4568 		mutex_unlock(&instance->reset_mutex);
4569 		rc = megasas_reset_fusion(instance->host, MFI_IO_TIMEOUT_OCR);
4570 		mutex_lock(&instance->reset_mutex);
4571 		return rc;
4572 	}
4573 
4574 	mpi_reply = (struct MPI2_SCSI_TASK_MANAGE_REPLY *) &mr_request->TMReply;
4575 	megasas_tm_response_code(instance, mpi_reply);
4576 
4577 	megasas_return_cmd(instance, cmd_mfi);
4578 	rc = SUCCESS;
4579 	switch (type) {
4580 	case MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK:
4581 		scsi_lookup = fusion->cmd_list[smid_task - 1];
4582 
4583 		if (scsi_lookup->scmd == NULL)
4584 			break;
4585 		else {
4586 			instance->instancet->disable_intr(instance);
4587 			megasas_sync_irqs((unsigned long)instance);
4588 			instance->instancet->enable_intr(instance);
4589 			megasas_enable_irq_poll(instance);
4590 			if (scsi_lookup->scmd == NULL)
4591 				break;
4592 		}
4593 		rc = FAILED;
4594 		break;
4595 
4596 	case MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
4597 		if ((channel == 0xFFFFFFFF) && (id == 0xFFFFFFFF))
4598 			break;
4599 		instance->instancet->disable_intr(instance);
4600 		megasas_sync_irqs((unsigned long)instance);
4601 		rc = megasas_track_scsiio(instance, id, channel);
4602 		instance->instancet->enable_intr(instance);
4603 		megasas_enable_irq_poll(instance);
4604 
4605 		break;
4606 	case MPI2_SCSITASKMGMT_TASKTYPE_ABRT_TASK_SET:
4607 	case MPI2_SCSITASKMGMT_TASKTYPE_QUERY_TASK:
4608 		break;
4609 	default:
4610 		rc = FAILED;
4611 		break;
4612 	}
4613 
4614 	return rc;
4615 
4616 }
4617 
4618 /*
4619  * megasas_fusion_smid_lookup : Look for fusion command correpspodning to SCSI
4620  * @instance: per adapter struct
4621  *
4622  * Return Non Zero index, if SMID found in outstanding commands
4623  */
4624 static u16 megasas_fusion_smid_lookup(struct scsi_cmnd *scmd)
4625 {
4626 	int i, ret = 0;
4627 	struct megasas_instance *instance;
4628 	struct megasas_cmd_fusion *cmd_fusion;
4629 	struct fusion_context *fusion;
4630 
4631 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4632 
4633 	fusion = instance->ctrl_context;
4634 
4635 	for (i = 0; i < instance->max_scsi_cmds; i++) {
4636 		cmd_fusion = fusion->cmd_list[i];
4637 		if (cmd_fusion->scmd && (cmd_fusion->scmd == scmd)) {
4638 			scmd_printk(KERN_NOTICE, scmd, "Abort request is for"
4639 				" SMID: %d\n", cmd_fusion->index);
4640 			ret = cmd_fusion->index;
4641 			break;
4642 		}
4643 	}
4644 
4645 	return ret;
4646 }
4647 
4648 /*
4649 * megasas_get_tm_devhandle - Get devhandle for TM request
4650 * @sdev-		     OS provided scsi device
4651 *
4652 * Returns-		     devhandle/targetID of SCSI device
4653 */
4654 static u16 megasas_get_tm_devhandle(struct scsi_device *sdev)
4655 {
4656 	u16 pd_index = 0;
4657 	u32 device_id;
4658 	struct megasas_instance *instance;
4659 	struct fusion_context *fusion;
4660 	struct MR_PD_CFG_SEQ_NUM_SYNC *pd_sync;
4661 	u16 devhandle = (u16)ULONG_MAX;
4662 
4663 	instance = (struct megasas_instance *)sdev->host->hostdata;
4664 	fusion = instance->ctrl_context;
4665 
4666 	if (!MEGASAS_IS_LOGICAL(sdev)) {
4667 		if (instance->use_seqnum_jbod_fp) {
4668 			pd_index = (sdev->channel * MEGASAS_MAX_DEV_PER_CHANNEL)
4669 				    + sdev->id;
4670 			pd_sync = (void *)fusion->pd_seq_sync
4671 					[(instance->pd_seq_map_id - 1) & 1];
4672 			devhandle = pd_sync->seq[pd_index].devHandle;
4673 		} else
4674 			sdev_printk(KERN_ERR, sdev, "Firmware expose tmCapable"
4675 				" without JBOD MAP support from %s %d\n", __func__, __LINE__);
4676 	} else {
4677 		device_id = ((sdev->channel % 2) * MEGASAS_MAX_DEV_PER_CHANNEL)
4678 				+ sdev->id;
4679 		devhandle = device_id;
4680 	}
4681 
4682 	return devhandle;
4683 }
4684 
4685 /*
4686  * megasas_task_abort_fusion : SCSI task abort function for fusion adapters
4687  * @scmd : pointer to scsi command object
4688  *
4689  * Return SUCCESS, if command aborted else FAILED
4690  */
4691 
4692 int megasas_task_abort_fusion(struct scsi_cmnd *scmd)
4693 {
4694 	struct megasas_instance *instance;
4695 	u16 smid, devhandle;
4696 	int ret;
4697 	struct MR_PRIV_DEVICE *mr_device_priv_data;
4698 	mr_device_priv_data = scmd->device->hostdata;
4699 
4700 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4701 
4702 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4703 		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4704 		"SCSI host:%d\n", instance->host->host_no);
4705 		ret = FAILED;
4706 		return ret;
4707 	}
4708 
4709 	if (!mr_device_priv_data) {
4710 		sdev_printk(KERN_INFO, scmd->device, "device been deleted! "
4711 			"scmd(%p)\n", scmd);
4712 		scmd->result = DID_NO_CONNECT << 16;
4713 		ret = SUCCESS;
4714 		goto out;
4715 	}
4716 
4717 	if (!mr_device_priv_data->is_tm_capable) {
4718 		ret = FAILED;
4719 		goto out;
4720 	}
4721 
4722 	mutex_lock(&instance->reset_mutex);
4723 
4724 	smid = megasas_fusion_smid_lookup(scmd);
4725 
4726 	if (!smid) {
4727 		ret = SUCCESS;
4728 		scmd_printk(KERN_NOTICE, scmd, "Command for which abort is"
4729 			" issued is not found in outstanding commands\n");
4730 		mutex_unlock(&instance->reset_mutex);
4731 		goto out;
4732 	}
4733 
4734 	devhandle = megasas_get_tm_devhandle(scmd->device);
4735 
4736 	if (devhandle == (u16)ULONG_MAX) {
4737 		ret = SUCCESS;
4738 		sdev_printk(KERN_INFO, scmd->device,
4739 			"task abort issued for invalid devhandle\n");
4740 		mutex_unlock(&instance->reset_mutex);
4741 		goto out;
4742 	}
4743 	sdev_printk(KERN_INFO, scmd->device,
4744 		"attempting task abort! scmd(0x%p) tm_dev_handle 0x%x\n",
4745 		scmd, devhandle);
4746 
4747 	mr_device_priv_data->tm_busy = true;
4748 	ret = megasas_issue_tm(instance, devhandle,
4749 			scmd->device->channel, scmd->device->id, smid,
4750 			MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK,
4751 			mr_device_priv_data);
4752 	mr_device_priv_data->tm_busy = false;
4753 
4754 	mutex_unlock(&instance->reset_mutex);
4755 	scmd_printk(KERN_INFO, scmd, "task abort %s!! scmd(0x%p)\n",
4756 			((ret == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4757 out:
4758 	scsi_print_command(scmd);
4759 	if (megasas_dbg_lvl & TM_DEBUG)
4760 		megasas_dump_fusion_io(scmd);
4761 
4762 	return ret;
4763 }
4764 
4765 /*
4766  * megasas_reset_target_fusion : target reset function for fusion adapters
4767  * scmd: SCSI command pointer
4768  *
4769  * Returns SUCCESS if all commands associated with target aborted else FAILED
4770  */
4771 
4772 int megasas_reset_target_fusion(struct scsi_cmnd *scmd)
4773 {
4774 
4775 	struct megasas_instance *instance;
4776 	int ret = FAILED;
4777 	u16 devhandle;
4778 	struct MR_PRIV_DEVICE *mr_device_priv_data;
4779 	mr_device_priv_data = scmd->device->hostdata;
4780 
4781 	instance = (struct megasas_instance *)scmd->device->host->hostdata;
4782 
4783 	if (atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) {
4784 		dev_err(&instance->pdev->dev, "Controller is not OPERATIONAL,"
4785 		"SCSI host:%d\n", instance->host->host_no);
4786 		ret = FAILED;
4787 		return ret;
4788 	}
4789 
4790 	if (!mr_device_priv_data) {
4791 		sdev_printk(KERN_INFO, scmd->device,
4792 			    "device been deleted! scmd: (0x%p)\n", scmd);
4793 		scmd->result = DID_NO_CONNECT << 16;
4794 		ret = SUCCESS;
4795 		goto out;
4796 	}
4797 
4798 	if (!mr_device_priv_data->is_tm_capable) {
4799 		ret = FAILED;
4800 		goto out;
4801 	}
4802 
4803 	mutex_lock(&instance->reset_mutex);
4804 	devhandle = megasas_get_tm_devhandle(scmd->device);
4805 
4806 	if (devhandle == (u16)ULONG_MAX) {
4807 		ret = SUCCESS;
4808 		sdev_printk(KERN_INFO, scmd->device,
4809 			"target reset issued for invalid devhandle\n");
4810 		mutex_unlock(&instance->reset_mutex);
4811 		goto out;
4812 	}
4813 
4814 	sdev_printk(KERN_INFO, scmd->device,
4815 		"attempting target reset! scmd(0x%p) tm_dev_handle: 0x%x\n",
4816 		scmd, devhandle);
4817 	mr_device_priv_data->tm_busy = true;
4818 	ret = megasas_issue_tm(instance, devhandle,
4819 			scmd->device->channel, scmd->device->id, 0,
4820 			MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET,
4821 			mr_device_priv_data);
4822 	mr_device_priv_data->tm_busy = false;
4823 	mutex_unlock(&instance->reset_mutex);
4824 	scmd_printk(KERN_NOTICE, scmd, "target reset %s!!\n",
4825 		(ret == SUCCESS) ? "SUCCESS" : "FAILED");
4826 
4827 out:
4828 	return ret;
4829 }
4830 
4831 /*SRIOV get other instance in cluster if any*/
4832 static struct
4833 megasas_instance *megasas_get_peer_instance(struct megasas_instance *instance)
4834 {
4835 	int i;
4836 
4837 	for (i = 0; i < MAX_MGMT_ADAPTERS; i++) {
4838 		if (megasas_mgmt_info.instance[i] &&
4839 			(megasas_mgmt_info.instance[i] != instance) &&
4840 			 megasas_mgmt_info.instance[i]->requestorId &&
4841 			 megasas_mgmt_info.instance[i]->peerIsPresent &&
4842 			(memcmp((megasas_mgmt_info.instance[i]->clusterId),
4843 			instance->clusterId, MEGASAS_CLUSTER_ID_SIZE) == 0))
4844 			return megasas_mgmt_info.instance[i];
4845 	}
4846 	return NULL;
4847 }
4848 
4849 /* Check for a second path that is currently UP */
4850 int megasas_check_mpio_paths(struct megasas_instance *instance,
4851 	struct scsi_cmnd *scmd)
4852 {
4853 	struct megasas_instance *peer_instance = NULL;
4854 	int retval = (DID_REQUEUE << 16);
4855 
4856 	if (instance->peerIsPresent) {
4857 		peer_instance = megasas_get_peer_instance(instance);
4858 		if ((peer_instance) &&
4859 			(atomic_read(&peer_instance->adprecovery) ==
4860 			MEGASAS_HBA_OPERATIONAL))
4861 			retval = (DID_NO_CONNECT << 16);
4862 	}
4863 	return retval;
4864 }
4865 
4866 /* Core fusion reset function */
4867 int megasas_reset_fusion(struct Scsi_Host *shost, int reason)
4868 {
4869 	int retval = SUCCESS, i, j, convert = 0;
4870 	struct megasas_instance *instance;
4871 	struct megasas_cmd_fusion *cmd_fusion, *r1_cmd;
4872 	struct fusion_context *fusion;
4873 	u32 abs_state, status_reg, reset_adapter, fpio_count = 0;
4874 	u32 io_timeout_in_crash_mode = 0;
4875 	struct scsi_cmnd *scmd_local = NULL;
4876 	struct scsi_device *sdev;
4877 	int ret_target_prop = DCMD_FAILED;
4878 	bool is_target_prop = false;
4879 	bool do_adp_reset = true;
4880 	int max_reset_tries = MEGASAS_FUSION_MAX_RESET_TRIES;
4881 
4882 	instance = (struct megasas_instance *)shost->hostdata;
4883 	fusion = instance->ctrl_context;
4884 
4885 	mutex_lock(&instance->reset_mutex);
4886 
4887 	if (atomic_read(&instance->adprecovery) == MEGASAS_HW_CRITICAL_ERROR) {
4888 		dev_warn(&instance->pdev->dev, "Hardware critical error, "
4889 		       "returning FAILED for scsi%d.\n",
4890 			instance->host->host_no);
4891 		mutex_unlock(&instance->reset_mutex);
4892 		return FAILED;
4893 	}
4894 	status_reg = instance->instancet->read_fw_status_reg(instance);
4895 	abs_state = status_reg & MFI_STATE_MASK;
4896 
4897 	/* IO timeout detected, forcibly put FW in FAULT state */
4898 	if (abs_state != MFI_STATE_FAULT && instance->crash_dump_buf &&
4899 		instance->crash_dump_app_support && reason) {
4900 		dev_info(&instance->pdev->dev, "IO/DCMD timeout is detected, "
4901 			"forcibly FAULT Firmware\n");
4902 		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4903 		status_reg = megasas_readl(instance, &instance->reg_set->doorbell);
4904 		writel(status_reg | MFI_STATE_FORCE_OCR,
4905 			&instance->reg_set->doorbell);
4906 		readl(&instance->reg_set->doorbell);
4907 		mutex_unlock(&instance->reset_mutex);
4908 		do {
4909 			ssleep(3);
4910 			io_timeout_in_crash_mode++;
4911 			dev_dbg(&instance->pdev->dev, "waiting for [%d] "
4912 				"seconds for crash dump collection and OCR "
4913 				"to be done\n", (io_timeout_in_crash_mode * 3));
4914 		} while ((atomic_read(&instance->adprecovery) != MEGASAS_HBA_OPERATIONAL) &&
4915 			(io_timeout_in_crash_mode < 80));
4916 
4917 		if (atomic_read(&instance->adprecovery) == MEGASAS_HBA_OPERATIONAL) {
4918 			dev_info(&instance->pdev->dev, "OCR done for IO "
4919 				"timeout case\n");
4920 			retval = SUCCESS;
4921 		} else {
4922 			dev_info(&instance->pdev->dev, "Controller is not "
4923 				"operational after 240 seconds wait for IO "
4924 				"timeout case in FW crash dump mode\n do "
4925 				"OCR/kill adapter\n");
4926 			retval = megasas_reset_fusion(shost, 0);
4927 		}
4928 		return retval;
4929 	}
4930 
4931 	if (instance->requestorId && !instance->skip_heartbeat_timer_del)
4932 		del_timer_sync(&instance->sriov_heartbeat_timer);
4933 	set_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
4934 	set_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);
4935 	atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_POLLING);
4936 	instance->instancet->disable_intr(instance);
4937 	megasas_sync_irqs((unsigned long)instance);
4938 
4939 	/* First try waiting for commands to complete */
4940 	if (megasas_wait_for_outstanding_fusion(instance, reason,
4941 						&convert)) {
4942 		atomic_set(&instance->adprecovery, MEGASAS_ADPRESET_SM_INFAULT);
4943 		dev_warn(&instance->pdev->dev, "resetting fusion "
4944 		       "adapter scsi%d.\n", instance->host->host_no);
4945 		if (convert)
4946 			reason = 0;
4947 
4948 		if (megasas_dbg_lvl & OCR_DEBUG)
4949 			dev_info(&instance->pdev->dev, "\nPending SCSI commands:\n");
4950 
4951 		/* Now return commands back to the OS */
4952 		for (i = 0 ; i < instance->max_scsi_cmds; i++) {
4953 			cmd_fusion = fusion->cmd_list[i];
4954 			/*check for extra commands issued by driver*/
4955 			if (instance->adapter_type >= VENTURA_SERIES) {
4956 				r1_cmd = fusion->cmd_list[i + instance->max_fw_cmds];
4957 				megasas_return_cmd_fusion(instance, r1_cmd);
4958 			}
4959 			scmd_local = cmd_fusion->scmd;
4960 			if (cmd_fusion->scmd) {
4961 				if (megasas_dbg_lvl & OCR_DEBUG) {
4962 					sdev_printk(KERN_INFO,
4963 						cmd_fusion->scmd->device, "SMID: 0x%x\n",
4964 						cmd_fusion->index);
4965 					megasas_dump_fusion_io(cmd_fusion->scmd);
4966 				}
4967 
4968 				if (cmd_fusion->io_request->Function ==
4969 					MPI2_FUNCTION_SCSI_IO_REQUEST)
4970 					fpio_count++;
4971 
4972 				scmd_local->result =
4973 					megasas_check_mpio_paths(instance,
4974 							scmd_local);
4975 				if (instance->ldio_threshold &&
4976 					megasas_cmd_type(scmd_local) == READ_WRITE_LDIO)
4977 					atomic_dec(&instance->ldio_outstanding);
4978 				megasas_return_cmd_fusion(instance, cmd_fusion);
4979 				scsi_dma_unmap(scmd_local);
4980 				scmd_local->scsi_done(scmd_local);
4981 			}
4982 		}
4983 
4984 		dev_info(&instance->pdev->dev, "Outstanding fastpath IOs: %d\n",
4985 			fpio_count);
4986 
4987 		atomic_set(&instance->fw_outstanding, 0);
4988 
4989 		status_reg = instance->instancet->read_fw_status_reg(instance);
4990 		abs_state = status_reg & MFI_STATE_MASK;
4991 		reset_adapter = status_reg & MFI_RESET_ADAPTER;
4992 		if (instance->disableOnlineCtrlReset ||
4993 		    (abs_state == MFI_STATE_FAULT && !reset_adapter)) {
4994 			/* Reset not supported, kill adapter */
4995 			dev_warn(&instance->pdev->dev, "Reset not supported"
4996 			       ", killing adapter scsi%d.\n",
4997 				instance->host->host_no);
4998 			goto kill_hba;
4999 		}
5000 
5001 		/* Let SR-IOV VF & PF sync up if there was a HB failure */
5002 		if (instance->requestorId && !reason) {
5003 			msleep(MEGASAS_OCR_SETTLE_TIME_VF);
5004 			do_adp_reset = false;
5005 			max_reset_tries = MEGASAS_SRIOV_MAX_RESET_TRIES_VF;
5006 		}
5007 
5008 		/* Now try to reset the chip */
5009 		for (i = 0; i < max_reset_tries; i++) {
5010 			/*
5011 			 * Do adp reset and wait for
5012 			 * controller to transition to ready
5013 			 */
5014 			if (megasas_adp_reset_wait_for_ready(instance,
5015 				do_adp_reset, 1) == FAILED)
5016 				continue;
5017 
5018 			/* Wait for FW to become ready */
5019 			if (megasas_transition_to_ready(instance, 1)) {
5020 				dev_warn(&instance->pdev->dev,
5021 					"Failed to transition controller to ready for "
5022 					"scsi%d.\n", instance->host->host_no);
5023 				continue;
5024 			}
5025 			megasas_reset_reply_desc(instance);
5026 			megasas_fusion_update_can_queue(instance, OCR_CONTEXT);
5027 
5028 			if (megasas_ioc_init_fusion(instance)) {
5029 				continue;
5030 			}
5031 
5032 			if (megasas_get_ctrl_info(instance)) {
5033 				dev_info(&instance->pdev->dev,
5034 					"Failed from %s %d\n",
5035 					__func__, __LINE__);
5036 				goto kill_hba;
5037 			}
5038 
5039 			megasas_refire_mgmt_cmd(instance,
5040 						(i == (MEGASAS_FUSION_MAX_RESET_TRIES - 1)
5041 							? 1 : 0));
5042 
5043 			/* Reset load balance info */
5044 			if (fusion->load_balance_info)
5045 				memset(fusion->load_balance_info, 0,
5046 				       (sizeof(struct LD_LOAD_BALANCE_INFO) *
5047 				       MAX_LOGICAL_DRIVES_EXT));
5048 
5049 			if (!megasas_get_map_info(instance)) {
5050 				megasas_sync_map_info(instance);
5051 			} else {
5052 				/*
5053 				 * Return pending polled mode cmds before
5054 				 * retrying OCR
5055 				 */
5056 				megasas_return_polled_cmds(instance);
5057 				continue;
5058 			}
5059 
5060 			megasas_setup_jbod_map(instance);
5061 
5062 			/* reset stream detection array */
5063 			if (instance->adapter_type >= VENTURA_SERIES) {
5064 				for (j = 0; j < MAX_LOGICAL_DRIVES_EXT; ++j) {
5065 					memset(fusion->stream_detect_by_ld[j],
5066 					0, sizeof(struct LD_STREAM_DETECT));
5067 				 fusion->stream_detect_by_ld[j]->mru_bit_map
5068 						= MR_STREAM_BITMAP;
5069 				}
5070 			}
5071 
5072 			clear_bit(MEGASAS_FUSION_IN_RESET,
5073 				  &instance->reset_flags);
5074 			instance->instancet->enable_intr(instance);
5075 			megasas_enable_irq_poll(instance);
5076 			shost_for_each_device(sdev, shost) {
5077 				if ((instance->tgt_prop) &&
5078 				    (instance->nvme_page_size))
5079 					ret_target_prop = megasas_get_target_prop(instance, sdev);
5080 
5081 				is_target_prop = (ret_target_prop == DCMD_SUCCESS) ? true : false;
5082 				megasas_set_dynamic_target_properties(sdev, is_target_prop);
5083 			}
5084 
5085 			status_reg = instance->instancet->read_fw_status_reg
5086 					(instance);
5087 			abs_state = status_reg & MFI_STATE_MASK;
5088 			if (abs_state != MFI_STATE_OPERATIONAL) {
5089 				dev_info(&instance->pdev->dev,
5090 					 "Adapter is not OPERATIONAL, state 0x%x for scsi:%d\n",
5091 					 abs_state, instance->host->host_no);
5092 				goto out;
5093 			}
5094 			atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
5095 
5096 			dev_info(&instance->pdev->dev,
5097 				 "Adapter is OPERATIONAL for scsi:%d\n",
5098 				 instance->host->host_no);
5099 
5100 			/* Restart SR-IOV heartbeat */
5101 			if (instance->requestorId) {
5102 				if (!megasas_sriov_start_heartbeat(instance, 0))
5103 					megasas_start_timer(instance);
5104 				else
5105 					instance->skip_heartbeat_timer_del = 1;
5106 			}
5107 
5108 			if (instance->crash_dump_drv_support &&
5109 				instance->crash_dump_app_support)
5110 				megasas_set_crash_dump_params(instance,
5111 					MR_CRASH_BUF_TURN_ON);
5112 			else
5113 				megasas_set_crash_dump_params(instance,
5114 					MR_CRASH_BUF_TURN_OFF);
5115 
5116 			if (instance->snapdump_wait_time) {
5117 				megasas_get_snapdump_properties(instance);
5118 				dev_info(&instance->pdev->dev,
5119 					 "Snap dump wait time\t: %d\n",
5120 					 instance->snapdump_wait_time);
5121 			}
5122 
5123 			retval = SUCCESS;
5124 
5125 			/* Adapter reset completed successfully */
5126 			dev_warn(&instance->pdev->dev,
5127 				 "Reset successful for scsi%d.\n",
5128 				 instance->host->host_no);
5129 
5130 			goto out;
5131 		}
5132 		/* Reset failed, kill the adapter */
5133 		dev_warn(&instance->pdev->dev, "Reset failed, killing "
5134 		       "adapter scsi%d.\n", instance->host->host_no);
5135 		goto kill_hba;
5136 	} else {
5137 		/* For VF: Restart HB timer if we didn't OCR */
5138 		if (instance->requestorId) {
5139 			megasas_start_timer(instance);
5140 		}
5141 		clear_bit(MEGASAS_FUSION_IN_RESET, &instance->reset_flags);
5142 		instance->instancet->enable_intr(instance);
5143 		megasas_enable_irq_poll(instance);
5144 		atomic_set(&instance->adprecovery, MEGASAS_HBA_OPERATIONAL);
5145 		goto out;
5146 	}
5147 kill_hba:
5148 	megaraid_sas_kill_hba(instance);
5149 	megasas_enable_irq_poll(instance);
5150 	instance->skip_heartbeat_timer_del = 1;
5151 	retval = FAILED;
5152 out:
5153 	clear_bit(MEGASAS_FUSION_OCR_NOT_POSSIBLE, &instance->reset_flags);
5154 	mutex_unlock(&instance->reset_mutex);
5155 	return retval;
5156 }
5157 
5158 /* Fusion Crash dump collection */
5159 static void  megasas_fusion_crash_dump(struct megasas_instance *instance)
5160 {
5161 	u32 status_reg;
5162 	u8 partial_copy = 0;
5163 	int wait = 0;
5164 
5165 
5166 	status_reg = instance->instancet->read_fw_status_reg(instance);
5167 
5168 	/*
5169 	 * Allocate host crash buffers to copy data from 1 MB DMA crash buffer
5170 	 * to host crash buffers
5171 	 */
5172 	if (instance->drv_buf_index == 0) {
5173 		/* Buffer is already allocated for old Crash dump.
5174 		 * Do OCR and do not wait for crash dump collection
5175 		 */
5176 		if (instance->drv_buf_alloc) {
5177 			dev_info(&instance->pdev->dev, "earlier crash dump is "
5178 				"not yet copied by application, ignoring this "
5179 				"crash dump and initiating OCR\n");
5180 			status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5181 			writel(status_reg,
5182 				&instance->reg_set->outbound_scratch_pad_0);
5183 			readl(&instance->reg_set->outbound_scratch_pad_0);
5184 			return;
5185 		}
5186 		megasas_alloc_host_crash_buffer(instance);
5187 		dev_info(&instance->pdev->dev, "Number of host crash buffers "
5188 			"allocated: %d\n", instance->drv_buf_alloc);
5189 	}
5190 
5191 	while (!(status_reg & MFI_STATE_CRASH_DUMP_DONE) &&
5192 	       (wait < MEGASAS_WATCHDOG_WAIT_COUNT)) {
5193 		if (!(status_reg & MFI_STATE_DMADONE)) {
5194 			/*
5195 			 * Next crash dump buffer is not yet DMA'd by FW
5196 			 * Check after 10ms. Wait for 1 second for FW to
5197 			 * post the next buffer. If not bail out.
5198 			 */
5199 			wait++;
5200 			msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5201 			status_reg = instance->instancet->read_fw_status_reg(
5202 					instance);
5203 			continue;
5204 		}
5205 
5206 		wait = 0;
5207 		if (instance->drv_buf_index >= instance->drv_buf_alloc) {
5208 			dev_info(&instance->pdev->dev,
5209 				 "Driver is done copying the buffer: %d\n",
5210 				 instance->drv_buf_alloc);
5211 			status_reg |= MFI_STATE_CRASH_DUMP_DONE;
5212 			partial_copy = 1;
5213 			break;
5214 		} else {
5215 			memcpy(instance->crash_buf[instance->drv_buf_index],
5216 			       instance->crash_dump_buf, CRASH_DMA_BUF_SIZE);
5217 			instance->drv_buf_index++;
5218 			status_reg &= ~MFI_STATE_DMADONE;
5219 		}
5220 
5221 		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);
5222 		readl(&instance->reg_set->outbound_scratch_pad_0);
5223 
5224 		msleep(MEGASAS_WAIT_FOR_NEXT_DMA_MSECS);
5225 		status_reg = instance->instancet->read_fw_status_reg(instance);
5226 	}
5227 
5228 	if (status_reg & MFI_STATE_CRASH_DUMP_DONE) {
5229 		dev_info(&instance->pdev->dev, "Crash Dump is available,number "
5230 			"of copied buffers: %d\n", instance->drv_buf_index);
5231 		instance->fw_crash_buffer_size =  instance->drv_buf_index;
5232 		instance->fw_crash_state = AVAILABLE;
5233 		instance->drv_buf_index = 0;
5234 		writel(status_reg, &instance->reg_set->outbound_scratch_pad_0);
5235 		readl(&instance->reg_set->outbound_scratch_pad_0);
5236 		if (!partial_copy)
5237 			megasas_reset_fusion(instance->host, 0);
5238 	}
5239 }
5240 
5241 
5242 /* Fusion OCR work queue */
5243 void megasas_fusion_ocr_wq(struct work_struct *work)
5244 {
5245 	struct megasas_instance *instance =
5246 		container_of(work, struct megasas_instance, work_init);
5247 
5248 	megasas_reset_fusion(instance->host, 0);
5249 }
5250 
5251 /* Allocate fusion context */
5252 int
5253 megasas_alloc_fusion_context(struct megasas_instance *instance)
5254 {
5255 	struct fusion_context *fusion;
5256 
5257 	instance->ctrl_context = kzalloc(sizeof(struct fusion_context),
5258 					 GFP_KERNEL);
5259 	if (!instance->ctrl_context) {
5260 		dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5261 			__func__, __LINE__);
5262 		return -ENOMEM;
5263 	}
5264 
5265 	fusion = instance->ctrl_context;
5266 
5267 	fusion->log_to_span_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5268 					      sizeof(LD_SPAN_INFO));
5269 	fusion->log_to_span =
5270 		(PLD_SPAN_INFO)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5271 						fusion->log_to_span_pages);
5272 	if (!fusion->log_to_span) {
5273 		fusion->log_to_span =
5274 			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5275 					   sizeof(LD_SPAN_INFO)));
5276 		if (!fusion->log_to_span) {
5277 			dev_err(&instance->pdev->dev, "Failed from %s %d\n",
5278 				__func__, __LINE__);
5279 			kfree(instance->ctrl_context);
5280 			return -ENOMEM;
5281 		}
5282 	}
5283 
5284 	fusion->load_balance_info_pages = get_order(MAX_LOGICAL_DRIVES_EXT *
5285 		sizeof(struct LD_LOAD_BALANCE_INFO));
5286 	fusion->load_balance_info =
5287 		(struct LD_LOAD_BALANCE_INFO *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
5288 		fusion->load_balance_info_pages);
5289 	if (!fusion->load_balance_info) {
5290 		fusion->load_balance_info =
5291 			vzalloc(array_size(MAX_LOGICAL_DRIVES_EXT,
5292 					   sizeof(struct LD_LOAD_BALANCE_INFO)));
5293 		if (!fusion->load_balance_info)
5294 			dev_err(&instance->pdev->dev, "Failed to allocate load_balance_info, "
5295 				"continuing without Load Balance support\n");
5296 	}
5297 
5298 	return 0;
5299 }
5300 
5301 void
5302 megasas_free_fusion_context(struct megasas_instance *instance)
5303 {
5304 	struct fusion_context *fusion = instance->ctrl_context;
5305 
5306 	if (fusion) {
5307 		if (fusion->load_balance_info) {
5308 			if (is_vmalloc_addr(fusion->load_balance_info))
5309 				vfree(fusion->load_balance_info);
5310 			else
5311 				free_pages((ulong)fusion->load_balance_info,
5312 					fusion->load_balance_info_pages);
5313 		}
5314 
5315 		if (fusion->log_to_span) {
5316 			if (is_vmalloc_addr(fusion->log_to_span))
5317 				vfree(fusion->log_to_span);
5318 			else
5319 				free_pages((ulong)fusion->log_to_span,
5320 					   fusion->log_to_span_pages);
5321 		}
5322 
5323 		kfree(fusion);
5324 	}
5325 }
5326 
5327 struct megasas_instance_template megasas_instance_template_fusion = {
5328 	.enable_intr = megasas_enable_intr_fusion,
5329 	.disable_intr = megasas_disable_intr_fusion,
5330 	.clear_intr = megasas_clear_intr_fusion,
5331 	.read_fw_status_reg = megasas_read_fw_status_reg_fusion,
5332 	.adp_reset = megasas_adp_reset_fusion,
5333 	.check_reset = megasas_check_reset_fusion,
5334 	.service_isr = megasas_isr_fusion,
5335 	.tasklet = megasas_complete_cmd_dpc_fusion,
5336 	.init_adapter = megasas_init_adapter_fusion,
5337 	.build_and_issue_cmd = megasas_build_and_issue_cmd_fusion,
5338 	.issue_dcmd = megasas_issue_dcmd_fusion,
5339 };
5340