134 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc,
137 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc);
139 _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc);
157 mpt3sas_base_check_cmd_timeout(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_check_cmd_timeout()  argument
165 	ioc_err(ioc, "Command %s\n",  in mpt3sas_base_check_cmd_timeout()
183 	struct MPT3SAS_ADAPTER *ioc;  in _scsih_set_fwfault_debug()  local
191 	list_for_each_entry(ioc, &mpt3sas_ioc_list, list)  in _scsih_set_fwfault_debug()
192 		ioc->fwfault_debug = mpt3sas_fwfault_debug;  in _scsih_set_fwfault_debug()
248 _base_clone_reply_to_sys_mem(struct MPT3SAS_ADAPTER *ioc, u32 reply,  in _base_clone_reply_to_sys_mem()  argument
256 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_clone_reply_to_sys_mem()
257 	void __iomem *reply_free_iomem = (void __iomem *)ioc->chip +  in _base_clone_reply_to_sys_mem()
259 			(cmd_credit * ioc->request_sz) + (index * sizeof(u32));  in _base_clone_reply_to_sys_mem()
312 _base_get_chain(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_get_chain()  argument
316 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_chain()
318 	base_chain  = (void __iomem *)ioc->chip + MPI_FRAME_START_OFFSET +  in _base_get_chain()
319 		(cmd_credit * ioc->request_sz) +  in _base_get_chain()
321 	chain_virt = base_chain + (smid * ioc->facts.MaxChainDepth *  in _base_get_chain()
322 			ioc->request_sz) + (sge_chain_count * ioc->request_sz);  in _base_get_chain()
338 _base_get_chain_phys(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_get_chain_phys()  argument
342 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_chain_phys()
344 	base_chain_phys  = ioc->chip_phys + MPI_FRAME_START_OFFSET +  in _base_get_chain_phys()
345 		(cmd_credit * ioc->request_sz) +  in _base_get_chain_phys()
347 	chain_phys = base_chain_phys + (smid * ioc->facts.MaxChainDepth *  in _base_get_chain_phys()
348 			ioc->request_sz) + (sge_chain_count * ioc->request_sz);  in _base_get_chain_phys()
364 _base_get_buffer_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_get_buffer_bar0()  argument
366 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_buffer_bar0()
368 	void __iomem *chain_end = _base_get_chain(ioc,  in _base_get_buffer_bar0()
370 			ioc->facts.MaxChainDepth);  in _base_get_buffer_bar0()
385 _base_get_buffer_phys_bar0(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_get_buffer_phys_bar0()  argument
387 	u16 cmd_credit = ioc->facts.RequestCredit + 1;  in _base_get_buffer_phys_bar0()
388 	phys_addr_t chain_end_phys = _base_get_chain_phys(ioc,  in _base_get_buffer_phys_bar0()
390 			ioc->facts.MaxChainDepth);  in _base_get_buffer_phys_bar0()
406 _base_get_chain_buffer_dma_to_chain_buffer(struct MPT3SAS_ADAPTER *ioc,  in _base_get_chain_buffer_dma_to_chain_buffer()  argument
412 	for (index = 0; index < ioc->scsiio_depth; index++) {  in _base_get_chain_buffer_dma_to_chain_buffer()
413 		for (j = 0; j < ioc->chains_needed_per_io; j++) {  in _base_get_chain_buffer_dma_to_chain_buffer()
414 			ct = &ioc->chain_lookup[index].chains_per_smid[j];  in _base_get_chain_buffer_dma_to_chain_buffer()
419 	ioc_info(ioc, "Provided chain_buffer_dma address is not in the lookup list\n");  in _base_get_chain_buffer_dma_to_chain_buffer()
433 static void _clone_sg_entries(struct MPT3SAS_ADAPTER *ioc,  in _clone_sg_entries()  argument
473 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _clone_sg_entries()
475 			ioc_err(ioc, "scmd is NULL\n");  in _clone_sg_entries()
498 	buffer_iomem = _base_get_buffer_bar0(ioc, smid);  in _clone_sg_entries()
499 	buffer_iomem_phys = _base_get_buffer_phys_bar0(ioc, smid);  in _clone_sg_entries()
509 	for (i = 0; i < MPT_MIN_PHYS_SEGMENTS + ioc->facts.MaxChainDepth; i++) {  in _clone_sg_entries()
522 				_base_get_chain_buffer_dma_to_chain_buffer(ioc,  in _clone_sg_entries()
531 				_base_get_chain(ioc,  in _clone_sg_entries()
535 			dst_addr_phys = _base_get_chain_phys(ioc,  in _clone_sg_entries()
558 					    ioc->config_vaddr,  in _clone_sg_entries()
596 				src_chain_addr[i], ioc->request_sz);  in _clone_sg_entries()
610 	struct MPT3SAS_ADAPTER *ioc = (struct MPT3SAS_ADAPTER *)arg;  in mpt3sas_remove_dead_ioc_func()  local
613 	if (!ioc)  in mpt3sas_remove_dead_ioc_func()
616 	pdev = ioc->pdev;  in mpt3sas_remove_dead_ioc_func()
629 static void _base_sync_drv_fw_timestamp(struct MPT3SAS_ADAPTER *ioc)  in _base_sync_drv_fw_timestamp()  argument
638 	mutex_lock(&ioc->scsih_cmds.mutex);  in _base_sync_drv_fw_timestamp()
639 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _base_sync_drv_fw_timestamp()
640 		ioc_err(ioc, "scsih_cmd in use %s\n", __func__);  in _base_sync_drv_fw_timestamp()
643 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _base_sync_drv_fw_timestamp()
644 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _base_sync_drv_fw_timestamp()
646 		ioc_err(ioc, "Failed obtaining a smid %s\n", __func__);  in _base_sync_drv_fw_timestamp()
647 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _base_sync_drv_fw_timestamp()
650 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_sync_drv_fw_timestamp()
651 	ioc->scsih_cmds.smid = smid;  in _base_sync_drv_fw_timestamp()
660 	init_completion(&ioc->scsih_cmds.done);  in _base_sync_drv_fw_timestamp()
661 	ioc->put_smid_default(ioc, smid);  in _base_sync_drv_fw_timestamp()
662 	dinitprintk(ioc, ioc_info(ioc,  in _base_sync_drv_fw_timestamp()
665 	wait_for_completion_timeout(&ioc->scsih_cmds.done,  in _base_sync_drv_fw_timestamp()
667 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_sync_drv_fw_timestamp()
668 		mpt3sas_check_cmd_timeout(ioc,  in _base_sync_drv_fw_timestamp()
669 		    ioc->scsih_cmds.status, mpi_request,  in _base_sync_drv_fw_timestamp()
673 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _base_sync_drv_fw_timestamp()
674 		mpi_reply = ioc->scsih_cmds.reply;  in _base_sync_drv_fw_timestamp()
675 		dinitprintk(ioc, ioc_info(ioc,  in _base_sync_drv_fw_timestamp()
682 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in _base_sync_drv_fw_timestamp()
683 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _base_sync_drv_fw_timestamp()
685 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _base_sync_drv_fw_timestamp()
697 	struct MPT3SAS_ADAPTER *ioc =  in _base_fault_reset_work()  local
705 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
706 	if ((ioc->shost_recovery && (ioc->ioc_coredump_loop == 0)) ||  in _base_fault_reset_work()
707 			ioc->pci_error_recovery)  in _base_fault_reset_work()
709 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
711 	doorbell = mpt3sas_base_get_iocstate(ioc, 0);  in _base_fault_reset_work()
713 		ioc_err(ioc, "SAS host is non-operational !!!!\n");  in _base_fault_reset_work()
723 		if (ioc->non_operational_loop++ < 5) {  in _base_fault_reset_work()
724 			spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock,  in _base_fault_reset_work()
736 		mpt3sas_base_pause_mq_polling(ioc);  in _base_fault_reset_work()
737 		ioc->schedule_dead_ioc_flush_running_cmds(ioc);  in _base_fault_reset_work()
742 		ioc->remove_host = 1;  in _base_fault_reset_work()
744 		p = kthread_run(mpt3sas_remove_dead_ioc_func, ioc,  in _base_fault_reset_work()
745 		    "%s_dead_ioc_%d", ioc->driver_name, ioc->id);  in _base_fault_reset_work()
747 			ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread failed !!!!\n",  in _base_fault_reset_work()
750 			ioc_err(ioc, "%s: Running mpt3sas_dead_ioc thread success !!!!\n",  in _base_fault_reset_work()
756 		u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?  in _base_fault_reset_work()
757 		    ioc->manu_pg11.CoreDumpTOSec :  in _base_fault_reset_work()
762 		if (ioc->ioc_coredump_loop == 0) {  in _base_fault_reset_work()
763 			mpt3sas_print_coredump_info(ioc,  in _base_fault_reset_work()
767 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
768 			ioc->shost_recovery = 1;  in _base_fault_reset_work()
770 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
771 			mpt3sas_base_mask_interrupts(ioc);  in _base_fault_reset_work()
772 			mpt3sas_base_pause_mq_polling(ioc);  in _base_fault_reset_work()
773 			_base_clear_outstanding_commands(ioc);  in _base_fault_reset_work()
776 		ioc_info(ioc, "%s: CoreDump loop %d.",  in _base_fault_reset_work()
777 		    __func__, ioc->ioc_coredump_loop);  in _base_fault_reset_work()
780 		if (ioc->ioc_coredump_loop++ < timeout) {  in _base_fault_reset_work()
782 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
787 	if (ioc->ioc_coredump_loop) {  in _base_fault_reset_work()
789 			ioc_err(ioc, "%s: CoreDump completed. LoopCount: %d",  in _base_fault_reset_work()
790 			    __func__, ioc->ioc_coredump_loop);  in _base_fault_reset_work()
792 			ioc_err(ioc, "%s: CoreDump Timed out. LoopCount: %d",  in _base_fault_reset_work()
793 			    __func__, ioc->ioc_coredump_loop);  in _base_fault_reset_work()
794 		ioc->ioc_coredump_loop = MPT3SAS_COREDUMP_LOOP_DONE;  in _base_fault_reset_work()
796 	ioc->non_operational_loop = 0;  in _base_fault_reset_work()
798 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in _base_fault_reset_work()
799 		ioc_warn(ioc, "%s: hard reset: %s\n",  in _base_fault_reset_work()
801 		doorbell = mpt3sas_base_get_iocstate(ioc, 0);  in _base_fault_reset_work()
803 			mpt3sas_print_fault_code(ioc, doorbell &  in _base_fault_reset_work()
807 			mpt3sas_print_coredump_info(ioc, doorbell &  in _base_fault_reset_work()
813 	ioc->ioc_coredump_loop = 0;  in _base_fault_reset_work()
814 	if (ioc->time_sync_interval &&  in _base_fault_reset_work()
815 	    ++ioc->timestamp_update_count >= ioc->time_sync_interval) {  in _base_fault_reset_work()
816 		ioc->timestamp_update_count = 0;  in _base_fault_reset_work()
817 		_base_sync_drv_fw_timestamp(ioc);  in _base_fault_reset_work()
819 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
821 	if (ioc->fault_reset_work_q)  in _base_fault_reset_work()
822 		queue_delayed_work(ioc->fault_reset_work_q,  in _base_fault_reset_work()
823 		    &ioc->fault_reset_work,  in _base_fault_reset_work()
825 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in _base_fault_reset_work()
835 mpt3sas_base_start_watchdog(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_start_watchdog()  argument
839 	if (ioc->fault_reset_work_q)  in mpt3sas_base_start_watchdog()
842 	ioc->timestamp_update_count = 0;  in mpt3sas_base_start_watchdog()
845 	INIT_DELAYED_WORK(&ioc->fault_reset_work, _base_fault_reset_work);  in mpt3sas_base_start_watchdog()
846 	snprintf(ioc->fault_reset_work_q_name,  in mpt3sas_base_start_watchdog()
847 	    sizeof(ioc->fault_reset_work_q_name), "poll_%s%d_status",  in mpt3sas_base_start_watchdog()
848 	    ioc->driver_name, ioc->id);  in mpt3sas_base_start_watchdog()
849 	ioc->fault_reset_work_q =  in mpt3sas_base_start_watchdog()
850 		create_singlethread_workqueue(ioc->fault_reset_work_q_name);  in mpt3sas_base_start_watchdog()
851 	if (!ioc->fault_reset_work_q) {  in mpt3sas_base_start_watchdog()
852 		ioc_err(ioc, "%s: failed (line=%d)\n", __func__, __LINE__);  in mpt3sas_base_start_watchdog()
855 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_start_watchdog()
856 	if (ioc->fault_reset_work_q)  in mpt3sas_base_start_watchdog()
857 		queue_delayed_work(ioc->fault_reset_work_q,  in mpt3sas_base_start_watchdog()
858 		    &ioc->fault_reset_work,  in mpt3sas_base_start_watchdog()
860 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_start_watchdog()
870 mpt3sas_base_stop_watchdog(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_stop_watchdog()  argument
875 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_stop_watchdog()
876 	wq = ioc->fault_reset_work_q;  in mpt3sas_base_stop_watchdog()
877 	ioc->fault_reset_work_q = NULL;  in mpt3sas_base_stop_watchdog()
878 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_stop_watchdog()
880 		if (!cancel_delayed_work_sync(&ioc->fault_reset_work))  in mpt3sas_base_stop_watchdog()
892 mpt3sas_base_fault_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code)  in mpt3sas_base_fault_info()  argument
894 	ioc_err(ioc, "fault_state(0x%04x)!\n", fault_code);  in mpt3sas_base_fault_info()
905 mpt3sas_base_coredump_info(struct MPT3SAS_ADAPTER *ioc, u16 fault_code)  in mpt3sas_base_coredump_info()  argument
907 	ioc_err(ioc, "coredump_state(0x%04x)!\n", fault_code);  in mpt3sas_base_coredump_info()
919 mpt3sas_base_wait_for_coredump_completion(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_wait_for_coredump_completion()  argument
922 	u8 timeout = (ioc->manu_pg11.CoreDumpTOSec) ?  in mpt3sas_base_wait_for_coredump_completion()
923 			ioc->manu_pg11.CoreDumpTOSec :  in mpt3sas_base_wait_for_coredump_completion()
926 	int ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_FAULT,  in mpt3sas_base_wait_for_coredump_completion()
930 		ioc_err(ioc,  in mpt3sas_base_wait_for_coredump_completion()
934 		ioc_info(ioc,  in mpt3sas_base_wait_for_coredump_completion()
951 mpt3sas_halt_firmware(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_halt_firmware()  argument
955 	if (!ioc->fwfault_debug)  in mpt3sas_halt_firmware()
960 	doorbell = ioc->base_readl_ext_retry(&ioc->chip->Doorbell);  in mpt3sas_halt_firmware()
962 		mpt3sas_print_fault_code(ioc, doorbell &  in mpt3sas_halt_firmware()
966 		mpt3sas_print_coredump_info(ioc, doorbell &  in mpt3sas_halt_firmware()
969 		writel(0xC0FFEE00, &ioc->chip->Doorbell);  in mpt3sas_halt_firmware()
970 		ioc_err(ioc, "Firmware is halted due to command timeout\n");  in mpt3sas_halt_firmware()
973 	if (ioc->fwfault_debug == 2)  in mpt3sas_halt_firmware()
987 _base_sas_ioc_info(struct MPT3SAS_ADAPTER *ioc, MPI2DefaultReply_t *mpi_reply,  in _base_sas_ioc_info()  argument
1014 		    !(ioc->logging_level & MPT_DEBUG_CONFIG)) {  in _base_sas_ioc_info()
1173 		frame_sz = sizeof(Mpi2ConfigRequest_t) + ioc->sge_size;  in _base_sas_ioc_info()
1197 		frame_sz = sizeof(Mpi2SmpPassthroughRequest_t) + ioc->sge_size;  in _base_sas_ioc_info()
1202 		    ioc->sge_size;  in _base_sas_ioc_info()
1211 	ioc_warn(ioc, "ioc_status: %s(0x%04x), request(0x%p),(%s)\n",  in _base_sas_ioc_info()
1223 _base_display_event_data(struct MPT3SAS_ADAPTER *ioc,  in _base_display_event_data()  argument
1229 	if (!(ioc->logging_level & MPT_DEBUG_EVENTS))  in _base_display_event_data()
1251 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1258 		ioc_info(ioc, "Discovery: (%s)",  in _base_display_event_data()
1283 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1287 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1291 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1295 		if (!ioc->hide_ir_msg)  in _base_display_event_data()
1314 		ioc_info(ioc, "PCIE Enumeration: (%s)",  in _base_display_event_data()
1331 	ioc_info(ioc, "%s\n", desc);  in _base_display_event_data()
1340 _base_sas_log_info(struct MPT3SAS_ADAPTER *ioc, u32 log_info)  in _base_sas_log_info()  argument
1363 	if (ioc->ignore_loginfos && (log_info == 0x30050000 || log_info ==  in _base_sas_log_info()
1375 		if (!ioc->hide_ir_msg)  in _base_sas_log_info()
1382 	ioc_warn(ioc, "log_info(0x%08x): originator(%s), code(0x%02x), sub_code(0x%04x)\n",  in _base_sas_log_info()
1395 _base_display_reply_info(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in _base_display_reply_info()  argument
1402 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _base_display_reply_info()
1404 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _base_display_reply_info()
1411 	    (ioc->logging_level & MPT_DEBUG_REPLY)) {  in _base_display_reply_info()
1412 		_base_sas_ioc_info(ioc, mpi_reply,  in _base_display_reply_info()
1413 		   mpt3sas_base_get_msg_frame(ioc, smid));  in _base_display_reply_info()
1418 		_base_sas_log_info(ioc, loginfo);  in _base_display_reply_info()
1423 		mpt3sas_trigger_mpi(ioc, ioc_status, loginfo);  in _base_display_reply_info()
1439 mpt3sas_base_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in mpt3sas_base_done()  argument
1444 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in mpt3sas_base_done()
1446 		return mpt3sas_check_for_pending_internal_cmds(ioc, smid);  in mpt3sas_base_done()
1448 	if (ioc->base_cmds.status == MPT3_CMD_NOT_USED)  in mpt3sas_base_done()
1451 	ioc->base_cmds.status |= MPT3_CMD_COMPLETE;  in mpt3sas_base_done()
1453 		ioc->base_cmds.status |= MPT3_CMD_REPLY_VALID;  in mpt3sas_base_done()
1454 		memcpy(ioc->base_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);  in mpt3sas_base_done()
1456 	ioc->base_cmds.status &= ~MPT3_CMD_PENDING;  in mpt3sas_base_done()
1458 	complete(&ioc->base_cmds.done);  in mpt3sas_base_done()
1473 _base_async_event(struct MPT3SAS_ADAPTER *ioc, u8 msix_index, u32 reply)  in _base_async_event()  argument
1480 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _base_async_event()
1486 	_base_display_event_data(ioc, mpi_reply);  in _base_async_event()
1490 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in _base_async_event()
1500 				&ioc->delayed_event_ack_list);  in _base_async_event()
1501 		dewtprintk(ioc,  in _base_async_event()
1502 			   ioc_info(ioc, "DELAYED: EVENT ACK: event (0x%04x)\n",  in _base_async_event()
1507 	ack_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_async_event()
1514 	ioc->put_smid_default(ioc, smid);  in _base_async_event()
1519 	mpt3sas_scsih_event_callback(ioc, msix_index, reply);  in _base_async_event()
1522 	mpt3sas_ctl_event_callback(ioc, msix_index, reply);  in _base_async_event()
1528 _get_st_from_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _get_st_from_smid()  argument
1533 	    WARN_ON(smid >= ioc->hi_priority_smid))  in _get_st_from_smid()
1536 	cmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _get_st_from_smid()
1551 _base_get_cb_idx(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_get_cb_idx()  argument
1554 	u16 ctl_smid = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT + 1;  in _base_get_cb_idx()
1557 	if (smid < ioc->hi_priority_smid) {  in _base_get_cb_idx()
1561 			st = _get_st_from_smid(ioc, smid);  in _base_get_cb_idx()
1565 			cb_idx = ioc->ctl_cb_idx;  in _base_get_cb_idx()
1566 	} else if (smid < ioc->internal_smid) {  in _base_get_cb_idx()
1567 		i = smid - ioc->hi_priority_smid;  in _base_get_cb_idx()
1568 		cb_idx = ioc->hpr_lookup[i].cb_idx;  in _base_get_cb_idx()
1569 	} else if (smid <= ioc->hba_queue_depth) {  in _base_get_cb_idx()
1570 		i = smid - ioc->internal_smid;  in _base_get_cb_idx()
1571 		cb_idx = ioc->internal_lookup[i].cb_idx;  in _base_get_cb_idx()
1588 mpt3sas_base_pause_mq_polling(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_pause_mq_polling()  argument
1591 	    ioc->reply_queue_count - ioc->iopoll_q_start_index;  in mpt3sas_base_pause_mq_polling()
1595 		atomic_set(&ioc->io_uring_poll_queues[qid].pause, 1);  in mpt3sas_base_pause_mq_polling()
1601 		while (atomic_read(&ioc->io_uring_poll_queues[qid].busy)) {  in mpt3sas_base_pause_mq_polling()
1615 mpt3sas_base_resume_mq_polling(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_resume_mq_polling()  argument
1618 	    ioc->reply_queue_count - ioc->iopoll_q_start_index;  in mpt3sas_base_resume_mq_polling()
1622 		atomic_set(&ioc->io_uring_poll_queues[qid].pause, 0);  in mpt3sas_base_resume_mq_polling()
1632 mpt3sas_base_mask_interrupts(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_mask_interrupts()  argument
1636 	ioc->mask_interrupts = 1;  in mpt3sas_base_mask_interrupts()
1637 	him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);  in mpt3sas_base_mask_interrupts()
1639 	writel(him_register, &ioc->chip->HostInterruptMask);  in mpt3sas_base_mask_interrupts()
1640 	ioc->base_readl(&ioc->chip->HostInterruptMask);  in mpt3sas_base_mask_interrupts()
1650 mpt3sas_base_unmask_interrupts(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_unmask_interrupts()  argument
1654 	him_register = ioc->base_readl(&ioc->chip->HostInterruptMask);  in mpt3sas_base_unmask_interrupts()
1656 	writel(him_register, &ioc->chip->HostInterruptMask);  in mpt3sas_base_unmask_interrupts()
1657 	ioc->mask_interrupts = 0;  in mpt3sas_base_unmask_interrupts()
1696 	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;  in _base_process_reply_queue()  local
1725 			cb_idx = _base_get_cb_idx(ioc, smid);  in _base_process_reply_queue()
1728 				rc = mpt_callbacks[cb_idx](ioc, smid,  in _base_process_reply_queue()
1731 					mpt3sas_base_free_smid(ioc, smid);  in _base_process_reply_queue()
1737 			if (reply > ioc->reply_dma_max_address ||  in _base_process_reply_queue()
1738 			    reply < ioc->reply_dma_min_address)  in _base_process_reply_queue()
1741 				cb_idx = _base_get_cb_idx(ioc, smid);  in _base_process_reply_queue()
1744 					rc = mpt_callbacks[cb_idx](ioc, smid,  in _base_process_reply_queue()
1747 						_base_display_reply_info(ioc,  in _base_process_reply_queue()
1750 						mpt3sas_base_free_smid(ioc,  in _base_process_reply_queue()
1754 				_base_async_event(ioc, msix_index, reply);  in _base_process_reply_queue()
1759 				ioc->reply_free_host_index =  in _base_process_reply_queue()
1760 				    (ioc->reply_free_host_index ==  in _base_process_reply_queue()
1761 				    (ioc->reply_free_queue_depth - 1)) ?  in _base_process_reply_queue()
1762 				    0 : ioc->reply_free_host_index + 1;  in _base_process_reply_queue()
1763 				ioc->reply_free[ioc->reply_free_host_index] =  in _base_process_reply_queue()
1765 				if (ioc->is_mcpu_endpoint)  in _base_process_reply_queue()
1766 					_base_clone_reply_to_sys_mem(ioc,  in _base_process_reply_queue()
1768 						ioc->reply_free_host_index);  in _base_process_reply_queue()
1769 				writel(ioc->reply_free_host_index,  in _base_process_reply_queue()
1770 				    &ioc->chip->ReplyFreeHostIndex);  in _base_process_reply_queue()
1777 		    (ioc->reply_post_queue_depth - 1)) ? 0 :  in _base_process_reply_queue()
1788 		if (completed_cmds >= ioc->thresh_hold) {  in _base_process_reply_queue()
1789 			if (ioc->combined_reply_queue) {  in _base_process_reply_queue()
1793 				    ioc->replyPostRegisterIndex[msix_index/8]);  in _base_process_reply_queue()
1798 						&ioc->chip->ReplyPostHostIndex);  in _base_process_reply_queue()
1823 	if (ioc->is_warpdrive) {  in _base_process_reply_queue()
1825 		ioc->reply_post_host_index[msix_index]);  in _base_process_reply_queue()
1845 	if (ioc->combined_reply_queue)  in _base_process_reply_queue()
1848 			ioc->replyPostRegisterIndex[msix_index/8]);  in _base_process_reply_queue()
1852 			&ioc->chip->ReplyPostHostIndex);  in _base_process_reply_queue()
1866 	struct MPT3SAS_ADAPTER *ioc =  in mpt3sas_blk_mq_poll()  local
1870 	int qid = queue_num - ioc->iopoll_q_start_index;  in mpt3sas_blk_mq_poll()
1872 	if (atomic_read(&ioc->io_uring_poll_queues[qid].pause) ||  in mpt3sas_blk_mq_poll()
1873 	    !atomic_add_unless(&ioc->io_uring_poll_queues[qid].busy, 1, 1))  in mpt3sas_blk_mq_poll()
1876 	reply_q = ioc->io_uring_poll_queues[qid].reply_q;  in mpt3sas_blk_mq_poll()
1879 	atomic_dec(&ioc->io_uring_poll_queues[qid].busy);  in mpt3sas_blk_mq_poll()
1895 	struct MPT3SAS_ADAPTER *ioc = reply_q->ioc;  in _base_interrupt()  local
1897 	if (ioc->mask_interrupts)  in _base_interrupt()
1949 _base_init_irqpolls(struct MPT3SAS_ADAPTER *ioc)  in _base_init_irqpolls()  argument
1953 	if (list_empty(&ioc->reply_queue_list))  in _base_init_irqpolls()
1956 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {  in _base_init_irqpolls()
1960 			ioc->hba_queue_depth/4, _base_irqpoll);  in _base_init_irqpolls()
1963 		reply_q->os_irq = pci_irq_vector(ioc->pdev,  in _base_init_irqpolls()
1975 _base_is_controller_msix_enabled(struct MPT3SAS_ADAPTER *ioc)  in _base_is_controller_msix_enabled()  argument
1977 	return (ioc->facts.IOCCapabilities &  in _base_is_controller_msix_enabled()
1978 	    MPI2_IOCFACTS_CAPABILITY_MSI_X_INDEX) && ioc->msix_enable;  in _base_is_controller_msix_enabled()
1991 mpt3sas_base_sync_reply_irqs(struct MPT3SAS_ADAPTER *ioc, u8 poll)  in mpt3sas_base_sync_reply_irqs()  argument
1998 	if (!_base_is_controller_msix_enabled(ioc))  in mpt3sas_base_sync_reply_irqs()
2001 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in mpt3sas_base_sync_reply_irqs()
2002 		if (ioc->shost_recovery || ioc->remove_host ||  in mpt3sas_base_sync_reply_irqs()
2003 				ioc->pci_error_recovery)  in mpt3sas_base_sync_reply_irqs()
2014 		synchronize_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index));  in mpt3sas_base_sync_reply_irqs()
2088 _base_build_zero_len_sge(struct MPT3SAS_ADAPTER *ioc, void *paddr)  in _base_build_zero_len_sge()  argument
2094 	ioc->base_add_sg_single(paddr, flags_length, -1);  in _base_build_zero_len_sge()
2141 _base_get_chain_buffer_tracker(struct MPT3SAS_ADAPTER *ioc,  in _base_get_chain_buffer_tracker()  argument
2148 	   atomic_read(&ioc->chain_lookup[smid - 1].chain_offset);  in _base_get_chain_buffer_tracker()
2150 	if (chain_offset == ioc->chains_needed_per_io)  in _base_get_chain_buffer_tracker()
2153 	chain_req = &ioc->chain_lookup[smid - 1].chains_per_smid[chain_offset];  in _base_get_chain_buffer_tracker()
2154 	atomic_inc(&ioc->chain_lookup[smid - 1].chain_offset);  in _base_get_chain_buffer_tracker()
2169 _base_build_sg(struct MPT3SAS_ADAPTER *ioc, void *psge,  in _base_build_sg()  argument
2176 		_base_build_zero_len_sge(ioc, psge);  in _base_build_sg()
2185 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2189 		psge += ioc->sge_size;  in _base_build_sg()
2196 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2203 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2210 		ioc->base_add_sg_single(psge, sgl_flags |  in _base_build_sg()
2273 _base_build_nvme_prp(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_build_nvme_prp()  argument
2301 	prp_page = (__le64 *)mpt3sas_base_get_pcie_sgl(ioc, smid);  in _base_build_nvme_prp()
2302 	prp_page_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);  in _base_build_nvme_prp()
2308 	page_mask = ioc->page_size - 1;  in _base_build_nvme_prp()
2358 		entry_len = ioc->page_size - offset;  in _base_build_nvme_prp()
2378 			if (length > ioc->page_size) {  in _base_build_nvme_prp()
2440 base_make_prp_nvme(struct MPT3SAS_ADAPTER *ioc,  in base_make_prp_nvme()  argument
2455 	nvme_pg_size = max_t(u32, ioc->page_size, NVME_PRP_PAGE_SIZE);  in base_make_prp_nvme()
2494 	curr_buff = mpt3sas_base_get_pcie_sgl(ioc, smid);  in base_make_prp_nvme()
2495 	msg_dma = mpt3sas_base_get_pcie_sgl_dma(ioc, smid);  in base_make_prp_nvme()
2567 base_is_prp_possible(struct MPT3SAS_ADAPTER *ioc,  in base_is_prp_possible()  argument
2605 _base_check_pcie_native_sgl(struct MPT3SAS_ADAPTER *ioc,  in _base_check_pcie_native_sgl()  argument
2617 	if (!base_is_prp_possible(ioc, pcie_device,  in _base_check_pcie_native_sgl()
2626 	base_make_prp_nvme(ioc, scmd, mpi_request,  in _base_check_pcie_native_sgl()
2665 _base_build_zero_len_sge_ieee(struct MPT3SAS_ADAPTER *ioc, void *paddr)  in _base_build_zero_len_sge_ieee()  argument
2705 _base_build_sg_scmd(struct MPT3SAS_ADAPTER *ioc,  in _base_build_sg_scmd()  argument
2722 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_build_sg_scmd()
2741 	sges_in_segment = ioc->max_sges_in_main_message;  in _base_build_sg_scmd()
2746 	    (sges_in_segment * ioc->sge_size))/4;  in _base_build_sg_scmd()
2751 			ioc->base_add_sg_single(sg_local,  in _base_build_sg_scmd()
2755 			ioc->base_add_sg_single(sg_local, sgl_flags |  in _base_build_sg_scmd()
2758 		sg_local += ioc->sge_size;  in _base_build_sg_scmd()
2765 	chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd()
2772 		    ioc->max_sges_in_chain_message) ? sges_left :  in _base_build_sg_scmd()
2773 		    ioc->max_sges_in_chain_message;  in _base_build_sg_scmd()
2775 		    0 : (sges_in_segment * ioc->sge_size)/4;  in _base_build_sg_scmd()
2776 		chain_length = sges_in_segment * ioc->sge_size;  in _base_build_sg_scmd()
2780 			chain_length += ioc->sge_size;  in _base_build_sg_scmd()
2782 		ioc->base_add_sg_single(sg_local, chain_flags | chain_offset |  in _base_build_sg_scmd()
2791 				ioc->base_add_sg_single(sg_local,  in _base_build_sg_scmd()
2796 				ioc->base_add_sg_single(sg_local, sgl_flags |  in _base_build_sg_scmd()
2800 			sg_local += ioc->sge_size;  in _base_build_sg_scmd()
2805 		chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd()
2818 			ioc->base_add_sg_single(sg_local, sgl_flags_end_buffer |  in _base_build_sg_scmd()
2821 			ioc->base_add_sg_single(sg_local, sgl_flags |  in _base_build_sg_scmd()
2824 		sg_local += ioc->sge_size;  in _base_build_sg_scmd()
2846 _base_build_sg_scmd_ieee(struct MPT3SAS_ADAPTER *ioc,  in _base_build_sg_scmd_ieee()  argument
2862 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_build_sg_scmd_ieee()
2873 	if ((pcie_device) && (_base_check_pcie_native_sgl(ioc, mpi_request,  in _base_build_sg_scmd_ieee()
2885 	sges_in_segment = (ioc->request_sz -  in _base_build_sg_scmd_ieee()
2886 		   offsetof(Mpi25SCSIIORequest_t, SGL))/ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2891 	    (offsetof(Mpi25SCSIIORequest_t, SGL)/ioc->sge_size_ieee);  in _base_build_sg_scmd_ieee()
2898 		sg_local += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2904 	chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd_ieee()
2911 		    ioc->max_sges_in_chain_message) ? sges_left :  in _base_build_sg_scmd_ieee()
2912 		    ioc->max_sges_in_chain_message;  in _base_build_sg_scmd_ieee()
2915 		chain_length = sges_in_segment * ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2917 			chain_length += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2930 			sg_local += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2935 		chain_req = _base_get_chain_buffer_tracker(ioc, scmd);  in _base_build_sg_scmd_ieee()
2955 		sg_local += ioc->sge_size_ieee;  in _base_build_sg_scmd_ieee()
2972 _base_build_sg_ieee(struct MPT3SAS_ADAPTER *ioc, void *psge,  in _base_build_sg_ieee()  argument
2979 		_base_build_zero_len_sge_ieee(ioc, psge);  in _base_build_sg_ieee()
2991 		psge += ioc->sge_size_ieee;  in _base_build_sg_ieee()
3022 _base_config_dma_addressing(struct MPT3SAS_ADAPTER *ioc, struct pci_dev *pdev)  in _base_config_dma_addressing()  argument
3027 	if (ioc->is_mcpu_endpoint || sizeof(dma_addr_t) == 4) {  in _base_config_dma_addressing()
3028 		ioc->dma_mask = 32;  in _base_config_dma_addressing()
3031 	} else if (ioc->hba_mpi_version_belonged > MPI2_VERSION) {  in _base_config_dma_addressing()
3032 		ioc->dma_mask = 63;  in _base_config_dma_addressing()
3035 		ioc->dma_mask = 64;  in _base_config_dma_addressing()
3039 	if (ioc->use_32bit_dma)  in _base_config_dma_addressing()
3046 	if (ioc->dma_mask > 32) {  in _base_config_dma_addressing()
3047 		ioc->base_add_sg_single = &_base_add_sg_single_64;  in _base_config_dma_addressing()
3048 		ioc->sge_size = sizeof(Mpi2SGESimple64_t);  in _base_config_dma_addressing()
3050 		ioc->base_add_sg_single = &_base_add_sg_single_32;  in _base_config_dma_addressing()
3051 		ioc->sge_size = sizeof(Mpi2SGESimple32_t);  in _base_config_dma_addressing()
3055 	ioc_info(ioc, "%d BIT PCI BUS DMA ADDRESSING SUPPORTED, total mem (%ld kB)\n",  in _base_config_dma_addressing()
3056 		ioc->dma_mask, convert_to_kb(s.totalram));  in _base_config_dma_addressing()
3069 _base_check_enable_msix(struct MPT3SAS_ADAPTER *ioc)  in _base_check_enable_msix()  argument
3077 	if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 &&  in _base_check_enable_msix()
3078 	    ioc->pdev->revision == SAS2_PCI_DEVICE_B0_REVISION) {  in _base_check_enable_msix()
3082 	base = pci_find_capability(ioc->pdev, PCI_CAP_ID_MSIX);  in _base_check_enable_msix()
3084 		dfailprintk(ioc, ioc_info(ioc, "msix not supported\n"));  in _base_check_enable_msix()
3090 	if (ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2004 ||  in _base_check_enable_msix()
3091 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2008 ||  in _base_check_enable_msix()
3092 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_1 ||  in _base_check_enable_msix()
3093 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_2 ||  in _base_check_enable_msix()
3094 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2108_3 ||  in _base_check_enable_msix()
3095 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_1 ||  in _base_check_enable_msix()
3096 	    ioc->pdev->device == MPI2_MFGPAGE_DEVID_SAS2116_2)  in _base_check_enable_msix()
3097 		ioc->msix_vector_count = 1;  in _base_check_enable_msix()
3099 		pci_read_config_word(ioc->pdev, base + 2, &message_control);  in _base_check_enable_msix()
3100 		ioc->msix_vector_count = (message_control & 0x3FF) + 1;  in _base_check_enable_msix()
3102 	dinitprintk(ioc, ioc_info(ioc, "msix is supported, vector_count(%d)\n",  in _base_check_enable_msix()
3103 				  ioc->msix_vector_count));  in _base_check_enable_msix()
3114 mpt3sas_base_free_irq(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_free_irq()  argument
3119 	if (list_empty(&ioc->reply_queue_list))  in mpt3sas_base_free_irq()
3122 	list_for_each_entry_safe(reply_q, next, &ioc->reply_queue_list, list) {  in mpt3sas_base_free_irq()
3129 		if (ioc->smp_affinity_enable) {  in mpt3sas_base_free_irq()
3130 			irq = pci_irq_vector(ioc->pdev, reply_q->msix_index);  in mpt3sas_base_free_irq()
3133 		free_irq(pci_irq_vector(ioc->pdev, reply_q->msix_index),  in mpt3sas_base_free_irq()
3147 _base_request_irq(struct MPT3SAS_ADAPTER *ioc, u8 index)  in _base_request_irq()  argument
3149 	struct pci_dev *pdev = ioc->pdev;  in _base_request_irq()
3155 		ioc_err(ioc, "unable to allocate memory %zu!\n",  in _base_request_irq()
3159 	reply_q->ioc = ioc;  in _base_request_irq()
3164 	if (index >= ioc->iopoll_q_start_index) {  in _base_request_irq()
3165 		qid = index - ioc->iopoll_q_start_index;  in _base_request_irq()
3167 		    ioc->driver_name, ioc->id, qid);  in _base_request_irq()
3169 		ioc->io_uring_poll_queues[qid].reply_q = reply_q;  in _base_request_irq()
3174 	if (ioc->msix_enable)  in _base_request_irq()
3176 		    ioc->driver_name, ioc->id, index);  in _base_request_irq()
3179 		    ioc->driver_name, ioc->id);  in _base_request_irq()
3190 	list_add_tail(&reply_q->list, &ioc->reply_queue_list);  in _base_request_irq()
3201 _base_assign_reply_queues(struct MPT3SAS_ADAPTER *ioc)  in _base_assign_reply_queues()  argument
3205 	int iopoll_q_count = ioc->reply_queue_count -  in _base_assign_reply_queues()
3206 	    ioc->iopoll_q_start_index;  in _base_assign_reply_queues()
3209 	if (!_base_is_controller_msix_enabled(ioc))  in _base_assign_reply_queues()
3212 	if (ioc->msix_load_balance)  in _base_assign_reply_queues()
3215 	memset(ioc->cpu_msix_table, 0, ioc->cpu_msix_table_sz);  in _base_assign_reply_queues()
3218 	nr_msix = ioc->reply_queue_count = min(ioc->reply_queue_count,  in _base_assign_reply_queues()
3219 					       ioc->facts.MaxMSIxVectors);  in _base_assign_reply_queues()
3223 	if (ioc->smp_affinity_enable) {  in _base_assign_reply_queues()
3229 		if (ioc->high_iops_queues) {  in _base_assign_reply_queues()
3230 			mask = cpumask_of_node(dev_to_node(&ioc->pdev->dev));  in _base_assign_reply_queues()
3231 			for (index = 0; index < ioc->high_iops_queues;  in _base_assign_reply_queues()
3233 				irq = pci_irq_vector(ioc->pdev, index);  in _base_assign_reply_queues()
3238 		list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_assign_reply_queues()
3241 			if (reply_q->msix_index < ioc->high_iops_queues ||  in _base_assign_reply_queues()
3242 			    reply_q->msix_index >= ioc->iopoll_q_start_index)  in _base_assign_reply_queues()
3245 			mask = pci_irq_get_affinity(ioc->pdev,  in _base_assign_reply_queues()
3248 				ioc_warn(ioc, "no affinity for msi %x\n",  in _base_assign_reply_queues()
3254 				if (cpu >= ioc->cpu_msix_table_sz)  in _base_assign_reply_queues()
3256 				ioc->cpu_msix_table[cpu] = reply_q->msix_index;  in _base_assign_reply_queues()
3264 	nr_msix -= (ioc->high_iops_queues - iopoll_q_count);  in _base_assign_reply_queues()
3267 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_assign_reply_queues()
3270 		if (reply_q->msix_index < ioc->high_iops_queues ||  in _base_assign_reply_queues()
3271 		    reply_q->msix_index >= ioc->iopoll_q_start_index)  in _base_assign_reply_queues()
3281 			ioc->cpu_msix_table[cpu] = reply_q->msix_index;  in _base_assign_reply_queues()
3303 _base_check_and_enable_high_iops_queues(struct MPT3SAS_ADAPTER *ioc,  in _base_check_and_enable_high_iops_queues()  argument
3313 	    ioc->io_uring_poll_queues) {  in _base_check_and_enable_high_iops_queues()
3314 		ioc->high_iops_queues = 0;  in _base_check_and_enable_high_iops_queues()
3320 		pcie_capability_read_word(ioc->pdev, PCI_EXP_LNKSTA, &lnksta);  in _base_check_and_enable_high_iops_queues()
3324 			ioc->high_iops_queues = 0;  in _base_check_and_enable_high_iops_queues()
3329 	if (!reset_devices && ioc->is_aero_ioc &&  in _base_check_and_enable_high_iops_queues()
3333 		ioc->high_iops_queues = MPT3SAS_HIGH_IOPS_REPLY_QUEUES;  in _base_check_and_enable_high_iops_queues()
3335 		ioc->high_iops_queues = 0;  in _base_check_and_enable_high_iops_queues()
3344 mpt3sas_base_disable_msix(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_disable_msix()  argument
3346 	if (!ioc->msix_enable)  in mpt3sas_base_disable_msix()
3348 	pci_free_irq_vectors(ioc->pdev);  in mpt3sas_base_disable_msix()
3349 	ioc->msix_enable = 0;  in mpt3sas_base_disable_msix()
3350 	kfree(ioc->io_uring_poll_queues);  in mpt3sas_base_disable_msix()
3359 _base_alloc_irq_vectors(struct MPT3SAS_ADAPTER *ioc)  in _base_alloc_irq_vectors()  argument
3362 	struct irq_affinity desc = { .pre_vectors = ioc->high_iops_queues };  in _base_alloc_irq_vectors()
3368 	int nr_msix_vectors = ioc->iopoll_q_start_index;  in _base_alloc_irq_vectors()
3371 	if (ioc->smp_affinity_enable)  in _base_alloc_irq_vectors()
3376 	ioc_info(ioc, " %d %d %d\n", ioc->high_iops_queues,  in _base_alloc_irq_vectors()
3377 	    ioc->reply_queue_count, nr_msix_vectors);  in _base_alloc_irq_vectors()
3379 	i = pci_alloc_irq_vectors_affinity(ioc->pdev,  in _base_alloc_irq_vectors()
3380 	    ioc->high_iops_queues,  in _base_alloc_irq_vectors()
3392 _base_enable_msix(struct MPT3SAS_ADAPTER *ioc)  in _base_enable_msix()  argument
3399 	ioc->msix_load_balance = false;  in _base_enable_msix()
3407 	if (_base_check_enable_msix(ioc) != 0)  in _base_enable_msix()
3410 	ioc_info(ioc, "MSI-X vectors supported: %d\n", ioc->msix_vector_count);  in _base_enable_msix()
3412 		ioc->cpu_count, max_msix_vectors);  in _base_enable_msix()
3414 	ioc->reply_queue_count =  in _base_enable_msix()
3415 		min_t(int, ioc->cpu_count, ioc->msix_vector_count);  in _base_enable_msix()
3417 	if (!ioc->rdpq_array_enable && max_msix_vectors == -1)  in _base_enable_msix()
3429 	if (!ioc->combined_reply_queue &&  in _base_enable_msix()
3430 	    ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_enable_msix()
3431 		ioc_info(ioc,  in _base_enable_msix()
3433 		ioc->msix_load_balance = true;  in _base_enable_msix()
3440 	if (ioc->msix_load_balance)  in _base_enable_msix()
3441 		ioc->smp_affinity_enable = 0;  in _base_enable_msix()
3443 	if (!ioc->smp_affinity_enable || ioc->reply_queue_count <= 1)  in _base_enable_msix()
3444 		ioc->shost->host_tagset = 0;  in _base_enable_msix()
3449 	if (ioc->shost->host_tagset)  in _base_enable_msix()
3453 		ioc->io_uring_poll_queues = kcalloc(iopoll_q_count,  in _base_enable_msix()
3455 		if (!ioc->io_uring_poll_queues)  in _base_enable_msix()
3459 	if (ioc->is_aero_ioc)  in _base_enable_msix()
3460 		_base_check_and_enable_high_iops_queues(ioc,  in _base_enable_msix()
3461 		    ioc->msix_vector_count);  in _base_enable_msix()
3467 	ioc->reply_queue_count = min_t(int,  in _base_enable_msix()
3468 	    ioc->reply_queue_count + ioc->high_iops_queues,  in _base_enable_msix()
3469 	    ioc->msix_vector_count);  in _base_enable_msix()
3476 		ioc->reply_queue_count = min_t(int, local_max_msix_vectors,  in _base_enable_msix()
3477 		    ioc->reply_queue_count);  in _base_enable_msix()
3483 		if (ioc->reply_queue_count < (iopoll_q_count + MPT3_MIN_IRQS))  in _base_enable_msix()
3485 		ioc->reply_queue_count = min_t(int,  in _base_enable_msix()
3486 		    ioc->reply_queue_count + iopoll_q_count,  in _base_enable_msix()
3487 		    ioc->msix_vector_count);  in _base_enable_msix()
3493 	ioc->iopoll_q_start_index =  in _base_enable_msix()
3494 	    ioc->reply_queue_count - iopoll_q_count;  in _base_enable_msix()
3496 	r = _base_alloc_irq_vectors(ioc);  in _base_enable_msix()
3498 		ioc_info(ioc, "pci_alloc_irq_vectors failed (r=%d) !!!\n", r);  in _base_enable_msix()
3507 	if (r < ioc->iopoll_q_start_index) {  in _base_enable_msix()
3508 		ioc->reply_queue_count = r + iopoll_q_count;  in _base_enable_msix()
3509 		ioc->iopoll_q_start_index =  in _base_enable_msix()
3510 		    ioc->reply_queue_count - iopoll_q_count;  in _base_enable_msix()
3513 	ioc->msix_enable = 1;  in _base_enable_msix()
3514 	for (i = 0; i < ioc->reply_queue_count; i++) {  in _base_enable_msix()
3515 		r = _base_request_irq(ioc, i);  in _base_enable_msix()
3517 			mpt3sas_base_free_irq(ioc);  in _base_enable_msix()
3518 			mpt3sas_base_disable_msix(ioc);  in _base_enable_msix()
3523 	ioc_info(ioc, "High IOPs queues : %s\n",  in _base_enable_msix()
3524 			ioc->high_iops_queues ? "enabled" : "disabled");  in _base_enable_msix()
3530 	ioc->high_iops_queues = 0;  in _base_enable_msix()
3531 	ioc_info(ioc, "High IOPs queues : disabled\n");  in _base_enable_msix()
3532 	ioc->reply_queue_count = 1;  in _base_enable_msix()
3533 	ioc->iopoll_q_start_index = ioc->reply_queue_count - 0;  in _base_enable_msix()
3534 	r = pci_alloc_irq_vectors(ioc->pdev, 1, 1, PCI_IRQ_LEGACY);  in _base_enable_msix()
3536 		dfailprintk(ioc,  in _base_enable_msix()
3537 			    ioc_info(ioc, "pci_alloc_irq_vector(legacy) failed (r=%d) !!!\n",  in _base_enable_msix()
3540 		r = _base_request_irq(ioc, 0);  in _base_enable_msix()
3550 mpt3sas_base_unmap_resources(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_unmap_resources()  argument
3552 	struct pci_dev *pdev = ioc->pdev;  in mpt3sas_base_unmap_resources()
3554 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_unmap_resources()
3556 	mpt3sas_base_free_irq(ioc);  in mpt3sas_base_unmap_resources()
3557 	mpt3sas_base_disable_msix(ioc);  in mpt3sas_base_unmap_resources()
3559 	kfree(ioc->replyPostRegisterIndex);  in mpt3sas_base_unmap_resources()
3560 	ioc->replyPostRegisterIndex = NULL;  in mpt3sas_base_unmap_resources()
3563 	if (ioc->chip_phys) {  in mpt3sas_base_unmap_resources()
3564 		iounmap(ioc->chip);  in mpt3sas_base_unmap_resources()
3565 		ioc->chip_phys = 0;  in mpt3sas_base_unmap_resources()
3569 		pci_release_selected_regions(ioc->pdev, ioc->bars);  in mpt3sas_base_unmap_resources()
3575 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc);
3585 mpt3sas_base_check_for_fault_and_issue_reset(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_check_for_fault_and_issue_reset()  argument
3590 	dinitprintk(ioc, pr_info("%s\n", __func__));  in mpt3sas_base_check_for_fault_and_issue_reset()
3591 	if (ioc->pci_error_recovery)  in mpt3sas_base_check_for_fault_and_issue_reset()
3593 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_check_for_fault_and_issue_reset()
3594 	dhsprintk(ioc, pr_info("%s: ioc_state(0x%08x)\n", __func__, ioc_state));  in mpt3sas_base_check_for_fault_and_issue_reset()
3597 		mpt3sas_print_fault_code(ioc, ioc_state &  in mpt3sas_base_check_for_fault_and_issue_reset()
3599 		mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3600 		rc = _base_diag_reset(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3603 		mpt3sas_print_coredump_info(ioc, ioc_state &  in mpt3sas_base_check_for_fault_and_issue_reset()
3605 		mpt3sas_base_wait_for_coredump_completion(ioc, __func__);  in mpt3sas_base_check_for_fault_and_issue_reset()
3606 		mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3607 		rc = _base_diag_reset(ioc);  in mpt3sas_base_check_for_fault_and_issue_reset()
3620 mpt3sas_base_map_resources(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_map_resources()  argument
3622 	struct pci_dev *pdev = ioc->pdev;  in mpt3sas_base_map_resources()
3631 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_map_resources()
3633 	ioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);  in mpt3sas_base_map_resources()
3635 		ioc_warn(ioc, "pci_enable_device_mem: failed\n");  in mpt3sas_base_map_resources()
3636 		ioc->bars = 0;  in mpt3sas_base_map_resources()
3641 	if (pci_request_selected_regions(pdev, ioc->bars,  in mpt3sas_base_map_resources()
3642 	    ioc->driver_name)) {  in mpt3sas_base_map_resources()
3643 		ioc_warn(ioc, "pci_request_selected_regions: failed\n");  in mpt3sas_base_map_resources()
3644 		ioc->bars = 0;  in mpt3sas_base_map_resources()
3652 	if (_base_config_dma_addressing(ioc, pdev) != 0) {  in mpt3sas_base_map_resources()
3653 		ioc_warn(ioc, "no suitable DMA mask for %s\n", pci_name(pdev));  in mpt3sas_base_map_resources()
3668 			ioc->chip_phys = pci_resource_start(pdev, i);  in mpt3sas_base_map_resources()
3669 			chip_phys = ioc->chip_phys;  in mpt3sas_base_map_resources()
3671 			ioc->chip = ioremap(ioc->chip_phys, memap_sz);  in mpt3sas_base_map_resources()
3675 	if (ioc->chip == NULL) {  in mpt3sas_base_map_resources()
3676 		ioc_err(ioc,  in mpt3sas_base_map_resources()
3682 	mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_map_resources()
3684 	r = _base_get_ioc_facts(ioc);  in mpt3sas_base_map_resources()
3686 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in mpt3sas_base_map_resources()
3687 		if (rc || (_base_get_ioc_facts(ioc)))  in mpt3sas_base_map_resources()
3691 	if (!ioc->rdpq_array_enable_assigned) {  in mpt3sas_base_map_resources()
3692 		ioc->rdpq_array_enable = ioc->rdpq_array_capable;  in mpt3sas_base_map_resources()
3693 		ioc->rdpq_array_enable_assigned = 1;  in mpt3sas_base_map_resources()
3696 	r = _base_enable_msix(ioc);  in mpt3sas_base_map_resources()
3700 	iopoll_q_count = ioc->reply_queue_count - ioc->iopoll_q_start_index;  in mpt3sas_base_map_resources()
3702 		atomic_set(&ioc->io_uring_poll_queues[i].busy, 0);  in mpt3sas_base_map_resources()
3703 		atomic_set(&ioc->io_uring_poll_queues[i].pause, 0);  in mpt3sas_base_map_resources()
3706 	if (!ioc->is_driver_loading)  in mpt3sas_base_map_resources()
3707 		_base_init_irqpolls(ioc);  in mpt3sas_base_map_resources()
3711 	if (ioc->combined_reply_queue) {  in mpt3sas_base_map_resources()
3718 		ioc->replyPostRegisterIndex = kcalloc(  in mpt3sas_base_map_resources()
3719 		     ioc->combined_reply_index_count,  in mpt3sas_base_map_resources()
3721 		if (!ioc->replyPostRegisterIndex) {  in mpt3sas_base_map_resources()
3722 			ioc_err(ioc,  in mpt3sas_base_map_resources()
3728 		for (i = 0; i < ioc->combined_reply_index_count; i++) {  in mpt3sas_base_map_resources()
3729 			ioc->replyPostRegisterIndex[i] =  in mpt3sas_base_map_resources()
3731 				((u8 __force *)&ioc->chip->Doorbell +  in mpt3sas_base_map_resources()
3737 	if (ioc->is_warpdrive) {  in mpt3sas_base_map_resources()
3738 		ioc->reply_post_host_index[0] = (resource_size_t __iomem *)  in mpt3sas_base_map_resources()
3739 		    &ioc->chip->ReplyPostHostIndex;  in mpt3sas_base_map_resources()
3741 		for (i = 1; i < ioc->cpu_msix_table_sz; i++)  in mpt3sas_base_map_resources()
3742 			ioc->reply_post_host_index[i] =  in mpt3sas_base_map_resources()
3744 			((u8 __iomem *)&ioc->chip->Doorbell + (0x4000 + ((i - 1)  in mpt3sas_base_map_resources()
3748 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in mpt3sas_base_map_resources()
3749 		if (reply_q->msix_index >= ioc->iopoll_q_start_index) {  in mpt3sas_base_map_resources()
3757 			ioc->msix_enable ? "PCI-MSI-X" : "IO-APIC",  in mpt3sas_base_map_resources()
3758 			pci_irq_vector(ioc->pdev, reply_q->msix_index));  in mpt3sas_base_map_resources()
3761 	ioc_info(ioc, "iomem(%pap), mapped(0x%p), size(%d)\n",  in mpt3sas_base_map_resources()
3762 		 &chip_phys, ioc->chip, memap_sz);  in mpt3sas_base_map_resources()
3763 	ioc_info(ioc, "ioport(0x%016llx), size(%d)\n",  in mpt3sas_base_map_resources()
3771 	mpt3sas_base_unmap_resources(ioc);  in mpt3sas_base_map_resources()
3783 mpt3sas_base_get_msg_frame(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_msg_frame()  argument
3785 	return (void *)(ioc->request + (smid * ioc->request_sz));  in mpt3sas_base_get_msg_frame()
3796 mpt3sas_base_get_sense_buffer(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_sense_buffer()  argument
3798 	return (void *)(ioc->sense + ((smid - 1) * SCSI_SENSE_BUFFERSIZE));  in mpt3sas_base_get_sense_buffer()
3809 mpt3sas_base_get_sense_buffer_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_sense_buffer_dma()  argument
3811 	return cpu_to_le32(ioc->sense_dma + ((smid - 1) *  in mpt3sas_base_get_sense_buffer_dma()
3823 mpt3sas_base_get_pcie_sgl(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_pcie_sgl()  argument
3825 	return (void *)(ioc->pcie_sg_lookup[smid - 1].pcie_sgl);  in mpt3sas_base_get_pcie_sgl()
3836 mpt3sas_base_get_pcie_sgl_dma(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_get_pcie_sgl_dma()  argument
3838 	return ioc->pcie_sg_lookup[smid - 1].pcie_sgl_dma;  in mpt3sas_base_get_pcie_sgl_dma()
3849 mpt3sas_base_get_reply_virt_addr(struct MPT3SAS_ADAPTER *ioc, u32 phys_addr)  in mpt3sas_base_get_reply_virt_addr()  argument
3853 	return ioc->reply + (phys_addr - (u32)ioc->reply_dma);  in mpt3sas_base_get_reply_virt_addr()
3866 _base_get_msix_index(struct MPT3SAS_ADAPTER *ioc,  in _base_get_msix_index()  argument
3870 	if (ioc->msix_load_balance)  in _base_get_msix_index()
3871 		return ioc->reply_queue_count ?  in _base_get_msix_index()
3873 		    &ioc->total_io_cnt), ioc->reply_queue_count) : 0;  in _base_get_msix_index()
3875 	if (scmd && ioc->shost->nr_hw_queues > 1) {  in _base_get_msix_index()
3879 			ioc->high_iops_queues;  in _base_get_msix_index()
3882 	return ioc->cpu_msix_table[raw_smp_processor_id()];  in _base_get_msix_index()
3896 _base_get_high_iops_msix_index(struct MPT3SAS_ADAPTER *ioc,  in _base_get_high_iops_msix_index()  argument
3907 		    atomic64_add_return(1, &ioc->high_iops_outstanding) /  in _base_get_high_iops_msix_index()
3911 	return _base_get_msix_index(ioc, scmd);  in _base_get_high_iops_msix_index()
3922 mpt3sas_base_get_smid(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)  in mpt3sas_base_get_smid()  argument
3928 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid()
3929 	if (list_empty(&ioc->internal_free_list)) {  in mpt3sas_base_get_smid()
3930 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid()
3931 		ioc_err(ioc, "%s: smid not available\n", __func__);  in mpt3sas_base_get_smid()
3935 	request = list_entry(ioc->internal_free_list.next,  in mpt3sas_base_get_smid()
3940 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid()
3953 mpt3sas_base_get_smid_scsiio(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx,  in mpt3sas_base_get_smid_scsiio()  argument
3973 	ioc->io_queue_num[tag] = blk_mq_unique_tag_to_hwq(unique_tag);  in mpt3sas_base_get_smid_scsiio()
3991 mpt3sas_base_get_smid_hpr(struct MPT3SAS_ADAPTER *ioc, u8 cb_idx)  in mpt3sas_base_get_smid_hpr()  argument
3997 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid_hpr()
3998 	if (list_empty(&ioc->hpr_free_list)) {  in mpt3sas_base_get_smid_hpr()
3999 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid_hpr()
4003 	request = list_entry(ioc->hpr_free_list.next,  in mpt3sas_base_get_smid_hpr()
4008 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_get_smid_hpr()
4013 _base_recovery_check(struct MPT3SAS_ADAPTER *ioc)  in _base_recovery_check()  argument
4018 	if (ioc->shost_recovery && ioc->pending_io_count) {  in _base_recovery_check()
4019 		ioc->pending_io_count = scsi_host_busy(ioc->shost);  in _base_recovery_check()
4020 		if (ioc->pending_io_count == 0)  in _base_recovery_check()
4021 			wake_up(&ioc->reset_wq);  in _base_recovery_check()
4025 void mpt3sas_base_clear_st(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_clear_st()  argument
4033 	atomic_set(&ioc->chain_lookup[st->smid - 1].chain_offset, 0);  in mpt3sas_base_clear_st()
4043 mpt3sas_base_free_smid(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_free_smid()  argument
4048 	if (smid < ioc->hi_priority_smid) {  in mpt3sas_base_free_smid()
4052 		st = _get_st_from_smid(ioc, smid);  in mpt3sas_base_free_smid()
4054 			_base_recovery_check(ioc);  in mpt3sas_base_free_smid()
4059 		request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_base_free_smid()
4060 		memset(request, 0, ioc->request_sz);  in mpt3sas_base_free_smid()
4062 		mpt3sas_base_clear_st(ioc, st);  in mpt3sas_base_free_smid()
4063 		_base_recovery_check(ioc);  in mpt3sas_base_free_smid()
4064 		ioc->io_queue_num[smid - 1] = 0;  in mpt3sas_base_free_smid()
4068 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_free_smid()
4069 	if (smid < ioc->internal_smid) {  in mpt3sas_base_free_smid()
4071 		i = smid - ioc->hi_priority_smid;  in mpt3sas_base_free_smid()
4072 		ioc->hpr_lookup[i].cb_idx = 0xFF;  in mpt3sas_base_free_smid()
4073 		list_add(&ioc->hpr_lookup[i].tracker_list, &ioc->hpr_free_list);  in mpt3sas_base_free_smid()
4074 	} else if (smid <= ioc->hba_queue_depth) {  in mpt3sas_base_free_smid()
4076 		i = smid - ioc->internal_smid;  in mpt3sas_base_free_smid()
4077 		ioc->internal_lookup[i].cb_idx = 0xFF;  in mpt3sas_base_free_smid()
4078 		list_add(&ioc->internal_lookup[i].tracker_list,  in mpt3sas_base_free_smid()
4079 		    &ioc->internal_free_list);  in mpt3sas_base_free_smid()
4081 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in mpt3sas_base_free_smid()
4141 _base_set_and_get_msix_index(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_set_and_get_msix_index()  argument
4145 	if (smid < ioc->hi_priority_smid)  in _base_set_and_get_msix_index()
4146 		st = _get_st_from_smid(ioc, smid);  in _base_set_and_get_msix_index()
4149 		return  _base_get_msix_index(ioc, NULL);  in _base_set_and_get_msix_index()
4151 	st->msix_io = ioc->get_msix_index_for_smlio(ioc, st->scmd);  in _base_set_and_get_msix_index()
4162 _base_put_smid_mpi_ep_scsi_io(struct MPT3SAS_ADAPTER *ioc,  in _base_put_smid_mpi_ep_scsi_io()  argument
4168 	__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);  in _base_put_smid_mpi_ep_scsi_io()
4170 	_clone_sg_entries(ioc, (void *) mfp, smid);  in _base_put_smid_mpi_ep_scsi_io()
4171 	mpi_req_iomem = (void __force *)ioc->chip +  in _base_put_smid_mpi_ep_scsi_io()
4172 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);  in _base_put_smid_mpi_ep_scsi_io()
4174 					ioc->request_sz);  in _base_put_smid_mpi_ep_scsi_io()
4176 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_mpi_ep_scsi_io()
4180 	_base_mpi_ep_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_mpi_ep_scsi_io()
4181 	    &ioc->scsi_lookup_lock);  in _base_put_smid_mpi_ep_scsi_io()
4191 _base_put_smid_scsi_io(struct MPT3SAS_ADAPTER *ioc, u16 smid, u16 handle)  in _base_put_smid_scsi_io()  argument
4198 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_scsi_io()
4202 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_scsi_io()
4203 	    &ioc->scsi_lookup_lock);  in _base_put_smid_scsi_io()
4213 _base_put_smid_fast_path(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_fast_path()  argument
4221 	descriptor.SCSIIO.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_fast_path()
4225 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_fast_path()
4226 	    &ioc->scsi_lookup_lock);  in _base_put_smid_fast_path()
4236 _base_put_smid_hi_priority(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_hi_priority()  argument
4243 	if (ioc->is_mcpu_endpoint) {  in _base_put_smid_hi_priority()
4244 		__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);  in _base_put_smid_hi_priority()
4247 		mpi_req_iomem = (void __force *)ioc->chip  in _base_put_smid_hi_priority()
4249 					+ (smid * ioc->request_sz);  in _base_put_smid_hi_priority()
4251 							ioc->request_sz);  in _base_put_smid_hi_priority()
4262 	if (ioc->is_mcpu_endpoint)  in _base_put_smid_hi_priority()
4264 				&ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_hi_priority()
4265 				&ioc->scsi_lookup_lock);  in _base_put_smid_hi_priority()
4267 		_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_hi_priority()
4268 		    &ioc->scsi_lookup_lock);  in _base_put_smid_hi_priority()
4278 mpt3sas_base_put_smid_nvme_encap(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_base_put_smid_nvme_encap()  argument
4285 	descriptor.Default.MSIxIndex =  _base_set_and_get_msix_index(ioc, smid);  in mpt3sas_base_put_smid_nvme_encap()
4289 	_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in mpt3sas_base_put_smid_nvme_encap()
4290 	    &ioc->scsi_lookup_lock);  in mpt3sas_base_put_smid_nvme_encap()
4299 _base_put_smid_default(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_put_smid_default()  argument
4305 	if (ioc->is_mcpu_endpoint) {  in _base_put_smid_default()
4306 		__le32 *mfp = (__le32 *)mpt3sas_base_get_msg_frame(ioc, smid);  in _base_put_smid_default()
4308 		_clone_sg_entries(ioc, (void *) mfp, smid);  in _base_put_smid_default()
4310 		mpi_req_iomem = (void __force *)ioc->chip +  in _base_put_smid_default()
4311 			MPI_FRAME_START_OFFSET + (smid * ioc->request_sz);  in _base_put_smid_default()
4313 							ioc->request_sz);  in _base_put_smid_default()
4317 	descriptor.Default.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_default()
4321 	if (ioc->is_mcpu_endpoint)  in _base_put_smid_default()
4323 				&ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_default()
4324 				&ioc->scsi_lookup_lock);  in _base_put_smid_default()
4326 		_base_writeq(*request, &ioc->chip->RequestDescriptorPostLow,  in _base_put_smid_default()
4327 				&ioc->scsi_lookup_lock);  in _base_put_smid_default()
4340 _base_put_smid_scsi_io_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_scsi_io_atomic()  argument
4347 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_scsi_io_atomic()
4350 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_scsi_io_atomic()
4362 _base_put_smid_fast_path_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_fast_path_atomic()  argument
4369 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_fast_path_atomic()
4372 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_fast_path_atomic()
4385 _base_put_smid_hi_priority_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _base_put_smid_hi_priority_atomic()  argument
4395 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_hi_priority_atomic()
4407 _base_put_smid_default_atomic(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _base_put_smid_default_atomic()  argument
4413 	descriptor.MSIxIndex = _base_set_and_get_msix_index(ioc, smid);  in _base_put_smid_default_atomic()
4416 	writel(cpu_to_le32(*request), &ioc->chip->AtomicRequestDescriptorPost);  in _base_put_smid_default_atomic()
4424 _base_display_OEMs_branding(struct MPT3SAS_ADAPTER *ioc)  in _base_display_OEMs_branding()  argument
4426 	if (ioc->pdev->subsystem_vendor != PCI_VENDOR_ID_INTEL)  in _base_display_OEMs_branding()
4429 	switch (ioc->pdev->subsystem_vendor) {  in _base_display_OEMs_branding()
4431 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4433 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4435 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4439 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4443 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4447 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4448 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4453 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4455 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4459 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4463 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4467 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4471 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4475 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4479 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4483 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4484 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4489 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4491 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4496 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4500 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4504 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4508 				ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4509 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4514 			ioc_info(ioc, "Intel(R) Controller: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4515 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4520 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4522 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4524 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4528 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4532 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4536 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4540 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4544 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4548 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4552 				ioc_info(ioc, "Dell 6Gbps HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4553 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4558 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4560 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4564 				ioc_info(ioc, "Dell 12Gbps HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4565 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4570 			ioc_info(ioc, "Dell HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4571 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4576 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4578 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4580 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4584 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4588 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4592 				ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4593 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4598 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4600 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4604 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4608 				ioc_info(ioc, "Cisco 12Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4609 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4614 			ioc_info(ioc, "Cisco SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4615 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4620 		switch (ioc->pdev->device) {  in _base_display_OEMs_branding()
4622 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4624 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4628 				ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4629 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4634 			switch (ioc->pdev->subsystem_device) {  in _base_display_OEMs_branding()
4636 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4640 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4644 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4648 				ioc_info(ioc, "%s\n",  in _base_display_OEMs_branding()
4652 				ioc_info(ioc, "HP 6Gbps SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4653 					 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4658 			ioc_info(ioc, "HP SAS HBA: Subsystem ID: 0x%X\n",  in _base_display_OEMs_branding()
4659 				 ioc->pdev->subsystem_device);  in _base_display_OEMs_branding()
4676 _base_display_fwpkg_version(struct MPT3SAS_ADAPTER *ioc)  in _base_display_fwpkg_version()  argument
4689 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_display_fwpkg_version()
4691 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {  in _base_display_fwpkg_version()
4692 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in _base_display_fwpkg_version()
4697 	fwpkg_data = dma_alloc_coherent(&ioc->pdev->dev, data_length,  in _base_display_fwpkg_version()
4700 		ioc_err(ioc,  in _base_display_fwpkg_version()
4706 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in _base_display_fwpkg_version()
4708 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _base_display_fwpkg_version()
4713 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in _base_display_fwpkg_version()
4714 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_display_fwpkg_version()
4715 	ioc->base_cmds.smid = smid;  in _base_display_fwpkg_version()
4720 	ioc->build_sg(ioc, &mpi_request->SGL, 0, 0, fwpkg_data_dma,  in _base_display_fwpkg_version()
4722 	init_completion(&ioc->base_cmds.done);  in _base_display_fwpkg_version()
4723 	ioc->put_smid_default(ioc, smid);  in _base_display_fwpkg_version()
4725 	wait_for_completion_timeout(&ioc->base_cmds.done,  in _base_display_fwpkg_version()
4727 	ioc_info(ioc, "%s: complete\n", __func__);  in _base_display_fwpkg_version()
4728 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_display_fwpkg_version()
4729 		ioc_err(ioc, "%s: timeout\n", __func__);  in _base_display_fwpkg_version()
4735 		if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID) {  in _base_display_fwpkg_version()
4736 			memcpy(&mpi_reply, ioc->base_cmds.reply,  in _base_display_fwpkg_version()
4755 					ioc_info(ioc,  in _base_display_fwpkg_version()
4767 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in _base_display_fwpkg_version()
4770 		dma_free_coherent(&ioc->pdev->dev, data_length, fwpkg_data,  in _base_display_fwpkg_version()
4773 		if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED)  in _base_display_fwpkg_version()
4775 		if (mpt3sas_base_check_for_fault_and_issue_reset(ioc))  in _base_display_fwpkg_version()
4787 _base_display_ioc_capabilities(struct MPT3SAS_ADAPTER *ioc)  in _base_display_ioc_capabilities()  argument
4793 	strncpy(desc, ioc->manu_pg0.ChipName, 16);  in _base_display_ioc_capabilities()
4794 	ioc_info(ioc, "%s: FWVersion(%02d.%02d.%02d.%02d), ChipRevision(0x%02x)\n",  in _base_display_ioc_capabilities()
4796 		 (ioc->facts.FWVersion.Word & 0xFF000000) >> 24,  in _base_display_ioc_capabilities()
4797 		 (ioc->facts.FWVersion.Word & 0x00FF0000) >> 16,  in _base_display_ioc_capabilities()
4798 		 (ioc->facts.FWVersion.Word & 0x0000FF00) >> 8,  in _base_display_ioc_capabilities()
4799 		 ioc->facts.FWVersion.Word & 0x000000FF,  in _base_display_ioc_capabilities()
4800 		 ioc->pdev->revision);  in _base_display_ioc_capabilities()
4802 	_base_display_OEMs_branding(ioc);  in _base_display_ioc_capabilities()
4804 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {  in _base_display_ioc_capabilities()
4809 	ioc_info(ioc, "Protocol=(");  in _base_display_ioc_capabilities()
4811 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR) {  in _base_display_ioc_capabilities()
4816 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_TARGET) {  in _base_display_ioc_capabilities()
4824 	if (!ioc->hide_ir_msg) {  in _base_display_ioc_capabilities()
4825 		if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4832 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR) {  in _base_display_ioc_capabilities()
4837 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_MULTICAST) {  in _base_display_ioc_capabilities()
4842 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4848 	if (ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_EEDP) {  in _base_display_ioc_capabilities()
4853 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4859 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4865 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4871 	if (ioc->facts.IOCCapabilities &  in _base_display_ioc_capabilities()
4877 	iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);  in _base_display_ioc_capabilities()
4897 mpt3sas_base_update_missing_delay(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_update_missing_delay()  argument
4908 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);  in mpt3sas_base_update_missing_delay()
4916 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_base_update_missing_delay()
4920 	if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,  in mpt3sas_base_update_missing_delay()
4922 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_base_update_missing_delay()
4929 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_base_update_missing_delay()
4954 	if (!mpt3sas_config_set_sas_iounit_pg1(ioc, &mpi_reply, sas_iounit_pg1,  in mpt3sas_base_update_missing_delay()
4962 		ioc_info(ioc, "device_missing_delay: old(%d), new(%d)\n",  in mpt3sas_base_update_missing_delay()
4964 		ioc_info(ioc, "ioc_missing_delay: old(%d), new(%d)\n",  in mpt3sas_base_update_missing_delay()
4967 		ioc->device_missing_delay = dmd_new;  in mpt3sas_base_update_missing_delay()
4968 		ioc->io_missing_delay = io_missing_delay;  in mpt3sas_base_update_missing_delay()
4984 _base_update_ioc_page1_inlinewith_perf_mode(struct MPT3SAS_ADAPTER *ioc)  in _base_update_ioc_page1_inlinewith_perf_mode()  argument
4990 	rc = mpt3sas_config_get_ioc_pg1(ioc, &mpi_reply, &ioc->ioc_pg1_copy);  in _base_update_ioc_page1_inlinewith_perf_mode()
4993 	memcpy(&ioc_pg1, &ioc->ioc_pg1_copy, sizeof(Mpi2IOCPage1_t));  in _base_update_ioc_page1_inlinewith_perf_mode()
4998 		if (ioc->high_iops_queues) {  in _base_update_ioc_page1_inlinewith_perf_mode()
4999 			ioc_info(ioc,  in _base_update_ioc_page1_inlinewith_perf_mode()
5014 			rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);  in _base_update_ioc_page1_inlinewith_perf_mode()
5017 			ioc_info(ioc, "performance mode: balanced\n");  in _base_update_ioc_page1_inlinewith_perf_mode()
5029 		rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);  in _base_update_ioc_page1_inlinewith_perf_mode()
5032 		ioc_info(ioc, "performance mode: latency\n");  in _base_update_ioc_page1_inlinewith_perf_mode()
5038 		ioc_info(ioc,  in _base_update_ioc_page1_inlinewith_perf_mode()
5043 		rc = mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply, &ioc_pg1);  in _base_update_ioc_page1_inlinewith_perf_mode()
5059 _base_get_event_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_event_diag_triggers()  argument
5069 	r = mpt3sas_config_get_driver_trigger_pg2(ioc, &mpi_reply,  in _base_get_event_diag_triggers()
5077 		dinitprintk(ioc,  in _base_get_event_diag_triggers()
5078 		    ioc_err(ioc,  in _base_get_event_diag_triggers()
5087 		ioc->diag_trigger_event.ValidEntries = count;  in _base_get_event_diag_triggers()
5089 		event_tg = &ioc->diag_trigger_event.EventTriggerEntry[0];  in _base_get_event_diag_triggers()
5111 _base_get_scsi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_scsi_diag_triggers()  argument
5121 	r = mpt3sas_config_get_driver_trigger_pg3(ioc, &mpi_reply,  in _base_get_scsi_diag_triggers()
5129 		dinitprintk(ioc,  in _base_get_scsi_diag_triggers()
5130 		    ioc_err(ioc,  in _base_get_scsi_diag_triggers()
5139 		ioc->diag_trigger_scsi.ValidEntries = count;  in _base_get_scsi_diag_triggers()
5141 		scsi_tg = &ioc->diag_trigger_scsi.SCSITriggerEntry[0];  in _base_get_scsi_diag_triggers()
5163 _base_get_mpi_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_mpi_diag_triggers()  argument
5173 	r = mpt3sas_config_get_driver_trigger_pg4(ioc, &mpi_reply,  in _base_get_mpi_diag_triggers()
5181 		dinitprintk(ioc,  in _base_get_mpi_diag_triggers()
5182 		    ioc_err(ioc,  in _base_get_mpi_diag_triggers()
5191 		ioc->diag_trigger_mpi.ValidEntries = count;  in _base_get_mpi_diag_triggers()
5193 		status_tg = &ioc->diag_trigger_mpi.MPITriggerEntry[0];  in _base_get_mpi_diag_triggers()
5217 _base_get_master_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_master_diag_triggers()  argument
5224 	r = mpt3sas_config_get_driver_trigger_pg1(ioc, &mpi_reply,  in _base_get_master_diag_triggers()
5232 		dinitprintk(ioc,  in _base_get_master_diag_triggers()
5233 		    ioc_err(ioc,  in _base_get_master_diag_triggers()
5240 		ioc->diag_trigger_master.MasterData |=  in _base_get_master_diag_triggers()
5259 _base_check_for_trigger_pages_support(struct MPT3SAS_ADAPTER *ioc, u32 *trigger_flags)  in _base_check_for_trigger_pages_support()  argument
5266 	r = mpt3sas_config_get_driver_trigger_pg0(ioc, &mpi_reply,  in _base_check_for_trigger_pages_support()
5289 _base_get_diag_triggers(struct MPT3SAS_ADAPTER *ioc)  in _base_get_diag_triggers()  argument
5297 	ioc->diag_trigger_master.MasterData =  in _base_get_diag_triggers()
5300 	r = _base_check_for_trigger_pages_support(ioc, &trigger_flags);  in _base_get_diag_triggers()
5311 	ioc->supports_trigger_pages = 1;  in _base_get_diag_triggers()
5319 		r = _base_get_master_diag_triggers(ioc);  in _base_get_diag_triggers()
5330 		r = _base_get_event_diag_triggers(ioc);  in _base_get_diag_triggers()
5341 		r = _base_get_scsi_diag_triggers(ioc);  in _base_get_diag_triggers()
5351 		r = _base_get_mpi_diag_triggers(ioc);  in _base_get_diag_triggers()
5367 _base_update_diag_trigger_pages(struct MPT3SAS_ADAPTER *ioc)  in _base_update_diag_trigger_pages()  argument
5370 	if (ioc->diag_trigger_master.MasterData)  in _base_update_diag_trigger_pages()
5371 		mpt3sas_config_update_driver_trigger_pg1(ioc,  in _base_update_diag_trigger_pages()
5372 		    &ioc->diag_trigger_master, 1);  in _base_update_diag_trigger_pages()
5374 	if (ioc->diag_trigger_event.ValidEntries)  in _base_update_diag_trigger_pages()
5375 		mpt3sas_config_update_driver_trigger_pg2(ioc,  in _base_update_diag_trigger_pages()
5376 		    &ioc->diag_trigger_event, 1);  in _base_update_diag_trigger_pages()
5378 	if (ioc->diag_trigger_scsi.ValidEntries)  in _base_update_diag_trigger_pages()
5379 		mpt3sas_config_update_driver_trigger_pg3(ioc,  in _base_update_diag_trigger_pages()
5380 		    &ioc->diag_trigger_scsi, 1);  in _base_update_diag_trigger_pages()
5382 	if (ioc->diag_trigger_mpi.ValidEntries)  in _base_update_diag_trigger_pages()
5383 		mpt3sas_config_update_driver_trigger_pg4(ioc,  in _base_update_diag_trigger_pages()
5384 		    &ioc->diag_trigger_mpi, 1);  in _base_update_diag_trigger_pages()
5395 static int _base_assign_fw_reported_qd(struct MPT3SAS_ADAPTER *ioc)  in _base_assign_fw_reported_qd()  argument
5404 	ioc->max_wideport_qd = MPT3SAS_SAS_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5405 	ioc->max_narrowport_qd = MPT3SAS_SAS_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5406 	ioc->max_sata_qd = MPT3SAS_SATA_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5407 	ioc->max_nvme_qd = MPT3SAS_NVME_QUEUE_DEPTH;  in _base_assign_fw_reported_qd()
5408 	if (!ioc->is_gen35_ioc)  in _base_assign_fw_reported_qd()
5415 		    ioc->name, __FILE__, __LINE__, __func__);  in _base_assign_fw_reported_qd()
5418 	rc = mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,  in _base_assign_fw_reported_qd()
5422 		    ioc->name, __FILE__, __LINE__, __func__);  in _base_assign_fw_reported_qd()
5427 	ioc->max_wideport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH);  in _base_assign_fw_reported_qd()
5430 	ioc->max_narrowport_qd = (depth ? depth : MPT3SAS_SAS_QUEUE_DEPTH);  in _base_assign_fw_reported_qd()
5433 	ioc->max_sata_qd = (depth ? depth : MPT3SAS_SATA_QUEUE_DEPTH);  in _base_assign_fw_reported_qd()
5436 	rc = mpt3sas_config_get_pcie_iounit_pg1(ioc, &mpi_reply,  in _base_assign_fw_reported_qd()
5440 		    ioc->name, __FILE__, __LINE__, __func__);  in _base_assign_fw_reported_qd()
5443 	ioc->max_nvme_qd = (le16_to_cpu(pcie_iounit_pg1.NVMeMaxQueueDepth)) ?  in _base_assign_fw_reported_qd()
5447 	dinitprintk(ioc, pr_err(  in _base_assign_fw_reported_qd()
5449 	    ioc->max_wideport_qd, ioc->max_narrowport_qd,  in _base_assign_fw_reported_qd()
5450 	    ioc->max_sata_qd, ioc->max_nvme_qd));  in _base_assign_fw_reported_qd()
5463 mpt3sas_atto_validate_nvram(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_atto_validate_nvram()  argument
5481 		ioc_err(ioc, "Invalid ATTO NVRAM checksum\n");  in mpt3sas_atto_validate_nvram()
5491 		ioc_err(ioc, "Invalid ATTO NVRAM signature\n");  in mpt3sas_atto_validate_nvram()
5493 		ioc_info(ioc, "Invalid ATTO NVRAM version");  in mpt3sas_atto_validate_nvram()
5500 		ioc_err(ioc, "Invalid ATTO SAS address\n");  in mpt3sas_atto_validate_nvram()
5514 mpt3sas_atto_get_sas_addr(struct MPT3SAS_ADAPTER *ioc, union ATTO_SAS_ADDRESS *sas_addr)  in mpt3sas_atto_get_sas_addr()  argument
5522 	r = mpt3sas_config_get_manufacturing_pg1(ioc, &mpi_reply, &mfg_pg1);  in mpt3sas_atto_get_sas_addr()
5524 		ioc_err(ioc, "Failed to read manufacturing page 1\n");  in mpt3sas_atto_get_sas_addr()
5530 	r = mpt3sas_atto_validate_nvram(ioc, nvram);  in mpt3sas_atto_get_sas_addr()
5547 mpt3sas_atto_init(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_atto_init()  argument
5558 	r = mpt3sas_atto_get_sas_addr(ioc, &sas_addr);  in mpt3sas_atto_init()
5563 	r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, NULL, 0);  in mpt3sas_atto_init()
5565 		ioc_err(ioc, "Failed to read ATTO bios page 4 header.\n");  in mpt3sas_atto_init()
5572 		ioc_err(ioc, "Failed to allocate memory for ATTO bios page.\n");  in mpt3sas_atto_init()
5577 	r = mpt3sas_config_get_bios_pg4(ioc, &mpi_reply, bios_pg4, sz);  in mpt3sas_atto_init()
5579 		ioc_err(ioc, "Failed to read ATTO bios page 4\n");  in mpt3sas_atto_init()
5593 	r = mpt3sas_config_set_bios_pg4(ioc, &mpi_reply, bios_pg4, sz);  in mpt3sas_atto_init()
5605 _base_static_config_pages(struct MPT3SAS_ADAPTER *ioc)  in _base_static_config_pages()  argument
5611 	ioc->nvme_abort_timeout = 30;  in _base_static_config_pages()
5613 	rc = mpt3sas_config_get_manufacturing_pg0(ioc, &mpi_reply,  in _base_static_config_pages()
5614 	    &ioc->manu_pg0);  in _base_static_config_pages()
5617 	if (ioc->ir_firmware) {  in _base_static_config_pages()
5618 		rc = mpt3sas_config_get_manufacturing_pg10(ioc, &mpi_reply,  in _base_static_config_pages()
5619 		    &ioc->manu_pg10);  in _base_static_config_pages()
5624 	if (ioc->pdev->vendor == MPI2_MFGPAGE_VENDORID_ATTO) {  in _base_static_config_pages()
5625 		rc = mpt3sas_atto_init(ioc);  in _base_static_config_pages()
5634 	rc = mpt3sas_config_get_manufacturing_pg11(ioc, &mpi_reply,  in _base_static_config_pages()
5635 	    &ioc->manu_pg11);  in _base_static_config_pages()
5638 	if (!ioc->is_gen35_ioc && ioc->manu_pg11.EEDPTagMode == 0) {  in _base_static_config_pages()
5640 		    ioc->name);  in _base_static_config_pages()
5641 		ioc->manu_pg11.EEDPTagMode &= ~0x3;  in _base_static_config_pages()
5642 		ioc->manu_pg11.EEDPTagMode |= 0x1;  in _base_static_config_pages()
5643 		mpt3sas_config_set_manufacturing_pg11(ioc, &mpi_reply,  in _base_static_config_pages()
5644 		    &ioc->manu_pg11);  in _base_static_config_pages()
5646 	if (ioc->manu_pg11.AddlFlags2 & NVME_TASK_MNGT_CUSTOM_MASK)  in _base_static_config_pages()
5647 		ioc->tm_custom_handling = 1;  in _base_static_config_pages()
5649 		ioc->tm_custom_handling = 0;  in _base_static_config_pages()
5650 		if (ioc->manu_pg11.NVMeAbortTO < NVME_TASK_ABORT_MIN_TIMEOUT)  in _base_static_config_pages()
5651 			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MIN_TIMEOUT;  in _base_static_config_pages()
5652 		else if (ioc->manu_pg11.NVMeAbortTO >  in _base_static_config_pages()
5654 			ioc->nvme_abort_timeout = NVME_TASK_ABORT_MAX_TIMEOUT;  in _base_static_config_pages()
5656 			ioc->nvme_abort_timeout = ioc->manu_pg11.NVMeAbortTO;  in _base_static_config_pages()
5658 	ioc->time_sync_interval =  in _base_static_config_pages()
5659 	    ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_MASK;  in _base_static_config_pages()
5660 	if (ioc->time_sync_interval) {  in _base_static_config_pages()
5661 		if (ioc->manu_pg11.TimeSyncInterval & MPT3SAS_TIMESYNC_UNIT_MASK)  in _base_static_config_pages()
5662 			ioc->time_sync_interval =  in _base_static_config_pages()
5663 			    ioc->time_sync_interval * SECONDS_PER_HOUR;  in _base_static_config_pages()
5665 			ioc->time_sync_interval =  in _base_static_config_pages()
5666 			    ioc->time_sync_interval * SECONDS_PER_MIN;  in _base_static_config_pages()
5667 		dinitprintk(ioc, ioc_info(ioc,  in _base_static_config_pages()
5669 		    ioc->time_sync_interval, (ioc->manu_pg11.TimeSyncInterval &  in _base_static_config_pages()
5672 		if (ioc->is_gen35_ioc)  in _base_static_config_pages()
5673 			ioc_warn(ioc,  in _base_static_config_pages()
5676 	rc = _base_assign_fw_reported_qd(ioc);  in _base_static_config_pages()
5683 	if (ioc->pdev->vendor ==  MPI2_MFGPAGE_VENDORID_ATTO)  in _base_static_config_pages()
5684 		ioc->bios_pg3.BiosVersion = 0;  in _base_static_config_pages()
5686 		rc = mpt3sas_config_get_bios_pg2(ioc, &mpi_reply, &ioc->bios_pg2);  in _base_static_config_pages()
5689 		rc = mpt3sas_config_get_bios_pg3(ioc, &mpi_reply, &ioc->bios_pg3);  in _base_static_config_pages()
5694 	rc = mpt3sas_config_get_ioc_pg8(ioc, &mpi_reply, &ioc->ioc_pg8);  in _base_static_config_pages()
5697 	rc = mpt3sas_config_get_iounit_pg0(ioc, &mpi_reply, &ioc->iounit_pg0);  in _base_static_config_pages()
5700 	rc = mpt3sas_config_get_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);  in _base_static_config_pages()
5703 	rc = mpt3sas_config_get_iounit_pg8(ioc, &mpi_reply, &ioc->iounit_pg8);  in _base_static_config_pages()
5706 	_base_display_ioc_capabilities(ioc);  in _base_static_config_pages()
5712 	iounit_pg1_flags = le32_to_cpu(ioc->iounit_pg1.Flags);  in _base_static_config_pages()
5713 	if ((ioc->facts.IOCCapabilities &  in _base_static_config_pages()
5720 	ioc->iounit_pg1.Flags = cpu_to_le32(iounit_pg1_flags);  in _base_static_config_pages()
5721 	rc = mpt3sas_config_set_iounit_pg1(ioc, &mpi_reply, &ioc->iounit_pg1);  in _base_static_config_pages()
5725 	if (ioc->iounit_pg8.NumSensors)  in _base_static_config_pages()
5726 		ioc->temp_sensors_count = ioc->iounit_pg8.NumSensors;  in _base_static_config_pages()
5727 	if (ioc->is_aero_ioc) {  in _base_static_config_pages()
5728 		rc = _base_update_ioc_page1_inlinewith_perf_mode(ioc);  in _base_static_config_pages()
5732 	if (ioc->is_gen35_ioc) {  in _base_static_config_pages()
5733 		if (ioc->is_driver_loading) {  in _base_static_config_pages()
5734 			rc = _base_get_diag_triggers(ioc);  in _base_static_config_pages()
5749 			_base_check_for_trigger_pages_support(ioc, &tg_flags);  in _base_static_config_pages()
5750 			if (!ioc->supports_trigger_pages && tg_flags != -EFAULT)  in _base_static_config_pages()
5751 				_base_update_diag_trigger_pages(ioc);  in _base_static_config_pages()
5752 			else if (ioc->supports_trigger_pages &&  in _base_static_config_pages()
5754 				ioc->supports_trigger_pages = 0;  in _base_static_config_pages()
5767 mpt3sas_free_enclosure_list(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_free_enclosure_list()  argument
5773 			enclosure_dev_next, &ioc->enclosure_list, list) {  in mpt3sas_free_enclosure_list()
5786 _base_release_memory_pools(struct MPT3SAS_ADAPTER *ioc)  in _base_release_memory_pools()  argument
5792 	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;  in _base_release_memory_pools()
5794 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_release_memory_pools()
5796 	if (ioc->request) {  in _base_release_memory_pools()
5797 		dma_free_coherent(&ioc->pdev->dev, ioc->request_dma_sz,  in _base_release_memory_pools()
5798 		    ioc->request,  ioc->request_dma);  in _base_release_memory_pools()
5799 		dexitprintk(ioc,  in _base_release_memory_pools()
5800 			    ioc_info(ioc, "request_pool(0x%p): free\n",  in _base_release_memory_pools()
5801 				     ioc->request));  in _base_release_memory_pools()
5802 		ioc->request = NULL;  in _base_release_memory_pools()
5805 	if (ioc->sense) {  in _base_release_memory_pools()
5806 		dma_pool_free(ioc->sense_dma_pool, ioc->sense, ioc->sense_dma);  in _base_release_memory_pools()
5807 		dma_pool_destroy(ioc->sense_dma_pool);  in _base_release_memory_pools()
5808 		dexitprintk(ioc,  in _base_release_memory_pools()
5809 			    ioc_info(ioc, "sense_pool(0x%p): free\n",  in _base_release_memory_pools()
5810 				     ioc->sense));  in _base_release_memory_pools()
5811 		ioc->sense = NULL;  in _base_release_memory_pools()
5814 	if (ioc->reply) {  in _base_release_memory_pools()
5815 		dma_pool_free(ioc->reply_dma_pool, ioc->reply, ioc->reply_dma);  in _base_release_memory_pools()
5816 		dma_pool_destroy(ioc->reply_dma_pool);  in _base_release_memory_pools()
5817 		dexitprintk(ioc,  in _base_release_memory_pools()
5818 			    ioc_info(ioc, "reply_pool(0x%p): free\n",  in _base_release_memory_pools()
5819 				     ioc->reply));  in _base_release_memory_pools()
5820 		ioc->reply = NULL;  in _base_release_memory_pools()
5823 	if (ioc->reply_free) {  in _base_release_memory_pools()
5824 		dma_pool_free(ioc->reply_free_dma_pool, ioc->reply_free,  in _base_release_memory_pools()
5825 		    ioc->reply_free_dma);  in _base_release_memory_pools()
5826 		dma_pool_destroy(ioc->reply_free_dma_pool);  in _base_release_memory_pools()
5827 		dexitprintk(ioc,  in _base_release_memory_pools()
5828 			    ioc_info(ioc, "reply_free_pool(0x%p): free\n",  in _base_release_memory_pools()
5829 				     ioc->reply_free));  in _base_release_memory_pools()
5830 		ioc->reply_free = NULL;  in _base_release_memory_pools()
5833 	if (ioc->reply_post) {  in _base_release_memory_pools()
5839 				if (ioc->reply_post[i].reply_post_free) {  in _base_release_memory_pools()
5841 					    ioc->reply_post_free_dma_pool,  in _base_release_memory_pools()
5842 					    ioc->reply_post[i].reply_post_free,  in _base_release_memory_pools()
5843 					ioc->reply_post[i].reply_post_free_dma);  in _base_release_memory_pools()
5844 					dexitprintk(ioc, ioc_info(ioc,  in _base_release_memory_pools()
5846 					   ioc->reply_post[i].reply_post_free));  in _base_release_memory_pools()
5847 					ioc->reply_post[i].reply_post_free =  in _base_release_memory_pools()
5853 		dma_pool_destroy(ioc->reply_post_free_dma_pool);  in _base_release_memory_pools()
5854 		if (ioc->reply_post_free_array &&  in _base_release_memory_pools()
5855 			ioc->rdpq_array_enable) {  in _base_release_memory_pools()
5856 			dma_pool_free(ioc->reply_post_free_array_dma_pool,  in _base_release_memory_pools()
5857 			    ioc->reply_post_free_array,  in _base_release_memory_pools()
5858 			    ioc->reply_post_free_array_dma);  in _base_release_memory_pools()
5859 			ioc->reply_post_free_array = NULL;  in _base_release_memory_pools()
5861 		dma_pool_destroy(ioc->reply_post_free_array_dma_pool);  in _base_release_memory_pools()
5862 		kfree(ioc->reply_post);  in _base_release_memory_pools()
5865 	if (ioc->pcie_sgl_dma_pool) {  in _base_release_memory_pools()
5866 		for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_release_memory_pools()
5867 			dma_pool_free(ioc->pcie_sgl_dma_pool,  in _base_release_memory_pools()
5868 					ioc->pcie_sg_lookup[i].pcie_sgl,  in _base_release_memory_pools()
5869 					ioc->pcie_sg_lookup[i].pcie_sgl_dma);  in _base_release_memory_pools()
5870 			ioc->pcie_sg_lookup[i].pcie_sgl = NULL;  in _base_release_memory_pools()
5872 		dma_pool_destroy(ioc->pcie_sgl_dma_pool);  in _base_release_memory_pools()
5874 	kfree(ioc->pcie_sg_lookup);  in _base_release_memory_pools()
5875 	ioc->pcie_sg_lookup = NULL;  in _base_release_memory_pools()
5877 	if (ioc->config_page) {  in _base_release_memory_pools()
5878 		dexitprintk(ioc,  in _base_release_memory_pools()
5879 			    ioc_info(ioc, "config_page(0x%p): free\n",  in _base_release_memory_pools()
5880 				     ioc->config_page));  in _base_release_memory_pools()
5881 		dma_free_coherent(&ioc->pdev->dev, ioc->config_page_sz,  in _base_release_memory_pools()
5882 		    ioc->config_page, ioc->config_page_dma);  in _base_release_memory_pools()
5885 	kfree(ioc->hpr_lookup);  in _base_release_memory_pools()
5886 	ioc->hpr_lookup = NULL;  in _base_release_memory_pools()
5887 	kfree(ioc->internal_lookup);  in _base_release_memory_pools()
5888 	ioc->internal_lookup = NULL;  in _base_release_memory_pools()
5889 	if (ioc->chain_lookup) {  in _base_release_memory_pools()
5890 		for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_release_memory_pools()
5891 			for (j = ioc->chains_per_prp_buffer;  in _base_release_memory_pools()
5892 			    j < ioc->chains_needed_per_io; j++) {  in _base_release_memory_pools()
5893 				ct = &ioc->chain_lookup[i].chains_per_smid[j];  in _base_release_memory_pools()
5895 					dma_pool_free(ioc->chain_dma_pool,  in _base_release_memory_pools()
5899 			kfree(ioc->chain_lookup[i].chains_per_smid);  in _base_release_memory_pools()
5901 		dma_pool_destroy(ioc->chain_dma_pool);  in _base_release_memory_pools()
5902 		kfree(ioc->chain_lookup);  in _base_release_memory_pools()
5903 		ioc->chain_lookup = NULL;  in _base_release_memory_pools()
5906 	kfree(ioc->io_queue_num);  in _base_release_memory_pools()
5907 	ioc->io_queue_num = NULL;  in _base_release_memory_pools()
5939 _base_reduce_hba_queue_depth(struct MPT3SAS_ADAPTER *ioc)  in _base_reduce_hba_queue_depth()  argument
5943 	if ((ioc->hba_queue_depth - reduce_sz) >  in _base_reduce_hba_queue_depth()
5944 	    (ioc->internal_depth + INTERNAL_SCSIIO_CMDS_COUNT)) {  in _base_reduce_hba_queue_depth()
5945 		ioc->hba_queue_depth -= reduce_sz;  in _base_reduce_hba_queue_depth()
5961 _base_allocate_pcie_sgl_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_pcie_sgl_pool()  argument
5966 	ioc->pcie_sgl_dma_pool =  in _base_allocate_pcie_sgl_pool()
5967 	    dma_pool_create("PCIe SGL pool", &ioc->pdev->dev, sz,  in _base_allocate_pcie_sgl_pool()
5968 	    ioc->page_size, 0);  in _base_allocate_pcie_sgl_pool()
5969 	if (!ioc->pcie_sgl_dma_pool) {  in _base_allocate_pcie_sgl_pool()
5970 		ioc_err(ioc, "PCIe SGL pool: dma_pool_create failed\n");  in _base_allocate_pcie_sgl_pool()
5974 	ioc->chains_per_prp_buffer = sz/ioc->chain_segment_sz;  in _base_allocate_pcie_sgl_pool()
5975 	ioc->chains_per_prp_buffer =  in _base_allocate_pcie_sgl_pool()
5976 	    min(ioc->chains_per_prp_buffer, ioc->chains_needed_per_io);  in _base_allocate_pcie_sgl_pool()
5977 	for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_allocate_pcie_sgl_pool()
5978 		ioc->pcie_sg_lookup[i].pcie_sgl =  in _base_allocate_pcie_sgl_pool()
5979 		    dma_pool_alloc(ioc->pcie_sgl_dma_pool, GFP_KERNEL,  in _base_allocate_pcie_sgl_pool()
5980 		    &ioc->pcie_sg_lookup[i].pcie_sgl_dma);  in _base_allocate_pcie_sgl_pool()
5981 		if (!ioc->pcie_sg_lookup[i].pcie_sgl) {  in _base_allocate_pcie_sgl_pool()
5982 			ioc_err(ioc, "PCIe SGL pool: dma_pool_alloc failed\n");  in _base_allocate_pcie_sgl_pool()
5987 		    ioc->pcie_sg_lookup[i].pcie_sgl_dma, sz)) {  in _base_allocate_pcie_sgl_pool()
5988 			ioc_err(ioc, "PCIE SGLs are not in same 4G !! pcie sgl (0x%p) dma = (0x%llx)\n",  in _base_allocate_pcie_sgl_pool()
5989 			    ioc->pcie_sg_lookup[i].pcie_sgl,  in _base_allocate_pcie_sgl_pool()
5991 			    ioc->pcie_sg_lookup[i].pcie_sgl_dma);  in _base_allocate_pcie_sgl_pool()
5992 			ioc->use_32bit_dma = true;  in _base_allocate_pcie_sgl_pool()
5996 		for (j = 0; j < ioc->chains_per_prp_buffer; j++) {  in _base_allocate_pcie_sgl_pool()
5997 			ct = &ioc->chain_lookup[i].chains_per_smid[j];  in _base_allocate_pcie_sgl_pool()
5999 			    ioc->pcie_sg_lookup[i].pcie_sgl +  in _base_allocate_pcie_sgl_pool()
6000 			    (j * ioc->chain_segment_sz);  in _base_allocate_pcie_sgl_pool()
6002 			    ioc->pcie_sg_lookup[i].pcie_sgl_dma +  in _base_allocate_pcie_sgl_pool()
6003 			    (j * ioc->chain_segment_sz);  in _base_allocate_pcie_sgl_pool()
6006 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_pcie_sgl_pool()
6008 	    ioc->scsiio_depth, sz, (sz * ioc->scsiio_depth)/1024));  in _base_allocate_pcie_sgl_pool()
6009 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_pcie_sgl_pool()
6011 	    ioc->chains_per_prp_buffer));  in _base_allocate_pcie_sgl_pool()
6024 _base_allocate_chain_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_chain_dma_pool()  argument
6029 	ioc->chain_dma_pool = dma_pool_create("chain pool", &ioc->pdev->dev,  in _base_allocate_chain_dma_pool()
6030 	    ioc->chain_segment_sz, 16, 0);  in _base_allocate_chain_dma_pool()
6031 	if (!ioc->chain_dma_pool)  in _base_allocate_chain_dma_pool()
6034 	for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_allocate_chain_dma_pool()
6035 		for (j = ioc->chains_per_prp_buffer;  in _base_allocate_chain_dma_pool()
6036 		    j < ioc->chains_needed_per_io; j++) {  in _base_allocate_chain_dma_pool()
6037 			ctr = &ioc->chain_lookup[i].chains_per_smid[j];  in _base_allocate_chain_dma_pool()
6038 			ctr->chain_buffer = dma_pool_alloc(ioc->chain_dma_pool,  in _base_allocate_chain_dma_pool()
6043 			    ctr->chain_buffer_dma, ioc->chain_segment_sz)) {  in _base_allocate_chain_dma_pool()
6044 				ioc_err(ioc,  in _base_allocate_chain_dma_pool()
6048 				ioc->use_32bit_dma = true;  in _base_allocate_chain_dma_pool()
6053 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_chain_dma_pool()
6055 	    ioc->scsiio_depth, ioc->chain_segment_sz, ((ioc->scsiio_depth *  in _base_allocate_chain_dma_pool()
6056 	    (ioc->chains_needed_per_io - ioc->chains_per_prp_buffer) *  in _base_allocate_chain_dma_pool()
6057 	    ioc->chain_segment_sz))/1024));  in _base_allocate_chain_dma_pool()
6069 _base_allocate_sense_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_sense_dma_pool()  argument
6071 	ioc->sense_dma_pool =  in _base_allocate_sense_dma_pool()
6072 	    dma_pool_create("sense pool", &ioc->pdev->dev, sz, 4, 0);  in _base_allocate_sense_dma_pool()
6073 	if (!ioc->sense_dma_pool)  in _base_allocate_sense_dma_pool()
6075 	ioc->sense = dma_pool_alloc(ioc->sense_dma_pool,  in _base_allocate_sense_dma_pool()
6076 	    GFP_KERNEL, &ioc->sense_dma);  in _base_allocate_sense_dma_pool()
6077 	if (!ioc->sense)  in _base_allocate_sense_dma_pool()
6079 	if (!mpt3sas_check_same_4gb_region(ioc->sense_dma, sz)) {  in _base_allocate_sense_dma_pool()
6080 		dinitprintk(ioc, pr_err(  in _base_allocate_sense_dma_pool()
6082 		    ioc->sense, (unsigned long long) ioc->sense_dma));  in _base_allocate_sense_dma_pool()
6083 		ioc->use_32bit_dma = true;  in _base_allocate_sense_dma_pool()
6086 	ioc_info(ioc,  in _base_allocate_sense_dma_pool()
6088 	    ioc->sense, (unsigned long long)ioc->sense_dma,  in _base_allocate_sense_dma_pool()
6089 	    ioc->scsiio_depth, SCSI_SENSE_BUFFERSIZE, sz/1024);  in _base_allocate_sense_dma_pool()
6101 _base_allocate_reply_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_reply_pool()  argument
6104 	ioc->reply_dma_pool = dma_pool_create("reply pool",  in _base_allocate_reply_pool()
6105 	    &ioc->pdev->dev, sz, 4, 0);  in _base_allocate_reply_pool()
6106 	if (!ioc->reply_dma_pool)  in _base_allocate_reply_pool()
6108 	ioc->reply = dma_pool_alloc(ioc->reply_dma_pool, GFP_KERNEL,  in _base_allocate_reply_pool()
6109 	    &ioc->reply_dma);  in _base_allocate_reply_pool()
6110 	if (!ioc->reply)  in _base_allocate_reply_pool()
6112 	if (!mpt3sas_check_same_4gb_region(ioc->reply_dma, sz)) {  in _base_allocate_reply_pool()
6113 		dinitprintk(ioc, pr_err(  in _base_allocate_reply_pool()
6115 		    ioc->reply, (unsigned long long) ioc->reply_dma));  in _base_allocate_reply_pool()
6116 		ioc->use_32bit_dma = true;  in _base_allocate_reply_pool()
6119 	ioc->reply_dma_min_address = (u32)(ioc->reply_dma);  in _base_allocate_reply_pool()
6120 	ioc->reply_dma_max_address = (u32)(ioc->reply_dma) + sz;  in _base_allocate_reply_pool()
6121 	ioc_info(ioc,  in _base_allocate_reply_pool()
6123 	    ioc->reply, (unsigned long long)ioc->reply_dma,  in _base_allocate_reply_pool()
6124 	    ioc->reply_free_queue_depth, ioc->reply_sz, sz/1024);  in _base_allocate_reply_pool()
6136 _base_allocate_reply_free_dma_pool(struct MPT3SAS_ADAPTER *ioc, u32 sz)  in _base_allocate_reply_free_dma_pool()  argument
6139 	ioc->reply_free_dma_pool = dma_pool_create(  in _base_allocate_reply_free_dma_pool()
6140 	    "reply_free pool", &ioc->pdev->dev, sz, 16, 0);  in _base_allocate_reply_free_dma_pool()
6141 	if (!ioc->reply_free_dma_pool)  in _base_allocate_reply_free_dma_pool()
6143 	ioc->reply_free = dma_pool_alloc(ioc->reply_free_dma_pool,  in _base_allocate_reply_free_dma_pool()
6144 	    GFP_KERNEL, &ioc->reply_free_dma);  in _base_allocate_reply_free_dma_pool()
6145 	if (!ioc->reply_free)  in _base_allocate_reply_free_dma_pool()
6147 	if (!mpt3sas_check_same_4gb_region(ioc->reply_free_dma, sz)) {  in _base_allocate_reply_free_dma_pool()
6148 		dinitprintk(ioc,  in _base_allocate_reply_free_dma_pool()
6150 		    ioc->reply_free, (unsigned long long) ioc->reply_free_dma));  in _base_allocate_reply_free_dma_pool()
6151 		ioc->use_32bit_dma = true;  in _base_allocate_reply_free_dma_pool()
6154 	memset(ioc->reply_free, 0, sz);  in _base_allocate_reply_free_dma_pool()
6155 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_reply_free_dma_pool()
6157 	    ioc->reply_free, ioc->reply_free_queue_depth, 4, sz/1024));  in _base_allocate_reply_free_dma_pool()
6158 	dinitprintk(ioc, ioc_info(ioc,  in _base_allocate_reply_free_dma_pool()
6160 	    (unsigned long long)ioc->reply_free_dma));  in _base_allocate_reply_free_dma_pool()
6173 _base_allocate_reply_post_free_array(struct MPT3SAS_ADAPTER *ioc,  in _base_allocate_reply_post_free_array()  argument
6176 	ioc->reply_post_free_array_dma_pool =  in _base_allocate_reply_post_free_array()
6178 	    &ioc->pdev->dev, reply_post_free_array_sz, 16, 0);  in _base_allocate_reply_post_free_array()
6179 	if (!ioc->reply_post_free_array_dma_pool)  in _base_allocate_reply_post_free_array()
6181 	ioc->reply_post_free_array =  in _base_allocate_reply_post_free_array()
6182 	    dma_pool_alloc(ioc->reply_post_free_array_dma_pool,  in _base_allocate_reply_post_free_array()
6183 	    GFP_KERNEL, &ioc->reply_post_free_array_dma);  in _base_allocate_reply_post_free_array()
6184 	if (!ioc->reply_post_free_array)  in _base_allocate_reply_post_free_array()
6186 	if (!mpt3sas_check_same_4gb_region(ioc->reply_post_free_array_dma,  in _base_allocate_reply_post_free_array()
6188 		dinitprintk(ioc, pr_err(  in _base_allocate_reply_post_free_array()
6190 		    ioc->reply_free,  in _base_allocate_reply_post_free_array()
6191 		    (unsigned long long) ioc->reply_free_dma));  in _base_allocate_reply_post_free_array()
6192 		ioc->use_32bit_dma = true;  in _base_allocate_reply_post_free_array()
6205 base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz)  in base_alloc_rdpq_dma_pool()  argument
6209 	int reply_post_free_sz = ioc->reply_post_queue_depth *  in base_alloc_rdpq_dma_pool()
6211 	int count = ioc->rdpq_array_enable ? ioc->reply_queue_count : 1;  in base_alloc_rdpq_dma_pool()
6213 	ioc->reply_post = kcalloc(count, sizeof(struct reply_post_struct),  in base_alloc_rdpq_dma_pool()
6215 	if (!ioc->reply_post)  in base_alloc_rdpq_dma_pool()
6228 	ioc->reply_post_free_dma_pool =  in base_alloc_rdpq_dma_pool()
6230 		    &ioc->pdev->dev, sz, 16, 0);  in base_alloc_rdpq_dma_pool()
6231 	if (!ioc->reply_post_free_dma_pool)  in base_alloc_rdpq_dma_pool()
6235 			ioc->reply_post[i].reply_post_free =  in base_alloc_rdpq_dma_pool()
6236 			    dma_pool_zalloc(ioc->reply_post_free_dma_pool,  in base_alloc_rdpq_dma_pool()
6238 				&ioc->reply_post[i].reply_post_free_dma);  in base_alloc_rdpq_dma_pool()
6239 			if (!ioc->reply_post[i].reply_post_free)  in base_alloc_rdpq_dma_pool()
6251 				ioc->reply_post[i].reply_post_free_dma, sz)) {  in base_alloc_rdpq_dma_pool()
6252 				dinitprintk(ioc,  in base_alloc_rdpq_dma_pool()
6253 				    ioc_err(ioc, "bad Replypost free pool(0x%p)"  in base_alloc_rdpq_dma_pool()
6255 				    ioc->reply_post[i].reply_post_free,  in base_alloc_rdpq_dma_pool()
6257 				    ioc->reply_post[i].reply_post_free_dma));  in base_alloc_rdpq_dma_pool()
6263 			ioc->reply_post[i].reply_post_free =  in base_alloc_rdpq_dma_pool()
6265 			    ((long)ioc->reply_post[i-1].reply_post_free  in base_alloc_rdpq_dma_pool()
6267 			ioc->reply_post[i].reply_post_free_dma =  in base_alloc_rdpq_dma_pool()
6269 			    (ioc->reply_post[i-1].reply_post_free_dma +  in base_alloc_rdpq_dma_pool()
6283 _base_allocate_memory_pools(struct MPT3SAS_ADAPTER *ioc)  in _base_allocate_memory_pools()  argument
6297 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_allocate_memory_pools()
6301 	facts = &ioc->facts;  in _base_allocate_memory_pools()
6307 		if (ioc->hba_mpi_version_belonged == MPI2_VERSION)  in _base_allocate_memory_pools()
6318 	if (ioc->is_mcpu_endpoint)  in _base_allocate_memory_pools()
6319 		ioc->shost->sg_tablesize = MPT_MIN_PHYS_SEGMENTS;  in _base_allocate_memory_pools()
6326 			ioc_warn(ioc, "sg_tablesize(%u) is bigger than kernel defined SG_CHUNK_SIZE(%u)\n",  in _base_allocate_memory_pools()
6329 		ioc->shost->sg_tablesize = sg_tablesize;  in _base_allocate_memory_pools()
6332 	ioc->internal_depth = min_t(int, (facts->HighPriorityCredit + (5)),  in _base_allocate_memory_pools()
6334 	if (ioc->internal_depth < INTERNAL_CMDS_COUNT) {  in _base_allocate_memory_pools()
6337 			ioc_err(ioc, "IOC doesn't have enough Request Credits, it has just %d number of credits\n",  in _base_allocate_memory_pools()
6341 		ioc->internal_depth = 10;  in _base_allocate_memory_pools()
6344 	ioc->hi_priority_depth = ioc->internal_depth - (5);  in _base_allocate_memory_pools()
6348 			ioc->internal_depth, facts->RequestCredit);  in _base_allocate_memory_pools()
6353 		    (MPT3SAS_KDUMP_SCSI_IO_DEPTH + ioc->internal_depth));  in _base_allocate_memory_pools()
6362 	ioc->hba_queue_depth = max_request_credit + ioc->hi_priority_depth;  in _base_allocate_memory_pools()
6365 	ioc->request_sz = facts->IOCRequestFrameSize * 4;  in _base_allocate_memory_pools()
6368 	ioc->reply_sz = facts->ReplyFrameSize * 4;  in _base_allocate_memory_pools()
6371 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_allocate_memory_pools()
6373 			ioc->chain_segment_sz =  in _base_allocate_memory_pools()
6378 			ioc->chain_segment_sz = DEFAULT_NUM_FWCHAIN_ELEMTS *  in _base_allocate_memory_pools()
6381 		ioc->chain_segment_sz = ioc->request_sz;  in _base_allocate_memory_pools()
6384 	sge_size = max_t(u16, ioc->sge_size, ioc->sge_size_ieee);  in _base_allocate_memory_pools()
6389 	max_sge_elements = ioc->request_sz - ((sizeof(Mpi2SCSIIORequest_t) -  in _base_allocate_memory_pools()
6391 	ioc->max_sges_in_main_message = max_sge_elements/sge_size;  in _base_allocate_memory_pools()
6394 	max_sge_elements = ioc->chain_segment_sz - sge_size;  in _base_allocate_memory_pools()
6395 	ioc->max_sges_in_chain_message = max_sge_elements/sge_size;  in _base_allocate_memory_pools()
6400 	chains_needed_per_io = ((ioc->shost->sg_tablesize -  in _base_allocate_memory_pools()
6401 	   ioc->max_sges_in_main_message)/ioc->max_sges_in_chain_message)  in _base_allocate_memory_pools()
6405 		ioc->shost->sg_tablesize = min_t(u16,  in _base_allocate_memory_pools()
6406 		ioc->max_sges_in_main_message + (ioc->max_sges_in_chain_message  in _base_allocate_memory_pools()
6407 		* chains_needed_per_io), ioc->shost->sg_tablesize);  in _base_allocate_memory_pools()
6409 	ioc->chains_needed_per_io = chains_needed_per_io;  in _base_allocate_memory_pools()
6412 	ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;  in _base_allocate_memory_pools()
6415 	if (ioc->is_mcpu_endpoint)  in _base_allocate_memory_pools()
6416 		ioc->reply_post_queue_depth = ioc->reply_free_queue_depth;  in _base_allocate_memory_pools()
6419 		ioc->reply_post_queue_depth = ioc->hba_queue_depth +  in _base_allocate_memory_pools()
6420 			ioc->reply_free_queue_depth +  1;  in _base_allocate_memory_pools()
6422 		if (ioc->reply_post_queue_depth % 16)  in _base_allocate_memory_pools()
6423 			ioc->reply_post_queue_depth += 16 -  in _base_allocate_memory_pools()
6424 				(ioc->reply_post_queue_depth % 16);  in _base_allocate_memory_pools()
6427 	if (ioc->reply_post_queue_depth >  in _base_allocate_memory_pools()
6429 		ioc->reply_post_queue_depth =  in _base_allocate_memory_pools()
6432 		ioc->hba_queue_depth =  in _base_allocate_memory_pools()
6433 				((ioc->reply_post_queue_depth - 64) / 2) - 1;  in _base_allocate_memory_pools()
6434 		ioc->reply_free_queue_depth = ioc->hba_queue_depth + 64;  in _base_allocate_memory_pools()
6437 	ioc_info(ioc,  in _base_allocate_memory_pools()
6440 	    ioc->max_sges_in_main_message,  in _base_allocate_memory_pools()
6441 	    ioc->max_sges_in_chain_message,  in _base_allocate_memory_pools()
6442 	    ioc->shost->sg_tablesize,  in _base_allocate_memory_pools()
6443 	    ioc->chains_needed_per_io);  in _base_allocate_memory_pools()
6446 	reply_post_free_sz = ioc->reply_post_queue_depth *  in _base_allocate_memory_pools()
6449 	if ((_base_is_controller_msix_enabled(ioc) && !ioc->rdpq_array_enable)  in _base_allocate_memory_pools()
6450 	    || (ioc->reply_queue_count < RDPQ_MAX_INDEX_IN_ONE_CHUNK))  in _base_allocate_memory_pools()
6451 		rdpq_sz = reply_post_free_sz * ioc->reply_queue_count;  in _base_allocate_memory_pools()
6452 	ret = base_alloc_rdpq_dma_pool(ioc, rdpq_sz);  in _base_allocate_memory_pools()
6458 		_base_release_memory_pools(ioc);  in _base_allocate_memory_pools()
6459 		ioc->use_32bit_dma = true;  in _base_allocate_memory_pools()
6460 		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {  in _base_allocate_memory_pools()
6461 			ioc_err(ioc,  in _base_allocate_memory_pools()
6462 			    "32 DMA mask failed %s\n", pci_name(ioc->pdev));  in _base_allocate_memory_pools()
6465 		if (base_alloc_rdpq_dma_pool(ioc, rdpq_sz))  in _base_allocate_memory_pools()
6469 	total_sz = rdpq_sz * (!ioc->rdpq_array_enable ? 1 :  in _base_allocate_memory_pools()
6470 	    DIV_ROUND_UP(ioc->reply_queue_count, RDPQ_MAX_INDEX_IN_ONE_CHUNK));  in _base_allocate_memory_pools()
6471 	ioc->scsiio_depth = ioc->hba_queue_depth -  in _base_allocate_memory_pools()
6472 	    ioc->hi_priority_depth - ioc->internal_depth;  in _base_allocate_memory_pools()
6477 	ioc->shost->can_queue = ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT;  in _base_allocate_memory_pools()
6478 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6479 		    ioc_info(ioc, "scsi host: can_queue depth (%d)\n",  in _base_allocate_memory_pools()
6480 			     ioc->shost->can_queue));  in _base_allocate_memory_pools()
6485 	ioc->chain_depth = ioc->chains_needed_per_io * ioc->scsiio_depth;  in _base_allocate_memory_pools()
6486 	sz = ((ioc->scsiio_depth + 1) * ioc->request_sz);  in _base_allocate_memory_pools()
6489 	sz += (ioc->hi_priority_depth * ioc->request_sz);  in _base_allocate_memory_pools()
6492 	sz += (ioc->internal_depth * ioc->request_sz);  in _base_allocate_memory_pools()
6494 	ioc->request_dma_sz = sz;  in _base_allocate_memory_pools()
6495 	ioc->request = dma_alloc_coherent(&ioc->pdev->dev, sz,  in _base_allocate_memory_pools()
6496 			&ioc->request_dma, GFP_KERNEL);  in _base_allocate_memory_pools()
6497 	if (!ioc->request) {  in _base_allocate_memory_pools()
6498 …ioc_err(ioc, "request pool: dma_alloc_coherent failed: hba_depth(%d), chains_per_io(%d), frame_sz(…  in _base_allocate_memory_pools()
6499 			ioc->hba_queue_depth, ioc->chains_needed_per_io,  in _base_allocate_memory_pools()
6500 			ioc->request_sz, sz / 1024);  in _base_allocate_memory_pools()
6501 		if (ioc->scsiio_depth < MPT3SAS_SAS_QUEUE_DEPTH)  in _base_allocate_memory_pools()
6504 		ioc->hba_queue_depth -= retry_sz;  in _base_allocate_memory_pools()
6505 		_base_release_memory_pools(ioc);  in _base_allocate_memory_pools()
6510 …ioc_err(ioc, "request pool: dma_alloc_coherent succeed: hba_depth(%d), chains_per_io(%d), frame_sz…  in _base_allocate_memory_pools()
6511 			ioc->hba_queue_depth, ioc->chains_needed_per_io,  in _base_allocate_memory_pools()
6512 			ioc->request_sz, sz / 1024);  in _base_allocate_memory_pools()
6515 	ioc->hi_priority = ioc->request + ((ioc->scsiio_depth + 1) *  in _base_allocate_memory_pools()
6516 	    ioc->request_sz);  in _base_allocate_memory_pools()
6517 	ioc->hi_priority_dma = ioc->request_dma + ((ioc->scsiio_depth + 1) *  in _base_allocate_memory_pools()
6518 	    ioc->request_sz);  in _base_allocate_memory_pools()
6521 	ioc->internal = ioc->hi_priority + (ioc->hi_priority_depth *  in _base_allocate_memory_pools()
6522 	    ioc->request_sz);  in _base_allocate_memory_pools()
6523 	ioc->internal_dma = ioc->hi_priority_dma + (ioc->hi_priority_depth *  in _base_allocate_memory_pools()
6524 	    ioc->request_sz);  in _base_allocate_memory_pools()
6526 	ioc_info(ioc,  in _base_allocate_memory_pools()
6529 	    ioc->request, (unsigned long long) ioc->request_dma,  in _base_allocate_memory_pools()
6530 	    ioc->hba_queue_depth, ioc->request_sz,  in _base_allocate_memory_pools()
6531 	    (ioc->hba_queue_depth * ioc->request_sz) / 1024);  in _base_allocate_memory_pools()
6535 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6536 		    ioc_info(ioc, "scsiio(0x%p): depth(%d)\n",  in _base_allocate_memory_pools()
6537 			     ioc->request, ioc->scsiio_depth));  in _base_allocate_memory_pools()
6539 	ioc->chain_depth = min_t(u32, ioc->chain_depth, MAX_CHAIN_DEPTH);  in _base_allocate_memory_pools()
6540 	sz = ioc->scsiio_depth * sizeof(struct chain_lookup);  in _base_allocate_memory_pools()
6541 	ioc->chain_lookup = kzalloc(sz, GFP_KERNEL);  in _base_allocate_memory_pools()
6542 	if (!ioc->chain_lookup) {  in _base_allocate_memory_pools()
6543 		ioc_err(ioc, "chain_lookup: __get_free_pages failed\n");  in _base_allocate_memory_pools()
6547 	sz = ioc->chains_needed_per_io * sizeof(struct chain_tracker);  in _base_allocate_memory_pools()
6548 	for (i = 0; i < ioc->scsiio_depth; i++) {  in _base_allocate_memory_pools()
6549 		ioc->chain_lookup[i].chains_per_smid = kzalloc(sz, GFP_KERNEL);  in _base_allocate_memory_pools()
6550 		if (!ioc->chain_lookup[i].chains_per_smid) {  in _base_allocate_memory_pools()
6551 			ioc_err(ioc, "chain_lookup: kzalloc failed\n");  in _base_allocate_memory_pools()
6557 	ioc->hpr_lookup = kcalloc(ioc->hi_priority_depth,  in _base_allocate_memory_pools()
6559 	if (!ioc->hpr_lookup) {  in _base_allocate_memory_pools()
6560 		ioc_err(ioc, "hpr_lookup: kcalloc failed\n");  in _base_allocate_memory_pools()
6563 	ioc->hi_priority_smid = ioc->scsiio_depth + 1;  in _base_allocate_memory_pools()
6564 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6565 		    ioc_info(ioc, "hi_priority(0x%p): depth(%d), start smid(%d)\n",  in _base_allocate_memory_pools()
6566 			     ioc->hi_priority,  in _base_allocate_memory_pools()
6567 			     ioc->hi_priority_depth, ioc->hi_priority_smid));  in _base_allocate_memory_pools()
6570 	ioc->internal_lookup = kcalloc(ioc->internal_depth,  in _base_allocate_memory_pools()
6572 	if (!ioc->internal_lookup) {  in _base_allocate_memory_pools()
6573 		ioc_err(ioc, "internal_lookup: kcalloc failed\n");  in _base_allocate_memory_pools()
6576 	ioc->internal_smid = ioc->hi_priority_smid + ioc->hi_priority_depth;  in _base_allocate_memory_pools()
6577 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6578 		    ioc_info(ioc, "internal(0x%p): depth(%d), start smid(%d)\n",  in _base_allocate_memory_pools()
6579 			     ioc->internal,  in _base_allocate_memory_pools()
6580 			     ioc->internal_depth, ioc->internal_smid));  in _base_allocate_memory_pools()
6582 	ioc->io_queue_num = kcalloc(ioc->scsiio_depth,  in _base_allocate_memory_pools()
6584 	if (!ioc->io_queue_num)  in _base_allocate_memory_pools()
6600 	ioc->chains_per_prp_buffer = 0;  in _base_allocate_memory_pools()
6601 	if (ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_NVME_DEVICES) {  in _base_allocate_memory_pools()
6603 			(ioc->shost->sg_tablesize * NVME_PRP_SIZE) - 1;  in _base_allocate_memory_pools()
6604 		nvme_blocks_needed /= (ioc->page_size - NVME_PRP_SIZE);  in _base_allocate_memory_pools()
6607 		sz = sizeof(struct pcie_sg_list) * ioc->scsiio_depth;  in _base_allocate_memory_pools()
6608 		ioc->pcie_sg_lookup = kzalloc(sz, GFP_KERNEL);  in _base_allocate_memory_pools()
6609 		if (!ioc->pcie_sg_lookup) {  in _base_allocate_memory_pools()
6610 			ioc_info(ioc, "PCIe SGL lookup: kzalloc failed\n");  in _base_allocate_memory_pools()
6613 		sz = nvme_blocks_needed * ioc->page_size;  in _base_allocate_memory_pools()
6614 		rc = _base_allocate_pcie_sgl_pool(ioc, sz);  in _base_allocate_memory_pools()
6619 		total_sz += sz * ioc->scsiio_depth;  in _base_allocate_memory_pools()
6622 	rc = _base_allocate_chain_dma_pool(ioc, ioc->chain_segment_sz);  in _base_allocate_memory_pools()
6627 	total_sz += ioc->chain_segment_sz * ((ioc->chains_needed_per_io -  in _base_allocate_memory_pools()
6628 		ioc->chains_per_prp_buffer) * ioc->scsiio_depth);  in _base_allocate_memory_pools()
6629 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6630 	    ioc_info(ioc, "chain pool depth(%d), frame_size(%d), pool_size(%d kB)\n",  in _base_allocate_memory_pools()
6631 	    ioc->chain_depth, ioc->chain_segment_sz,  in _base_allocate_memory_pools()
6632 	    (ioc->chain_depth * ioc->chain_segment_sz) / 1024));  in _base_allocate_memory_pools()
6634 	sense_sz = ioc->scsiio_depth * SCSI_SENSE_BUFFERSIZE;  in _base_allocate_memory_pools()
6635 	rc = _base_allocate_sense_dma_pool(ioc, sense_sz);  in _base_allocate_memory_pools()
6642 	sz = ioc->reply_free_queue_depth * ioc->reply_sz;  in _base_allocate_memory_pools()
6643 	rc = _base_allocate_reply_pool(ioc, sz);  in _base_allocate_memory_pools()
6651 	sz = ioc->reply_free_queue_depth * 4;  in _base_allocate_memory_pools()
6652 	rc = _base_allocate_reply_free_dma_pool(ioc, sz);  in _base_allocate_memory_pools()
6657 	dinitprintk(ioc,  in _base_allocate_memory_pools()
6658 		    ioc_info(ioc, "reply_free_dma (0x%llx)\n",  in _base_allocate_memory_pools()
6659 			     (unsigned long long)ioc->reply_free_dma));  in _base_allocate_memory_pools()
6661 	if (ioc->rdpq_array_enable) {  in _base_allocate_memory_pools()
6662 		reply_post_free_array_sz = ioc->reply_queue_count *  in _base_allocate_memory_pools()
6664 		rc = _base_allocate_reply_post_free_array(ioc,  in _base_allocate_memory_pools()
6671 	ioc->config_page_sz = 512;  in _base_allocate_memory_pools()
6672 	ioc->config_page = dma_alloc_coherent(&ioc->pdev->dev,  in _base_allocate_memory_pools()
6673 			ioc->config_page_sz, &ioc->config_page_dma, GFP_KERNEL);  in _base_allocate_memory_pools()
6674 	if (!ioc->config_page) {  in _base_allocate_memory_pools()
6675 		ioc_err(ioc, "config page: dma_pool_alloc failed\n");  in _base_allocate_memory_pools()
6679 	ioc_info(ioc, "config page(0x%p) - dma(0x%llx): size(%d)\n",  in _base_allocate_memory_pools()
6680 	    ioc->config_page, (unsigned long long)ioc->config_page_dma,  in _base_allocate_memory_pools()
6681 	    ioc->config_page_sz);  in _base_allocate_memory_pools()
6682 	total_sz += ioc->config_page_sz;  in _base_allocate_memory_pools()
6684 	ioc_info(ioc, "Allocated physical memory: size(%d kB)\n",  in _base_allocate_memory_pools()
6686 	ioc_info(ioc, "Current Controller Queue Depth(%d),Max Controller Queue Depth(%d)\n",  in _base_allocate_memory_pools()
6687 		 ioc->shost->can_queue, facts->RequestCredit);  in _base_allocate_memory_pools()
6688 	ioc_info(ioc, "Scatter Gather Elements per IO(%d)\n",  in _base_allocate_memory_pools()
6689 		 ioc->shost->sg_tablesize);  in _base_allocate_memory_pools()
6693 	_base_release_memory_pools(ioc);  in _base_allocate_memory_pools()
6694 	if (ioc->use_32bit_dma && (ioc->dma_mask > 32)) {  in _base_allocate_memory_pools()
6696 		if (_base_config_dma_addressing(ioc, ioc->pdev) != 0) {  in _base_allocate_memory_pools()
6698 			    pci_name(ioc->pdev));  in _base_allocate_memory_pools()
6701 	} else if (_base_reduce_hba_queue_depth(ioc) != 0)  in _base_allocate_memory_pools()
6718 mpt3sas_base_get_iocstate(struct MPT3SAS_ADAPTER *ioc, int cooked)  in mpt3sas_base_get_iocstate()  argument
6722 	s = ioc->base_readl_ext_retry(&ioc->chip->Doorbell);  in mpt3sas_base_get_iocstate()
6736 _base_wait_on_iocstate(struct MPT3SAS_ADAPTER *ioc, u32 ioc_state, int timeout)  in _base_wait_on_iocstate()  argument
6744 		current_state = mpt3sas_base_get_iocstate(ioc, 1);  in _base_wait_on_iocstate()
6766 _base_dump_reg_set(struct MPT3SAS_ADAPTER *ioc)  in _base_dump_reg_set()  argument
6769 	u32 __iomem *reg = (u32 __iomem *)ioc->chip;  in _base_dump_reg_set()
6771 	ioc_info(ioc, "System Register set:\n");  in _base_dump_reg_set()
6788 _base_wait_for_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_doorbell_int()  argument
6796 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);  in _base_wait_for_doorbell_int()
6798 			dhsprintk(ioc,  in _base_wait_for_doorbell_int()
6799 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_wait_for_doorbell_int()
6808 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",  in _base_wait_for_doorbell_int()
6814 _base_spin_on_doorbell_int(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_spin_on_doorbell_int()  argument
6822 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);  in _base_spin_on_doorbell_int()
6824 			dhsprintk(ioc,  in _base_spin_on_doorbell_int()
6825 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_spin_on_doorbell_int()
6834 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",  in _base_spin_on_doorbell_int()
6851 _base_wait_for_doorbell_ack(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_doorbell_ack()  argument
6860 		int_status = ioc->base_readl(&ioc->chip->HostInterruptStatus);  in _base_wait_for_doorbell_ack()
6862 			dhsprintk(ioc,  in _base_wait_for_doorbell_ack()
6863 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_wait_for_doorbell_ack()
6867 			doorbell = ioc->base_readl_ext_retry(&ioc->chip->Doorbell);  in _base_wait_for_doorbell_ack()
6870 				mpt3sas_print_fault_code(ioc, doorbell);  in _base_wait_for_doorbell_ack()
6875 				mpt3sas_print_coredump_info(ioc, doorbell);  in _base_wait_for_doorbell_ack()
6886 	ioc_err(ioc, "%s: failed due to timeout count(%d), int_status(%x)!\n",  in _base_wait_for_doorbell_ack()
6899 _base_wait_for_doorbell_not_used(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_doorbell_not_used()  argument
6907 		doorbell_reg = ioc->base_readl_ext_retry(&ioc->chip->Doorbell);  in _base_wait_for_doorbell_not_used()
6909 			dhsprintk(ioc,  in _base_wait_for_doorbell_not_used()
6910 				  ioc_info(ioc, "%s: successful count(%d), timeout(%d)\n",  in _base_wait_for_doorbell_not_used()
6919 	ioc_err(ioc, "%s: failed due to timeout count(%d), doorbell_reg(%x)!\n",  in _base_wait_for_doorbell_not_used()
6933 _base_send_ioc_reset(struct MPT3SAS_ADAPTER *ioc, u8 reset_type, int timeout)  in _base_send_ioc_reset()  argument
6940 		ioc_err(ioc, "%s: unknown reset_type\n", __func__);  in _base_send_ioc_reset()
6944 	if (!(ioc->facts.IOCCapabilities &  in _base_send_ioc_reset()
6948 	ioc_info(ioc, "sending message unit reset !!\n");  in _base_send_ioc_reset()
6951 	    &ioc->chip->Doorbell);  in _base_send_ioc_reset()
6952 	if ((_base_wait_for_doorbell_ack(ioc, 15))) {  in _base_send_ioc_reset()
6957 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);  in _base_send_ioc_reset()
6959 		ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in _base_send_ioc_reset()
6966 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in _base_send_ioc_reset()
6967 		spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6973 		    MPI2_IOC_STATE_COREDUMP && (ioc->is_driver_loading == 1 ||  in _base_send_ioc_reset()
6974 		    ioc->fault_reset_work_q == NULL)) {  in _base_send_ioc_reset()
6976 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6977 			mpt3sas_print_coredump_info(ioc, ioc_state);  in _base_send_ioc_reset()
6978 			mpt3sas_base_wait_for_coredump_completion(ioc,  in _base_send_ioc_reset()
6981 			    &ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6983 		spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in _base_send_ioc_reset()
6985 	ioc_info(ioc, "message unit reset: %s\n",  in _base_send_ioc_reset()
7001 mpt3sas_wait_for_ioc(struct MPT3SAS_ADAPTER *ioc, int timeout)  in mpt3sas_wait_for_ioc()  argument
7007 		ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in mpt3sas_wait_for_ioc()
7018 		if (ioc->is_driver_loading)  in mpt3sas_wait_for_ioc()
7022 		ioc_info(ioc, "%s: waiting for operational state(count=%d)\n",  in mpt3sas_wait_for_ioc()
7026 		ioc_err(ioc, "%s: failed due to ioc not operational\n", __func__);  in mpt3sas_wait_for_ioc()
7030 		ioc_info(ioc, "ioc is operational\n");  in mpt3sas_wait_for_ioc()
7046 _base_handshake_req_reply_wait(struct MPT3SAS_ADAPTER *ioc, int request_bytes,  in _base_handshake_req_reply_wait()  argument
7055 	if ((ioc->base_readl_ext_retry(&ioc->chip->Doorbell) & MPI2_DOORBELL_USED)) {  in _base_handshake_req_reply_wait()
7056 		ioc_err(ioc, "doorbell is in use (line=%d)\n", __LINE__);  in _base_handshake_req_reply_wait()
7061 	if (ioc->base_readl(&ioc->chip->HostInterruptStatus) &  in _base_handshake_req_reply_wait()
7063 		writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7068 	    &ioc->chip->Doorbell);  in _base_handshake_req_reply_wait()
7070 	if ((_base_spin_on_doorbell_int(ioc, 5))) {  in _base_handshake_req_reply_wait()
7071 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7075 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7077 	if ((_base_wait_for_doorbell_ack(ioc, 5))) {  in _base_handshake_req_reply_wait()
7078 		ioc_err(ioc, "doorbell handshake ack failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7085 		writel(cpu_to_le32(request[i]), &ioc->chip->Doorbell);  in _base_handshake_req_reply_wait()
7086 		if ((_base_wait_for_doorbell_ack(ioc, 5)))  in _base_handshake_req_reply_wait()
7091 		ioc_err(ioc, "doorbell handshake sending request failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7097 	if ((_base_wait_for_doorbell_int(ioc, timeout))) {  in _base_handshake_req_reply_wait()
7098 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7104 	reply[0] = le16_to_cpu(ioc->base_readl_ext_retry(&ioc->chip->Doorbell)  in _base_handshake_req_reply_wait()
7106 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7107 	if ((_base_wait_for_doorbell_int(ioc, 5))) {  in _base_handshake_req_reply_wait()
7108 		ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7112 	reply[1] = le16_to_cpu(ioc->base_readl_ext_retry(&ioc->chip->Doorbell)  in _base_handshake_req_reply_wait()
7114 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7117 		if ((_base_wait_for_doorbell_int(ioc, 5))) {  in _base_handshake_req_reply_wait()
7118 			ioc_err(ioc, "doorbell handshake int failed (line=%d)\n",  in _base_handshake_req_reply_wait()
7123 			ioc->base_readl_ext_retry(&ioc->chip->Doorbell);  in _base_handshake_req_reply_wait()
7126 			    ioc->base_readl_ext_retry(&ioc->chip->Doorbell)  in _base_handshake_req_reply_wait()
7128 		writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7131 	_base_wait_for_doorbell_int(ioc, 5);  in _base_handshake_req_reply_wait()
7132 	if (_base_wait_for_doorbell_not_used(ioc, 5) != 0) {  in _base_handshake_req_reply_wait()
7133 		dhsprintk(ioc,  in _base_handshake_req_reply_wait()
7134 			  ioc_info(ioc, "doorbell is in use (line=%d)\n",  in _base_handshake_req_reply_wait()
7137 	writel(0, &ioc->chip->HostInterruptStatus);  in _base_handshake_req_reply_wait()
7139 	if (ioc->logging_level & MPT_DEBUG_INIT) {  in _base_handshake_req_reply_wait()
7143 			ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,  in _base_handshake_req_reply_wait()
7164 mpt3sas_base_sas_iounit_control(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_sas_iounit_control()  argument
7173 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_sas_iounit_control()
7175 	mutex_lock(&ioc->base_cmds.mutex);  in mpt3sas_base_sas_iounit_control()
7177 	if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {  in mpt3sas_base_sas_iounit_control()
7178 		ioc_err(ioc, "%s: base_cmd in use\n", __func__);  in mpt3sas_base_sas_iounit_control()
7183 	rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);  in mpt3sas_base_sas_iounit_control()
7187 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in mpt3sas_base_sas_iounit_control()
7189 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_base_sas_iounit_control()
7195 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_base_sas_iounit_control()
7196 	request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_base_sas_iounit_control()
7197 	ioc->base_cmds.smid = smid;  in mpt3sas_base_sas_iounit_control()
7201 		ioc->ioc_link_reset_in_progress = 1;  in mpt3sas_base_sas_iounit_control()
7202 	init_completion(&ioc->base_cmds.done);  in mpt3sas_base_sas_iounit_control()
7203 	ioc->put_smid_default(ioc, smid);  in mpt3sas_base_sas_iounit_control()
7204 	wait_for_completion_timeout(&ioc->base_cmds.done,  in mpt3sas_base_sas_iounit_control()
7208 	    ioc->ioc_link_reset_in_progress)  in mpt3sas_base_sas_iounit_control()
7209 		ioc->ioc_link_reset_in_progress = 0;  in mpt3sas_base_sas_iounit_control()
7210 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in mpt3sas_base_sas_iounit_control()
7211 		mpt3sas_check_cmd_timeout(ioc, ioc->base_cmds.status,  in mpt3sas_base_sas_iounit_control()
7216 	if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)  in mpt3sas_base_sas_iounit_control()
7217 		memcpy(mpi_reply, ioc->base_cmds.reply,  in mpt3sas_base_sas_iounit_control()
7221 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_sas_iounit_control()
7226 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_base_sas_iounit_control()
7227 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_sas_iounit_control()
7230 	mutex_unlock(&ioc->base_cmds.mutex);  in mpt3sas_base_sas_iounit_control()
7246 mpt3sas_base_scsi_enclosure_processor(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_scsi_enclosure_processor()  argument
7254 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_scsi_enclosure_processor()
7256 	mutex_lock(&ioc->base_cmds.mutex);  in mpt3sas_base_scsi_enclosure_processor()
7258 	if (ioc->base_cmds.status != MPT3_CMD_NOT_USED) {  in mpt3sas_base_scsi_enclosure_processor()
7259 		ioc_err(ioc, "%s: base_cmd in use\n", __func__);  in mpt3sas_base_scsi_enclosure_processor()
7264 	rc = mpt3sas_wait_for_ioc(ioc, IOC_OPERATIONAL_WAIT_COUNT);  in mpt3sas_base_scsi_enclosure_processor()
7268 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in mpt3sas_base_scsi_enclosure_processor()
7270 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_base_scsi_enclosure_processor()
7276 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_base_scsi_enclosure_processor()
7277 	request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_base_scsi_enclosure_processor()
7278 	ioc->base_cmds.smid = smid;  in mpt3sas_base_scsi_enclosure_processor()
7279 	memset(request, 0, ioc->request_sz);  in mpt3sas_base_scsi_enclosure_processor()
7281 	init_completion(&ioc->base_cmds.done);  in mpt3sas_base_scsi_enclosure_processor()
7282 	ioc->put_smid_default(ioc, smid);  in mpt3sas_base_scsi_enclosure_processor()
7283 	wait_for_completion_timeout(&ioc->base_cmds.done,  in mpt3sas_base_scsi_enclosure_processor()
7285 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in mpt3sas_base_scsi_enclosure_processor()
7286 		mpt3sas_check_cmd_timeout(ioc,  in mpt3sas_base_scsi_enclosure_processor()
7287 		    ioc->base_cmds.status, mpi_request,  in mpt3sas_base_scsi_enclosure_processor()
7291 	if (ioc->base_cmds.status & MPT3_CMD_REPLY_VALID)  in mpt3sas_base_scsi_enclosure_processor()
7292 		memcpy(mpi_reply, ioc->base_cmds.reply,  in mpt3sas_base_scsi_enclosure_processor()
7296 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_scsi_enclosure_processor()
7301 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_base_scsi_enclosure_processor()
7302 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_scsi_enclosure_processor()
7305 	mutex_unlock(&ioc->base_cmds.mutex);  in mpt3sas_base_scsi_enclosure_processor()
7317 _base_get_port_facts(struct MPT3SAS_ADAPTER *ioc, int port)  in _base_get_port_facts()  argument
7324 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_get_port_facts()
7331 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,  in _base_get_port_facts()
7335 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);  in _base_get_port_facts()
7339 	pfacts = &ioc->pfacts[port];  in _base_get_port_facts()
7358 _base_wait_for_iocstate(struct MPT3SAS_ADAPTER *ioc, int timeout)  in _base_wait_for_iocstate()  argument
7363 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_wait_for_iocstate()
7365 	if (ioc->pci_error_recovery) {  in _base_wait_for_iocstate()
7366 		dfailprintk(ioc,  in _base_wait_for_iocstate()
7367 			    ioc_info(ioc, "%s: host in pci error recovery\n",  in _base_wait_for_iocstate()
7372 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in _base_wait_for_iocstate()
7373 	dhsprintk(ioc,  in _base_wait_for_iocstate()
7374 		  ioc_info(ioc, "%s: ioc_state(0x%08x)\n",  in _base_wait_for_iocstate()
7382 		dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));  in _base_wait_for_iocstate()
7387 		mpt3sas_print_fault_code(ioc, ioc_state &  in _base_wait_for_iocstate()
7392 		ioc_info(ioc,  in _base_wait_for_iocstate()
7398 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, timeout);  in _base_wait_for_iocstate()
7400 		dfailprintk(ioc,  in _base_wait_for_iocstate()
7401 			    ioc_info(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in _base_wait_for_iocstate()
7409 	rc = _base_diag_reset(ioc);  in _base_wait_for_iocstate()
7420 _base_get_ioc_facts(struct MPT3SAS_ADAPTER *ioc)  in _base_get_ioc_facts()  argument
7427 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_get_ioc_facts()
7429 	r = _base_wait_for_iocstate(ioc, 10);  in _base_get_ioc_facts()
7431 		dfailprintk(ioc,  in _base_get_ioc_facts()
7432 			    ioc_info(ioc, "%s: failed getting to correct state\n",  in _base_get_ioc_facts()
7440 	r = _base_handshake_req_reply_wait(ioc, mpi_request_sz,  in _base_get_ioc_facts()
7444 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);  in _base_get_ioc_facts()
7448 	facts = &ioc->facts;  in _base_get_ioc_facts()
7459 	if (ioc->msix_enable && (facts->MaxMSIxVectors <=  in _base_get_ioc_facts()
7460 	    MAX_COMBINED_MSIX_VECTORS(ioc->is_gen35_ioc)))  in _base_get_ioc_facts()
7461 		ioc->combined_reply_queue = 0;  in _base_get_ioc_facts()
7468 		ioc->ir_firmware = 1;  in _base_get_ioc_facts()
7471 		ioc->rdpq_array_capable = 1;  in _base_get_ioc_facts()
7473 	    && ioc->is_aero_ioc)  in _base_get_ioc_facts()
7474 		ioc->atomic_desc_capable = 1;  in _base_get_ioc_facts()
7478 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_get_ioc_facts()
7484 	ioc->shost->max_id = -1;  in _base_get_ioc_facts()
7497 	ioc->page_size = 1 << facts->CurrentHostPageSize;  in _base_get_ioc_facts()
7498 	if (ioc->page_size == 1) {  in _base_get_ioc_facts()
7499 		ioc_info(ioc, "CurrentHostPageSize is 0: Setting default host page size to 4k\n");  in _base_get_ioc_facts()
7500 		ioc->page_size = 1 << MPT3SAS_HOST_PAGE_SIZE_4K;  in _base_get_ioc_facts()
7502 	dinitprintk(ioc,  in _base_get_ioc_facts()
7503 		    ioc_info(ioc, "CurrentHostPageSize(%d)\n",  in _base_get_ioc_facts()
7506 	dinitprintk(ioc,  in _base_get_ioc_facts()
7507 		    ioc_info(ioc, "hba queue depth(%d), max chains per io(%d)\n",  in _base_get_ioc_facts()
7509 	dinitprintk(ioc,  in _base_get_ioc_facts()
7510 		    ioc_info(ioc, "request frame size(%d), reply frame size(%d)\n",  in _base_get_ioc_facts()
7523 _base_send_ioc_init(struct MPT3SAS_ADAPTER *ioc)  in _base_send_ioc_init()  argument
7532 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_send_ioc_init()
7539 	mpi_request.MsgVersion = cpu_to_le16(ioc->hba_mpi_version_belonged);  in _base_send_ioc_init()
7543 	if (_base_is_controller_msix_enabled(ioc))  in _base_send_ioc_init()
7544 		mpi_request.HostMSIxVectors = ioc->reply_queue_count;  in _base_send_ioc_init()
7545 	mpi_request.SystemRequestFrameSize = cpu_to_le16(ioc->request_sz/4);  in _base_send_ioc_init()
7547 	    cpu_to_le16(ioc->reply_post_queue_depth);  in _base_send_ioc_init()
7549 	    cpu_to_le16(ioc->reply_free_queue_depth);  in _base_send_ioc_init()
7552 	    cpu_to_le32((u64)ioc->sense_dma >> 32);  in _base_send_ioc_init()
7554 	    cpu_to_le32((u64)ioc->reply_dma >> 32);  in _base_send_ioc_init()
7556 	    cpu_to_le64((u64)ioc->request_dma);  in _base_send_ioc_init()
7558 	    cpu_to_le64((u64)ioc->reply_free_dma);  in _base_send_ioc_init()
7560 	if (ioc->rdpq_array_enable) {  in _base_send_ioc_init()
7561 		reply_post_free_array_sz = ioc->reply_queue_count *  in _base_send_ioc_init()
7563 		memset(ioc->reply_post_free_array, 0, reply_post_free_array_sz);  in _base_send_ioc_init()
7564 		for (i = 0; i < ioc->reply_queue_count; i++)  in _base_send_ioc_init()
7565 			ioc->reply_post_free_array[i].RDPQBaseAddress =  in _base_send_ioc_init()
7567 				(u64)ioc->reply_post[i].reply_post_free_dma);  in _base_send_ioc_init()
7570 		    cpu_to_le64((u64)ioc->reply_post_free_array_dma);  in _base_send_ioc_init()
7573 		    cpu_to_le64((u64)ioc->reply_post[0].reply_post_free_dma);  in _base_send_ioc_init()
7588 	if (ioc->logging_level & MPT_DEBUG_INIT) {  in _base_send_ioc_init()
7593 		ioc_info(ioc, "\toffset:data\n");  in _base_send_ioc_init()
7595 			ioc_info(ioc, "\t[0x%02x]:%08x\n", i*4,  in _base_send_ioc_init()
7599 	r = _base_handshake_req_reply_wait(ioc,  in _base_send_ioc_init()
7604 		ioc_err(ioc, "%s: handshake failed (r=%d)\n", __func__, r);  in _base_send_ioc_init()
7611 		ioc_err(ioc, "%s: failed\n", __func__);  in _base_send_ioc_init()
7616 	ioc->timestamp_update_count = 0;  in _base_send_ioc_init()
7631 mpt3sas_port_enable_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in mpt3sas_port_enable_done()  argument
7637 	if (ioc->port_enable_cmds.status == MPT3_CMD_NOT_USED)  in mpt3sas_port_enable_done()
7640 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in mpt3sas_port_enable_done()
7647 	ioc->port_enable_cmds.status &= ~MPT3_CMD_PENDING;  in mpt3sas_port_enable_done()
7648 	ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE;  in mpt3sas_port_enable_done()
7649 	ioc->port_enable_cmds.status |= MPT3_CMD_REPLY_VALID;  in mpt3sas_port_enable_done()
7650 	memcpy(ioc->port_enable_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);  in mpt3sas_port_enable_done()
7653 		ioc->port_enable_failed = 1;  in mpt3sas_port_enable_done()
7655 	if (ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE_ASYNC) {  in mpt3sas_port_enable_done()
7656 		ioc->port_enable_cmds.status &= ~MPT3_CMD_COMPLETE_ASYNC;  in mpt3sas_port_enable_done()
7658 			mpt3sas_port_enable_complete(ioc);  in mpt3sas_port_enable_done()
7661 			ioc->start_scan_failed = ioc_status;  in mpt3sas_port_enable_done()
7662 			ioc->start_scan = 0;  in mpt3sas_port_enable_done()
7666 	complete(&ioc->port_enable_cmds.done);  in mpt3sas_port_enable_done()
7677 _base_send_port_enable(struct MPT3SAS_ADAPTER *ioc)  in _base_send_port_enable()  argument
7685 	ioc_info(ioc, "sending port enable !!\n");  in _base_send_port_enable()
7687 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {  in _base_send_port_enable()
7688 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in _base_send_port_enable()
7692 	smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);  in _base_send_port_enable()
7694 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _base_send_port_enable()
7698 	ioc->port_enable_cmds.status = MPT3_CMD_PENDING;  in _base_send_port_enable()
7699 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_send_port_enable()
7700 	ioc->port_enable_cmds.smid = smid;  in _base_send_port_enable()
7704 	init_completion(&ioc->port_enable_cmds.done);  in _base_send_port_enable()
7705 	ioc->put_smid_default(ioc, smid);  in _base_send_port_enable()
7706 	wait_for_completion_timeout(&ioc->port_enable_cmds.done, 300*HZ);  in _base_send_port_enable()
7707 	if (!(ioc->port_enable_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_send_port_enable()
7708 		ioc_err(ioc, "%s: timeout\n", __func__);  in _base_send_port_enable()
7711 		if (ioc->port_enable_cmds.status & MPT3_CMD_RESET)  in _base_send_port_enable()
7718 	mpi_reply = ioc->port_enable_cmds.reply;  in _base_send_port_enable()
7721 		ioc_err(ioc, "%s: failed with (ioc_status=0x%08x)\n",  in _base_send_port_enable()
7728 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in _base_send_port_enable()
7729 	ioc_info(ioc, "port enable: %s\n", r == 0 ? "SUCCESS" : "FAILED");  in _base_send_port_enable()
7740 mpt3sas_port_enable(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_port_enable()  argument
7745 	ioc_info(ioc, "sending port enable !!\n");  in mpt3sas_port_enable()
7747 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {  in mpt3sas_port_enable()
7748 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in mpt3sas_port_enable()
7752 	smid = mpt3sas_base_get_smid(ioc, ioc->port_enable_cb_idx);  in mpt3sas_port_enable()
7754 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_port_enable()
7757 	ioc->drv_internal_flags |= MPT_DRV_INTERNAL_FIRST_PE_ISSUED;  in mpt3sas_port_enable()
7758 	ioc->port_enable_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_port_enable()
7759 	ioc->port_enable_cmds.status |= MPT3_CMD_COMPLETE_ASYNC;  in mpt3sas_port_enable()
7760 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_port_enable()
7761 	ioc->port_enable_cmds.smid = smid;  in mpt3sas_port_enable()
7765 	ioc->put_smid_default(ioc, smid);  in mpt3sas_port_enable()
7779 _base_determine_wait_on_discovery(struct MPT3SAS_ADAPTER *ioc)  in _base_determine_wait_on_discovery()  argument
7787 	if (ioc->ir_firmware)  in _base_determine_wait_on_discovery()
7791 	if (!ioc->bios_pg3.BiosVersion)  in _base_determine_wait_on_discovery()
7801 	if ((ioc->bios_pg2.CurrentBootDeviceForm &  in _base_determine_wait_on_discovery()
7805 	   (ioc->bios_pg2.ReqBootDeviceForm &  in _base_determine_wait_on_discovery()
7809 	   (ioc->bios_pg2.ReqAltBootDeviceForm &  in _base_determine_wait_on_discovery()
7825 _base_unmask_events(struct MPT3SAS_ADAPTER *ioc, u16 event)  in _base_unmask_events()  argument
7835 		ioc->event_masks[0] &= ~desired_event;  in _base_unmask_events()
7837 		ioc->event_masks[1] &= ~desired_event;  in _base_unmask_events()
7839 		ioc->event_masks[2] &= ~desired_event;  in _base_unmask_events()
7841 		ioc->event_masks[3] &= ~desired_event;  in _base_unmask_events()
7851 _base_event_notification(struct MPT3SAS_ADAPTER *ioc)  in _base_event_notification()  argument
7858 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_event_notification()
7860 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {  in _base_event_notification()
7861 		ioc_err(ioc, "%s: internal command already in use\n", __func__);  in _base_event_notification()
7865 	smid = mpt3sas_base_get_smid(ioc, ioc->base_cb_idx);  in _base_event_notification()
7867 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _base_event_notification()
7870 	ioc->base_cmds.status = MPT3_CMD_PENDING;  in _base_event_notification()
7871 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _base_event_notification()
7872 	ioc->base_cmds.smid = smid;  in _base_event_notification()
7879 		    cpu_to_le32(ioc->event_masks[i]);  in _base_event_notification()
7880 	init_completion(&ioc->base_cmds.done);  in _base_event_notification()
7881 	ioc->put_smid_default(ioc, smid);  in _base_event_notification()
7882 	wait_for_completion_timeout(&ioc->base_cmds.done, 30*HZ);  in _base_event_notification()
7883 	if (!(ioc->base_cmds.status & MPT3_CMD_COMPLETE)) {  in _base_event_notification()
7884 		ioc_err(ioc, "%s: timeout\n", __func__);  in _base_event_notification()
7887 		if (ioc->base_cmds.status & MPT3_CMD_RESET)  in _base_event_notification()
7893 		dinitprintk(ioc, ioc_info(ioc, "%s: complete\n", __func__));  in _base_event_notification()
7894 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in _base_event_notification()
7897 		if (ioc->drv_internal_flags & MPT_DRV_INTERNAL_FIRST_PE_ISSUED)  in _base_event_notification()
7899 		if (mpt3sas_base_check_for_fault_and_issue_reset(ioc))  in _base_event_notification()
7915 mpt3sas_base_validate_event_type(struct MPT3SAS_ADAPTER *ioc, u32 *event_type)  in mpt3sas_base_validate_event_type()  argument
7927 			    (ioc->event_masks[i] & desired_event)) {  in mpt3sas_base_validate_event_type()
7928 				ioc->event_masks[i] &= ~desired_event;  in mpt3sas_base_validate_event_type()
7938 	mutex_lock(&ioc->base_cmds.mutex);  in mpt3sas_base_validate_event_type()
7939 	_base_event_notification(ioc);  in mpt3sas_base_validate_event_type()
7940 	mutex_unlock(&ioc->base_cmds.mutex);  in mpt3sas_base_validate_event_type()
7950 _base_diag_reset(struct MPT3SAS_ADAPTER *ioc)  in _base_diag_reset()  argument
7957 	ioc_info(ioc, "sending diag reset !!\n");  in _base_diag_reset()
7959 	pci_cfg_access_lock(ioc->pdev);  in _base_diag_reset()
7961 	drsprintk(ioc, ioc_info(ioc, "clear interrupts\n"));  in _base_diag_reset()
7968 		drsprintk(ioc, ioc_info(ioc, "write magic sequence\n"));  in _base_diag_reset()
7969 		writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7970 		writel(MPI2_WRSEQ_1ST_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7971 		writel(MPI2_WRSEQ_2ND_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7972 		writel(MPI2_WRSEQ_3RD_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7973 		writel(MPI2_WRSEQ_4TH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7974 		writel(MPI2_WRSEQ_5TH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7975 		writel(MPI2_WRSEQ_6TH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
7981 			ioc_info(ioc,  in _base_diag_reset()
7983 			_base_dump_reg_set(ioc);  in _base_diag_reset()
7987 		host_diagnostic = ioc->base_readl_ext_retry(&ioc->chip->HostDiagnostic);  in _base_diag_reset()
7988 		drsprintk(ioc,  in _base_diag_reset()
7989 			  ioc_info(ioc, "wrote magic sequence: count(%d), host_diagnostic(0x%08x)\n",  in _base_diag_reset()
7994 	hcb_size = ioc->base_readl(&ioc->chip->HCBSize);  in _base_diag_reset()
7996 	drsprintk(ioc, ioc_info(ioc, "diag reset: issued\n"));  in _base_diag_reset()
7998 	     &ioc->chip->HostDiagnostic);  in _base_diag_reset()
8007 		host_diagnostic = ioc->base_readl_ext_retry(&ioc->chip->HostDiagnostic);  in _base_diag_reset()
8010 			ioc_info(ioc,  in _base_diag_reset()
8012 			_base_dump_reg_set(ioc);  in _base_diag_reset()
8023 		drsprintk(ioc,  in _base_diag_reset()
8024 			  ioc_info(ioc, "restart the adapter assuming the HCB Address points to good F/W\n"));  in _base_diag_reset()
8027 		writel(host_diagnostic, &ioc->chip->HostDiagnostic);  in _base_diag_reset()
8029 		drsprintk(ioc, ioc_info(ioc, "re-enable the HCDW\n"));  in _base_diag_reset()
8031 		    &ioc->chip->HCBSize);  in _base_diag_reset()
8034 	drsprintk(ioc, ioc_info(ioc, "restart the adapter\n"));  in _base_diag_reset()
8036 	    &ioc->chip->HostDiagnostic);  in _base_diag_reset()
8038 	drsprintk(ioc,  in _base_diag_reset()
8039 		  ioc_info(ioc, "disable writes to the diagnostic register\n"));  in _base_diag_reset()
8040 	writel(MPI2_WRSEQ_FLUSH_KEY_VALUE, &ioc->chip->WriteSequence);  in _base_diag_reset()
8042 	drsprintk(ioc, ioc_info(ioc, "Wait for FW to go to the READY state\n"));  in _base_diag_reset()
8043 	ioc_state = _base_wait_on_iocstate(ioc, MPI2_IOC_STATE_READY, 20);  in _base_diag_reset()
8045 		ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in _base_diag_reset()
8047 		_base_dump_reg_set(ioc);  in _base_diag_reset()
8051 	pci_cfg_access_unlock(ioc->pdev);  in _base_diag_reset()
8052 	ioc_info(ioc, "diag reset: SUCCESS\n");  in _base_diag_reset()
8056 	pci_cfg_access_unlock(ioc->pdev);  in _base_diag_reset()
8057 	ioc_err(ioc, "diag reset: FAILED\n");  in _base_diag_reset()
8069 mpt3sas_base_make_ioc_ready(struct MPT3SAS_ADAPTER *ioc, enum reset_type type)  in mpt3sas_base_make_ioc_ready()  argument
8075 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_make_ioc_ready()
8077 	if (ioc->pci_error_recovery)  in mpt3sas_base_make_ioc_ready()
8080 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_make_ioc_ready()
8081 	dhsprintk(ioc,  in mpt3sas_base_make_ioc_ready()
8082 		  ioc_info(ioc, "%s: ioc_state(0x%08x)\n",  in mpt3sas_base_make_ioc_ready()
8091 				ioc_err(ioc, "%s: failed going to ready state (ioc_state=0x%x)\n",  in mpt3sas_base_make_ioc_ready()
8096 			ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_make_ioc_ready()
8104 		ioc_info(ioc, "unexpected doorbell active!\n");  in mpt3sas_base_make_ioc_ready()
8109 		mpt3sas_print_fault_code(ioc, ioc_state &  in mpt3sas_base_make_ioc_ready()
8122 		if (ioc->ioc_coredump_loop != MPT3SAS_COREDUMP_LOOP_DONE) {  in mpt3sas_base_make_ioc_ready()
8123 			mpt3sas_print_coredump_info(ioc, ioc_state &  in mpt3sas_base_make_ioc_ready()
8125 			mpt3sas_base_wait_for_coredump_completion(ioc,  in mpt3sas_base_make_ioc_ready()
8135 		if (!(_base_send_ioc_reset(ioc,  in mpt3sas_base_make_ioc_ready()
8141 	rc = _base_diag_reset(ioc);  in mpt3sas_base_make_ioc_ready()
8152 _base_make_ioc_operational(struct MPT3SAS_ADAPTER *ioc)  in _base_make_ioc_operational()  argument
8165 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in _base_make_ioc_operational()
8169 	    &ioc->delayed_tr_list, list) {  in _base_make_ioc_operational()
8176 	    &ioc->delayed_tr_volume_list, list) {  in _base_make_ioc_operational()
8182 	    &ioc->delayed_sc_list, list) {  in _base_make_ioc_operational()
8188 	    &ioc->delayed_event_ack_list, list) {  in _base_make_ioc_operational()
8193 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _base_make_ioc_operational()
8196 	INIT_LIST_HEAD(&ioc->hpr_free_list);  in _base_make_ioc_operational()
8197 	smid = ioc->hi_priority_smid;  in _base_make_ioc_operational()
8198 	for (i = 0; i < ioc->hi_priority_depth; i++, smid++) {  in _base_make_ioc_operational()
8199 		ioc->hpr_lookup[i].cb_idx = 0xFF;  in _base_make_ioc_operational()
8200 		ioc->hpr_lookup[i].smid = smid;  in _base_make_ioc_operational()
8201 		list_add_tail(&ioc->hpr_lookup[i].tracker_list,  in _base_make_ioc_operational()
8202 		    &ioc->hpr_free_list);  in _base_make_ioc_operational()
8206 	INIT_LIST_HEAD(&ioc->internal_free_list);  in _base_make_ioc_operational()
8207 	smid = ioc->internal_smid;  in _base_make_ioc_operational()
8208 	for (i = 0; i < ioc->internal_depth; i++, smid++) {  in _base_make_ioc_operational()
8209 		ioc->internal_lookup[i].cb_idx = 0xFF;  in _base_make_ioc_operational()
8210 		ioc->internal_lookup[i].smid = smid;  in _base_make_ioc_operational()
8211 		list_add_tail(&ioc->internal_lookup[i].tracker_list,  in _base_make_ioc_operational()
8212 		    &ioc->internal_free_list);  in _base_make_ioc_operational()
8215 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _base_make_ioc_operational()
8218 	for (i = 0, reply_address = (u32)ioc->reply_dma ;  in _base_make_ioc_operational()
8219 	    i < ioc->reply_free_queue_depth ; i++, reply_address +=  in _base_make_ioc_operational()
8220 	    ioc->reply_sz) {  in _base_make_ioc_operational()
8221 		ioc->reply_free[i] = cpu_to_le32(reply_address);  in _base_make_ioc_operational()
8222 		if (ioc->is_mcpu_endpoint)  in _base_make_ioc_operational()
8223 			_base_clone_reply_to_sys_mem(ioc,  in _base_make_ioc_operational()
8228 	if (ioc->is_driver_loading)  in _base_make_ioc_operational()
8229 		_base_assign_reply_queues(ioc);  in _base_make_ioc_operational()
8233 	reply_post_free_contig = ioc->reply_post[0].reply_post_free;  in _base_make_ioc_operational()
8234 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_make_ioc_operational()
8239 		if (ioc->rdpq_array_enable) {  in _base_make_ioc_operational()
8241 				ioc->reply_post[index++].reply_post_free;  in _base_make_ioc_operational()
8244 			reply_post_free_contig += ioc->reply_post_queue_depth;  in _base_make_ioc_operational()
8248 		for (i = 0; i < ioc->reply_post_queue_depth; i++)  in _base_make_ioc_operational()
8251 		if (!_base_is_controller_msix_enabled(ioc))  in _base_make_ioc_operational()
8256 	r = _base_send_ioc_init(ioc);  in _base_make_ioc_operational()
8263 		if (!ioc->is_driver_loading)  in _base_make_ioc_operational()
8266 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in _base_make_ioc_operational()
8267 		if (rc || (_base_send_ioc_init(ioc)))  in _base_make_ioc_operational()
8272 	ioc->reply_free_host_index = ioc->reply_free_queue_depth - 1;  in _base_make_ioc_operational()
8273 	writel(ioc->reply_free_host_index, &ioc->chip->ReplyFreeHostIndex);  in _base_make_ioc_operational()
8276 	list_for_each_entry(reply_q, &ioc->reply_queue_list, list) {  in _base_make_ioc_operational()
8277 		if (ioc->combined_reply_queue)  in _base_make_ioc_operational()
8280 			   ioc->replyPostRegisterIndex[reply_q->msix_index/8]);  in _base_make_ioc_operational()
8284 				&ioc->chip->ReplyPostHostIndex);  in _base_make_ioc_operational()
8286 		if (!_base_is_controller_msix_enabled(ioc))  in _base_make_ioc_operational()
8292 	mpt3sas_base_unmask_interrupts(ioc);  in _base_make_ioc_operational()
8294 	if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _base_make_ioc_operational()
8295 		r = _base_display_fwpkg_version(ioc);  in _base_make_ioc_operational()
8300 	r = _base_static_config_pages(ioc);  in _base_make_ioc_operational()
8304 	r = _base_event_notification(ioc);  in _base_make_ioc_operational()
8308 	if (!ioc->shost_recovery) {  in _base_make_ioc_operational()
8310 		if (ioc->is_warpdrive && ioc->manu_pg10.OEMIdentifier  in _base_make_ioc_operational()
8313 			    le32_to_cpu(ioc->manu_pg10.OEMSpecificFlags0) &  in _base_make_ioc_operational()
8316 				ioc->mfg_pg10_hide_flag = hide_flag;  in _base_make_ioc_operational()
8319 		ioc->wait_for_discovery_to_complete =  in _base_make_ioc_operational()
8320 		    _base_determine_wait_on_discovery(ioc);  in _base_make_ioc_operational()
8325 	r = _base_send_port_enable(ioc);  in _base_make_ioc_operational()
8337 mpt3sas_base_free_resources(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_free_resources()  argument
8339 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_free_resources()
8342 	mutex_lock(&ioc->pci_access_mutex);  in mpt3sas_base_free_resources()
8343 	if (ioc->chip_phys && ioc->chip) {  in mpt3sas_base_free_resources()
8344 		mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_free_resources()
8345 		ioc->shost_recovery = 1;  in mpt3sas_base_free_resources()
8346 		mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);  in mpt3sas_base_free_resources()
8347 		ioc->shost_recovery = 0;  in mpt3sas_base_free_resources()
8350 	mpt3sas_base_unmap_resources(ioc);  in mpt3sas_base_free_resources()
8351 	mutex_unlock(&ioc->pci_access_mutex);  in mpt3sas_base_free_resources()
8362 mpt3sas_base_attach(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_attach()  argument
8367 	dinitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_attach()
8370 	ioc->cpu_count = num_online_cpus();  in mpt3sas_base_attach()
8373 	ioc->cpu_msix_table_sz = last_cpu_id + 1;  in mpt3sas_base_attach()
8374 	ioc->cpu_msix_table = kzalloc(ioc->cpu_msix_table_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8375 	ioc->reply_queue_count = 1;  in mpt3sas_base_attach()
8376 	if (!ioc->cpu_msix_table) {  in mpt3sas_base_attach()
8377 		ioc_info(ioc, "Allocation for cpu_msix_table failed!!!\n");  in mpt3sas_base_attach()
8382 	if (ioc->is_warpdrive) {  in mpt3sas_base_attach()
8383 		ioc->reply_post_host_index = kcalloc(ioc->cpu_msix_table_sz,  in mpt3sas_base_attach()
8385 		if (!ioc->reply_post_host_index) {  in mpt3sas_base_attach()
8386 			ioc_info(ioc, "Allocation for reply_post_host_index failed!!!\n");  in mpt3sas_base_attach()
8392 	ioc->smp_affinity_enable = smp_affinity_enable;  in mpt3sas_base_attach()
8394 	ioc->rdpq_array_enable_assigned = 0;  in mpt3sas_base_attach()
8395 	ioc->use_32bit_dma = false;  in mpt3sas_base_attach()
8396 	ioc->dma_mask = 64;  in mpt3sas_base_attach()
8397 	if (ioc->is_aero_ioc) {  in mpt3sas_base_attach()
8398 		ioc->base_readl = &_base_readl_aero;  in mpt3sas_base_attach()
8399 		ioc->base_readl_ext_retry = &_base_readl_ext_retry;  in mpt3sas_base_attach()
8401 		ioc->base_readl = &_base_readl;  in mpt3sas_base_attach()
8402 		ioc->base_readl_ext_retry = &_base_readl;  in mpt3sas_base_attach()
8404 	r = mpt3sas_base_map_resources(ioc);  in mpt3sas_base_attach()
8408 	pci_set_drvdata(ioc->pdev, ioc->shost);  in mpt3sas_base_attach()
8409 	r = _base_get_ioc_facts(ioc);  in mpt3sas_base_attach()
8411 		rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in mpt3sas_base_attach()
8412 		if (rc || (_base_get_ioc_facts(ioc)))  in mpt3sas_base_attach()
8416 	switch (ioc->hba_mpi_version_belonged) {  in mpt3sas_base_attach()
8418 		ioc->build_sg_scmd = &_base_build_sg_scmd;  in mpt3sas_base_attach()
8419 		ioc->build_sg = &_base_build_sg;  in mpt3sas_base_attach()
8420 		ioc->build_zero_len_sge = &_base_build_zero_len_sge;  in mpt3sas_base_attach()
8421 		ioc->get_msix_index_for_smlio = &_base_get_msix_index;  in mpt3sas_base_attach()
8431 		ioc->build_sg_scmd = &_base_build_sg_scmd_ieee;  in mpt3sas_base_attach()
8432 		ioc->build_sg = &_base_build_sg_ieee;  in mpt3sas_base_attach()
8433 		ioc->build_nvme_prp = &_base_build_nvme_prp;  in mpt3sas_base_attach()
8434 		ioc->build_zero_len_sge = &_base_build_zero_len_sge_ieee;  in mpt3sas_base_attach()
8435 		ioc->sge_size_ieee = sizeof(Mpi2IeeeSgeSimple64_t);  in mpt3sas_base_attach()
8436 		if (ioc->high_iops_queues)  in mpt3sas_base_attach()
8437 			ioc->get_msix_index_for_smlio =  in mpt3sas_base_attach()
8440 			ioc->get_msix_index_for_smlio = &_base_get_msix_index;  in mpt3sas_base_attach()
8443 	if (ioc->atomic_desc_capable) {  in mpt3sas_base_attach()
8444 		ioc->put_smid_default = &_base_put_smid_default_atomic;  in mpt3sas_base_attach()
8445 		ioc->put_smid_scsi_io = &_base_put_smid_scsi_io_atomic;  in mpt3sas_base_attach()
8446 		ioc->put_smid_fast_path =  in mpt3sas_base_attach()
8448 		ioc->put_smid_hi_priority =  in mpt3sas_base_attach()
8451 		ioc->put_smid_default = &_base_put_smid_default;  in mpt3sas_base_attach()
8452 		ioc->put_smid_fast_path = &_base_put_smid_fast_path;  in mpt3sas_base_attach()
8453 		ioc->put_smid_hi_priority = &_base_put_smid_hi_priority;  in mpt3sas_base_attach()
8454 		if (ioc->is_mcpu_endpoint)  in mpt3sas_base_attach()
8455 			ioc->put_smid_scsi_io =  in mpt3sas_base_attach()
8458 			ioc->put_smid_scsi_io = &_base_put_smid_scsi_io;  in mpt3sas_base_attach()
8466 	ioc->build_sg_mpi = &_base_build_sg;  in mpt3sas_base_attach()
8467 	ioc->build_zero_len_sge_mpi = &_base_build_zero_len_sge;  in mpt3sas_base_attach()
8469 	r = mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);  in mpt3sas_base_attach()
8473 	ioc->pfacts = kcalloc(ioc->facts.NumberOfPorts,  in mpt3sas_base_attach()
8475 	if (!ioc->pfacts) {  in mpt3sas_base_attach()
8480 	for (i = 0 ; i < ioc->facts.NumberOfPorts; i++) {  in mpt3sas_base_attach()
8481 		r = _base_get_port_facts(ioc, i);  in mpt3sas_base_attach()
8483 			rc = mpt3sas_base_check_for_fault_and_issue_reset(ioc);  in mpt3sas_base_attach()
8484 			if (rc || (_base_get_port_facts(ioc, i)))  in mpt3sas_base_attach()
8489 	r = _base_allocate_memory_pools(ioc);  in mpt3sas_base_attach()
8494 		ioc->thresh_hold = irqpoll_weight;  in mpt3sas_base_attach()
8496 		ioc->thresh_hold = ioc->hba_queue_depth/4;  in mpt3sas_base_attach()
8498 	_base_init_irqpolls(ioc);  in mpt3sas_base_attach()
8499 	init_waitqueue_head(&ioc->reset_wq);  in mpt3sas_base_attach()
8502 	ioc->pd_handles_sz = (ioc->facts.MaxDevHandle / 8);  in mpt3sas_base_attach()
8503 	if (ioc->facts.MaxDevHandle % 8)  in mpt3sas_base_attach()
8504 		ioc->pd_handles_sz++;  in mpt3sas_base_attach()
8509 	ioc->pd_handles_sz = ALIGN(ioc->pd_handles_sz, sizeof(unsigned long));  in mpt3sas_base_attach()
8511 	ioc->pd_handles = kzalloc(ioc->pd_handles_sz,  in mpt3sas_base_attach()
8513 	if (!ioc->pd_handles) {  in mpt3sas_base_attach()
8517 	ioc->blocking_handles = kzalloc(ioc->pd_handles_sz,  in mpt3sas_base_attach()
8519 	if (!ioc->blocking_handles) {  in mpt3sas_base_attach()
8525 	ioc->pend_os_device_add_sz = (ioc->facts.MaxDevHandle / 8);  in mpt3sas_base_attach()
8526 	if (ioc->facts.MaxDevHandle % 8)  in mpt3sas_base_attach()
8527 		ioc->pend_os_device_add_sz++;  in mpt3sas_base_attach()
8533 	ioc->pend_os_device_add_sz = ALIGN(ioc->pend_os_device_add_sz,  in mpt3sas_base_attach()
8535 	ioc->pend_os_device_add = kzalloc(ioc->pend_os_device_add_sz,  in mpt3sas_base_attach()
8537 	if (!ioc->pend_os_device_add) {  in mpt3sas_base_attach()
8542 	ioc->device_remove_in_progress_sz = ioc->pend_os_device_add_sz;  in mpt3sas_base_attach()
8543 	ioc->device_remove_in_progress =  in mpt3sas_base_attach()
8544 		kzalloc(ioc->device_remove_in_progress_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8545 	if (!ioc->device_remove_in_progress) {  in mpt3sas_base_attach()
8550 	ioc->fwfault_debug = mpt3sas_fwfault_debug;  in mpt3sas_base_attach()
8553 	mutex_init(&ioc->base_cmds.mutex);  in mpt3sas_base_attach()
8554 	ioc->base_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8555 	ioc->base_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8558 	ioc->port_enable_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8559 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8562 	ioc->transport_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8563 	ioc->transport_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8564 	mutex_init(&ioc->transport_cmds.mutex);  in mpt3sas_base_attach()
8567 	ioc->scsih_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8568 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8569 	mutex_init(&ioc->scsih_cmds.mutex);  in mpt3sas_base_attach()
8572 	ioc->tm_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8573 	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8574 	mutex_init(&ioc->tm_cmds.mutex);  in mpt3sas_base_attach()
8577 	ioc->config_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8578 	ioc->config_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8579 	mutex_init(&ioc->config_cmds.mutex);  in mpt3sas_base_attach()
8582 	ioc->ctl_cmds.reply = kzalloc(ioc->reply_sz, GFP_KERNEL);  in mpt3sas_base_attach()
8583 	ioc->ctl_cmds.sense = kzalloc(SCSI_SENSE_BUFFERSIZE, GFP_KERNEL);  in mpt3sas_base_attach()
8584 	ioc->ctl_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_base_attach()
8585 	mutex_init(&ioc->ctl_cmds.mutex);  in mpt3sas_base_attach()
8587 	if (!ioc->base_cmds.reply || !ioc->port_enable_cmds.reply ||  in mpt3sas_base_attach()
8588 	    !ioc->transport_cmds.reply || !ioc->scsih_cmds.reply ||  in mpt3sas_base_attach()
8589 	    !ioc->tm_cmds.reply || !ioc->config_cmds.reply ||  in mpt3sas_base_attach()
8590 	    !ioc->ctl_cmds.reply || !ioc->ctl_cmds.sense) {  in mpt3sas_base_attach()
8596 		ioc->event_masks[i] = -1;  in mpt3sas_base_attach()
8599 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DISCOVERY);  in mpt3sas_base_attach()
8600 	_base_unmask_events(ioc, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);  in mpt3sas_base_attach()
8601 	_base_unmask_events(ioc, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);  in mpt3sas_base_attach()
8602 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);  in mpt3sas_base_attach()
8603 	_base_unmask_events(ioc, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);  in mpt3sas_base_attach()
8604 	_base_unmask_events(ioc, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);  in mpt3sas_base_attach()
8605 	_base_unmask_events(ioc, MPI2_EVENT_IR_VOLUME);  in mpt3sas_base_attach()
8606 	_base_unmask_events(ioc, MPI2_EVENT_IR_PHYSICAL_DISK);  in mpt3sas_base_attach()
8607 	_base_unmask_events(ioc, MPI2_EVENT_IR_OPERATION_STATUS);  in mpt3sas_base_attach()
8608 	_base_unmask_events(ioc, MPI2_EVENT_LOG_ENTRY_ADDED);  in mpt3sas_base_attach()
8609 	_base_unmask_events(ioc, MPI2_EVENT_TEMP_THRESHOLD);  in mpt3sas_base_attach()
8610 	_base_unmask_events(ioc, MPI2_EVENT_ACTIVE_CABLE_EXCEPTION);  in mpt3sas_base_attach()
8611 	_base_unmask_events(ioc, MPI2_EVENT_SAS_DEVICE_DISCOVERY_ERROR);  in mpt3sas_base_attach()
8612 	if (ioc->hba_mpi_version_belonged == MPI26_VERSION) {  in mpt3sas_base_attach()
8613 		if (ioc->is_gen35_ioc) {  in mpt3sas_base_attach()
8614 			_base_unmask_events(ioc,  in mpt3sas_base_attach()
8616 			_base_unmask_events(ioc, MPI2_EVENT_PCIE_ENUMERATION);  in mpt3sas_base_attach()
8617 			_base_unmask_events(ioc,  in mpt3sas_base_attach()
8621 	r = _base_make_ioc_operational(ioc);  in mpt3sas_base_attach()
8623 		r = _base_make_ioc_operational(ioc);  in mpt3sas_base_attach()
8632 	memcpy(&ioc->prev_fw_facts, &ioc->facts,  in mpt3sas_base_attach()
8635 	ioc->non_operational_loop = 0;  in mpt3sas_base_attach()
8636 	ioc->ioc_coredump_loop = 0;  in mpt3sas_base_attach()
8637 	ioc->got_task_abort_from_ioctl = 0;  in mpt3sas_base_attach()
8642 	ioc->remove_host = 1;  in mpt3sas_base_attach()
8644 	mpt3sas_base_free_resources(ioc);  in mpt3sas_base_attach()
8645 	_base_release_memory_pools(ioc);  in mpt3sas_base_attach()
8646 	pci_set_drvdata(ioc->pdev, NULL);  in mpt3sas_base_attach()
8647 	kfree(ioc->cpu_msix_table);  in mpt3sas_base_attach()
8648 	if (ioc->is_warpdrive)  in mpt3sas_base_attach()
8649 		kfree(ioc->reply_post_host_index);  in mpt3sas_base_attach()
8650 	kfree(ioc->pd_handles);  in mpt3sas_base_attach()
8651 	kfree(ioc->blocking_handles);  in mpt3sas_base_attach()
8652 	kfree(ioc->device_remove_in_progress);  in mpt3sas_base_attach()
8653 	kfree(ioc->pend_os_device_add);  in mpt3sas_base_attach()
8654 	kfree(ioc->tm_cmds.reply);  in mpt3sas_base_attach()
8655 	kfree(ioc->transport_cmds.reply);  in mpt3sas_base_attach()
8656 	kfree(ioc->scsih_cmds.reply);  in mpt3sas_base_attach()
8657 	kfree(ioc->config_cmds.reply);  in mpt3sas_base_attach()
8658 	kfree(ioc->base_cmds.reply);  in mpt3sas_base_attach()
8659 	kfree(ioc->port_enable_cmds.reply);  in mpt3sas_base_attach()
8660 	kfree(ioc->ctl_cmds.reply);  in mpt3sas_base_attach()
8661 	kfree(ioc->ctl_cmds.sense);  in mpt3sas_base_attach()
8662 	kfree(ioc->pfacts);  in mpt3sas_base_attach()
8663 	ioc->ctl_cmds.reply = NULL;  in mpt3sas_base_attach()
8664 	ioc->base_cmds.reply = NULL;  in mpt3sas_base_attach()
8665 	ioc->tm_cmds.reply = NULL;  in mpt3sas_base_attach()
8666 	ioc->scsih_cmds.reply = NULL;  in mpt3sas_base_attach()
8667 	ioc->transport_cmds.reply = NULL;  in mpt3sas_base_attach()
8668 	ioc->config_cmds.reply = NULL;  in mpt3sas_base_attach()
8669 	ioc->pfacts = NULL;  in mpt3sas_base_attach()
8679 mpt3sas_base_detach(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_base_detach()  argument
8681 	dexitprintk(ioc, ioc_info(ioc, "%s\n", __func__));  in mpt3sas_base_detach()
8683 	mpt3sas_base_stop_watchdog(ioc);  in mpt3sas_base_detach()
8684 	mpt3sas_base_free_resources(ioc);  in mpt3sas_base_detach()
8685 	_base_release_memory_pools(ioc);  in mpt3sas_base_detach()
8686 	mpt3sas_free_enclosure_list(ioc);  in mpt3sas_base_detach()
8687 	pci_set_drvdata(ioc->pdev, NULL);  in mpt3sas_base_detach()
8688 	kfree(ioc->cpu_msix_table);  in mpt3sas_base_detach()
8689 	if (ioc->is_warpdrive)  in mpt3sas_base_detach()
8690 		kfree(ioc->reply_post_host_index);  in mpt3sas_base_detach()
8691 	kfree(ioc->pd_handles);  in mpt3sas_base_detach()
8692 	kfree(ioc->blocking_handles);  in mpt3sas_base_detach()
8693 	kfree(ioc->device_remove_in_progress);  in mpt3sas_base_detach()
8694 	kfree(ioc->pend_os_device_add);  in mpt3sas_base_detach()
8695 	kfree(ioc->pfacts);  in mpt3sas_base_detach()
8696 	kfree(ioc->ctl_cmds.reply);  in mpt3sas_base_detach()
8697 	kfree(ioc->ctl_cmds.sense);  in mpt3sas_base_detach()
8698 	kfree(ioc->base_cmds.reply);  in mpt3sas_base_detach()
8699 	kfree(ioc->port_enable_cmds.reply);  in mpt3sas_base_detach()
8700 	kfree(ioc->tm_cmds.reply);  in mpt3sas_base_detach()
8701 	kfree(ioc->transport_cmds.reply);  in mpt3sas_base_detach()
8702 	kfree(ioc->scsih_cmds.reply);  in mpt3sas_base_detach()
8703 	kfree(ioc->config_cmds.reply);  in mpt3sas_base_detach()
8710 static void _base_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)  in _base_pre_reset_handler()  argument
8712 	mpt3sas_scsih_pre_reset_handler(ioc);  in _base_pre_reset_handler()
8713 	mpt3sas_ctl_pre_reset_handler(ioc);  in _base_pre_reset_handler()
8714 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));  in _base_pre_reset_handler()
8722 _base_clear_outstanding_mpt_commands(struct MPT3SAS_ADAPTER *ioc)  in _base_clear_outstanding_mpt_commands()  argument
8724 	dtmprintk(ioc,  in _base_clear_outstanding_mpt_commands()
8725 	    ioc_info(ioc, "%s: clear outstanding mpt cmds\n", __func__));  in _base_clear_outstanding_mpt_commands()
8726 	if (ioc->transport_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8727 		ioc->transport_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8728 		mpt3sas_base_free_smid(ioc, ioc->transport_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8729 		complete(&ioc->transport_cmds.done);  in _base_clear_outstanding_mpt_commands()
8731 	if (ioc->base_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8732 		ioc->base_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8733 		mpt3sas_base_free_smid(ioc, ioc->base_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8734 		complete(&ioc->base_cmds.done);  in _base_clear_outstanding_mpt_commands()
8736 	if (ioc->port_enable_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8737 		ioc->port_enable_failed = 1;  in _base_clear_outstanding_mpt_commands()
8738 		ioc->port_enable_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8739 		mpt3sas_base_free_smid(ioc, ioc->port_enable_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8740 		if (ioc->is_driver_loading) {  in _base_clear_outstanding_mpt_commands()
8741 			ioc->start_scan_failed =  in _base_clear_outstanding_mpt_commands()
8743 			ioc->start_scan = 0;  in _base_clear_outstanding_mpt_commands()
8745 			complete(&ioc->port_enable_cmds.done);  in _base_clear_outstanding_mpt_commands()
8748 	if (ioc->config_cmds.status & MPT3_CMD_PENDING) {  in _base_clear_outstanding_mpt_commands()
8749 		ioc->config_cmds.status |= MPT3_CMD_RESET;  in _base_clear_outstanding_mpt_commands()
8750 		mpt3sas_base_free_smid(ioc, ioc->config_cmds.smid);  in _base_clear_outstanding_mpt_commands()
8751 		ioc->config_cmds.smid = USHRT_MAX;  in _base_clear_outstanding_mpt_commands()
8752 		complete(&ioc->config_cmds.done);  in _base_clear_outstanding_mpt_commands()
8760 static void _base_clear_outstanding_commands(struct MPT3SAS_ADAPTER *ioc)  in _base_clear_outstanding_commands()  argument
8762 	mpt3sas_scsih_clear_outstanding_scsi_tm_commands(ioc);  in _base_clear_outstanding_commands()
8763 	mpt3sas_ctl_clear_outstanding_ioctls(ioc);  in _base_clear_outstanding_commands()
8764 	_base_clear_outstanding_mpt_commands(ioc);  in _base_clear_outstanding_commands()
8771 static void _base_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)  in _base_reset_done_handler()  argument
8773 	mpt3sas_scsih_reset_done_handler(ioc);  in _base_reset_done_handler()
8774 	mpt3sas_ctl_reset_done_handler(ioc);  in _base_reset_done_handler()
8775 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));  in _base_reset_done_handler()
8786 mpt3sas_wait_for_commands_to_complete(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_wait_for_commands_to_complete()  argument
8790 	ioc->pending_io_count = 0;  in mpt3sas_wait_for_commands_to_complete()
8792 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_wait_for_commands_to_complete()
8797 	ioc->pending_io_count = scsi_host_busy(ioc->shost);  in mpt3sas_wait_for_commands_to_complete()
8799 	if (!ioc->pending_io_count)  in mpt3sas_wait_for_commands_to_complete()
8803 	wait_event_timeout(ioc->reset_wq, ioc->pending_io_count == 0, 10 * HZ);  in mpt3sas_wait_for_commands_to_complete()
8814 _base_check_ioc_facts_changes(struct MPT3SAS_ADAPTER *ioc)  in _base_check_ioc_facts_changes()  argument
8819 	struct mpt3sas_facts *old_facts = &ioc->prev_fw_facts;  in _base_check_ioc_facts_changes()
8821 	if (ioc->facts.MaxDevHandle > old_facts->MaxDevHandle) {  in _base_check_ioc_facts_changes()
8822 		pd_handles_sz = (ioc->facts.MaxDevHandle / 8);  in _base_check_ioc_facts_changes()
8823 		if (ioc->facts.MaxDevHandle % 8)  in _base_check_ioc_facts_changes()
8832 		pd_handles = krealloc(ioc->pd_handles, pd_handles_sz,  in _base_check_ioc_facts_changes()
8835 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8840 		memset(pd_handles + ioc->pd_handles_sz, 0,  in _base_check_ioc_facts_changes()
8841 		    (pd_handles_sz - ioc->pd_handles_sz));  in _base_check_ioc_facts_changes()
8842 		ioc->pd_handles = pd_handles;  in _base_check_ioc_facts_changes()
8844 		blocking_handles = krealloc(ioc->blocking_handles,  in _base_check_ioc_facts_changes()
8847 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8853 		memset(blocking_handles + ioc->pd_handles_sz, 0,  in _base_check_ioc_facts_changes()
8854 		    (pd_handles_sz - ioc->pd_handles_sz));  in _base_check_ioc_facts_changes()
8855 		ioc->blocking_handles = blocking_handles;  in _base_check_ioc_facts_changes()
8856 		ioc->pd_handles_sz = pd_handles_sz;  in _base_check_ioc_facts_changes()
8858 		pend_os_device_add = krealloc(ioc->pend_os_device_add,  in _base_check_ioc_facts_changes()
8861 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8866 		memset(pend_os_device_add + ioc->pend_os_device_add_sz, 0,  in _base_check_ioc_facts_changes()
8867 		    (pd_handles_sz - ioc->pend_os_device_add_sz));  in _base_check_ioc_facts_changes()
8868 		ioc->pend_os_device_add = pend_os_device_add;  in _base_check_ioc_facts_changes()
8869 		ioc->pend_os_device_add_sz = pd_handles_sz;  in _base_check_ioc_facts_changes()
8872 		    ioc->device_remove_in_progress, pd_handles_sz, GFP_KERNEL);  in _base_check_ioc_facts_changes()
8874 			ioc_info(ioc,  in _base_check_ioc_facts_changes()
8881 		    ioc->device_remove_in_progress_sz, 0,  in _base_check_ioc_facts_changes()
8882 		    (pd_handles_sz - ioc->device_remove_in_progress_sz));  in _base_check_ioc_facts_changes()
8883 		ioc->device_remove_in_progress = device_remove_in_progress;  in _base_check_ioc_facts_changes()
8884 		ioc->device_remove_in_progress_sz = pd_handles_sz;  in _base_check_ioc_facts_changes()
8887 	memcpy(&ioc->prev_fw_facts, &ioc->facts, sizeof(struct mpt3sas_facts));  in _base_check_ioc_facts_changes()
8899 mpt3sas_base_hard_reset_handler(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_base_hard_reset_handler()  argument
8907 	dtmprintk(ioc, ioc_info(ioc, "%s: enter\n", __func__));  in mpt3sas_base_hard_reset_handler()
8909 	if (ioc->pci_error_recovery) {  in mpt3sas_base_hard_reset_handler()
8910 		ioc_err(ioc, "%s: pci error recovery reset\n", __func__);  in mpt3sas_base_hard_reset_handler()
8916 		mpt3sas_halt_firmware(ioc);  in mpt3sas_base_hard_reset_handler()
8919 	mutex_lock(&ioc->reset_in_progress_mutex);  in mpt3sas_base_hard_reset_handler()
8921 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8922 	ioc->shost_recovery = 1;  in mpt3sas_base_hard_reset_handler()
8923 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8925 	if ((ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &  in mpt3sas_base_hard_reset_handler()
8927 	    (!(ioc->diag_buffer_status[MPI2_DIAG_BUF_TYPE_TRACE] &  in mpt3sas_base_hard_reset_handler()
8930 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_base_hard_reset_handler()
8935 			ioc->htb_rel.trigger_info_dwords[1] =  in mpt3sas_base_hard_reset_handler()
8939 	_base_pre_reset_handler(ioc);  in mpt3sas_base_hard_reset_handler()
8940 	mpt3sas_wait_for_commands_to_complete(ioc);  in mpt3sas_base_hard_reset_handler()
8941 	mpt3sas_base_mask_interrupts(ioc);  in mpt3sas_base_hard_reset_handler()
8942 	mpt3sas_base_pause_mq_polling(ioc);  in mpt3sas_base_hard_reset_handler()
8943 	r = mpt3sas_base_make_ioc_ready(ioc, type);  in mpt3sas_base_hard_reset_handler()
8946 	_base_clear_outstanding_commands(ioc);  in mpt3sas_base_hard_reset_handler()
8951 	if (ioc->is_driver_loading && ioc->port_enable_failed) {  in mpt3sas_base_hard_reset_handler()
8952 		ioc->remove_host = 1;  in mpt3sas_base_hard_reset_handler()
8956 	r = _base_get_ioc_facts(ioc);  in mpt3sas_base_hard_reset_handler()
8960 	r = _base_check_ioc_facts_changes(ioc);  in mpt3sas_base_hard_reset_handler()
8962 		ioc_info(ioc,  in mpt3sas_base_hard_reset_handler()
8967 	if (ioc->rdpq_array_enable && !ioc->rdpq_array_capable)  in mpt3sas_base_hard_reset_handler()
8970 		      " firmware version is running\n", ioc->name);  in mpt3sas_base_hard_reset_handler()
8972 	r = _base_make_ioc_operational(ioc);  in mpt3sas_base_hard_reset_handler()
8974 		_base_reset_done_handler(ioc);  in mpt3sas_base_hard_reset_handler()
8977 	ioc_info(ioc, "%s: %s\n", __func__, r == 0 ? "SUCCESS" : "FAILED");  in mpt3sas_base_hard_reset_handler()
8979 	spin_lock_irqsave(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8980 	ioc->shost_recovery = 0;  in mpt3sas_base_hard_reset_handler()
8981 	spin_unlock_irqrestore(&ioc->ioc_reset_in_progress_lock, flags);  in mpt3sas_base_hard_reset_handler()
8982 	ioc->ioc_reset_count++;  in mpt3sas_base_hard_reset_handler()
8983 	mutex_unlock(&ioc->reset_in_progress_mutex);  in mpt3sas_base_hard_reset_handler()
8984 	mpt3sas_base_resume_mq_polling(ioc);  in mpt3sas_base_hard_reset_handler()
8989 			mpt3sas_trigger_master(ioc, MASTER_TRIGGER_FW_FAULT);  in mpt3sas_base_hard_reset_handler()
8991 			mpt3sas_trigger_master(ioc,  in mpt3sas_base_hard_reset_handler()
8994 	dtmprintk(ioc, ioc_info(ioc, "%s: exit\n", __func__));  in mpt3sas_base_hard_reset_handler()