66 static void _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,
70 static void _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,
72 static int _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle,
74 static int _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
75 static void _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,
78 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle);
79 static u8 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid);
80 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc);
218 	struct MPT3SAS_ADAPTER *ioc;  member
314 	struct MPT3SAS_ADAPTER *ioc;  in _scsih_set_debug_level()  local
321 	list_for_each_entry(ioc, &mpt3sas_ioc_list, list)  in _scsih_set_debug_level()
322 		ioc->logging_level = logging_level;  in _scsih_set_debug_level()
386 mpt3sas_get_port_by_id(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_port_by_id()  argument
396 	if (!ioc->multipath_on_hba)  in mpt3sas_get_port_by_id()
400 	    &ioc->port_table_list, list) {  in mpt3sas_get_port_by_id()
415 	if (!ioc->multipath_on_hba) {  in mpt3sas_get_port_by_id()
421 		ioc_info(ioc,  in mpt3sas_get_port_by_id()
425 		    &ioc->port_table_list);  in mpt3sas_get_port_by_id()
440 mpt3sas_get_vphy_by_phy(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_vphy_by_phy()  argument
509 _scsih_get_sas_address(struct MPT3SAS_ADAPTER *ioc, u16 handle,  in _scsih_get_sas_address()  argument
518 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_get_sas_address()
520 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_sas_address()
530 		if ((handle <= ioc->sas_hba.num_phys) &&  in _scsih_get_sas_address()
533 			*sas_address = ioc->sas_hba.sas_address;  in _scsih_get_sas_address()
544 	ioc_err(ioc, "handle(0x%04x), ioc_status(0x%04x), failure at %s:%d/%s()!\n",  in _scsih_get_sas_address()
563 _scsih_determine_boot_device(struct MPT3SAS_ADAPTER *ioc, void *device,  in _scsih_determine_boot_device()  argument
575 	if (!ioc->is_driver_loading)  in _scsih_determine_boot_device()
579 	if (!ioc->bios_pg3.BiosVersion)  in _scsih_determine_boot_device()
602 	if (!ioc->req_boot_device.device) {  in _scsih_determine_boot_device()
605 		    (ioc->bios_pg2.ReqBootDeviceForm &  in _scsih_determine_boot_device()
607 		    &ioc->bios_pg2.RequestedBootDevice)) {  in _scsih_determine_boot_device()
608 			dinitprintk(ioc,  in _scsih_determine_boot_device()
609 				    ioc_info(ioc, "%s: req_boot_device(0x%016llx)\n",  in _scsih_determine_boot_device()
611 			ioc->req_boot_device.device = device;  in _scsih_determine_boot_device()
612 			ioc->req_boot_device.channel = channel;  in _scsih_determine_boot_device()
616 	if (!ioc->req_alt_boot_device.device) {  in _scsih_determine_boot_device()
619 		    (ioc->bios_pg2.ReqAltBootDeviceForm &  in _scsih_determine_boot_device()
621 		    &ioc->bios_pg2.RequestedAltBootDevice)) {  in _scsih_determine_boot_device()
622 			dinitprintk(ioc,  in _scsih_determine_boot_device()
623 				    ioc_info(ioc, "%s: req_alt_boot_device(0x%016llx)\n",  in _scsih_determine_boot_device()
625 			ioc->req_alt_boot_device.device = device;  in _scsih_determine_boot_device()
626 			ioc->req_alt_boot_device.channel = channel;  in _scsih_determine_boot_device()
630 	if (!ioc->current_boot_device.device) {  in _scsih_determine_boot_device()
633 		    (ioc->bios_pg2.CurrentBootDeviceForm &  in _scsih_determine_boot_device()
635 		    &ioc->bios_pg2.CurrentBootDevice)) {  in _scsih_determine_boot_device()
636 			dinitprintk(ioc,  in _scsih_determine_boot_device()
637 				    ioc_info(ioc, "%s: current_boot_device(0x%016llx)\n",  in _scsih_determine_boot_device()
639 			ioc->current_boot_device.device = device;  in _scsih_determine_boot_device()
640 			ioc->current_boot_device.channel = channel;  in _scsih_determine_boot_device()
646 __mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_sdev_from_target()  argument
651 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_from_target()
661 mpt3sas_get_sdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_sdev_from_target()  argument
667 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_from_target()
668 	ret = __mpt3sas_get_sdev_from_target(ioc, tgt_priv);  in mpt3sas_get_sdev_from_target()
669 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_from_target()
675 __mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_pdev_from_target()  argument
680 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_from_target()
700 mpt3sas_get_pdev_from_target(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_pdev_from_target()  argument
706 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_from_target()
707 	ret = __mpt3sas_get_pdev_from_target(ioc, tgt_priv);  in mpt3sas_get_pdev_from_target()
708 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_from_target()
726 __mpt3sas_get_sdev_by_rphy(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_sdev_by_rphy()  argument
731 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_by_rphy()
733 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {  in __mpt3sas_get_sdev_by_rphy()
741 	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {  in __mpt3sas_get_sdev_by_rphy()
762 __mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,  in __mpt3sas_get_sdev_by_addr()  argument
770 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_by_addr()
772 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {  in __mpt3sas_get_sdev_by_addr()
781 	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list) {  in __mpt3sas_get_sdev_by_addr()
804 mpt3sas_get_sdev_by_addr(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_get_sdev_by_addr()  argument
810 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_addr()
811 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in mpt3sas_get_sdev_by_addr()
813 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_addr()
819 __mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in __mpt3sas_get_sdev_by_handle()  argument
823 	assert_spin_locked(&ioc->sas_device_lock);  in __mpt3sas_get_sdev_by_handle()
825 	list_for_each_entry(sas_device, &ioc->sas_device_list, list)  in __mpt3sas_get_sdev_by_handle()
829 	list_for_each_entry(sas_device, &ioc->sas_device_init_list, list)  in __mpt3sas_get_sdev_by_handle()
850 mpt3sas_get_sdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_get_sdev_by_handle()  argument
855 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_handle()
856 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in mpt3sas_get_sdev_by_handle()
857 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_get_sdev_by_handle()
870 _scsih_display_enclosure_chassis_info(struct MPT3SAS_ADAPTER *ioc,  in _scsih_display_enclosure_chassis_info()  argument
907 			ioc_info(ioc, "enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_display_enclosure_chassis_info()
911 			ioc_info(ioc, "enclosure level(0x%04x), connector name( %s)\n",  in _scsih_display_enclosure_chassis_info()
915 			ioc_info(ioc, "chassis slot(0x%04x)\n",  in _scsih_display_enclosure_chassis_info()
929 _scsih_sas_device_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_remove()  argument
936 	ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_sas_device_remove()
939 	_scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);  in _scsih_sas_device_remove()
945 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_remove()
950 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_remove()
959 _scsih_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_device_remove_by_handle()  argument
964 	if (ioc->shost_recovery)  in _scsih_device_remove_by_handle()
967 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_device_remove_by_handle()
968 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_device_remove_by_handle()
973 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_device_remove_by_handle()
975 		_scsih_remove_device(ioc, sas_device);  in _scsih_device_remove_by_handle()
990 mpt3sas_device_remove_by_sas_address(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_device_remove_by_sas_address()  argument
996 	if (ioc->shost_recovery)  in mpt3sas_device_remove_by_sas_address()
999 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in mpt3sas_device_remove_by_sas_address()
1000 	sas_device = __mpt3sas_get_sdev_by_addr(ioc, sas_address, port);  in mpt3sas_device_remove_by_sas_address()
1005 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in mpt3sas_device_remove_by_sas_address()
1007 		_scsih_remove_device(ioc, sas_device);  in mpt3sas_device_remove_by_sas_address()
1021 _scsih_sas_device_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_add()  argument
1026 	dewtprintk(ioc,  in _scsih_sas_device_add()
1027 		   ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_sas_device_add()
1031 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_sas_device_add()
1034 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_add()
1036 	list_add_tail(&sas_device->list, &ioc->sas_device_list);  in _scsih_sas_device_add()
1037 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_add()
1039 	if (ioc->hide_drives) {  in _scsih_sas_device_add()
1040 		clear_bit(sas_device->handle, ioc->pend_os_device_add);  in _scsih_sas_device_add()
1044 	if (!mpt3sas_transport_port_add(ioc, sas_device->handle,  in _scsih_sas_device_add()
1046 		_scsih_sas_device_remove(ioc, sas_device);  in _scsih_sas_device_add()
1053 		if (!ioc->is_driver_loading) {  in _scsih_sas_device_add()
1054 			mpt3sas_transport_port_remove(ioc,  in _scsih_sas_device_add()
1058 			_scsih_sas_device_remove(ioc, sas_device);  in _scsih_sas_device_add()
1061 		clear_bit(sas_device->handle, ioc->pend_os_device_add);  in _scsih_sas_device_add()
1073 _scsih_sas_device_init_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_init_add()  argument
1078 	dewtprintk(ioc,  in _scsih_sas_device_init_add()
1079 		   ioc_info(ioc, "%s: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_sas_device_init_add()
1083 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_sas_device_init_add()
1086 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_init_add()
1088 	list_add_tail(&sas_device->list, &ioc->sas_device_init_list);  in _scsih_sas_device_init_add()
1089 	_scsih_determine_boot_device(ioc, sas_device, 0);  in _scsih_sas_device_init_add()
1090 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_init_add()
1095 __mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)  in __mpt3sas_get_pdev_by_wwid()  argument
1099 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_by_wwid()
1101 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)  in __mpt3sas_get_pdev_by_wwid()
1105 	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)  in __mpt3sas_get_pdev_by_wwid()
1128 mpt3sas_get_pdev_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)  in mpt3sas_get_pdev_by_wwid()  argument
1133 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_wwid()
1134 	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);  in mpt3sas_get_pdev_by_wwid()
1135 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_wwid()
1142 __mpt3sas_get_pdev_by_idchannel(struct MPT3SAS_ADAPTER *ioc, int id,  in __mpt3sas_get_pdev_by_idchannel()  argument
1147 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_by_idchannel()
1149 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)  in __mpt3sas_get_pdev_by_idchannel()
1153 	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)  in __mpt3sas_get_pdev_by_idchannel()
1165 __mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in __mpt3sas_get_pdev_by_handle()  argument
1169 	assert_spin_locked(&ioc->pcie_device_lock);  in __mpt3sas_get_pdev_by_handle()
1171 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list)  in __mpt3sas_get_pdev_by_handle()
1175 	list_for_each_entry(pcie_device, &ioc->pcie_device_init_list, list)  in __mpt3sas_get_pdev_by_handle()
1199 mpt3sas_get_pdev_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_get_pdev_by_handle()  argument
1204 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_handle()
1205 	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in mpt3sas_get_pdev_by_handle()
1206 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in mpt3sas_get_pdev_by_handle()
1221 _scsih_set_nvme_max_shutdown_latency(struct MPT3SAS_ADAPTER *ioc)  in _scsih_set_nvme_max_shutdown_latency()  argument
1227 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_set_nvme_max_shutdown_latency()
1228 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {  in _scsih_set_nvme_max_shutdown_latency()
1235 	ioc->max_shutdown_latency = shutdown_latency;  in _scsih_set_nvme_max_shutdown_latency()
1236 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_set_nvme_max_shutdown_latency()
1248 _scsih_pcie_device_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_remove()  argument
1257 	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove()
1260 		ioc_info(ioc, "removing enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove()
1264 		ioc_info(ioc, "removing enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_remove()
1268 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove()
1273 	if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)  in _scsih_pcie_device_remove()
1275 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove()
1287 		_scsih_set_nvme_max_shutdown_latency(ioc);  in _scsih_pcie_device_remove()
1297 _scsih_pcie_device_remove_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_pcie_device_remove_by_handle()  argument
1304 	if (ioc->shost_recovery)  in _scsih_pcie_device_remove_by_handle()
1307 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove_by_handle()
1308 	pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in _scsih_pcie_device_remove_by_handle()
1315 		if (pcie_device->shutdown_latency == ioc->max_shutdown_latency)  in _scsih_pcie_device_remove_by_handle()
1318 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_remove_by_handle()
1320 		_scsih_pcie_device_remove_from_sml(ioc, pcie_device);  in _scsih_pcie_device_remove_by_handle()
1330 		_scsih_set_nvme_max_shutdown_latency(ioc);  in _scsih_pcie_device_remove_by_handle()
1341 _scsih_pcie_device_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_add()  argument
1346 	dewtprintk(ioc,  in _scsih_pcie_device_add()
1347 		   ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_add()
1351 		dewtprintk(ioc,  in _scsih_pcie_device_add()
1352 			   ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",  in _scsih_pcie_device_add()
1357 		dewtprintk(ioc,  in _scsih_pcie_device_add()
1358 			   ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_add()
1362 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_add()
1364 	list_add_tail(&pcie_device->list, &ioc->pcie_device_list);  in _scsih_pcie_device_add()
1365 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_add()
1369 		clear_bit(pcie_device->handle, ioc->pend_os_device_add);  in _scsih_pcie_device_add()
1372 	if (scsi_add_device(ioc->shost, PCIE_CHANNEL, pcie_device->id, 0)) {  in _scsih_pcie_device_add()
1373 		_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_pcie_device_add()
1375 		if (!ioc->is_driver_loading) {  in _scsih_pcie_device_add()
1377 			clear_bit(pcie_device->handle, ioc->pend_os_device_add);  in _scsih_pcie_device_add()
1380 		clear_bit(pcie_device->handle, ioc->pend_os_device_add);  in _scsih_pcie_device_add()
1392 _scsih_pcie_device_init_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_init_add()  argument
1397 	dewtprintk(ioc,  in _scsih_pcie_device_init_add()
1398 		   ioc_info(ioc, "%s: handle (0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_init_add()
1402 		dewtprintk(ioc,  in _scsih_pcie_device_init_add()
1403 			   ioc_info(ioc, "%s: enclosure logical id(0x%016llx), slot( %d)\n",  in _scsih_pcie_device_init_add()
1408 		dewtprintk(ioc,  in _scsih_pcie_device_init_add()
1409 			   ioc_info(ioc, "%s: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_init_add()
1413 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_init_add()
1415 	list_add_tail(&pcie_device->list, &ioc->pcie_device_init_list);  in _scsih_pcie_device_init_add()
1418 		_scsih_determine_boot_device(ioc, pcie_device, PCIE_CHANNEL);  in _scsih_pcie_device_init_add()
1419 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_init_add()
1432 _scsih_raid_device_find_by_id(struct MPT3SAS_ADAPTER *ioc, int id, int channel)  in _scsih_raid_device_find_by_id()  argument
1437 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in _scsih_raid_device_find_by_id()
1458 mpt3sas_raid_device_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_raid_device_find_by_handle()  argument
1463 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in mpt3sas_raid_device_find_by_handle()
1484 _scsih_raid_device_find_by_wwid(struct MPT3SAS_ADAPTER *ioc, u64 wwid)  in _scsih_raid_device_find_by_wwid()  argument
1489 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in _scsih_raid_device_find_by_wwid()
1508 _scsih_raid_device_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_raid_device_add()  argument
1513 	dewtprintk(ioc,  in _scsih_raid_device_add()
1514 		   ioc_info(ioc, "%s: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_raid_device_add()
1518 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_raid_device_add()
1519 	list_add_tail(&raid_device->list, &ioc->raid_device_list);  in _scsih_raid_device_add()
1520 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_raid_device_add()
1530 _scsih_raid_device_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_raid_device_remove()  argument
1535 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_raid_device_remove()
1538 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_raid_device_remove()
1551 mpt3sas_scsih_expander_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_expander_find_by_handle()  argument
1556 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {  in mpt3sas_scsih_expander_find_by_handle()
1576 mpt3sas_scsih_enclosure_find_by_handle(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_enclosure_find_by_handle()  argument
1581 	list_for_each_entry(enclosure_dev, &ioc->enclosure_list, list) {  in mpt3sas_scsih_enclosure_find_by_handle()
1601 mpt3sas_scsih_expander_find_by_sas_address(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_scsih_expander_find_by_sas_address()  argument
1609 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {  in mpt3sas_scsih_expander_find_by_sas_address()
1630 _scsih_expander_node_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_expander_node_add()  argument
1635 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_node_add()
1636 	list_add_tail(&sas_expander->list, &ioc->sas_expander_list);  in _scsih_expander_node_add()
1637 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_node_add()
1690 _scsih_scsi_lookup_find_by_target(struct MPT3SAS_ADAPTER *ioc, int id,  in _scsih_scsi_lookup_find_by_target()  argument
1697 	     smid <= ioc->shost->can_queue; smid++) {  in _scsih_scsi_lookup_find_by_target()
1698 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_scsi_lookup_find_by_target()
1720 _scsih_scsi_lookup_find_by_lun(struct MPT3SAS_ADAPTER *ioc, int id,  in _scsih_scsi_lookup_find_by_lun()  argument
1726 	for (smid = 1; smid <= ioc->shost->can_queue; smid++) {  in _scsih_scsi_lookup_find_by_lun()
1728 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_scsi_lookup_find_by_lun()
1748 mpt3sas_scsih_scsi_lookup_get(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_scsih_scsi_lookup_get()  argument
1756 	    smid <= ioc->scsiio_depth - INTERNAL_SCSIIO_CMDS_COUNT) {  in mpt3sas_scsih_scsi_lookup_get()
1758 		    ioc->io_queue_num[tag] << BLK_MQ_UNIQUE_TAG_BITS | tag;  in mpt3sas_scsih_scsi_lookup_get()
1760 		mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_scsih_scsi_lookup_get()
1771 		scmd = scsi_host_find_tag(ioc->shost, unique_tag);  in mpt3sas_scsih_scsi_lookup_get()
1793 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_change_queue_depth()  local
1805 	if (ioc->enable_sdev_max_qd || ioc->is_gen35_ioc)  in scsih_change_queue_depth()
1817 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_change_queue_depth()
1818 	sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);  in scsih_change_queue_depth()
1825 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_change_queue_depth()
1852 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in mpt3sas_scsih_change_queue_depth()  local
1854 	if (ioc->enable_sdev_max_qd)  in mpt3sas_scsih_change_queue_depth()
1871 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_target_alloc()  local
1890 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_target_alloc()
1891 		raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,  in scsih_target_alloc()
1897 			if (ioc->is_warpdrive)  in scsih_target_alloc()
1901 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_target_alloc()
1907 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_target_alloc()
1908 		pcie_device = __mpt3sas_get_pdev_by_idchannel(ioc, starget->id,  in scsih_target_alloc()
1924 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_target_alloc()
1929 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_target_alloc()
1931 	sas_device = __mpt3sas_get_sdev_by_rphy(ioc, rphy);  in scsih_target_alloc()
1941 		if (test_bit(sas_device->handle, ioc->pd_handles))  in scsih_target_alloc()
1948 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_target_alloc()
1961 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_target_destroy()  local
1973 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_target_destroy()
1974 		raid_device = _scsih_raid_device_find_by_id(ioc, starget->id,  in scsih_target_destroy()
1980 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_target_destroy()
1985 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_target_destroy()
1986 		pcie_device = __mpt3sas_get_pdev_from_target(ioc,  in scsih_target_destroy()
2001 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_target_destroy()
2005 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_target_destroy()
2006 	sas_device = __mpt3sas_get_sdev_from_target(ioc, sas_target_priv_data);  in scsih_target_destroy()
2021 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_target_destroy()
2039 	struct MPT3SAS_ADAPTER *ioc;  in scsih_slave_alloc()  local
2065 	ioc = shost_priv(shost);  in scsih_slave_alloc()
2067 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_slave_alloc()
2068 		raid_device = _scsih_raid_device_find_by_id(ioc,  in scsih_slave_alloc()
2072 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_slave_alloc()
2075 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_slave_alloc()
2076 		pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,  in scsih_slave_alloc()
2087 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_alloc()
2090 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_slave_alloc()
2091 		sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in scsih_slave_alloc()
2104 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_alloc()
2120 	struct MPT3SAS_ADAPTER *ioc;  in scsih_slave_destroy()  local
2133 	ioc = shost_priv(shost);  in scsih_slave_destroy()
2136 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_slave_destroy()
2137 		pcie_device = __mpt3sas_get_pdev_from_target(ioc,  in scsih_slave_destroy()
2145 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_destroy()
2148 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_slave_destroy()
2149 		sas_device = __mpt3sas_get_sdev_from_target(ioc,  in scsih_slave_destroy()
2156 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_destroy()
2170 _scsih_display_sata_capabilities(struct MPT3SAS_ADAPTER *ioc,  in _scsih_display_sata_capabilities()  argument
2179 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_display_sata_capabilities()
2181 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_display_sata_capabilities()
2189 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_display_sata_capabilities()
2224 	struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);  in scsih_is_raid()  local
2226 	if (ioc->is_warpdrive)  in scsih_is_raid()
2247 	struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);  in scsih_get_resync()  local
2258 	if (ioc->is_warpdrive)  in scsih_get_resync()
2261 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_get_resync()
2262 	raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,  in scsih_get_resync()
2268 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_get_resync()
2273 	if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,  in scsih_get_resync()
2276 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in scsih_get_resync()
2289 	switch (ioc->hba_mpi_version_belonged) {  in scsih_get_resync()
2308 	struct MPT3SAS_ADAPTER *ioc = shost_priv(sdev->host);  in scsih_get_state()  local
2317 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_get_state()
2318 	raid_device = _scsih_raid_device_find_by_id(ioc, sdev->id,  in scsih_get_state()
2322 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_get_state()
2327 	if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, &vol_pg0,  in scsih_get_state()
2330 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in scsih_get_state()
2355 	switch (ioc->hba_mpi_version_belonged) {  in scsih_get_state()
2373 _scsih_set_level(struct MPT3SAS_ADAPTER *ioc,  in _scsih_set_level()  argument
2393 	switch (ioc->hba_mpi_version_belonged) {  in _scsih_set_level()
2415 _scsih_get_volume_capabilities(struct MPT3SAS_ADAPTER *ioc,  in _scsih_get_volume_capabilities()  argument
2425 	if ((mpt3sas_config_get_number_pds(ioc, raid_device->handle,  in _scsih_get_volume_capabilities()
2427 		dfailprintk(ioc,  in _scsih_get_volume_capabilities()
2428 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_volume_capabilities()
2438 		dfailprintk(ioc,  in _scsih_get_volume_capabilities()
2439 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_volume_capabilities()
2444 	if ((mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply, vol_pg0,  in _scsih_get_volume_capabilities()
2446 		dfailprintk(ioc,  in _scsih_get_volume_capabilities()
2447 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_volume_capabilities()
2458 	if (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,  in _scsih_get_volume_capabilities()
2461 		if (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_get_volume_capabilities()
2483 _scsih_enable_tlr(struct MPT3SAS_ADAPTER *ioc, struct scsi_device *sdev)  in _scsih_enable_tlr()  argument
2490 	if (!(ioc->facts.IOCCapabilities & MPI2_IOCFACTS_CAPABILITY_TLR))  in _scsih_enable_tlr()
2511 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_slave_configure()  local
2535 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in scsih_slave_configure()
2536 		raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in scsih_slave_configure()
2537 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in scsih_slave_configure()
2539 			dfailprintk(ioc,  in scsih_slave_configure()
2540 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2545 		if (_scsih_get_volume_capabilities(ioc, raid_device)) {  in scsih_slave_configure()
2546 			dfailprintk(ioc,  in scsih_slave_configure()
2547 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2555 		mpt3sas_init_warpdrive_properties(ioc, raid_device);  in scsih_slave_configure()
2581 			if (ioc->manu_pg10.OEMIdentifier &&  in scsih_slave_configure()
2582 			    (le32_to_cpu(ioc->manu_pg10.GenericFlags0) &  in scsih_slave_configure()
2604 		if (!ioc->hide_ir_msg)  in scsih_slave_configure()
2623 		if (!ioc->is_warpdrive)  in scsih_slave_configure()
2624 			_scsih_set_level(ioc, sdev, raid_device->volume_type);  in scsih_slave_configure()
2630 		if (mpt3sas_config_get_volume_handle(ioc, handle,  in scsih_slave_configure()
2632 			dfailprintk(ioc,  in scsih_slave_configure()
2633 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2637 		if (volume_handle && mpt3sas_config_get_volume_wwid(ioc,  in scsih_slave_configure()
2639 			dfailprintk(ioc,  in scsih_slave_configure()
2640 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2648 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in scsih_slave_configure()
2649 		pcie_device = __mpt3sas_get_pdev_by_wwid(ioc,  in scsih_slave_configure()
2652 			spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_configure()
2653 			dfailprintk(ioc,  in scsih_slave_configure()
2654 				    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2659 		qdepth = ioc->max_nvme_qd;  in scsih_slave_configure()
2683 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in scsih_slave_configure()
2692 				ioc->page_size - 1);  in scsih_slave_configure()
2696 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in scsih_slave_configure()
2697 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in scsih_slave_configure()
2701 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_configure()
2702 		dfailprintk(ioc,  in scsih_slave_configure()
2703 			    ioc_warn(ioc, "failure at %s:%d/%s()!\n",  in scsih_slave_configure()
2712 			ioc->max_wideport_qd : ioc->max_narrowport_qd;  in scsih_slave_configure()
2724 		qdepth = ioc->max_sata_qd;  in scsih_slave_configure()
2740 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in scsih_slave_configure()
2743 		_scsih_display_sata_capabilities(ioc, handle, sdev);  in scsih_slave_configure()
2750 		_scsih_enable_tlr(ioc, sdev);  in scsih_slave_configure()
2808 _scsih_response_code(struct MPT3SAS_ADAPTER *ioc, u8 response_code)  in _scsih_response_code()  argument
2841 	ioc_warn(ioc, "response_code(0x%01x): %s\n", response_code, desc);  in _scsih_response_code()
2858 _scsih_tm_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)  in _scsih_tm_done()  argument
2862 	if (ioc->tm_cmds.status == MPT3_CMD_NOT_USED)  in _scsih_tm_done()
2864 	if (ioc->tm_cmds.smid != smid)  in _scsih_tm_done()
2866 	ioc->tm_cmds.status |= MPT3_CMD_COMPLETE;  in _scsih_tm_done()
2867 	mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_tm_done()
2869 		memcpy(ioc->tm_cmds.reply, mpi_reply, mpi_reply->MsgLength*4);  in _scsih_tm_done()
2870 		ioc->tm_cmds.status |= MPT3_CMD_REPLY_VALID;  in _scsih_tm_done()
2872 	ioc->tm_cmds.status &= ~MPT3_CMD_PENDING;  in _scsih_tm_done()
2873 	complete(&ioc->tm_cmds.done);  in _scsih_tm_done()
2885 mpt3sas_scsih_set_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_set_tm_flag()  argument
2891 	shost_for_each_device(sdev, ioc->shost) {  in mpt3sas_scsih_set_tm_flag()
2900 			ioc->ignore_loginfos = 1;  in mpt3sas_scsih_set_tm_flag()
2913 mpt3sas_scsih_clear_tm_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in mpt3sas_scsih_clear_tm_flag()  argument
2919 	shost_for_each_device(sdev, ioc->shost) {  in mpt3sas_scsih_clear_tm_flag()
2928 			ioc->ignore_loginfos = 0;  in mpt3sas_scsih_clear_tm_flag()
2946 scsih_tm_cmd_map_status(struct MPT3SAS_ADAPTER *ioc, uint channel,  in scsih_tm_cmd_map_status()  argument
2950 	if (smid_task <= ioc->shost->can_queue) {  in scsih_tm_cmd_map_status()
2953 			if (!(_scsih_scsi_lookup_find_by_target(ioc,  in scsih_tm_cmd_map_status()
2959 			if (!(_scsih_scsi_lookup_find_by_lun(ioc, id,  in scsih_tm_cmd_map_status()
2966 	} else if (smid_task == ioc->scsih_cmds.smid) {  in scsih_tm_cmd_map_status()
2967 		if ((ioc->scsih_cmds.status & MPT3_CMD_COMPLETE) ||  in scsih_tm_cmd_map_status()
2968 		    (ioc->scsih_cmds.status & MPT3_CMD_NOT_USED))  in scsih_tm_cmd_map_status()
2970 	} else if (smid_task == ioc->ctl_cmds.smid) {  in scsih_tm_cmd_map_status()
2971 		if ((ioc->ctl_cmds.status & MPT3_CMD_COMPLETE) ||  in scsih_tm_cmd_map_status()
2972 		    (ioc->ctl_cmds.status & MPT3_CMD_NOT_USED))  in scsih_tm_cmd_map_status()
2997 scsih_tm_post_processing(struct MPT3SAS_ADAPTER *ioc, u16 handle,  in scsih_tm_post_processing()  argument
3002 	rc = scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);  in scsih_tm_post_processing()
3006 	ioc_info(ioc,  in scsih_tm_post_processing()
3016 	mpt3sas_base_mask_interrupts(ioc);  in scsih_tm_post_processing()
3017 	mpt3sas_base_sync_reply_irqs(ioc, 1);  in scsih_tm_post_processing()
3018 	mpt3sas_base_unmask_interrupts(ioc);  in scsih_tm_post_processing()
3020 	return scsih_tm_cmd_map_status(ioc, channel, id, lun, type, smid_task);  in scsih_tm_post_processing()
3045 mpt3sas_scsih_issue_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle, uint channel,  in mpt3sas_scsih_issue_tm()  argument
3057 	lockdep_assert_held(&ioc->tm_cmds.mutex);  in mpt3sas_scsih_issue_tm()
3059 	if (ioc->tm_cmds.status != MPT3_CMD_NOT_USED) {  in mpt3sas_scsih_issue_tm()
3060 		ioc_info(ioc, "%s: tm_cmd busy!!!\n", __func__);  in mpt3sas_scsih_issue_tm()
3064 	if (ioc->shost_recovery || ioc->remove_host ||  in mpt3sas_scsih_issue_tm()
3065 	    ioc->pci_error_recovery) {  in mpt3sas_scsih_issue_tm()
3066 		ioc_info(ioc, "%s: host reset in progress!\n", __func__);  in mpt3sas_scsih_issue_tm()
3070 	ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in mpt3sas_scsih_issue_tm()
3072 		dhsprintk(ioc, ioc_info(ioc, "unexpected doorbell active!\n"));  in mpt3sas_scsih_issue_tm()
3073 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_scsih_issue_tm()
3078 		mpt3sas_print_fault_code(ioc, ioc_state &  in mpt3sas_scsih_issue_tm()
3080 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_scsih_issue_tm()
3084 		mpt3sas_print_coredump_info(ioc, ioc_state &  in mpt3sas_scsih_issue_tm()
3086 		rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in mpt3sas_scsih_issue_tm()
3090 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_cb_idx);  in mpt3sas_scsih_issue_tm()
3092 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in mpt3sas_scsih_issue_tm()
3096 	dtmprintk(ioc,  in mpt3sas_scsih_issue_tm()
3097 …ioc_info(ioc, "sending tm: handle(0x%04x), task_type(0x%02x), smid(%d), timeout(%d), tr_method(0x%…  in mpt3sas_scsih_issue_tm()
3099 	ioc->tm_cmds.status = MPT3_CMD_PENDING;  in mpt3sas_scsih_issue_tm()
3100 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in mpt3sas_scsih_issue_tm()
3101 	ioc->tm_cmds.smid = smid;  in mpt3sas_scsih_issue_tm()
3103 	memset(ioc->tm_cmds.reply, 0, sizeof(Mpi2SCSITaskManagementReply_t));  in mpt3sas_scsih_issue_tm()
3112 	mpt3sas_scsih_set_tm_flag(ioc, handle);  in mpt3sas_scsih_issue_tm()
3113 	init_completion(&ioc->tm_cmds.done);  in mpt3sas_scsih_issue_tm()
3114 	ioc->put_smid_hi_priority(ioc, smid, msix_task);  in mpt3sas_scsih_issue_tm()
3115 	wait_for_completion_timeout(&ioc->tm_cmds.done, timeout*HZ);  in mpt3sas_scsih_issue_tm()
3116 	if (!(ioc->tm_cmds.status & MPT3_CMD_COMPLETE)) {  in mpt3sas_scsih_issue_tm()
3117 		mpt3sas_check_cmd_timeout(ioc,  in mpt3sas_scsih_issue_tm()
3118 		    ioc->tm_cmds.status, mpi_request,  in mpt3sas_scsih_issue_tm()
3121 			rc = mpt3sas_base_hard_reset_handler(ioc,  in mpt3sas_scsih_issue_tm()
3129 	mpt3sas_base_sync_reply_irqs(ioc, 0);  in mpt3sas_scsih_issue_tm()
3131 	if (ioc->tm_cmds.status & MPT3_CMD_REPLY_VALID) {  in mpt3sas_scsih_issue_tm()
3132 		mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);  in mpt3sas_scsih_issue_tm()
3133 		mpi_reply = ioc->tm_cmds.reply;  in mpt3sas_scsih_issue_tm()
3134 		dtmprintk(ioc,  in mpt3sas_scsih_issue_tm()
3135 			  ioc_info(ioc, "complete tm: ioc_status(0x%04x), loginfo(0x%08x), term_count(0x%08x)\n",  in mpt3sas_scsih_issue_tm()
3139 		if (ioc->logging_level & MPT_DEBUG_TM) {  in mpt3sas_scsih_issue_tm()
3140 			_scsih_response_code(ioc, mpi_reply->ResponseCode);  in mpt3sas_scsih_issue_tm()
3161 		request = mpt3sas_base_get_msg_frame(ioc, smid_task);  in mpt3sas_scsih_issue_tm()
3165 		ioc_info(ioc, "Task abort tm failed: handle(0x%04x),"  in mpt3sas_scsih_issue_tm()
3174 		rc = scsih_tm_post_processing(ioc, handle, channel, id, lun,  in mpt3sas_scsih_issue_tm()
3186 	mpt3sas_scsih_clear_tm_flag(ioc, handle);  in mpt3sas_scsih_issue_tm()
3187 	ioc->tm_cmds.status = MPT3_CMD_NOT_USED;  in mpt3sas_scsih_issue_tm()
3191 int mpt3sas_scsih_issue_locked_tm(struct MPT3SAS_ADAPTER *ioc, u16 handle,  in mpt3sas_scsih_issue_locked_tm()  argument
3197 	mutex_lock(&ioc->tm_cmds.mutex);  in mpt3sas_scsih_issue_locked_tm()
3198 	ret = mpt3sas_scsih_issue_tm(ioc, handle, channel, id, lun, type,  in mpt3sas_scsih_issue_locked_tm()
3200 	mutex_unlock(&ioc->tm_cmds.mutex);  in mpt3sas_scsih_issue_locked_tm()
3213 _scsih_tm_display_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd)  in _scsih_tm_display_info()  argument
3224 	if (ioc->hide_ir_msg)  in _scsih_tm_display_info()
3237 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_tm_display_info()
3238 		pcie_device = __mpt3sas_get_pdev_from_target(ioc, priv_target);  in _scsih_tm_display_info()
3258 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_tm_display_info()
3261 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_tm_display_info()
3262 		sas_device = __mpt3sas_get_sdev_from_target(ioc, priv_target);  in _scsih_tm_display_info()
3283 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_tm_display_info()
3296 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_abort()  local
3308 	_scsih_tm_display_info(ioc, scmd);  in scsih_abort()
3312 	    ioc->remove_host) {  in scsih_abort()
3339 	mpt3sas_halt_firmware(ioc);  in scsih_abort()
3342 	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);  in scsih_abort()
3343 	if (pcie_device && (!ioc->tm_custom_handling) &&  in scsih_abort()
3345 		timeout = ioc->nvme_abort_timeout;  in scsih_abort()
3346 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,  in scsih_abort()
3370 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_dev_reset()  local
3384 	_scsih_tm_display_info(ioc, scmd);  in scsih_dev_reset()
3388 	    ioc->remove_host) {  in scsih_dev_reset()
3401 		sas_device = mpt3sas_get_sdev_from_target(ioc,  in scsih_dev_reset()
3414 	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);  in scsih_dev_reset()
3416 	if (pcie_device && (!ioc->tm_custom_handling) &&  in scsih_dev_reset()
3423 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,  in scsih_dev_reset()
3451 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_target_reset()  local
3464 	_scsih_tm_display_info(ioc, scmd);  in scsih_target_reset()
3468 	    ioc->remove_host) {  in scsih_target_reset()
3481 		sas_device = mpt3sas_get_sdev_from_target(ioc,  in scsih_target_reset()
3494 	pcie_device = mpt3sas_get_pdev_by_handle(ioc, handle);  in scsih_target_reset()
3496 	if (pcie_device && (!ioc->tm_custom_handling) &&  in scsih_target_reset()
3502 	r = mpt3sas_scsih_issue_locked_tm(ioc, handle, scmd->device->channel,  in scsih_target_reset()
3530 	struct MPT3SAS_ADAPTER *ioc = shost_priv(scmd->device->host);  in scsih_host_reset()  local
3533 	ioc_info(ioc, "attempting host reset! scmd(0x%p)\n", scmd);  in scsih_host_reset()
3536 	if (ioc->is_driver_loading || ioc->remove_host) {  in scsih_host_reset()
3537 		ioc_info(ioc, "Blocking the host reset\n");  in scsih_host_reset()
3542 	retval = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in scsih_host_reset()
3545 	ioc_info(ioc, "host reset: %s scmd(0x%p)\n",  in scsih_host_reset()
3561 _scsih_fw_event_add(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)  in _scsih_fw_event_add()  argument
3565 	if (ioc->firmware_event_thread == NULL)  in _scsih_fw_event_add()
3568 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_fw_event_add()
3571 	list_add_tail(&fw_event->list, &ioc->fw_event_list);  in _scsih_fw_event_add()
3574 	queue_work(ioc->firmware_event_thread, &fw_event->work);  in _scsih_fw_event_add()
3575 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_fw_event_add()
3587 _scsih_fw_event_del_from_list(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work  in _scsih_fw_event_del_from_list()  argument
3592 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_fw_event_del_from_list()
3597 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_fw_event_del_from_list()
3607 mpt3sas_send_trigger_data_event(struct MPT3SAS_ADAPTER *ioc,  in mpt3sas_send_trigger_data_event()  argument
3613 	if (ioc->is_driver_loading)  in mpt3sas_send_trigger_data_event()
3620 	fw_event->ioc = ioc;  in mpt3sas_send_trigger_data_event()
3622 	_scsih_fw_event_add(ioc, fw_event);  in mpt3sas_send_trigger_data_event()
3631 _scsih_error_recovery_delete_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_error_recovery_delete_devices()  argument
3639 	fw_event->ioc = ioc;  in _scsih_error_recovery_delete_devices()
3640 	_scsih_fw_event_add(ioc, fw_event);  in _scsih_error_recovery_delete_devices()
3649 mpt3sas_port_enable_complete(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_port_enable_complete()  argument
3657 	fw_event->ioc = ioc;  in mpt3sas_port_enable_complete()
3658 	_scsih_fw_event_add(ioc, fw_event);  in mpt3sas_port_enable_complete()
3662 static struct fw_event_work *dequeue_next_fw_event(struct MPT3SAS_ADAPTER *ioc)  in dequeue_next_fw_event()  argument
3667 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in dequeue_next_fw_event()
3668 	if (!list_empty(&ioc->fw_event_list)) {  in dequeue_next_fw_event()
3669 		fw_event = list_first_entry(&ioc->fw_event_list,  in dequeue_next_fw_event()
3674 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in dequeue_next_fw_event()
3689 _scsih_fw_event_cleanup_queue(struct MPT3SAS_ADAPTER *ioc)  in _scsih_fw_event_cleanup_queue()  argument
3693 	if ((list_empty(&ioc->fw_event_list) && !ioc->current_event) ||  in _scsih_fw_event_cleanup_queue()
3694 	    !ioc->firmware_event_thread)  in _scsih_fw_event_cleanup_queue()
3702 	if (ioc->shost_recovery && ioc->current_event)  in _scsih_fw_event_cleanup_queue()
3703 		ioc->current_event->ignore = 1;  in _scsih_fw_event_cleanup_queue()
3705 	ioc->fw_events_cleanup = 1;  in _scsih_fw_event_cleanup_queue()
3706 	while ((fw_event = dequeue_next_fw_event(ioc)) ||  in _scsih_fw_event_cleanup_queue()
3707 	     (fw_event = ioc->current_event)) {  in _scsih_fw_event_cleanup_queue()
3723 		if (fw_event == ioc->current_event &&  in _scsih_fw_event_cleanup_queue()
3724 		    ioc->current_event->event !=  in _scsih_fw_event_cleanup_queue()
3726 			ioc->current_event = NULL;  in _scsih_fw_event_cleanup_queue()
3739 			ioc->port_enable_cmds.status |= MPT3_CMD_RESET;  in _scsih_fw_event_cleanup_queue()
3740 			ioc->start_scan = 0;  in _scsih_fw_event_cleanup_queue()
3755 	ioc->fw_events_cleanup = 0;  in _scsih_fw_event_cleanup_queue()
3834 _scsih_ublock_io_all_device(struct MPT3SAS_ADAPTER *ioc)  in _scsih_ublock_io_all_device()  argument
3839 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_ublock_io_all_device()
3846 		dewtprintk(ioc, sdev_printk(KERN_INFO, sdev,  in _scsih_ublock_io_all_device()
3863 _scsih_ublock_io_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_ublock_io_device()  argument
3869 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_ublock_io_device()
3891 _scsih_block_io_all_device(struct MPT3SAS_ADAPTER *ioc)  in _scsih_block_io_all_device()  argument
3896 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_block_io_all_device()
3920 _scsih_block_io_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_block_io_device()  argument
3926 	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_block_io_device()
3928 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_block_io_device()
3961 _scsih_block_io_to_children_attached_to_ex(struct MPT3SAS_ADAPTER *ioc,  in _scsih_block_io_to_children_attached_to_ex()  argument
3976 			spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_block_io_to_children_attached_to_ex()
3977 			sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in _scsih_block_io_to_children_attached_to_ex()
3982 						ioc->blocking_handles);  in _scsih_block_io_to_children_attached_to_ex()
3985 			spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_block_io_to_children_attached_to_ex()
3998 			    ioc, mpt3sas_port->remote_identify.sas_address,  in _scsih_block_io_to_children_attached_to_ex()
4000 			_scsih_block_io_to_children_attached_to_ex(ioc,  in _scsih_block_io_to_children_attached_to_ex()
4015 _scsih_block_io_to_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,  in _scsih_block_io_to_children_attached_directly()  argument
4029 			_scsih_block_io_device(ioc, handle);  in _scsih_block_io_to_children_attached_directly()
4042 _scsih_block_io_to_pcie_children_attached_directly(struct MPT3SAS_ADAPTER *ioc,  in _scsih_block_io_to_pcie_children_attached_directly()  argument
4057 			_scsih_block_io_device(ioc, handle);  in _scsih_block_io_to_pcie_children_attached_directly()
4076 _scsih_tm_tr_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_tm_tr_send()  argument
4090 	if (ioc->pci_error_recovery) {  in _scsih_tm_tr_send()
4091 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4092 			   ioc_info(ioc, "%s: host in pci error recovery: handle(0x%04x)\n",  in _scsih_tm_tr_send()
4096 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in _scsih_tm_tr_send()
4098 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4099 			   ioc_info(ioc, "%s: host is not operational: handle(0x%04x)\n",  in _scsih_tm_tr_send()
4105 	if (test_bit(handle, ioc->pd_handles))  in _scsih_tm_tr_send()
4108 	clear_bit(handle, ioc->pend_os_device_add);  in _scsih_tm_tr_send()
4110 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_tm_tr_send()
4111 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_tm_tr_send()
4119 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_tm_tr_send()
4121 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_tm_tr_send()
4122 		pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in _scsih_tm_tr_send()
4129 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_tm_tr_send()
4130 		if (pcie_device && (!ioc->tm_custom_handling) &&  in _scsih_tm_tr_send()
4139 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4140 			   ioc_info(ioc, "setting delete flag: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_tm_tr_send()
4144 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4145 					   ioc_info(ioc, "setting delete flag:enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_tm_tr_send()
4149 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4150 					   ioc_info(ioc, "setting delete flag: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_tm_tr_send()
4155 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4156 					   ioc_info(ioc, "setting delete flag: logical id(0x%016llx), slot(%d)\n",  in _scsih_tm_tr_send()
4160 				dewtprintk(ioc,  in _scsih_tm_tr_send()
4161 					   ioc_info(ioc, "setting delete flag:, enclosure level(0x%04x), connector name( %s)\n",  in _scsih_tm_tr_send()
4165 		_scsih_ublock_io_device(ioc, sas_address, port);  in _scsih_tm_tr_send()
4169 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_cb_idx);  in _scsih_tm_tr_send()
4176 		list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);  in _scsih_tm_tr_send()
4177 		dewtprintk(ioc,  in _scsih_tm_tr_send()
4178 			   ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",  in _scsih_tm_tr_send()
4183 	dewtprintk(ioc,  in _scsih_tm_tr_send()
4184 		   ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_tm_tr_send()
4185 			    handle, smid, ioc->tm_tr_cb_idx));  in _scsih_tm_tr_send()
4186 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_tr_send()
4192 	set_bit(handle, ioc->device_remove_in_progress);  in _scsih_tm_tr_send()
4193 	ioc->put_smid_hi_priority(ioc, smid, 0);  in _scsih_tm_tr_send()
4194 	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_DEVICE_REMOVAL);  in _scsih_tm_tr_send()
4220 _scsih_tm_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index,  in _scsih_tm_tr_complete()  argument
4226 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_tm_tr_complete()
4232 	if (ioc->pci_error_recovery) {  in _scsih_tm_tr_complete()
4233 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4234 			   ioc_info(ioc, "%s: host in pci error recovery\n",  in _scsih_tm_tr_complete()
4238 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in _scsih_tm_tr_complete()
4240 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4241 			   ioc_info(ioc, "%s: host is not operational\n",  in _scsih_tm_tr_complete()
4246 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _scsih_tm_tr_complete()
4250 	mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_tr_complete()
4253 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4254 			   ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",  in _scsih_tm_tr_complete()
4260 	mpt3sas_trigger_master(ioc, MASTER_TRIGGER_TASK_MANAGMENT);  in _scsih_tm_tr_complete()
4261 	dewtprintk(ioc,  in _scsih_tm_tr_complete()
4262 …ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), c…  in _scsih_tm_tr_complete()
4267 	smid_sas_ctrl = mpt3sas_base_get_smid(ioc, ioc->tm_sas_control_cb_idx);  in _scsih_tm_tr_complete()
4271 			return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_tr_complete()
4274 		list_add_tail(&delayed_sc->list, &ioc->delayed_sc_list);  in _scsih_tm_tr_complete()
4275 		dewtprintk(ioc,  in _scsih_tm_tr_complete()
4276 			   ioc_info(ioc, "DELAYED:sc:handle(0x%04x), (open)\n",  in _scsih_tm_tr_complete()
4278 		return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_tr_complete()
4281 	dewtprintk(ioc,  in _scsih_tm_tr_complete()
4282 		   ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_tm_tr_complete()
4283 			    handle, smid_sas_ctrl, ioc->tm_sas_control_cb_idx));  in _scsih_tm_tr_complete()
4284 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid_sas_ctrl);  in _scsih_tm_tr_complete()
4289 	ioc->put_smid_default(ioc, smid_sas_ctrl);  in _scsih_tm_tr_complete()
4291 	return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_tr_complete()
4301 inline bool _scsih_allow_scmd_to_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_allow_scmd_to_device()  argument
4305 	if (ioc->pci_error_recovery)  in _scsih_allow_scmd_to_device()
4308 	if (ioc->hba_mpi_version_belonged == MPI2_VERSION) {  in _scsih_allow_scmd_to_device()
4309 		if (ioc->remove_host)  in _scsih_allow_scmd_to_device()
4315 	if (ioc->remove_host) {  in _scsih_allow_scmd_to_device()
4345 _scsih_sas_control_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _scsih_sas_control_complete()  argument
4349 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_sas_control_complete()
4352 		dewtprintk(ioc,  in _scsih_sas_control_complete()
4353 …ioc_info(ioc, "sc_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_sas_control_complete()
4360 			    ioc->device_remove_in_progress);  in _scsih_sas_control_complete()
4363 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _scsih_sas_control_complete()
4366 	return mpt3sas_check_for_pending_internal_cmds(ioc, smid);  in _scsih_sas_control_complete()
4380 _scsih_tm_tr_volume_send(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_tm_tr_volume_send()  argument
4386 	if (ioc->pci_error_recovery) {  in _scsih_tm_tr_volume_send()
4387 		dewtprintk(ioc,  in _scsih_tm_tr_volume_send()
4388 			   ioc_info(ioc, "%s: host reset in progress!\n",  in _scsih_tm_tr_volume_send()
4393 	smid = mpt3sas_base_get_smid_hpr(ioc, ioc->tm_tr_volume_cb_idx);  in _scsih_tm_tr_volume_send()
4400 		list_add_tail(&delayed_tr->list, &ioc->delayed_tr_volume_list);  in _scsih_tm_tr_volume_send()
4401 		dewtprintk(ioc,  in _scsih_tm_tr_volume_send()
4402 			   ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",  in _scsih_tm_tr_volume_send()
4407 	dewtprintk(ioc,  in _scsih_tm_tr_volume_send()
4408 		   ioc_info(ioc, "tr_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_tm_tr_volume_send()
4409 			    handle, smid, ioc->tm_tr_volume_cb_idx));  in _scsih_tm_tr_volume_send()
4410 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_tr_volume_send()
4415 	ioc->put_smid_hi_priority(ioc, smid, 0);  in _scsih_tm_tr_volume_send()
4430 _scsih_tm_volume_tr_complete(struct MPT3SAS_ADAPTER *ioc, u16 smid,  in _scsih_tm_volume_tr_complete()  argument
4436 	    mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_tm_volume_tr_complete()
4438 	if (ioc->shost_recovery || ioc->pci_error_recovery) {  in _scsih_tm_volume_tr_complete()
4439 		dewtprintk(ioc,  in _scsih_tm_volume_tr_complete()
4440 			   ioc_info(ioc, "%s: host reset in progress!\n",  in _scsih_tm_volume_tr_complete()
4445 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in _scsih_tm_volume_tr_complete()
4450 	mpi_request_tm = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_tm_volume_tr_complete()
4453 		dewtprintk(ioc,  in _scsih_tm_volume_tr_complete()
4454 			   ioc_err(ioc, "spurious interrupt: handle(0x%04x:0x%04x), smid(%d)!!!\n",  in _scsih_tm_volume_tr_complete()
4460 	dewtprintk(ioc,  in _scsih_tm_volume_tr_complete()
4461 …ioc_info(ioc, "tr_complete:handle(0x%04x), (open) smid(%d), ioc_status(0x%04x), loginfo(0x%08x), c…  in _scsih_tm_volume_tr_complete()
4466 	return _scsih_check_for_pending_tm(ioc, smid);  in _scsih_tm_volume_tr_complete()
4479 _scsih_issue_delayed_event_ack(struct MPT3SAS_ADAPTER *ioc, u16 smid, U16 event,  in _scsih_issue_delayed_event_ack()  argument
4483 	int i = smid - ioc->internal_smid;  in _scsih_issue_delayed_event_ack()
4490 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_event_ack()
4491 	ioc->internal_lookup[i].cb_idx = ioc->base_cb_idx;  in _scsih_issue_delayed_event_ack()
4492 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_event_ack()
4494 	dewtprintk(ioc,  in _scsih_issue_delayed_event_ack()
4495 		   ioc_info(ioc, "EVENT ACK: event(0x%04x), smid(%d), cb(%d)\n",  in _scsih_issue_delayed_event_ack()
4496 			    le16_to_cpu(event), smid, ioc->base_cb_idx));  in _scsih_issue_delayed_event_ack()
4497 	ack_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_issue_delayed_event_ack()
4504 	ioc->put_smid_default(ioc, smid);  in _scsih_issue_delayed_event_ack()
4517 _scsih_issue_delayed_sas_io_unit_ctrl(struct MPT3SAS_ADAPTER *ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()  argument
4522 	int i = smid - ioc->internal_smid;  in _scsih_issue_delayed_sas_io_unit_ctrl()
4525 	if (ioc->remove_host) {  in _scsih_issue_delayed_sas_io_unit_ctrl()
4526 		dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4527 			   ioc_info(ioc, "%s: host has been removed\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4530 	} else if (ioc->pci_error_recovery) {  in _scsih_issue_delayed_sas_io_unit_ctrl()
4531 		dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4532 			   ioc_info(ioc, "%s: host in pci error recovery\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4536 	ioc_state = mpt3sas_base_get_iocstate(ioc, 1);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4538 		dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4539 			   ioc_info(ioc, "%s: host is not operational\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4548 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4549 	ioc->internal_lookup[i].cb_idx = ioc->tm_sas_control_cb_idx;  in _scsih_issue_delayed_sas_io_unit_ctrl()
4550 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4552 	dewtprintk(ioc,  in _scsih_issue_delayed_sas_io_unit_ctrl()
4553 		   ioc_info(ioc, "sc_send:handle(0x%04x), (open), smid(%d), cb(%d)\n",  in _scsih_issue_delayed_sas_io_unit_ctrl()
4554 			    handle, smid, ioc->tm_sas_control_cb_idx));  in _scsih_issue_delayed_sas_io_unit_ctrl()
4555 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4560 	ioc->put_smid_default(ioc, smid);  in _scsih_issue_delayed_sas_io_unit_ctrl()
4577 mpt3sas_check_for_pending_internal_cmds(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in mpt3sas_check_for_pending_internal_cmds()  argument
4582 	if (!list_empty(&ioc->delayed_event_ack_list)) {  in mpt3sas_check_for_pending_internal_cmds()
4583 		delayed_event_ack = list_entry(ioc->delayed_event_ack_list.next,  in mpt3sas_check_for_pending_internal_cmds()
4585 		_scsih_issue_delayed_event_ack(ioc, smid,  in mpt3sas_check_for_pending_internal_cmds()
4592 	if (!list_empty(&ioc->delayed_sc_list)) {  in mpt3sas_check_for_pending_internal_cmds()
4593 		delayed_sc = list_entry(ioc->delayed_sc_list.next,  in mpt3sas_check_for_pending_internal_cmds()
4595 		_scsih_issue_delayed_sas_io_unit_ctrl(ioc, smid,  in mpt3sas_check_for_pending_internal_cmds()
4616 _scsih_check_for_pending_tm(struct MPT3SAS_ADAPTER *ioc, u16 smid)  in _scsih_check_for_pending_tm()  argument
4620 	if (!list_empty(&ioc->delayed_tr_volume_list)) {  in _scsih_check_for_pending_tm()
4621 		delayed_tr = list_entry(ioc->delayed_tr_volume_list.next,  in _scsih_check_for_pending_tm()
4623 		mpt3sas_base_free_smid(ioc, smid);  in _scsih_check_for_pending_tm()
4624 		_scsih_tm_tr_volume_send(ioc, delayed_tr->handle);  in _scsih_check_for_pending_tm()
4630 	if (!list_empty(&ioc->delayed_tr_list)) {  in _scsih_check_for_pending_tm()
4631 		delayed_tr = list_entry(ioc->delayed_tr_list.next,  in _scsih_check_for_pending_tm()
4633 		mpt3sas_base_free_smid(ioc, smid);  in _scsih_check_for_pending_tm()
4634 		_scsih_tm_tr_send(ioc, delayed_tr->handle);  in _scsih_check_for_pending_tm()
4655 _scsih_check_topo_delete_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_topo_delete_events()  argument
4673 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_topo_delete_events()
4677 	if (expander_handle < ioc->sas_hba.num_phys) {  in _scsih_check_topo_delete_events()
4678 		_scsih_block_io_to_children_attached_directly(ioc, event_data);  in _scsih_check_topo_delete_events()
4684 		spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_check_topo_delete_events()
4685 		sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,  in _scsih_check_topo_delete_events()
4687 		_scsih_block_io_to_children_attached_to_ex(ioc, sas_expander);  in _scsih_check_topo_delete_events()
4688 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_check_topo_delete_events()
4690 			handle = find_first_bit(ioc->blocking_handles,  in _scsih_check_topo_delete_events()
4691 			    ioc->facts.MaxDevHandle);  in _scsih_check_topo_delete_events()
4692 			if (handle < ioc->facts.MaxDevHandle)  in _scsih_check_topo_delete_events()
4693 				_scsih_block_io_device(ioc, handle);  in _scsih_check_topo_delete_events()
4694 		} while (test_and_clear_bit(handle, ioc->blocking_handles));  in _scsih_check_topo_delete_events()
4696 		_scsih_block_io_to_children_attached_directly(ioc, event_data);  in _scsih_check_topo_delete_events()
4702 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_check_topo_delete_events()
4703 	list_for_each_entry(fw_event, &ioc->fw_event_list, list) {  in _scsih_check_topo_delete_events()
4715 				dewtprintk(ioc,  in _scsih_check_topo_delete_events()
4716 					   ioc_info(ioc, "setting ignoring flag\n"));  in _scsih_check_topo_delete_events()
4721 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_check_topo_delete_events()
4736 _scsih_check_pcie_topo_remove_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_pcie_topo_remove_events()  argument
4752 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_pcie_topo_remove_events()
4758 							ioc, event_data);  in _scsih_check_pcie_topo_remove_events()
4767 							ioc, event_data);  in _scsih_check_pcie_topo_remove_events()
4773 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in _scsih_check_pcie_topo_remove_events()
4774 	list_for_each_entry(fw_event, &ioc->fw_event_list, list) {  in _scsih_check_pcie_topo_remove_events()
4787 				dewtprintk(ioc,  in _scsih_check_pcie_topo_remove_events()
4788 					   ioc_info(ioc, "setting ignoring flag for switch event\n"));  in _scsih_check_pcie_topo_remove_events()
4793 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in _scsih_check_pcie_topo_remove_events()
4804 _scsih_set_volume_delete_flag(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_set_volume_delete_flag()  argument
4810 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_set_volume_delete_flag()
4811 	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_set_volume_delete_flag()
4817 		dewtprintk(ioc,  in _scsih_set_volume_delete_flag()
4818 			   ioc_info(ioc, "setting delete flag: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_set_volume_delete_flag()
4821 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_set_volume_delete_flag()
4858 _scsih_check_ir_config_unhide_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_ir_config_unhide_events()  argument
4869 	if (ioc->is_warpdrive)  in _scsih_check_ir_config_unhide_events()
4883 			_scsih_set_volume_delete_flag(ioc, volume_handle);  in _scsih_check_ir_config_unhide_events()
4901 		_scsih_tm_tr_volume_send(ioc, a);  in _scsih_check_ir_config_unhide_events()
4903 		_scsih_tm_tr_volume_send(ioc, b);  in _scsih_check_ir_config_unhide_events()
4912 		clear_bit(handle, ioc->pd_handles);  in _scsih_check_ir_config_unhide_events()
4914 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_ir_config_unhide_events()
4920 			list_add_tail(&delayed_tr->list, &ioc->delayed_tr_list);  in _scsih_check_ir_config_unhide_events()
4921 			dewtprintk(ioc,  in _scsih_check_ir_config_unhide_events()
4922 				   ioc_info(ioc, "DELAYED:tr:handle(0x%04x), (open)\n",  in _scsih_check_ir_config_unhide_events()
4925 			_scsih_tm_tr_send(ioc, handle);  in _scsih_check_ir_config_unhide_events()
4941 _scsih_check_volume_delete_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_volume_delete_events()  argument
4951 		_scsih_set_volume_delete_flag(ioc,  in _scsih_check_volume_delete_events()
4962 _scsih_temp_threshold_events(struct MPT3SAS_ADAPTER *ioc,  in _scsih_temp_threshold_events()  argument
4966 	if (ioc->temp_sensors_count >= event_data->SensorNum) {  in _scsih_temp_threshold_events()
4967 		ioc_err(ioc, "Temperature Threshold flags %s%s%s%s exceeded for Sensor: %d !!!\n",  in _scsih_temp_threshold_events()
4973 		ioc_err(ioc, "Current Temp In Celsius: %d\n",  in _scsih_temp_threshold_events()
4975 		if (ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _scsih_temp_threshold_events()
4976 			doorbell = mpt3sas_base_get_iocstate(ioc, 0);  in _scsih_temp_threshold_events()
4979 				mpt3sas_print_fault_code(ioc,  in _scsih_temp_threshold_events()
4983 				mpt3sas_print_coredump_info(ioc,  in _scsih_temp_threshold_events()
5012 _scsih_flush_running_cmds(struct MPT3SAS_ADAPTER *ioc)  in _scsih_flush_running_cmds()  argument
5019 	for (smid = 1; smid <= ioc->scsiio_depth; smid++) {  in _scsih_flush_running_cmds()
5020 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_flush_running_cmds()
5026 		mpt3sas_base_clear_st(ioc, st);  in _scsih_flush_running_cmds()
5028 		if (ioc->pci_error_recovery || ioc->remove_host)  in _scsih_flush_running_cmds()
5034 	dtmprintk(ioc, ioc_info(ioc, "completing %d cmds\n", count));  in _scsih_flush_running_cmds()
5046 _scsih_setup_eedp(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,  in _scsih_setup_eedp()  argument
5079 	if (ioc->is_gen35_ioc)  in _scsih_setup_eedp()
5126 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_qcmd()  local
5138 	if (ioc->logging_level & MPT_DEBUG_SCSI)  in scsih_qcmd()
5148 	if (!(_scsih_allow_scmd_to_device(ioc, scmd))) {  in scsih_qcmd()
5178 	if (ioc->shost_recovery || ioc->ioc_link_reset_in_progress) {  in scsih_qcmd()
5220 	if (((!ioc->is_warpdrive && !scsih_is_raid(&scmd->device->sdev_gendev))  in scsih_qcmd()
5225 	smid = mpt3sas_base_get_smid_scsiio(ioc, ioc->scsi_io_cb_idx, scmd);  in scsih_qcmd()
5227 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in scsih_qcmd()
5231 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in scsih_qcmd()
5232 	memset(mpi_request, 0, ioc->request_sz);  in scsih_qcmd()
5233 	_scsih_setup_eedp(ioc, scmd, mpi_request);  in scsih_qcmd()
5250 	    mpt3sas_base_get_sense_buffer_dma(ioc, smid);  in scsih_qcmd()
5258 		if (ioc->build_sg_scmd(ioc, scmd, smid, pcie_device)) {  in scsih_qcmd()
5259 			mpt3sas_base_free_smid(ioc, smid);  in scsih_qcmd()
5264 		ioc->build_zero_len_sge(ioc, &mpi_request->SGL);  in scsih_qcmd()
5268 		mpt3sas_setup_direct_io(ioc, scmd,  in scsih_qcmd()
5275 			ioc->put_smid_fast_path(ioc, smid, handle);  in scsih_qcmd()
5277 			ioc->put_smid_scsi_io(ioc, smid,  in scsih_qcmd()
5280 		ioc->put_smid_default(ioc, smid);  in scsih_qcmd()
5320 _scsih_scsi_ioc_info(struct MPT3SAS_ADAPTER *ioc, struct scsi_cmnd *scmd,  in _scsih_scsi_ioc_info()  argument
5331 	char *desc_scsi_state = ioc->tmp_string;  in _scsih_scsi_ioc_info()
5341 	if (ioc->hide_ir_msg)  in _scsih_scsi_ioc_info()
5465 		ioc_warn(ioc, "\t%s wwid(0x%016llx)\n",  in _scsih_scsi_ioc_info()
5468 		pcie_device = mpt3sas_get_pdev_from_target(ioc, priv_target);  in _scsih_scsi_ioc_info()
5470 			ioc_info(ioc, "\twwid(0x%016llx), port(%d)\n",  in _scsih_scsi_ioc_info()
5473 				ioc_info(ioc, "\tenclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_scsi_ioc_info()
5477 				ioc_info(ioc, "\tenclosure level(0x%04x), connector name( %s)\n",  in _scsih_scsi_ioc_info()
5483 		sas_device = mpt3sas_get_sdev_from_target(ioc, priv_target);  in _scsih_scsi_ioc_info()
5485 			ioc_warn(ioc, "\tsas_address(0x%016llx), phy(%d)\n",  in _scsih_scsi_ioc_info()
5488 			_scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_scsi_ioc_info()
5495 	ioc_warn(ioc, "\thandle(0x%04x), ioc_status(%s)(0x%04x), smid(%d)\n",  in _scsih_scsi_ioc_info()
5498 	ioc_warn(ioc, "\trequest_len(%d), underflow(%d), resid(%d)\n",  in _scsih_scsi_ioc_info()
5500 	ioc_warn(ioc, "\ttag(%d), transfer_count(%d), sc->result(0x%08x)\n",  in _scsih_scsi_ioc_info()
5503 	ioc_warn(ioc, "\tscsi_status(%s)(0x%02x), scsi_state(%s)(0x%02x)\n",  in _scsih_scsi_ioc_info()
5509 		ioc_warn(ioc, "\t[sense_key,asc,ascq]: [0x%02x,0x%02x,0x%02x], count(%d)\n",  in _scsih_scsi_ioc_info()
5516 		_scsih_response_code(ioc, response_bytes[0]);  in _scsih_scsi_ioc_info()
5527 _scsih_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_turn_on_pfa_led()  argument
5533 	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_turn_on_pfa_led()
5544 	if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,  in _scsih_turn_on_pfa_led()
5546 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_turn_on_pfa_led()
5553 		dewtprintk(ioc,  in _scsih_turn_on_pfa_led()
5554 			   ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",  in _scsih_turn_on_pfa_led()
5570 _scsih_turn_off_pfa_led(struct MPT3SAS_ADAPTER *ioc,  in _scsih_turn_off_pfa_led()  argument
5584 	if ((mpt3sas_base_scsi_enclosure_processor(ioc, &mpi_reply,  in _scsih_turn_off_pfa_led()
5586 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_turn_off_pfa_led()
5592 		dewtprintk(ioc,  in _scsih_turn_off_pfa_led()
5593 			   ioc_info(ioc, "enclosure_processor: ioc_status (0x%04x), loginfo(0x%08x)\n",  in _scsih_turn_off_pfa_led()
5607 _scsih_send_event_to_turn_on_pfa_led(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_send_event_to_turn_on_pfa_led()  argument
5616 	fw_event->ioc = ioc;  in _scsih_send_event_to_turn_on_pfa_led()
5617 	_scsih_fw_event_add(ioc, fw_event);  in _scsih_send_event_to_turn_on_pfa_led()
5628 _scsih_smart_predicted_fault(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_smart_predicted_fault()  argument
5639 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_smart_predicted_fault()
5640 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_smart_predicted_fault()
5653 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_smart_predicted_fault()
5655 	if (ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM)  in _scsih_smart_predicted_fault()
5656 		_scsih_send_event_to_turn_on_pfa_led(ioc, handle);  in _scsih_smart_predicted_fault()
5663 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_smart_predicted_fault()
5680 	mpt3sas_ctl_add_to_event_log(ioc, event_reply);  in _scsih_smart_predicted_fault()
5688 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_smart_predicted_fault()
5705 _scsih_io_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)  in _scsih_io_done()  argument
5719 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_io_done()
5721 	scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_io_done()
5727 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_io_done()
5755 		ioc->put_smid_scsi_io(ioc, smid,  in _scsih_io_done()
5766 		if ((!ioc->is_warpdrive &&  in _scsih_io_done()
5794 		const void *sense_data = mpt3sas_base_get_sense_buffer(ioc,  in _scsih_io_done()
5802 			_scsih_smart_predicted_fault(ioc,  in _scsih_io_done()
5804 		mpt3sas_trigger_scsi(ioc, data.skey, data.asc, data.ascq);  in _scsih_io_done()
5806 		if ((ioc->logging_level & MPT_DEBUG_REPLY) &&  in _scsih_io_done()
5810 			_scsih_scsi_ioc_info(ioc, scmd, mpi_reply, smid);  in _scsih_io_done()
5920 	if (scmd->result && (ioc->logging_level & MPT_DEBUG_REPLY))  in _scsih_io_done()
5921 		_scsih_scsi_ioc_info(ioc , scmd, mpi_reply, smid);  in _scsih_io_done()
5926 	mpt3sas_base_free_smid(ioc, smid);  in _scsih_io_done()
5939 _scsih_update_vphys_after_reset(struct MPT3SAS_ADAPTER *ioc)  in _scsih_update_vphys_after_reset()  argument
5957 	    &ioc->port_table_list, list) {  in _scsih_update_vphys_after_reset()
5970 	    (ioc->sas_hba.num_phys * sizeof(Mpi2SasIOUnit0PhyData_t));  in _scsih_update_vphys_after_reset()
5973 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_update_vphys_after_reset()
5977 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_update_vphys_after_reset()
5986 	for (i = 0; i < ioc->sas_hba.num_phys; i++) {  in _scsih_update_vphys_after_reset()
6005 		if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,  in _scsih_update_vphys_after_reset()
6007 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_update_vphys_after_reset()
6020 		if (_scsih_get_sas_address(ioc, attached_handle,  in _scsih_update_vphys_after_reset()
6022 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_update_vphys_after_reset()
6034 		    port_next, &ioc->port_table_list, list) {  in _scsih_update_vphys_after_reset()
6067 				mport = mpt3sas_get_port_by_id(ioc, port_id, 1);  in _scsih_update_vphys_after_reset()
6074 					ioc_info(ioc,  in _scsih_update_vphys_after_reset()
6078 						&ioc->port_table_list);  in _scsih_update_vphys_after_reset()
6096 					    ioc, attached_sas_addr, port);  in _scsih_update_vphys_after_reset()
6137 _scsih_get_port_table_after_reset(struct MPT3SAS_ADAPTER *ioc,  in _scsih_get_port_table_after_reset()  argument
6148 	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys  in _scsih_get_port_table_after_reset()
6152 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_port_table_after_reset()
6157 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_get_port_table_after_reset()
6163 	for (i = 0; i < ioc->sas_hba.num_phys; i++) {  in _scsih_get_port_table_after_reset()
6171 		    ioc, attached_handle, &attached_sas_addr) != 0) {  in _scsih_get_port_table_after_reset()
6172 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_get_port_table_after_reset()
6221 _scsih_look_and_get_matched_port_entry(struct MPT3SAS_ADAPTER *ioc,  in _scsih_look_and_get_matched_port_entry()  argument
6230 	list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {  in _scsih_look_and_get_matched_port_entry()
6288 _scsih_del_phy_part_of_anther_port(struct MPT3SAS_ADAPTER *ioc,  in _scsih_del_phy_part_of_anther_port()  argument
6292 	struct _sas_node *sas_node = &ioc->sas_hba;  in _scsih_del_phy_part_of_anther_port()
6301 			    ioc, sas_node, &sas_node->phy[offset]);  in _scsih_del_phy_part_of_anther_port()
6321 _scsih_add_or_del_phys_from_existing_port(struct MPT3SAS_ADAPTER *ioc,  in _scsih_add_or_del_phys_from_existing_port()  argument
6326 	struct _sas_node *sas_node = &ioc->sas_hba;  in _scsih_add_or_del_phys_from_existing_port()
6330 	for (offset = 0; offset < ioc->sas_hba.num_phys; offset++) {  in _scsih_add_or_del_phys_from_existing_port()
6334 				    ioc, port_table, index, port_count,  in _scsih_add_or_del_phys_from_existing_port()
6340 				    ioc, sas_node, &sas_node->phy[offset]);  in _scsih_add_or_del_phys_from_existing_port()
6342 			    ioc, sas_node, &sas_node->phy[offset],  in _scsih_add_or_del_phys_from_existing_port()
6356 _scsih_del_dirty_vphy(struct MPT3SAS_ADAPTER *ioc)  in _scsih_del_dirty_vphy()  argument
6362 	    &ioc->port_table_list, list) {  in _scsih_del_dirty_vphy()
6368 				drsprintk(ioc, ioc_info(ioc,  in _scsih_del_dirty_vphy()
6389 _scsih_del_dirty_port_entries(struct MPT3SAS_ADAPTER *ioc)  in _scsih_del_dirty_port_entries()  argument
6394 	    &ioc->port_table_list, list) {  in _scsih_del_dirty_port_entries()
6399 		drsprintk(ioc, ioc_info(ioc,  in _scsih_del_dirty_port_entries()
6412 _scsih_sas_port_refresh(struct MPT3SAS_ADAPTER *ioc)  in _scsih_sas_port_refresh()  argument
6423 	drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6425 	    (unsigned long long)ioc->sas_hba.sas_address));  in _scsih_sas_port_refresh()
6427 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);  in _scsih_sas_port_refresh()
6429 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_port_refresh()
6434 	if (num_phys > ioc->sas_hba.nr_phys_allocated) {  in _scsih_sas_port_refresh()
6435 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_port_refresh()
6439 	ioc->sas_hba.num_phys = num_phys;  in _scsih_sas_port_refresh()
6441 	port_table = kcalloc(ioc->sas_hba.num_phys,  in _scsih_sas_port_refresh()
6446 	port_count = _scsih_get_port_table_after_reset(ioc, port_table);  in _scsih_sas_port_refresh()
6450 	drsprintk(ioc, ioc_info(ioc, "New Port table\n"));  in _scsih_sas_port_refresh()
6452 		drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6457 	list_for_each_entry(port_table_entry, &ioc->port_table_list, list)  in _scsih_sas_port_refresh()
6460 	drsprintk(ioc, ioc_info(ioc, "Old Port table\n"));  in _scsih_sas_port_refresh()
6462 	list_for_each_entry(port_table_entry, &ioc->port_table_list, list) {  in _scsih_sas_port_refresh()
6463 		drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6471 		ret = _scsih_look_and_get_matched_port_entry(ioc,  in _scsih_sas_port_refresh()
6474 			drsprintk(ioc, ioc_info(ioc,  in _scsih_sas_port_refresh()
6484 			_scsih_add_or_del_phys_from_existing_port(ioc,  in _scsih_sas_port_refresh()
6497 				_scsih_add_or_del_phys_from_existing_port(ioc,  in _scsih_sas_port_refresh()
6522 _scsih_alloc_vphy(struct MPT3SAS_ADAPTER *ioc, u8 port_id, u8 phy_num)  in _scsih_alloc_vphy()  argument
6527 	port = mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_alloc_vphy()
6531 	vphy = mpt3sas_get_vphy_by_phy(ioc, port, phy_num);  in _scsih_alloc_vphy()
6549 		ioc_info(ioc,  in _scsih_alloc_vphy()
6566 _scsih_sas_host_refresh(struct MPT3SAS_ADAPTER *ioc)  in _scsih_sas_host_refresh()  argument
6578 	dtmprintk(ioc,  in _scsih_sas_host_refresh()
6579 		  ioc_info(ioc, "updating handles for sas_host(0x%016llx)\n",  in _scsih_sas_host_refresh()
6580 			   (u64)ioc->sas_hba.sas_address));  in _scsih_sas_host_refresh()
6582 	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys  in _scsih_sas_host_refresh()
6586 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_refresh()
6591 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_sas_host_refresh()
6597 	for (i = 0; i < ioc->sas_hba.num_phys ; i++) {  in _scsih_sas_host_refresh()
6600 			ioc->sas_hba.handle = le16_to_cpu(  in _scsih_sas_host_refresh()
6603 		if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {  in _scsih_sas_host_refresh()
6609 			ioc_info(ioc,  in _scsih_sas_host_refresh()
6612 			if (ioc->shost_recovery)  in _scsih_sas_host_refresh()
6614 			list_add_tail(&port->list, &ioc->port_table_list);  in _scsih_sas_host_refresh()
6622 			if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,  in _scsih_sas_host_refresh()
6624 				ioc_err(ioc,  in _scsih_sas_host_refresh()
6636 			if (!_scsih_alloc_vphy(ioc, port_id, i))  in _scsih_sas_host_refresh()
6638 			ioc->sas_hba.phy[i].hba_vphy = 1;  in _scsih_sas_host_refresh()
6645 		if (!ioc->sas_hba.phy[i].phy) {  in _scsih_sas_host_refresh()
6646 			if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply,  in _scsih_sas_host_refresh()
6648 				ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_refresh()
6655 				ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_refresh()
6659 			ioc->sas_hba.phy[i].phy_id = i;  in _scsih_sas_host_refresh()
6660 			mpt3sas_transport_add_host_phy(ioc,  in _scsih_sas_host_refresh()
6661 				&ioc->sas_hba.phy[i], phy_pg0,  in _scsih_sas_host_refresh()
6662 				ioc->sas_hba.parent_dev);  in _scsih_sas_host_refresh()
6665 		ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;  in _scsih_sas_host_refresh()
6670 		ioc->sas_hba.phy[i].port =  in _scsih_sas_host_refresh()
6671 		    mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_sas_host_refresh()
6672 		mpt3sas_transport_update_links(ioc, ioc->sas_hba.sas_address,  in _scsih_sas_host_refresh()
6674 		    ioc->sas_hba.phy[i].port);  in _scsih_sas_host_refresh()
6680 	for (i = ioc->sas_hba.num_phys;  in _scsih_sas_host_refresh()
6681 	     i < ioc->sas_hba.nr_phys_allocated; i++) {  in _scsih_sas_host_refresh()
6682 		if (ioc->sas_hba.phy[i].phy &&  in _scsih_sas_host_refresh()
6683 		    ioc->sas_hba.phy[i].phy->negotiated_linkrate >=  in _scsih_sas_host_refresh()
6685 			mpt3sas_transport_update_links(ioc,  in _scsih_sas_host_refresh()
6686 				ioc->sas_hba.sas_address, 0, i,  in _scsih_sas_host_refresh()
6700 _scsih_sas_host_add(struct MPT3SAS_ADAPTER *ioc)  in _scsih_sas_host_add()  argument
6715 	mpt3sas_config_get_number_hba_phys(ioc, &num_phys);  in _scsih_sas_host_add()
6717 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6722 	ioc->sas_hba.nr_phys_allocated = max_t(u8,  in _scsih_sas_host_add()
6724 	ioc->sas_hba.phy = kcalloc(ioc->sas_hba.nr_phys_allocated,  in _scsih_sas_host_add()
6726 	if (!ioc->sas_hba.phy) {  in _scsih_sas_host_add()
6727 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6731 	ioc->sas_hba.num_phys = num_phys;  in _scsih_sas_host_add()
6734 	sz = offsetof(Mpi2SasIOUnitPage0_t, PhyData) + (ioc->sas_hba.num_phys *  in _scsih_sas_host_add()
6738 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6742 	if ((mpt3sas_config_get_sas_iounit_pg0(ioc, &mpi_reply,  in _scsih_sas_host_add()
6744 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6751 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6757 	sz = offsetof(Mpi2SasIOUnitPage1_t, PhyData) + (ioc->sas_hba.num_phys *  in _scsih_sas_host_add()
6761 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6765 	if ((mpt3sas_config_get_sas_iounit_pg1(ioc, &mpi_reply,  in _scsih_sas_host_add()
6767 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6774 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6779 	ioc->io_missing_delay =  in _scsih_sas_host_add()
6784 		ioc->device_missing_delay = (device_missing_delay &  in _scsih_sas_host_add()
6787 		ioc->device_missing_delay = device_missing_delay &  in _scsih_sas_host_add()
6790 	ioc->sas_hba.parent_dev = &ioc->shost->shost_gendev;  in _scsih_sas_host_add()
6791 	for (i = 0; i < ioc->sas_hba.num_phys ; i++) {  in _scsih_sas_host_add()
6792 		if ((mpt3sas_config_get_phy_pg0(ioc, &mpi_reply, &phy_pg0,  in _scsih_sas_host_add()
6794 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6801 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6807 			ioc->sas_hba.handle = le16_to_cpu(sas_iounit_pg0->  in _scsih_sas_host_add()
6811 		if (!(mpt3sas_get_port_by_id(ioc, port_id, 0))) {  in _scsih_sas_host_add()
6817 			ioc_info(ioc,  in _scsih_sas_host_add()
6821 			    &ioc->port_table_list);  in _scsih_sas_host_add()
6834 			if (!_scsih_alloc_vphy(ioc, port_id, i))  in _scsih_sas_host_add()
6836 			ioc->sas_hba.phy[i].hba_vphy = 1;  in _scsih_sas_host_add()
6839 		ioc->sas_hba.phy[i].handle = ioc->sas_hba.handle;  in _scsih_sas_host_add()
6840 		ioc->sas_hba.phy[i].phy_id = i;  in _scsih_sas_host_add()
6841 		ioc->sas_hba.phy[i].port =  in _scsih_sas_host_add()
6842 		    mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_sas_host_add()
6843 		mpt3sas_transport_add_host_phy(ioc, &ioc->sas_hba.phy[i],  in _scsih_sas_host_add()
6844 		    phy_pg0, ioc->sas_hba.parent_dev);  in _scsih_sas_host_add()
6846 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_sas_host_add()
6847 	    MPI2_SAS_DEVICE_PGAD_FORM_HANDLE, ioc->sas_hba.handle))) {  in _scsih_sas_host_add()
6848 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_host_add()
6852 	ioc->sas_hba.enclosure_handle =  in _scsih_sas_host_add()
6854 	ioc->sas_hba.sas_address = le64_to_cpu(sas_device_pg0.SASAddress);  in _scsih_sas_host_add()
6855 	ioc_info(ioc, "host_add: handle(0x%04x), sas_addr(0x%016llx), phys(%d)\n",  in _scsih_sas_host_add()
6856 		 ioc->sas_hba.handle,  in _scsih_sas_host_add()
6857 		 (u64)ioc->sas_hba.sas_address,  in _scsih_sas_host_add()
6858 		 ioc->sas_hba.num_phys);  in _scsih_sas_host_add()
6860 	if (ioc->sas_hba.enclosure_handle) {  in _scsih_sas_host_add()
6861 		if (!(mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,  in _scsih_sas_host_add()
6863 		   ioc->sas_hba.enclosure_handle)))  in _scsih_sas_host_add()
6864 			ioc->sas_hba.enclosure_logical_id =  in _scsih_sas_host_add()
6883 _scsih_expander_add(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_expander_add()  argument
6903 	if (ioc->shost_recovery || ioc->pci_error_recovery)  in _scsih_expander_add()
6906 	if ((mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,  in _scsih_expander_add()
6908 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6916 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6923 	if (_scsih_get_sas_address(ioc, parent_handle, &sas_address_parent)  in _scsih_expander_add()
6925 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6931 	if (sas_address_parent != ioc->sas_hba.sas_address) {  in _scsih_expander_add()
6932 		spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6933 		sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,  in _scsih_expander_add()
6935 		    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_expander_add()
6936 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6938 			rc = _scsih_expander_add(ioc, parent_handle);  in _scsih_expander_add()
6944 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6946 	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,  in _scsih_expander_add()
6947 	    sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_expander_add()
6948 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_add()
6956 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6965 	sas_expander->port = mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_expander_add()
6967 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6973 	ioc_info(ioc, "expander_add: handle(0x%04x), parent(0x%04x), sas_addr(0x%016llx), phys(%d)\n",  in _scsih_expander_add()
6984 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
6991 	mpt3sas_port = mpt3sas_transport_port_add(ioc, handle,  in _scsih_expander_add()
6994 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
7003 		if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,  in _scsih_expander_add()
7005 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
7013 		    mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_expander_add()
7015 		if ((mpt3sas_transport_add_expander_phy(ioc,  in _scsih_expander_add()
7018 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_expander_add()
7027 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_expander_add()
7034 	_scsih_expander_node_add(ioc, sas_expander);  in _scsih_expander_add()
7040 		mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,  in _scsih_expander_add()
7053 mpt3sas_expander_remove(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,  in mpt3sas_expander_remove()  argument
7059 	if (ioc->shost_recovery)  in mpt3sas_expander_remove()
7065 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in mpt3sas_expander_remove()
7066 	sas_expander = mpt3sas_scsih_expander_find_by_sas_address(ioc,  in mpt3sas_expander_remove()
7068 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in mpt3sas_expander_remove()
7070 		_scsih_expander_node_remove(ioc, sas_expander);  in mpt3sas_expander_remove()
7087 _scsih_done(struct MPT3SAS_ADAPTER *ioc, u16 smid, u8 msix_index, u32 reply)  in _scsih_done()  argument
7091 	mpi_reply =  mpt3sas_base_get_reply_virt_addr(ioc, reply);  in _scsih_done()
7092 	if (ioc->scsih_cmds.status == MPT3_CMD_NOT_USED)  in _scsih_done()
7094 	if (ioc->scsih_cmds.smid != smid)  in _scsih_done()
7096 	ioc->scsih_cmds.status |= MPT3_CMD_COMPLETE;  in _scsih_done()
7098 		memcpy(ioc->scsih_cmds.reply, mpi_reply,  in _scsih_done()
7100 		ioc->scsih_cmds.status |= MPT3_CMD_REPLY_VALID;  in _scsih_done()
7102 	ioc->scsih_cmds.status &= ~MPT3_CMD_PENDING;  in _scsih_done()
7103 	complete(&ioc->scsih_cmds.done);  in _scsih_done()
7123 _scsih_check_access_status(struct MPT3SAS_ADAPTER *ioc, u64 sas_address,  in _scsih_check_access_status()  argument
7171 	ioc_err(ioc, "discovery errors(%s): sas_address(0x%016llx), handle(0x%04x)\n",  in _scsih_check_access_status()
7185 _scsih_check_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_check_device()  argument
7200 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_check_device()
7219 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_check_device()
7221 	port = mpt3sas_get_port_by_id(ioc, sas_device_pg0.PhysicalPort, 0);  in _scsih_check_device()
7224 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in _scsih_check_device()
7253 		enclosure_dev = mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_check_device()
7270 		ioc_err(ioc, "device is not present handle(0x%04x), flags!!!\n",  in _scsih_check_device()
7276 	if (_scsih_check_access_status(ioc, sas_address, handle,  in _scsih_check_device()
7280 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_check_device()
7281 	_scsih_ublock_io_device(ioc, sas_address, port);  in _scsih_check_device()
7288 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_check_device()
7305 _scsih_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phy_num,  in _scsih_add_device()  argument
7317 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_add_device()
7319 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7327 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7336 	set_bit(handle, ioc->pend_os_device_add);  in _scsih_add_device()
7342 		ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",  in _scsih_add_device()
7348 	if (_scsih_check_access_status(ioc, sas_address, handle,  in _scsih_add_device()
7353 	sas_device = mpt3sas_get_sdev_by_addr(ioc,  in _scsih_add_device()
7354 	    sas_address, mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_add_device()
7356 		clear_bit(handle, ioc->pend_os_device_add);  in _scsih_add_device()
7363 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_add_device()
7366 			ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",  in _scsih_add_device()
7373 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7380 	if (_scsih_get_sas_address(ioc,  in _scsih_add_device()
7383 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7395 	sas_device->port = mpt3sas_get_port_by_id(ioc, port_id, 0);  in _scsih_add_device()
7397 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_add_device()
7429 	ioc_info(ioc,  in _scsih_add_device()
7433 	if (ioc->wait_for_discovery_to_complete)  in _scsih_add_device()
7434 		_scsih_sas_device_init_add(ioc, sas_device);  in _scsih_add_device()
7436 		_scsih_sas_device_add(ioc, sas_device);  in _scsih_add_device()
7449 _scsih_remove_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_remove_device()  argument
7454 	if ((ioc->pdev->subsystem_vendor == PCI_VENDOR_ID_IBM) &&  in _scsih_remove_device()
7456 		_scsih_turn_off_pfa_led(ioc, sas_device);  in _scsih_remove_device()
7460 	dewtprintk(ioc,  in _scsih_remove_device()
7461 		   ioc_info(ioc, "%s: enter: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_remove_device()
7465 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_remove_device()
7471 		_scsih_ublock_io_device(ioc, sas_device->sas_address,  in _scsih_remove_device()
7477 	if (!ioc->hide_drives)  in _scsih_remove_device()
7478 		mpt3sas_transport_port_remove(ioc,  in _scsih_remove_device()
7483 	ioc_info(ioc, "removing handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_remove_device()
7486 	_scsih_display_enclosure_chassis_info(ioc, sas_device, NULL, NULL);  in _scsih_remove_device()
7488 	dewtprintk(ioc,  in _scsih_remove_device()
7489 		   ioc_info(ioc, "%s: exit: handle(0x%04x), sas_addr(0x%016llx)\n",  in _scsih_remove_device()
7492 	dewtprintk(ioc, _scsih_display_enclosure_chassis_info(ioc, sas_device,  in _scsih_remove_device()
7503 _scsih_sas_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_topology_change_event_debug()  argument
7531 	ioc_info(ioc, "sas topology change: (%s)\n", status_str);  in _scsih_sas_topology_change_event_debug()
7581 _scsih_sas_topology_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_topology_change_event()  argument
7597 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_sas_topology_change_event()
7598 		_scsih_sas_topology_change_event_debug(ioc, event_data);  in _scsih_sas_topology_change_event()
7600 	if (ioc->shost_recovery || ioc->remove_host || ioc->pci_error_recovery)  in _scsih_sas_topology_change_event()
7603 	if (!ioc->sas_hba.num_phys)  in _scsih_sas_topology_change_event()
7604 		_scsih_sas_host_add(ioc);  in _scsih_sas_topology_change_event()
7606 		_scsih_sas_host_refresh(ioc);  in _scsih_sas_topology_change_event()
7609 		dewtprintk(ioc, ioc_info(ioc, "ignoring expander event\n"));  in _scsih_sas_topology_change_event()
7614 	port = mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0);  in _scsih_sas_topology_change_event()
7618 		if (_scsih_expander_add(ioc, parent_handle) != 0)  in _scsih_sas_topology_change_event()
7621 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_sas_topology_change_event()
7622 	sas_expander = mpt3sas_scsih_expander_find_by_handle(ioc,  in _scsih_sas_topology_change_event()
7628 	} else if (parent_handle < ioc->sas_hba.num_phys) {  in _scsih_sas_topology_change_event()
7629 		sas_address = ioc->sas_hba.sas_address;  in _scsih_sas_topology_change_event()
7630 		max_phys = ioc->sas_hba.num_phys;  in _scsih_sas_topology_change_event()
7632 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_sas_topology_change_event()
7635 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_sas_topology_change_event()
7640 			dewtprintk(ioc,  in _scsih_sas_topology_change_event()
7641 				   ioc_info(ioc, "ignoring expander event\n"));  in _scsih_sas_topology_change_event()
7644 		if (ioc->remove_host || ioc->pci_error_recovery)  in _scsih_sas_topology_change_event()
7663 			if (ioc->shost_recovery)  in _scsih_sas_topology_change_event()
7669 			mpt3sas_transport_update_links(ioc, sas_address,  in _scsih_sas_topology_change_event()
7675 			_scsih_check_device(ioc, sas_address, handle,  in _scsih_sas_topology_change_event()
7678 			if (!test_bit(handle, ioc->pend_os_device_add))  in _scsih_sas_topology_change_event()
7685 			if (ioc->shost_recovery)  in _scsih_sas_topology_change_event()
7688 			mpt3sas_transport_update_links(ioc, sas_address,  in _scsih_sas_topology_change_event()
7691 			_scsih_add_device(ioc, handle, phy_number, 0);  in _scsih_sas_topology_change_event()
7696 			_scsih_device_remove_by_handle(ioc, handle);  in _scsih_sas_topology_change_event()
7704 		mpt3sas_expander_remove(ioc, sas_address, port);  in _scsih_sas_topology_change_event()
7716 _scsih_sas_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_status_change_event_debug()  argument
7765 	ioc_info(ioc, "device status change: (%s)\thandle(0x%04x), sas address(0x%016llx), tag(%d)",  in _scsih_sas_device_status_change_event_debug()
7782 _scsih_sas_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_status_change_event()  argument
7793 	if ((ioc->facts.HeaderVersion >> 8) < 0xC)  in _scsih_sas_device_status_change_event()
7802 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_device_status_change_event()
7804 	sas_device = __mpt3sas_get_sdev_by_addr(ioc,  in _scsih_sas_device_status_change_event()
7806 	    mpt3sas_get_port_by_id(ioc, event_data->PhysicalPort, 0));  in _scsih_sas_device_status_change_event()
7821 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_sas_device_status_change_event()
7822 		ioc_info(ioc,  in _scsih_sas_device_status_change_event()
7831 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_device_status_change_event()
7845 _scsih_check_pcie_access_status(struct MPT3SAS_ADAPTER *ioc, u64 wwid,  in _scsih_check_pcie_access_status()  argument
7861 		ioc_info(ioc,  in _scsih_check_pcie_access_status()
7910 		ioc_err(ioc, "NVMe discovery error(0x%02x): wwid(0x%016llx), handle(0x%04x)\n",  in _scsih_check_pcie_access_status()
7918 	ioc_info(ioc, "NVMe discovery error(%s): wwid(0x%016llx), handle(0x%04x)\n",  in _scsih_check_pcie_access_status()
7930 _scsih_pcie_device_remove_from_sml(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_remove_from_sml()  argument
7935 	dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7936 		   ioc_info(ioc, "%s: enter: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove_from_sml()
7940 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7941 			   ioc_info(ioc, "%s: enter: enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove_from_sml()
7946 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7947 			   ioc_info(ioc, "%s: enter: enclosure level(0x%04x), connector name(%s)\n",  in _scsih_pcie_device_remove_from_sml()
7955 		_scsih_ublock_io_device(ioc, pcie_device->wwid, NULL);  in _scsih_pcie_device_remove_from_sml()
7959 	ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove_from_sml()
7962 		ioc_info(ioc, "removing : enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove_from_sml()
7966 		ioc_info(ioc, "removing: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_remove_from_sml()
7973 	dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7974 		   ioc_info(ioc, "%s: exit: handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_pcie_device_remove_from_sml()
7978 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7979 			   ioc_info(ioc, "%s: exit: enclosure logical id(0x%016llx), slot(%d)\n",  in _scsih_pcie_device_remove_from_sml()
7984 		dewtprintk(ioc,  in _scsih_pcie_device_remove_from_sml()
7985 			   ioc_info(ioc, "%s: exit: enclosure level(0x%04x), connector name( %s)\n",  in _scsih_pcie_device_remove_from_sml()
8000 _scsih_pcie_check_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_pcie_check_device()  argument
8012 	if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_pcie_check_device()
8026 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8027 	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);  in _scsih_pcie_check_device()
8030 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8059 		ioc_info(ioc, "device is not present handle(0x%04x), flags!!!\n",  in _scsih_pcie_check_device()
8061 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8067 	if (_scsih_check_pcie_access_status(ioc, wwid, handle,  in _scsih_pcie_check_device()
8069 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8074 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_check_device()
8077 	_scsih_ublock_io_device(ioc, wwid, NULL);  in _scsih_pcie_check_device()
8092 _scsih_pcie_add_device(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_pcie_add_device()  argument
8102 	if ((mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_pcie_add_device()
8104 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_pcie_add_device()
8111 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_pcie_add_device()
8116 	set_bit(handle, ioc->pend_os_device_add);  in _scsih_pcie_add_device()
8122 		ioc_err(ioc, "device is not present handle(0x04%x)!!!\n",  in _scsih_pcie_add_device()
8128 	if (_scsih_check_pcie_access_status(ioc, wwid, handle,  in _scsih_pcie_add_device()
8136 	pcie_device = mpt3sas_get_pdev_by_wwid(ioc, wwid);  in _scsih_pcie_add_device()
8138 		clear_bit(handle, ioc->pend_os_device_add);  in _scsih_pcie_add_device()
8149 		if (mpt3sas_config_get_pcie_device_pg2(ioc, &mpi_reply,  in _scsih_pcie_add_device()
8152 			ioc_err(ioc,  in _scsih_pcie_add_device()
8161 			ioc_err(ioc,  in _scsih_pcie_add_device()
8170 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_pcie_add_device()
8176 	pcie_device->id = ioc->pcie_target_id++;  in _scsih_pcie_add_device()
8204 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_pcie_add_device()
8222 		if (pcie_device->shutdown_latency > ioc->max_shutdown_latency)  in _scsih_pcie_add_device()
8223 			ioc->max_shutdown_latency =  in _scsih_pcie_add_device()
8233 	if (ioc->wait_for_discovery_to_complete)  in _scsih_pcie_add_device()
8234 		_scsih_pcie_device_init_add(ioc, pcie_device);  in _scsih_pcie_add_device()
8236 		_scsih_pcie_device_add(ioc, pcie_device);  in _scsih_pcie_add_device()
8250 _scsih_pcie_topology_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_topology_change_event_debug()  argument
8278 	ioc_info(ioc, "pcie topology change: (%s)\n", status_str);  in _scsih_pcie_topology_change_event_debug()
8330 _scsih_pcie_topology_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_topology_change_event()  argument
8343 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_pcie_topology_change_event()
8344 		_scsih_pcie_topology_change_event_debug(ioc, event_data);  in _scsih_pcie_topology_change_event()
8346 	if (ioc->shost_recovery || ioc->remove_host ||  in _scsih_pcie_topology_change_event()
8347 		ioc->pci_error_recovery)  in _scsih_pcie_topology_change_event()
8351 		dewtprintk(ioc, ioc_info(ioc, "ignoring switch event\n"));  in _scsih_pcie_topology_change_event()
8358 			dewtprintk(ioc,  in _scsih_pcie_topology_change_event()
8359 				   ioc_info(ioc, "ignoring switch event\n"));  in _scsih_pcie_topology_change_event()
8362 		if (ioc->remove_host || ioc->pci_error_recovery)  in _scsih_pcie_topology_change_event()
8377 			if (ioc->shost_recovery)  in _scsih_pcie_topology_change_event()
8384 			_scsih_pcie_check_device(ioc, handle);  in _scsih_pcie_topology_change_event()
8392 			spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_topology_change_event()
8393 			pcie_device = __mpt3sas_get_pdev_by_handle(ioc, handle);  in _scsih_pcie_topology_change_event()
8394 			spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_topology_change_event()
8401 			if (!test_bit(handle, ioc->pend_os_device_add))  in _scsih_pcie_topology_change_event()
8404 			dewtprintk(ioc,  in _scsih_pcie_topology_change_event()
8405 				   ioc_info(ioc, "handle(0x%04x) device not found: convert event to a device add\n",  in _scsih_pcie_topology_change_event()
8412 			if (ioc->shost_recovery)  in _scsih_pcie_topology_change_event()
8417 			rc = _scsih_pcie_add_device(ioc, handle);  in _scsih_pcie_topology_change_event()
8429 			_scsih_pcie_device_remove_by_handle(ioc, handle);  in _scsih_pcie_topology_change_event()
8442 _scsih_pcie_device_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_status_change_event_debug()  argument
8489 	ioc_info(ioc, "PCIE device status change: (%s)\n"  in _scsih_pcie_device_status_change_event_debug()
8508 _scsih_pcie_device_status_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_device_status_change_event()  argument
8517 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_pcie_device_status_change_event()
8518 		_scsih_pcie_device_status_change_event_debug(ioc,  in _scsih_pcie_device_status_change_event()
8527 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_status_change_event()
8529 	pcie_device = __mpt3sas_get_pdev_by_wwid(ioc, wwid);  in _scsih_pcie_device_status_change_event()
8547 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_pcie_device_status_change_event()
8558 _scsih_sas_enclosure_dev_status_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_enclosure_dev_status_change_event_debug()  argument
8575 	ioc_info(ioc, "enclosure status change: (%s)\n"  in _scsih_sas_enclosure_dev_status_change_event_debug()
8590 _scsih_sas_enclosure_dev_status_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_enclosure_dev_status_change_event()  argument
8600 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _scsih_sas_enclosure_dev_status_change_event()
8601 		_scsih_sas_enclosure_dev_status_change_event_debug(ioc,  in _scsih_sas_enclosure_dev_status_change_event()
8604 	if (ioc->shost_recovery)  in _scsih_sas_enclosure_dev_status_change_event()
8609 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_sas_enclosure_dev_status_change_event()
8618 				ioc_info(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_enclosure_dev_status_change_event()
8622 			rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,  in _scsih_sas_enclosure_dev_status_change_event()
8634 							&ioc->enclosure_list);  in _scsih_sas_enclosure_dev_status_change_event()
8655 _scsih_sas_broadcast_primitive_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_broadcast_primitive_event()  argument
8676 	mutex_lock(&ioc->tm_cmds.mutex);  in _scsih_sas_broadcast_primitive_event()
8677 	ioc_info(ioc, "%s: enter: phy number(%d), width(%d)\n",  in _scsih_sas_broadcast_primitive_event()
8680 	_scsih_block_io_all_device(ioc);  in _scsih_sas_broadcast_primitive_event()
8682 	spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8683 	mpi_reply = ioc->tm_cmds.reply;  in _scsih_sas_broadcast_primitive_event()
8688 		dewtprintk(ioc, ioc_info(ioc, "%s: giving up\n", __func__));  in _scsih_sas_broadcast_primitive_event()
8691 		dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8692 			   ioc_info(ioc, "%s: %d retry\n",  in _scsih_sas_broadcast_primitive_event()
8697 	for (smid = 1; smid <= ioc->scsiio_depth; smid++) {  in _scsih_sas_broadcast_primitive_event()
8698 		if (ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8700 		scmd = mpt3sas_scsih_scsi_lookup_get(ioc, smid);  in _scsih_sas_broadcast_primitive_event()
8725 		if (ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8728 		spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8729 		r = mpt3sas_scsih_issue_tm(ioc, handle, 0, 0, lun,  in _scsih_sas_broadcast_primitive_event()
8736 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8745 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8754 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8760 			dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8761 				   ioc_info(ioc, "%s: ABORT_TASK: giving up\n",  in _scsih_sas_broadcast_primitive_event()
8763 			spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8767 		if (ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8770 		r = mpt3sas_scsih_issue_tm(ioc, handle, sdev->channel, sdev->id,  in _scsih_sas_broadcast_primitive_event()
8787 		spin_lock_irqsave(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8790 	if (ioc->broadcast_aen_pending) {  in _scsih_sas_broadcast_primitive_event()
8791 		dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8792 			   ioc_info(ioc,  in _scsih_sas_broadcast_primitive_event()
8795 		 ioc->broadcast_aen_pending = 0;  in _scsih_sas_broadcast_primitive_event()
8800 	spin_unlock_irqrestore(&ioc->scsi_lookup_lock, flags);  in _scsih_sas_broadcast_primitive_event()
8803 	dewtprintk(ioc,  in _scsih_sas_broadcast_primitive_event()
8804 		   ioc_info(ioc, "%s - exit, query_count = %d termination_count = %d\n",  in _scsih_sas_broadcast_primitive_event()
8807 	ioc->broadcast_aen_busy = 0;  in _scsih_sas_broadcast_primitive_event()
8808 	if (!ioc->shost_recovery)  in _scsih_sas_broadcast_primitive_event()
8809 		_scsih_ublock_io_all_device(ioc);  in _scsih_sas_broadcast_primitive_event()
8810 	mutex_unlock(&ioc->tm_cmds.mutex);  in _scsih_sas_broadcast_primitive_event()
8820 _scsih_sas_discovery_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_discovery_event()  argument
8826 	if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) {  in _scsih_sas_discovery_event()
8827 		ioc_info(ioc, "discovery event: (%s)",  in _scsih_sas_discovery_event()
8837 	    !ioc->sas_hba.num_phys) {  in _scsih_sas_discovery_event()
8838 		if (disable_discovery > 0 && ioc->shost_recovery) {  in _scsih_sas_discovery_event()
8840 			while (ioc->shost_recovery)  in _scsih_sas_discovery_event()
8843 		_scsih_sas_host_add(ioc);  in _scsih_sas_discovery_event()
8855 _scsih_sas_device_discovery_error_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_device_discovery_error_event()  argument
8863 …ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_po…  in _scsih_sas_device_discovery_error_event()
8869 …ioc_warn(ioc, "SMP command sent to the expander (handle:0x%04x, sas_address:0x%016llx, physical_po…  in _scsih_sas_device_discovery_error_event()
8886 _scsih_pcie_enumeration_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_pcie_enumeration_event()  argument
8892 	if (!(ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK))  in _scsih_pcie_enumeration_event()
8895 	ioc_info(ioc, "pcie enumeration event: (%s) Flag 0x%02x",  in _scsih_pcie_enumeration_event()
8914 _scsih_ir_fastpath(struct MPT3SAS_ADAPTER *ioc, u16 handle, u8 phys_disk_num)  in _scsih_ir_fastpath()  argument
8924 	if (ioc->hba_mpi_version_belonged == MPI2_VERSION)  in _scsih_ir_fastpath()
8927 	mutex_lock(&ioc->scsih_cmds.mutex);  in _scsih_ir_fastpath()
8929 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _scsih_ir_fastpath()
8930 		ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);  in _scsih_ir_fastpath()
8934 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _scsih_ir_fastpath()
8936 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _scsih_ir_fastpath()
8938 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _scsih_ir_fastpath()
8939 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_fastpath()
8944 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_ir_fastpath()
8945 	ioc->scsih_cmds.smid = smid;  in _scsih_ir_fastpath()
8952 	dewtprintk(ioc,  in _scsih_ir_fastpath()
8953 … ioc_info(ioc, "IR RAID_ACTION: turning fast path on for handle(0x%04x), phys_disk_num (0x%02x)\n",  in _scsih_ir_fastpath()
8956 	init_completion(&ioc->scsih_cmds.done);  in _scsih_ir_fastpath()
8957 	ioc->put_smid_default(ioc, smid);  in _scsih_ir_fastpath()
8958 	wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);  in _scsih_ir_fastpath()
8960 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _scsih_ir_fastpath()
8961 		mpt3sas_check_cmd_timeout(ioc,  in _scsih_ir_fastpath()
8962 		    ioc->scsih_cmds.status, mpi_request,  in _scsih_ir_fastpath()
8968 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _scsih_ir_fastpath()
8970 		mpi_reply = ioc->scsih_cmds.reply;  in _scsih_ir_fastpath()
8978 			dewtprintk(ioc,  in _scsih_ir_fastpath()
8979 				   ioc_info(ioc, "IR RAID_ACTION: failed: ioc_status(0x%04x), loginfo(0x%08x)!!!\n",  in _scsih_ir_fastpath()
8983 			dewtprintk(ioc,  in _scsih_ir_fastpath()
8984 				   ioc_info(ioc, "IR RAID_ACTION: completed successfully\n"));  in _scsih_ir_fastpath()
8988 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_fastpath()
8989 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _scsih_ir_fastpath()
8992 		mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in _scsih_ir_fastpath()
9018 _scsih_sas_volume_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_volume_add()  argument
9027 	mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);  in _scsih_sas_volume_add()
9029 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_volume_add()
9034 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9035 	raid_device = _scsih_raid_device_find_by_wwid(ioc, wwid);  in _scsih_sas_volume_add()
9036 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9043 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_volume_add()
9048 	raid_device->id = ioc->sas_id++;  in _scsih_sas_volume_add()
9052 	_scsih_raid_device_add(ioc, raid_device);  in _scsih_sas_volume_add()
9053 	if (!ioc->wait_for_discovery_to_complete) {  in _scsih_sas_volume_add()
9054 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_sas_volume_add()
9057 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_sas_volume_add()
9059 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9060 		_scsih_determine_boot_device(ioc, raid_device, 1);  in _scsih_sas_volume_add()
9061 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_add()
9072 _scsih_sas_volume_delete(struct MPT3SAS_ADAPTER *ioc, u16 handle)  in _scsih_sas_volume_delete()  argument
9079 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_delete()
9080 	raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_sas_volume_delete()
9087 		ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in _scsih_sas_volume_delete()
9092 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_volume_delete()
9104 _scsih_sas_pd_expose(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_expose()  argument
9113 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_expose()
9114 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_pd_expose()
9118 		clear_bit(handle, ioc->pd_handles);  in _scsih_sas_pd_expose()
9126 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_expose()
9144 _scsih_sas_pd_hide(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_hide()  argument
9155 	mpt3sas_config_get_volume_handle(ioc, handle, &volume_handle);  in _scsih_sas_pd_hide()
9157 		mpt3sas_config_get_volume_wwid(ioc, volume_handle,  in _scsih_sas_pd_hide()
9160 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_hide()
9161 	sas_device = __mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_pd_hide()
9163 		set_bit(handle, ioc->pd_handles);  in _scsih_sas_pd_hide()
9173 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_sas_pd_hide()
9178 	_scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);  in _scsih_sas_pd_hide()
9193 _scsih_sas_pd_delete(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_delete()  argument
9198 	_scsih_device_remove_by_handle(ioc, handle);  in _scsih_sas_pd_delete()
9208 _scsih_sas_pd_add(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_pd_add()  argument
9219 	set_bit(handle, ioc->pd_handles);  in _scsih_sas_pd_add()
9221 	sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_pd_add()
9223 		_scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);  in _scsih_sas_pd_add()
9228 	if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply, &sas_device_pg0,  in _scsih_sas_pd_add()
9230 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_pd_add()
9238 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_pd_add()
9244 	if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))  in _scsih_sas_pd_add()
9245 		mpt3sas_transport_update_links(ioc, sas_address, handle,  in _scsih_sas_pd_add()
9247 		    mpt3sas_get_port_by_id(ioc,  in _scsih_sas_pd_add()
9250 	_scsih_ir_fastpath(ioc, handle, element->PhysDiskNum);  in _scsih_sas_pd_add()
9251 	_scsih_add_device(ioc, handle, 0, 1);  in _scsih_sas_pd_add()
9261 _scsih_sas_ir_config_change_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_config_change_event_debug()  argument
9271 	ioc_info(ioc, "raid config change: (%s), elements(%d)\n",  in _scsih_sas_ir_config_change_event_debug()
9339 _scsih_sas_ir_config_change_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_config_change_event()  argument
9349 	if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&  in _scsih_sas_ir_config_change_event()
9350 	     (!ioc->hide_ir_msg))  in _scsih_sas_ir_config_change_event()
9351 		_scsih_sas_ir_config_change_event_debug(ioc, event_data);  in _scsih_sas_ir_config_change_event()
9357 	if (ioc->shost_recovery &&  in _scsih_sas_ir_config_change_event()
9358 	    ioc->hba_mpi_version_belonged != MPI2_VERSION) {  in _scsih_sas_ir_config_change_event()
9361 				_scsih_ir_fastpath(ioc,  in _scsih_sas_ir_config_change_event()
9374 				_scsih_sas_volume_add(ioc, element);  in _scsih_sas_ir_config_change_event()
9379 				_scsih_sas_volume_delete(ioc,  in _scsih_sas_ir_config_change_event()
9383 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9384 				_scsih_sas_pd_hide(ioc, element);  in _scsih_sas_ir_config_change_event()
9387 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9388 				_scsih_sas_pd_expose(ioc, element);  in _scsih_sas_ir_config_change_event()
9391 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9392 				_scsih_sas_pd_add(ioc, element);  in _scsih_sas_ir_config_change_event()
9395 			if (!ioc->is_warpdrive)  in _scsih_sas_ir_config_change_event()
9396 				_scsih_sas_pd_delete(ioc, element);  in _scsih_sas_ir_config_change_event()
9409 _scsih_sas_ir_volume_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_volume_event()  argument
9421 	if (ioc->shost_recovery)  in _scsih_sas_ir_volume_event()
9429 	if (!ioc->hide_ir_msg)  in _scsih_sas_ir_volume_event()
9430 		dewtprintk(ioc,  in _scsih_sas_ir_volume_event()
9431 			   ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",  in _scsih_sas_ir_volume_event()
9438 		_scsih_sas_volume_delete(ioc, handle);  in _scsih_sas_ir_volume_event()
9445 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_volume_event()
9446 		raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_sas_ir_volume_event()
9447 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_volume_event()
9452 		mpt3sas_config_get_volume_wwid(ioc, handle, &wwid);  in _scsih_sas_ir_volume_event()
9454 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_volume_event()
9461 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_volume_event()
9466 		raid_device->id = ioc->sas_id++;  in _scsih_sas_ir_volume_event()
9470 		_scsih_raid_device_add(ioc, raid_device);  in _scsih_sas_ir_volume_event()
9471 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_sas_ir_volume_event()
9474 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_sas_ir_volume_event()
9490 _scsih_sas_ir_physical_disk_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_physical_disk_event()  argument
9503 	if (ioc->shost_recovery)  in _scsih_sas_ir_physical_disk_event()
9512 	if (!ioc->hide_ir_msg)  in _scsih_sas_ir_physical_disk_event()
9513 		dewtprintk(ioc,  in _scsih_sas_ir_physical_disk_event()
9514 			   ioc_info(ioc, "%s: handle(0x%04x), old(0x%08x), new(0x%08x)\n",  in _scsih_sas_ir_physical_disk_event()
9526 		if (!ioc->is_warpdrive)  in _scsih_sas_ir_physical_disk_event()
9527 			set_bit(handle, ioc->pd_handles);  in _scsih_sas_ir_physical_disk_event()
9529 		sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_sas_ir_physical_disk_event()
9535 		if ((mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_sas_ir_physical_disk_event()
9538 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_physical_disk_event()
9546 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_sas_ir_physical_disk_event()
9552 		if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address))  in _scsih_sas_ir_physical_disk_event()
9553 			mpt3sas_transport_update_links(ioc, sas_address, handle,  in _scsih_sas_ir_physical_disk_event()
9555 			    mpt3sas_get_port_by_id(ioc,  in _scsih_sas_ir_physical_disk_event()
9558 		_scsih_add_device(ioc, handle, 0, 1);  in _scsih_sas_ir_physical_disk_event()
9577 _scsih_sas_ir_operation_status_event_debug(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_operation_status_event_debug()  argument
9603 	ioc_info(ioc, "raid operational status: (%s)\thandle(0x%04x), percent complete(%d)\n",  in _scsih_sas_ir_operation_status_event_debug()
9616 _scsih_sas_ir_operation_status_event(struct MPT3SAS_ADAPTER *ioc,  in _scsih_sas_ir_operation_status_event()  argument
9626 	if ((ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK) &&  in _scsih_sas_ir_operation_status_event()
9627 	    (!ioc->hide_ir_msg))  in _scsih_sas_ir_operation_status_event()
9628 		_scsih_sas_ir_operation_status_event_debug(ioc,  in _scsih_sas_ir_operation_status_event()
9634 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_operation_status_event()
9636 		raid_device = mpt3sas_raid_device_find_by_handle(ioc, handle);  in _scsih_sas_ir_operation_status_event()
9640 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_sas_ir_operation_status_event()
9652 _scsih_prep_device_scan(struct MPT3SAS_ADAPTER *ioc)  in _scsih_prep_device_scan()  argument
9657 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_prep_device_scan()
9670 _scsih_update_device_qdepth(struct MPT3SAS_ADAPTER *ioc)  in _scsih_update_device_qdepth()  argument
9678 	ioc_info(ioc, "Update devices with firmware reported queue depth\n");  in _scsih_update_device_qdepth()
9679 	shost_for_each_device(sdev, ioc->shost) {  in _scsih_update_device_qdepth()
9685 				qdepth = ioc->max_nvme_qd;  in _scsih_update_device_qdepth()
9689 				    ioc->max_wideport_qd : ioc->max_narrowport_qd;  in _scsih_update_device_qdepth()
9692 				qdepth = ioc->max_sata_qd;  in _scsih_update_device_qdepth()
9709 _scsih_mark_responding_sas_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_mark_responding_sas_device()  argument
9718 	    ioc, sas_device_pg0->PhysicalPort, 0);  in _scsih_mark_responding_sas_device()
9722 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_mark_responding_sas_device()
9725 			ioc_info(ioc, "Enclosure handle(0x%04x) doesn't match with enclosure device!\n",  in _scsih_mark_responding_sas_device()
9728 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_mark_responding_sas_device()
9729 	list_for_each_entry(sas_device, &ioc->sas_device_list, list) {  in _scsih_mark_responding_sas_device()
9797 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_mark_responding_sas_device()
9806 _scsih_create_enclosure_list_after_reset(struct MPT3SAS_ADAPTER *ioc)  in _scsih_create_enclosure_list_after_reset()  argument
9814 	mpt3sas_free_enclosure_list(ioc);  in _scsih_create_enclosure_list_after_reset()
9822 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_create_enclosure_list_after_reset()
9826 		rc = mpt3sas_config_get_enclosure_pg0(ioc, &mpi_reply,  in _scsih_create_enclosure_list_after_reset()
9837 						&ioc->enclosure_list);  in _scsih_create_enclosure_list_after_reset()
9851 _scsih_search_responding_sas_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_sas_devices()  argument
9859 	ioc_info(ioc, "search for end-devices: start\n");  in _scsih_search_responding_sas_devices()
9861 	if (list_empty(&ioc->sas_device_list))  in _scsih_search_responding_sas_devices()
9865 	while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_search_responding_sas_devices()
9876 		_scsih_mark_responding_sas_device(ioc, &sas_device_pg0);  in _scsih_search_responding_sas_devices()
9880 	ioc_info(ioc, "search for end-devices: complete\n");  in _scsih_search_responding_sas_devices()
9892 _scsih_mark_responding_pcie_device(struct MPT3SAS_ADAPTER *ioc,  in _scsih_mark_responding_pcie_device()  argument
9900 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_mark_responding_pcie_device()
9901 	list_for_each_entry(pcie_device, &ioc->pcie_device_list, list) {  in _scsih_mark_responding_pcie_device()
9931 			    (ioc->hba_mpi_version_belonged != MPI2_VERSION)) {  in _scsih_mark_responding_pcie_device()
9956 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_mark_responding_pcie_device()
9967 _scsih_search_responding_pcie_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_pcie_devices()  argument
9975 	ioc_info(ioc, "search for end-devices: start\n");  in _scsih_search_responding_pcie_devices()
9977 	if (list_empty(&ioc->pcie_device_list))  in _scsih_search_responding_pcie_devices()
9981 	while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_search_responding_pcie_devices()
9987 			ioc_info(ioc, "\tbreak from %s: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_search_responding_pcie_devices()
9996 		_scsih_mark_responding_pcie_device(ioc, &pcie_device_pg0);  in _scsih_search_responding_pcie_devices()
9999 	ioc_info(ioc, "search for PCIe end-devices: complete\n");  in _scsih_search_responding_pcie_devices()
10012 _scsih_mark_responding_raid_device(struct MPT3SAS_ADAPTER *ioc, u64 wwid,  in _scsih_mark_responding_raid_device()  argument
10020 	spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10021 	list_for_each_entry(raid_device, &ioc->raid_device_list, list) {  in _scsih_mark_responding_raid_device()
10030 			spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10040 			mpt3sas_init_warpdrive_properties(ioc, raid_device);  in _scsih_mark_responding_raid_device()
10041 			spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10043 				spin_unlock_irqrestore(&ioc->raid_device_lock,  in _scsih_mark_responding_raid_device()
10052 			spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10056 	spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_mark_responding_raid_device()
10067 _scsih_search_responding_raid_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_raid_devices()  argument
10077 	if (!ioc->ir_firmware)  in _scsih_search_responding_raid_devices()
10080 	ioc_info(ioc, "search for raid volumes: start\n");  in _scsih_search_responding_raid_devices()
10082 	if (list_empty(&ioc->raid_device_list))  in _scsih_search_responding_raid_devices()
10086 	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,  in _scsih_search_responding_raid_devices()
10094 		if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,  in _scsih_search_responding_raid_devices()
10102 			_scsih_mark_responding_raid_device(ioc,  in _scsih_search_responding_raid_devices()
10107 	if (!ioc->is_warpdrive) {  in _scsih_search_responding_raid_devices()
10109 		memset(ioc->pd_handles, 0, ioc->pd_handles_sz);  in _scsih_search_responding_raid_devices()
10110 		while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,  in _scsih_search_responding_raid_devices()
10119 			set_bit(handle, ioc->pd_handles);  in _scsih_search_responding_raid_devices()
10123 	ioc_info(ioc, "search for responding raid volumes: complete\n");  in _scsih_search_responding_raid_devices()
10135 _scsih_mark_responding_expander(struct MPT3SAS_ADAPTER *ioc,  in _scsih_mark_responding_expander()  argument
10146 	    ioc, expander_pg0->PhysicalPort, 0);  in _scsih_mark_responding_expander()
10150 			mpt3sas_scsih_enclosure_find_by_handle(ioc,  in _scsih_mark_responding_expander()
10153 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_mark_responding_expander()
10154 	list_for_each_entry(sas_expander, &ioc->sas_expander_list, list) {  in _scsih_mark_responding_expander()
10180 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_mark_responding_expander()
10191 _scsih_search_responding_expanders(struct MPT3SAS_ADAPTER *ioc)  in _scsih_search_responding_expanders()  argument
10200 	ioc_info(ioc, "search for expanders: start\n");  in _scsih_search_responding_expanders()
10202 	if (list_empty(&ioc->sas_expander_list))  in _scsih_search_responding_expanders()
10206 	while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,  in _scsih_search_responding_expanders()
10220 		    (ioc->multipath_on_hba ?  in _scsih_search_responding_expanders()
10222 		_scsih_mark_responding_expander(ioc, &expander_pg0);  in _scsih_search_responding_expanders()
10226 	ioc_info(ioc, "search for expanders: complete\n");  in _scsih_search_responding_expanders()
10234 _scsih_remove_unresponding_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_remove_unresponding_devices()  argument
10244 	ioc_info(ioc, "removing unresponding devices: start\n");  in _scsih_remove_unresponding_devices()
10247 	ioc_info(ioc, "removing unresponding devices: end-devices\n");  in _scsih_remove_unresponding_devices()
10252 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_remove_unresponding_devices()
10259 	    &ioc->sas_device_init_list, list) {  in _scsih_remove_unresponding_devices()
10265 	    &ioc->sas_device_list, list) {  in _scsih_remove_unresponding_devices()
10271 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_remove_unresponding_devices()
10277 		_scsih_remove_device(ioc, sas_device);  in _scsih_remove_unresponding_devices()
10282 	ioc_info(ioc, "Removing unresponding devices: pcie end-devices\n");  in _scsih_remove_unresponding_devices()
10284 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_remove_unresponding_devices()
10290 	    &ioc->pcie_device_init_list, list) {  in _scsih_remove_unresponding_devices()
10296 	    &ioc->pcie_device_list, list) {  in _scsih_remove_unresponding_devices()
10302 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_remove_unresponding_devices()
10305 		_scsih_pcie_device_remove_from_sml(ioc, pcie_device);  in _scsih_remove_unresponding_devices()
10311 	if (ioc->ir_firmware) {  in _scsih_remove_unresponding_devices()
10312 		ioc_info(ioc, "removing unresponding devices: volumes\n");  in _scsih_remove_unresponding_devices()
10314 		    &ioc->raid_device_list, list) {  in _scsih_remove_unresponding_devices()
10316 				_scsih_sas_volume_delete(ioc,  in _scsih_remove_unresponding_devices()
10324 	ioc_info(ioc, "removing unresponding devices: expanders\n");  in _scsih_remove_unresponding_devices()
10325 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_remove_unresponding_devices()
10328 	    &ioc->sas_expander_list, list) {  in _scsih_remove_unresponding_devices()
10334 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_remove_unresponding_devices()
10337 		_scsih_expander_node_remove(ioc, sas_expander);  in _scsih_remove_unresponding_devices()
10340 	ioc_info(ioc, "removing unresponding devices: complete\n");  in _scsih_remove_unresponding_devices()
10343 	_scsih_ublock_io_all_device(ioc);  in _scsih_remove_unresponding_devices()
10347 _scsih_refresh_expander_links(struct MPT3SAS_ADAPTER *ioc,  in _scsih_refresh_expander_links()  argument
10355 		if ((mpt3sas_config_get_expander_pg1(ioc, &mpi_reply,  in _scsih_refresh_expander_links()
10357 			ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_refresh_expander_links()
10362 		mpt3sas_transport_update_links(ioc, sas_expander->sas_address,  in _scsih_refresh_expander_links()
10374 _scsih_scan_for_devices_after_reset(struct MPT3SAS_ADAPTER *ioc)  in _scsih_scan_for_devices_after_reset()  argument
10405 	ioc_info(ioc, "scan devices: start\n");  in _scsih_scan_for_devices_after_reset()
10407 	_scsih_sas_host_refresh(ioc);  in _scsih_scan_for_devices_after_reset()
10409 	ioc_info(ioc, "\tscan devices: expanders start\n");  in _scsih_scan_for_devices_after_reset()
10413 	while (!(mpt3sas_config_get_expander_pg0(ioc, &mpi_reply, &expander_pg0,  in _scsih_scan_for_devices_after_reset()
10418 			ioc_info(ioc, "\tbreak from expander scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10423 		spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_scan_for_devices_after_reset()
10426 		    ioc, le64_to_cpu(expander_pg0.SASAddress),  in _scsih_scan_for_devices_after_reset()
10427 		    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10428 		spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_scan_for_devices_after_reset()
10430 			_scsih_refresh_expander_links(ioc, expander_device,  in _scsih_scan_for_devices_after_reset()
10433 			ioc_info(ioc, "\tBEFORE adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10436 			_scsih_expander_add(ioc, handle);  in _scsih_scan_for_devices_after_reset()
10437 			ioc_info(ioc, "\tAFTER adding expander: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10443 	ioc_info(ioc, "\tscan devices: expanders complete\n");  in _scsih_scan_for_devices_after_reset()
10445 	if (!ioc->ir_firmware)  in _scsih_scan_for_devices_after_reset()
10448 	ioc_info(ioc, "\tscan devices: phys disk start\n");  in _scsih_scan_for_devices_after_reset()
10452 	while (!(mpt3sas_config_get_phys_disk_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10458 			ioc_info(ioc, "\tbreak from phys disk scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10464 		sas_device = mpt3sas_get_sdev_by_handle(ioc, handle);  in _scsih_scan_for_devices_after_reset()
10469 		if (mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10476 			ioc_info(ioc, "\tbreak from phys disk scan ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10481 		if (!_scsih_get_sas_address(ioc, parent_handle,  in _scsih_scan_for_devices_after_reset()
10483 			ioc_info(ioc, "\tBEFORE adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10487 			mpt3sas_transport_update_links(ioc, sas_address,  in _scsih_scan_for_devices_after_reset()
10490 			    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10491 			set_bit(handle, ioc->pd_handles);  in _scsih_scan_for_devices_after_reset()
10497 			while (_scsih_add_device(ioc, handle, retry_count++,  in _scsih_scan_for_devices_after_reset()
10501 			ioc_info(ioc, "\tAFTER adding phys disk: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10507 	ioc_info(ioc, "\tscan devices: phys disk complete\n");  in _scsih_scan_for_devices_after_reset()
10509 	ioc_info(ioc, "\tscan devices: volumes start\n");  in _scsih_scan_for_devices_after_reset()
10513 	while (!(mpt3sas_config_get_raid_volume_pg1(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10518 			ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10523 		spin_lock_irqsave(&ioc->raid_device_lock, flags);  in _scsih_scan_for_devices_after_reset()
10524 		raid_device = _scsih_raid_device_find_by_wwid(ioc,  in _scsih_scan_for_devices_after_reset()
10526 		spin_unlock_irqrestore(&ioc->raid_device_lock, flags);  in _scsih_scan_for_devices_after_reset()
10529 		if (mpt3sas_config_get_raid_volume_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10536 			ioc_info(ioc, "\tbreak from volume scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10546 			ioc_info(ioc, "\tBEFORE adding volume: handle (0x%04x)\n",  in _scsih_scan_for_devices_after_reset()
10548 			_scsih_sas_volume_add(ioc, &element);  in _scsih_scan_for_devices_after_reset()
10549 			ioc_info(ioc, "\tAFTER adding volume: handle (0x%04x)\n",  in _scsih_scan_for_devices_after_reset()
10554 	ioc_info(ioc, "\tscan devices: volumes complete\n");  in _scsih_scan_for_devices_after_reset()
10558 	ioc_info(ioc, "\tscan devices: end devices start\n");  in _scsih_scan_for_devices_after_reset()
10562 	while (!(mpt3sas_config_get_sas_device_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10568 			ioc_info(ioc, "\tbreak from end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10577 		sas_device = mpt3sas_get_sdev_by_addr(ioc,  in _scsih_scan_for_devices_after_reset()
10579 		    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10585 		if (!_scsih_get_sas_address(ioc, parent_handle, &sas_address)) {  in _scsih_scan_for_devices_after_reset()
10586 			ioc_info(ioc, "\tBEFORE adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10589 			mpt3sas_transport_update_links(ioc, sas_address, handle,  in _scsih_scan_for_devices_after_reset()
10591 			    mpt3sas_get_port_by_id(ioc, port_id, 0));  in _scsih_scan_for_devices_after_reset()
10597 			while (_scsih_add_device(ioc, handle, retry_count++,  in _scsih_scan_for_devices_after_reset()
10601 			ioc_info(ioc, "\tAFTER adding end device: handle (0x%04x), sas_addr(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10606 	ioc_info(ioc, "\tscan devices: end devices complete\n");  in _scsih_scan_for_devices_after_reset()
10607 	ioc_info(ioc, "\tscan devices: pcie end devices start\n");  in _scsih_scan_for_devices_after_reset()
10611 	while (!(mpt3sas_config_get_pcie_device_pg0(ioc, &mpi_reply,  in _scsih_scan_for_devices_after_reset()
10617 			ioc_info(ioc, "\tbreak from pcie end device scan: ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_scan_for_devices_after_reset()
10625 		pcie_device = mpt3sas_get_pdev_by_wwid(ioc,  in _scsih_scan_for_devices_after_reset()
10633 		_scsih_pcie_add_device(ioc, handle);  in _scsih_scan_for_devices_after_reset()
10635 		ioc_info(ioc, "\tAFTER adding pcie end device: handle (0x%04x), wwid(0x%016llx)\n",  in _scsih_scan_for_devices_after_reset()
10642 	ioc_info(ioc, "\tpcie devices: pcie end devices complete\n");  in _scsih_scan_for_devices_after_reset()
10643 	ioc_info(ioc, "scan devices: complete\n");  in _scsih_scan_for_devices_after_reset()
10652 void mpt3sas_scsih_pre_reset_handler(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_scsih_pre_reset_handler()  argument
10654 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_PRE_RESET\n", __func__));  in mpt3sas_scsih_pre_reset_handler()
10665 mpt3sas_scsih_clear_outstanding_scsi_tm_commands(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()  argument
10667 	dtmprintk(ioc,  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10668 	    ioc_info(ioc, "%s: clear outstanding scsi & tm cmds\n", __func__));  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10669 	if (ioc->scsih_cmds.status & MPT3_CMD_PENDING) {  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10670 		ioc->scsih_cmds.status |= MPT3_CMD_RESET;  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10671 		mpt3sas_base_free_smid(ioc, ioc->scsih_cmds.smid);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10672 		complete(&ioc->scsih_cmds.done);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10674 	if (ioc->tm_cmds.status & MPT3_CMD_PENDING) {  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10675 		ioc->tm_cmds.status |= MPT3_CMD_RESET;  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10676 		mpt3sas_base_free_smid(ioc, ioc->tm_cmds.smid);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10677 		complete(&ioc->tm_cmds.done);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10680 	memset(ioc->pend_os_device_add, 0, ioc->pend_os_device_add_sz);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10681 	memset(ioc->device_remove_in_progress, 0,  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10682 	       ioc->device_remove_in_progress_sz);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10683 	_scsih_fw_event_cleanup_queue(ioc);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10684 	_scsih_flush_running_cmds(ioc);  in mpt3sas_scsih_clear_outstanding_scsi_tm_commands()
10694 mpt3sas_scsih_reset_done_handler(struct MPT3SAS_ADAPTER *ioc)  in mpt3sas_scsih_reset_done_handler()  argument
10696 	dtmprintk(ioc, ioc_info(ioc, "%s: MPT3_IOC_DONE_RESET\n", __func__));  in mpt3sas_scsih_reset_done_handler()
10697 	if (!(disable_discovery > 0 && !ioc->sas_hba.num_phys)) {  in mpt3sas_scsih_reset_done_handler()
10698 		if (ioc->multipath_on_hba) {  in mpt3sas_scsih_reset_done_handler()
10699 			_scsih_sas_port_refresh(ioc);  in mpt3sas_scsih_reset_done_handler()
10700 			_scsih_update_vphys_after_reset(ioc);  in mpt3sas_scsih_reset_done_handler()
10702 		_scsih_prep_device_scan(ioc);  in mpt3sas_scsih_reset_done_handler()
10703 		_scsih_create_enclosure_list_after_reset(ioc);  in mpt3sas_scsih_reset_done_handler()
10704 		_scsih_search_responding_sas_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10705 		_scsih_search_responding_pcie_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10706 		_scsih_search_responding_raid_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10707 		_scsih_search_responding_expanders(ioc);  in mpt3sas_scsih_reset_done_handler()
10708 		_scsih_error_recovery_delete_devices(ioc);  in mpt3sas_scsih_reset_done_handler()
10719 _mpt3sas_fw_work(struct MPT3SAS_ADAPTER *ioc, struct fw_event_work *fw_event)  in _mpt3sas_fw_work()  argument
10721 	ioc->current_event = fw_event;  in _mpt3sas_fw_work()
10722 	_scsih_fw_event_del_from_list(ioc, fw_event);  in _mpt3sas_fw_work()
10725 	if (ioc->remove_host || ioc->pci_error_recovery) {  in _mpt3sas_fw_work()
10727 		ioc->current_event = NULL;  in _mpt3sas_fw_work()
10733 		mpt3sas_process_trigger_data(ioc,  in _mpt3sas_fw_work()
10738 		while (scsi_host_in_recovery(ioc->shost) ||  in _mpt3sas_fw_work()
10739 					 ioc->shost_recovery) {  in _mpt3sas_fw_work()
10744 			if (ioc->remove_host || ioc->fw_events_cleanup)  in _mpt3sas_fw_work()
10748 		_scsih_remove_unresponding_devices(ioc);  in _mpt3sas_fw_work()
10749 		_scsih_del_dirty_vphy(ioc);  in _mpt3sas_fw_work()
10750 		_scsih_del_dirty_port_entries(ioc);  in _mpt3sas_fw_work()
10751 		if (ioc->is_gen35_ioc)  in _mpt3sas_fw_work()
10752 			_scsih_update_device_qdepth(ioc);  in _mpt3sas_fw_work()
10753 		_scsih_scan_for_devices_after_reset(ioc);  in _mpt3sas_fw_work()
10764 		if (ioc->is_driver_loading)  in _mpt3sas_fw_work()
10765 			_scsih_complete_devices_scanning(ioc);  in _mpt3sas_fw_work()
10766 		_scsih_set_nvme_max_shutdown_latency(ioc);  in _mpt3sas_fw_work()
10769 		ioc->start_scan = 0;  in _mpt3sas_fw_work()
10771 			mpt3sas_base_update_missing_delay(ioc, missing_delay[0],  in _mpt3sas_fw_work()
10773 		dewtprintk(ioc,  in _mpt3sas_fw_work()
10774 			   ioc_info(ioc, "port enable: complete from worker thread\n"));  in _mpt3sas_fw_work()
10777 		_scsih_turn_on_pfa_led(ioc, fw_event->device_handle);  in _mpt3sas_fw_work()
10780 		_scsih_sas_topology_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10783 		if (ioc->logging_level & MPT_DEBUG_EVENT_WORK_TASK)  in _mpt3sas_fw_work()
10784 			_scsih_sas_device_status_change_event_debug(ioc,  in _mpt3sas_fw_work()
10789 		_scsih_sas_discovery_event(ioc, fw_event);  in _mpt3sas_fw_work()
10792 		_scsih_sas_device_discovery_error_event(ioc, fw_event);  in _mpt3sas_fw_work()
10795 		_scsih_sas_broadcast_primitive_event(ioc, fw_event);  in _mpt3sas_fw_work()
10798 		_scsih_sas_enclosure_dev_status_change_event(ioc,  in _mpt3sas_fw_work()
10802 		_scsih_sas_ir_config_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10805 		_scsih_sas_ir_volume_event(ioc, fw_event);  in _mpt3sas_fw_work()
10808 		_scsih_sas_ir_physical_disk_event(ioc, fw_event);  in _mpt3sas_fw_work()
10811 		_scsih_sas_ir_operation_status_event(ioc, fw_event);  in _mpt3sas_fw_work()
10814 		_scsih_pcie_device_status_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10817 		_scsih_pcie_enumeration_event(ioc, fw_event);  in _mpt3sas_fw_work()
10820 		_scsih_pcie_topology_change_event(ioc, fw_event);  in _mpt3sas_fw_work()
10821 		ioc->current_event = NULL;  in _mpt3sas_fw_work()
10826 	ioc->current_event = NULL;  in _mpt3sas_fw_work()
10843 	_mpt3sas_fw_work(fw_event->ioc, fw_event);  in _firmware_event_work()
10860 mpt3sas_scsih_event_callback(struct MPT3SAS_ADAPTER *ioc, u8 msix_index,  in mpt3sas_scsih_event_callback()  argument
10870 	if (ioc->pci_error_recovery)  in mpt3sas_scsih_event_callback()
10873 	mpi_reply = mpt3sas_base_get_reply_virt_addr(ioc, reply);  in mpt3sas_scsih_event_callback()
10876 		ioc_err(ioc, "mpi_reply not valid at %s:%d/%s()!\n",  in mpt3sas_scsih_event_callback()
10884 		mpt3sas_trigger_event(ioc, event, 0);  in mpt3sas_scsih_event_callback()
10898 		if (ioc->broadcast_aen_busy) {  in mpt3sas_scsih_event_callback()
10899 			ioc->broadcast_aen_pending++;  in mpt3sas_scsih_event_callback()
10902 			ioc->broadcast_aen_busy = 1;  in mpt3sas_scsih_event_callback()
10907 		_scsih_check_topo_delete_events(ioc,  in mpt3sas_scsih_event_callback()
10918 		if (ioc->shost_recovery)  in mpt3sas_scsih_event_callback()
10922 	_scsih_check_pcie_topo_remove_events(ioc,  in mpt3sas_scsih_event_callback()
10925 		if (ioc->shost_recovery)  in mpt3sas_scsih_event_callback()
10929 		_scsih_check_ir_config_unhide_events(ioc,  in mpt3sas_scsih_event_callback()
10934 		_scsih_check_volume_delete_events(ioc,  in mpt3sas_scsih_event_callback()
10943 		if (!ioc->is_warpdrive)  in mpt3sas_scsih_event_callback()
10956 …ioc_warn(ioc, "WarpDrive Warning: IO Throttling has occurred in the WarpDrive subsystem. Check War…  in mpt3sas_scsih_event_callback()
10959 …ioc_warn(ioc, "WarpDrive Warning: Program/Erase Cycles for the WarpDrive subsystem in degraded ran…  in mpt3sas_scsih_event_callback()
10962 …ioc_err(ioc, "WarpDrive Fatal Error: There are no Program/Erase Cycles for the WarpDrive subsystem…  in mpt3sas_scsih_event_callback()
10965 …ioc_err(ioc, "WarpDrive Fatal Error: The Backup Rail Monitor has failed on the WarpDrive subsystem…  in mpt3sas_scsih_event_callback()
10972 		_scsih_sas_device_status_change_event(ioc,  in mpt3sas_scsih_event_callback()
10986 		_scsih_temp_threshold_events(ioc,  in mpt3sas_scsih_event_callback()
10995 			ioc_notice(ioc, "Currently an active cable with ReceptacleID %d\n",  in mpt3sas_scsih_event_callback()
11005 			ioc_notice(ioc, "Currently a cable with ReceptacleID %d\n",  in mpt3sas_scsih_event_callback()
11021 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in mpt3sas_scsih_event_callback()
11027 	fw_event->ioc = ioc;  in mpt3sas_scsih_event_callback()
11031 	_scsih_fw_event_add(ioc, fw_event);  in mpt3sas_scsih_event_callback()
11045 _scsih_expander_node_remove(struct MPT3SAS_ADAPTER *ioc,  in _scsih_expander_node_remove()  argument
11055 		if (ioc->shost_recovery)  in _scsih_expander_node_remove()
11059 			mpt3sas_device_remove_by_sas_address(ioc,  in _scsih_expander_node_remove()
11066 			mpt3sas_expander_remove(ioc,  in _scsih_expander_node_remove()
11073 	mpt3sas_transport_port_remove(ioc, sas_expander->sas_address,  in _scsih_expander_node_remove()
11076 	ioc_info(ioc,  in _scsih_expander_node_remove()
11082 	spin_lock_irqsave(&ioc->sas_node_lock, flags);  in _scsih_expander_node_remove()
11084 	spin_unlock_irqrestore(&ioc->sas_node_lock, flags);  in _scsih_expander_node_remove()
11099 _scsih_nvme_shutdown(struct MPT3SAS_ADAPTER *ioc)  in _scsih_nvme_shutdown()  argument
11106 	if (list_empty(&ioc->pcie_device_list))  in _scsih_nvme_shutdown()
11109 	mutex_lock(&ioc->scsih_cmds.mutex);  in _scsih_nvme_shutdown()
11111 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _scsih_nvme_shutdown()
11112 		ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);  in _scsih_nvme_shutdown()
11116 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _scsih_nvme_shutdown()
11118 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _scsih_nvme_shutdown()
11120 		ioc_err(ioc,  in _scsih_nvme_shutdown()
11122 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_nvme_shutdown()
11126 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_nvme_shutdown()
11127 	ioc->scsih_cmds.smid = smid;  in _scsih_nvme_shutdown()
11132 	init_completion(&ioc->scsih_cmds.done);  in _scsih_nvme_shutdown()
11133 	ioc->put_smid_default(ioc, smid);  in _scsih_nvme_shutdown()
11135 	ioc_info(ioc,  in _scsih_nvme_shutdown()
11137 		ioc->max_shutdown_latency);  in _scsih_nvme_shutdown()
11138 	wait_for_completion_timeout(&ioc->scsih_cmds.done,  in _scsih_nvme_shutdown()
11139 			ioc->max_shutdown_latency*HZ);  in _scsih_nvme_shutdown()
11141 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _scsih_nvme_shutdown()
11142 		ioc_err(ioc, "%s: timeout\n", __func__);  in _scsih_nvme_shutdown()
11146 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _scsih_nvme_shutdown()
11147 		mpi_reply = ioc->scsih_cmds.reply;  in _scsih_nvme_shutdown()
11148 		ioc_info(ioc, "Io Unit Control shutdown (complete):"  in _scsih_nvme_shutdown()
11154 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_nvme_shutdown()
11155 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _scsih_nvme_shutdown()
11167 _scsih_ir_shutdown(struct MPT3SAS_ADAPTER *ioc)  in _scsih_ir_shutdown()  argument
11174 	if (!ioc->ir_firmware)  in _scsih_ir_shutdown()
11178 	if (list_empty(&ioc->raid_device_list))  in _scsih_ir_shutdown()
11181 	mutex_lock(&ioc->scsih_cmds.mutex);  in _scsih_ir_shutdown()
11183 	if (ioc->scsih_cmds.status != MPT3_CMD_NOT_USED) {  in _scsih_ir_shutdown()
11184 		ioc_err(ioc, "%s: scsih_cmd in use\n", __func__);  in _scsih_ir_shutdown()
11187 	ioc->scsih_cmds.status = MPT3_CMD_PENDING;  in _scsih_ir_shutdown()
11189 	smid = mpt3sas_base_get_smid(ioc, ioc->scsih_cb_idx);  in _scsih_ir_shutdown()
11191 		ioc_err(ioc, "%s: failed obtaining a smid\n", __func__);  in _scsih_ir_shutdown()
11192 		ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_shutdown()
11196 	mpi_request = mpt3sas_base_get_msg_frame(ioc, smid);  in _scsih_ir_shutdown()
11197 	ioc->scsih_cmds.smid = smid;  in _scsih_ir_shutdown()
11203 	if (!ioc->hide_ir_msg)  in _scsih_ir_shutdown()
11204 		ioc_info(ioc, "IR shutdown (sending)\n");  in _scsih_ir_shutdown()
11205 	init_completion(&ioc->scsih_cmds.done);  in _scsih_ir_shutdown()
11206 	ioc->put_smid_default(ioc, smid);  in _scsih_ir_shutdown()
11207 	wait_for_completion_timeout(&ioc->scsih_cmds.done, 10*HZ);  in _scsih_ir_shutdown()
11209 	if (!(ioc->scsih_cmds.status & MPT3_CMD_COMPLETE)) {  in _scsih_ir_shutdown()
11210 		ioc_err(ioc, "%s: timeout\n", __func__);  in _scsih_ir_shutdown()
11214 	if (ioc->scsih_cmds.status & MPT3_CMD_REPLY_VALID) {  in _scsih_ir_shutdown()
11215 		mpi_reply = ioc->scsih_cmds.reply;  in _scsih_ir_shutdown()
11216 		if (!ioc->hide_ir_msg)  in _scsih_ir_shutdown()
11217 			ioc_info(ioc, "IR shutdown (complete): ioc_status(0x%04x), loginfo(0x%08x)\n",  in _scsih_ir_shutdown()
11223 	ioc->scsih_cmds.status = MPT3_CMD_NOT_USED;  in _scsih_ir_shutdown()
11224 	mutex_unlock(&ioc->scsih_cmds.mutex);  in _scsih_ir_shutdown()
11238 	struct Scsi_Host **shost, struct MPT3SAS_ADAPTER **ioc)  in _scsih_get_shost_and_ioc()  argument
11246 	*ioc = shost_priv(*shost);  in _scsih_get_shost_and_ioc()
11247 	if (*ioc == NULL) {  in _scsih_get_shost_and_ioc()
11264 	struct MPT3SAS_ADAPTER *ioc;  in scsih_remove()  local
11274 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_remove()
11277 	ioc->remove_host = 1;  in scsih_remove()
11280 		mpt3sas_base_pause_mq_polling(ioc);  in scsih_remove()
11281 		_scsih_flush_running_cmds(ioc);  in scsih_remove()
11284 	_scsih_fw_event_cleanup_queue(ioc);  in scsih_remove()
11286 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in scsih_remove()
11287 	wq = ioc->firmware_event_thread;  in scsih_remove()
11288 	ioc->firmware_event_thread = NULL;  in scsih_remove()
11289 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in scsih_remove()
11296 	if (ioc->is_aero_ioc)  in scsih_remove()
11297 		mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,  in scsih_remove()
11298 				&ioc->ioc_pg1_copy);  in scsih_remove()
11300 	_scsih_ir_shutdown(ioc);  in scsih_remove()
11301 	mpt3sas_destroy_debugfs(ioc);  in scsih_remove()
11303 	list_for_each_entry_safe(raid_device, next, &ioc->raid_device_list,  in scsih_remove()
11311 		ioc_info(ioc, "removing handle(0x%04x), wwid(0x%016llx)\n",  in scsih_remove()
11313 		_scsih_raid_device_remove(ioc, raid_device);  in scsih_remove()
11315 	list_for_each_entry_safe(pcie_device, pcienext, &ioc->pcie_device_list,  in scsih_remove()
11317 		_scsih_pcie_device_remove_from_sml(ioc, pcie_device);  in scsih_remove()
11324 	   &ioc->sas_hba.sas_port_list, port_list) {  in scsih_remove()
11327 			mpt3sas_device_remove_by_sas_address(ioc,  in scsih_remove()
11334 			mpt3sas_expander_remove(ioc,  in scsih_remove()
11340 	    &ioc->port_table_list, list) {  in scsih_remove()
11346 	if (ioc->sas_hba.num_phys) {  in scsih_remove()
11347 		kfree(ioc->sas_hba.phy);  in scsih_remove()
11348 		ioc->sas_hba.phy = NULL;  in scsih_remove()
11349 		ioc->sas_hba.num_phys = 0;  in scsih_remove()
11352 	mpt3sas_base_detach(ioc);  in scsih_remove()
11354 	list_del(&ioc->list);  in scsih_remove()
11367 	struct MPT3SAS_ADAPTER *ioc;  in scsih_shutdown()  local
11372 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_shutdown()
11375 	ioc->remove_host = 1;  in scsih_shutdown()
11378 		mpt3sas_base_pause_mq_polling(ioc);  in scsih_shutdown()
11379 		_scsih_flush_running_cmds(ioc);  in scsih_shutdown()
11382 	_scsih_fw_event_cleanup_queue(ioc);  in scsih_shutdown()
11384 	spin_lock_irqsave(&ioc->fw_event_lock, flags);  in scsih_shutdown()
11385 	wq = ioc->firmware_event_thread;  in scsih_shutdown()
11386 	ioc->firmware_event_thread = NULL;  in scsih_shutdown()
11387 	spin_unlock_irqrestore(&ioc->fw_event_lock, flags);  in scsih_shutdown()
11394 	if (ioc->is_aero_ioc)  in scsih_shutdown()
11395 		mpt3sas_config_set_ioc_pg1(ioc, &mpi_reply,  in scsih_shutdown()
11396 				&ioc->ioc_pg1_copy);  in scsih_shutdown()
11398 	_scsih_ir_shutdown(ioc);  in scsih_shutdown()
11399 	_scsih_nvme_shutdown(ioc);  in scsih_shutdown()
11400 	mpt3sas_base_mask_interrupts(ioc);  in scsih_shutdown()
11401 	mpt3sas_base_stop_watchdog(ioc);  in scsih_shutdown()
11402 	ioc->shost_recovery = 1;  in scsih_shutdown()
11403 	mpt3sas_base_make_ioc_ready(ioc, SOFT_RESET);  in scsih_shutdown()
11404 	ioc->shost_recovery = 0;  in scsih_shutdown()
11405 	mpt3sas_base_free_irq(ioc);  in scsih_shutdown()
11406 	mpt3sas_base_disable_msix(ioc);  in scsih_shutdown()
11419 _scsih_probe_boot_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_boot_devices()  argument
11435 	if (!ioc->bios_pg3.BiosVersion)  in _scsih_probe_boot_devices()
11439 	if (ioc->req_boot_device.device) {  in _scsih_probe_boot_devices()
11440 		device =  ioc->req_boot_device.device;  in _scsih_probe_boot_devices()
11441 		channel = ioc->req_boot_device.channel;  in _scsih_probe_boot_devices()
11442 	} else if (ioc->req_alt_boot_device.device) {  in _scsih_probe_boot_devices()
11443 		device =  ioc->req_alt_boot_device.device;  in _scsih_probe_boot_devices()
11444 		channel = ioc->req_alt_boot_device.channel;  in _scsih_probe_boot_devices()
11445 	} else if (ioc->current_boot_device.device) {  in _scsih_probe_boot_devices()
11446 		device =  ioc->current_boot_device.device;  in _scsih_probe_boot_devices()
11447 		channel = ioc->current_boot_device.channel;  in _scsih_probe_boot_devices()
11462 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_probe_boot_devices()
11465 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_probe_boot_devices()
11475 		spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in _scsih_probe_boot_devices()
11477 		list_move_tail(&pcie_device->list, &ioc->pcie_device_list);  in _scsih_probe_boot_devices()
11478 		spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in _scsih_probe_boot_devices()
11479 		rc = scsi_add_device(ioc->shost, PCIE_CHANNEL, tid, 0);  in _scsih_probe_boot_devices()
11481 			_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_probe_boot_devices()
11491 		spin_lock_irqsave(&ioc->sas_device_lock, flags);  in _scsih_probe_boot_devices()
11496 		list_move_tail(&sas_device->list, &ioc->sas_device_list);  in _scsih_probe_boot_devices()
11497 		spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in _scsih_probe_boot_devices()
11499 		if (ioc->hide_drives)  in _scsih_probe_boot_devices()
11505 		if (!mpt3sas_transport_port_add(ioc, handle,  in _scsih_probe_boot_devices()
11507 			_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_boot_devices()
11509 			if (!ioc->is_driver_loading) {  in _scsih_probe_boot_devices()
11510 				mpt3sas_transport_port_remove(ioc,  in _scsih_probe_boot_devices()
11513 				_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_boot_devices()
11526 _scsih_probe_raid(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_raid()  argument
11532 	    &ioc->raid_device_list, list) {  in _scsih_probe_raid()
11535 		rc = scsi_add_device(ioc->shost, RAID_CHANNEL,  in _scsih_probe_raid()
11538 			_scsih_raid_device_remove(ioc, raid_device);  in _scsih_probe_raid()
11542 static struct _sas_device *get_next_sas_device(struct MPT3SAS_ADAPTER *ioc)  in get_next_sas_device()  argument
11547 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in get_next_sas_device()
11548 	if (!list_empty(&ioc->sas_device_init_list)) {  in get_next_sas_device()
11549 		sas_device = list_first_entry(&ioc->sas_device_init_list,  in get_next_sas_device()
11553 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in get_next_sas_device()
11558 static void sas_device_make_active(struct MPT3SAS_ADAPTER *ioc,  in sas_device_make_active()  argument
11563 	spin_lock_irqsave(&ioc->sas_device_lock, flags);  in sas_device_make_active()
11579 	list_add_tail(&sas_device->list, &ioc->sas_device_list);  in sas_device_make_active()
11581 	spin_unlock_irqrestore(&ioc->sas_device_lock, flags);  in sas_device_make_active()
11591 _scsih_probe_sas(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_sas()  argument
11595 	if (ioc->hide_drives)  in _scsih_probe_sas()
11598 	while ((sas_device = get_next_sas_device(ioc))) {  in _scsih_probe_sas()
11599 		if (!mpt3sas_transport_port_add(ioc, sas_device->handle,  in _scsih_probe_sas()
11601 			_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_sas()
11611 			if (!ioc->is_driver_loading) {  in _scsih_probe_sas()
11612 				mpt3sas_transport_port_remove(ioc,  in _scsih_probe_sas()
11616 				_scsih_sas_device_remove(ioc, sas_device);  in _scsih_probe_sas()
11621 		sas_device_make_active(ioc, sas_device);  in _scsih_probe_sas()
11635 static struct _pcie_device *get_next_pcie_device(struct MPT3SAS_ADAPTER *ioc)  in get_next_pcie_device()  argument
11640 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in get_next_pcie_device()
11641 	if (!list_empty(&ioc->pcie_device_init_list)) {  in get_next_pcie_device()
11642 		pcie_device = list_first_entry(&ioc->pcie_device_init_list,  in get_next_pcie_device()
11646 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in get_next_pcie_device()
11659 static void pcie_device_make_active(struct MPT3SAS_ADAPTER *ioc,  in pcie_device_make_active()  argument
11664 	spin_lock_irqsave(&ioc->pcie_device_lock, flags);  in pcie_device_make_active()
11671 	list_add_tail(&pcie_device->list, &ioc->pcie_device_list);  in pcie_device_make_active()
11673 	spin_unlock_irqrestore(&ioc->pcie_device_lock, flags);  in pcie_device_make_active()
11683 _scsih_probe_pcie(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_pcie()  argument
11689 	while ((pcie_device = get_next_pcie_device(ioc))) {  in _scsih_probe_pcie()
11696 			pcie_device_make_active(ioc, pcie_device);  in _scsih_probe_pcie()
11700 		rc = scsi_add_device(ioc->shost, PCIE_CHANNEL,  in _scsih_probe_pcie()
11703 			_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_probe_pcie()
11713 			if (!ioc->is_driver_loading) {  in _scsih_probe_pcie()
11717 				_scsih_pcie_device_remove(ioc, pcie_device);  in _scsih_probe_pcie()
11722 		pcie_device_make_active(ioc, pcie_device);  in _scsih_probe_pcie()
11734 _scsih_probe_devices(struct MPT3SAS_ADAPTER *ioc)  in _scsih_probe_devices()  argument
11738 	if (!(ioc->facts.ProtocolFlags & MPI2_IOCFACTS_PROTOCOL_SCSI_INITIATOR))  in _scsih_probe_devices()
11741 	_scsih_probe_boot_devices(ioc);  in _scsih_probe_devices()
11743 	if (ioc->ir_firmware) {  in _scsih_probe_devices()
11745 		    le16_to_cpu(ioc->ioc_pg8.IRVolumeMappingFlags) &  in _scsih_probe_devices()
11749 			_scsih_probe_raid(ioc);  in _scsih_probe_devices()
11750 			_scsih_probe_sas(ioc);  in _scsih_probe_devices()
11752 			_scsih_probe_sas(ioc);  in _scsih_probe_devices()
11753 			_scsih_probe_raid(ioc);  in _scsih_probe_devices()
11756 		_scsih_probe_sas(ioc);  in _scsih_probe_devices()
11757 		_scsih_probe_pcie(ioc);  in _scsih_probe_devices()
11772 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_scan_start()  local
11775 		mpt3sas_enable_diag_buffer(ioc, diag_buffer_enable);  in scsih_scan_start()
11776 	else if (ioc->manu_pg11.HostTraceBufferMaxSizeKB != 0)  in scsih_scan_start()
11777 		mpt3sas_enable_diag_buffer(ioc, 1);  in scsih_scan_start()
11782 	ioc->start_scan = 1;  in scsih_scan_start()
11783 	rc = mpt3sas_port_enable(ioc);  in scsih_scan_start()
11786 		ioc_info(ioc, "port enable: FAILED\n");  in scsih_scan_start()
11796 static void _scsih_complete_devices_scanning(struct MPT3SAS_ADAPTER *ioc)  in _scsih_complete_devices_scanning()  argument
11799 	if (ioc->wait_for_discovery_to_complete) {  in _scsih_complete_devices_scanning()
11800 		ioc->wait_for_discovery_to_complete = 0;  in _scsih_complete_devices_scanning()
11801 		_scsih_probe_devices(ioc);  in _scsih_complete_devices_scanning()
11804 	mpt3sas_base_start_watchdog(ioc);  in _scsih_complete_devices_scanning()
11805 	ioc->is_driver_loading = 0;  in _scsih_complete_devices_scanning()
11820 	struct MPT3SAS_ADAPTER *ioc = shost_priv(shost);  in scsih_scan_finished()  local
11825 		ioc->is_driver_loading = 0;  in scsih_scan_finished()
11826 		ioc->wait_for_discovery_to_complete = 0;  in scsih_scan_finished()
11831 		ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11832 		ioc_info(ioc, "port enable: FAILED with timeout (timeout=300s)\n");  in scsih_scan_finished()
11833 		ioc->is_driver_loading = 0;  in scsih_scan_finished()
11837 	if (ioc->start_scan) {  in scsih_scan_finished()
11838 		ioc_state = mpt3sas_base_get_iocstate(ioc, 0);  in scsih_scan_finished()
11840 			mpt3sas_print_fault_code(ioc, ioc_state &  in scsih_scan_finished()
11846 			mpt3sas_base_coredump_info(ioc, ioc_state &  in scsih_scan_finished()
11848 			mpt3sas_base_wait_for_coredump_completion(ioc, __func__);  in scsih_scan_finished()
11855 	if (ioc->port_enable_cmds.status & MPT3_CMD_RESET) {  in scsih_scan_finished()
11856 		ioc_info(ioc,  in scsih_scan_finished()
11858 		ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11861 	if (ioc->start_scan_failed) {  in scsih_scan_finished()
11862 		ioc_info(ioc, "port enable: FAILED with (ioc_status=0x%08x)\n",  in scsih_scan_finished()
11863 			 ioc->start_scan_failed);  in scsih_scan_finished()
11864 		ioc->is_driver_loading = 0;  in scsih_scan_finished()
11865 		ioc->wait_for_discovery_to_complete = 0;  in scsih_scan_finished()
11866 		ioc->remove_host = 1;  in scsih_scan_finished()
11870 	ioc_info(ioc, "port enable: SUCCESS\n");  in scsih_scan_finished()
11871 	ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11872 	_scsih_complete_devices_scanning(ioc);  in scsih_scan_finished()
11876 		ioc->port_enable_cmds.status = MPT3_CMD_NOT_USED;  in scsih_scan_finished()
11877 		if (mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET))  in scsih_scan_finished()
11878 			ioc->is_driver_loading = 0;  in scsih_scan_finished()
11889 	struct MPT3SAS_ADAPTER *ioc =  in scsih_map_queues()  local
11893 	int nr_msix_vectors = ioc->iopoll_q_start_index;  in scsih_map_queues()
11894 	int iopoll_q_count = ioc->reply_queue_count - nr_msix_vectors;  in scsih_map_queues()
11905 			    nr_msix_vectors - ioc->high_iops_queues;  in scsih_map_queues()
11906 			offset = ioc->high_iops_queues;  in scsih_map_queues()
11919 			blk_mq_pci_map_queues(map, ioc->pdev, offset);  in scsih_map_queues()
12088 	struct MPT3SAS_ADAPTER *ioc;  in _scsih_probe()  local
12121 		ioc = shost_priv(shost);  in _scsih_probe()
12122 		memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));  in _scsih_probe()
12123 		ioc->hba_mpi_version_belonged = hba_mpi_version;  in _scsih_probe()
12124 		ioc->id = mpt2_ids++;  in _scsih_probe()
12125 		sprintf(ioc->driver_name, "%s", MPT2SAS_DRIVER_NAME);  in _scsih_probe()
12128 			ioc->is_warpdrive = 1;  in _scsih_probe()
12129 			ioc->hide_ir_msg = 1;  in _scsih_probe()
12133 			ioc->is_mcpu_endpoint = 1;  in _scsih_probe()
12136 			ioc->mfg_pg10_hide_flag = MFG_PAGE10_EXPOSE_ALL_DISKS;  in _scsih_probe()
12141 			ioc->multipath_on_hba = 0;  in _scsih_probe()
12143 			ioc->multipath_on_hba = 1;  in _scsih_probe()
12153 		ioc = shost_priv(shost);  in _scsih_probe()
12154 		memset(ioc, 0, sizeof(struct MPT3SAS_ADAPTER));  in _scsih_probe()
12155 		ioc->hba_mpi_version_belonged = hba_mpi_version;  in _scsih_probe()
12156 		ioc->id = mpt3_ids++;  in _scsih_probe()
12157 		sprintf(ioc->driver_name, "%s", MPT3SAS_DRIVER_NAME);  in _scsih_probe()
12167 			ioc->is_gen35_ioc = 1;  in _scsih_probe()
12190 			ioc->is_aero_ioc = ioc->is_gen35_ioc = 1;  in _scsih_probe()
12193 			ioc->is_gen35_ioc = ioc->is_aero_ioc = 0;  in _scsih_probe()
12195 		if ((ioc->hba_mpi_version_belonged == MPI25_VERSION &&  in _scsih_probe()
12197 			(ioc->hba_mpi_version_belonged == MPI26_VERSION)) {  in _scsih_probe()
12198 			ioc->combined_reply_queue = 1;  in _scsih_probe()
12199 			if (ioc->is_gen35_ioc)  in _scsih_probe()
12200 				ioc->combined_reply_index_count =  in _scsih_probe()
12203 				ioc->combined_reply_index_count =  in _scsih_probe()
12207 		switch (ioc->is_gen35_ioc) {  in _scsih_probe()
12210 				ioc->multipath_on_hba = 0;  in _scsih_probe()
12212 				ioc->multipath_on_hba = 1;  in _scsih_probe()
12216 				ioc->multipath_on_hba = 1;  in _scsih_probe()
12218 				ioc->multipath_on_hba = 0;  in _scsih_probe()
12229 	INIT_LIST_HEAD(&ioc->list);  in _scsih_probe()
12231 	list_add_tail(&ioc->list, &mpt3sas_ioc_list);  in _scsih_probe()
12233 	ioc->shost = shost;  in _scsih_probe()
12234 	ioc->pdev = pdev;  in _scsih_probe()
12235 	ioc->scsi_io_cb_idx = scsi_io_cb_idx;  in _scsih_probe()
12236 	ioc->tm_cb_idx = tm_cb_idx;  in _scsih_probe()
12237 	ioc->ctl_cb_idx = ctl_cb_idx;  in _scsih_probe()
12238 	ioc->base_cb_idx = base_cb_idx;  in _scsih_probe()
12239 	ioc->port_enable_cb_idx = port_enable_cb_idx;  in _scsih_probe()
12240 	ioc->transport_cb_idx = transport_cb_idx;  in _scsih_probe()
12241 	ioc->scsih_cb_idx = scsih_cb_idx;  in _scsih_probe()
12242 	ioc->config_cb_idx = config_cb_idx;  in _scsih_probe()
12243 	ioc->tm_tr_cb_idx = tm_tr_cb_idx;  in _scsih_probe()
12244 	ioc->tm_tr_volume_cb_idx = tm_tr_volume_cb_idx;  in _scsih_probe()
12245 	ioc->tm_sas_control_cb_idx = tm_sas_control_cb_idx;  in _scsih_probe()
12246 	ioc->logging_level = logging_level;  in _scsih_probe()
12247 	ioc->schedule_dead_ioc_flush_running_cmds = &_scsih_flush_running_cmds;  in _scsih_probe()
12249 	ioc->max_shutdown_latency = IO_UNIT_CONTROL_SHUTDOWN_TIMEOUT;  in _scsih_probe()
12253 	ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_MEMMOVE;  in _scsih_probe()
12255 	ioc->drv_support_bitmap |= MPT_DRV_SUPPORT_BITMAP_ADDNLQUERY;  in _scsih_probe()
12257 	ioc->enable_sdev_max_qd = enable_sdev_max_qd;  in _scsih_probe()
12260 	mutex_init(&ioc->reset_in_progress_mutex);  in _scsih_probe()
12262 	mutex_init(&ioc->pci_access_mutex);  in _scsih_probe()
12263 	spin_lock_init(&ioc->ioc_reset_in_progress_lock);  in _scsih_probe()
12264 	spin_lock_init(&ioc->scsi_lookup_lock);  in _scsih_probe()
12265 	spin_lock_init(&ioc->sas_device_lock);  in _scsih_probe()
12266 	spin_lock_init(&ioc->sas_node_lock);  in _scsih_probe()
12267 	spin_lock_init(&ioc->fw_event_lock);  in _scsih_probe()
12268 	spin_lock_init(&ioc->raid_device_lock);  in _scsih_probe()
12269 	spin_lock_init(&ioc->pcie_device_lock);  in _scsih_probe()
12270 	spin_lock_init(&ioc->diag_trigger_lock);  in _scsih_probe()
12272 	INIT_LIST_HEAD(&ioc->sas_device_list);  in _scsih_probe()
12273 	INIT_LIST_HEAD(&ioc->sas_device_init_list);  in _scsih_probe()
12274 	INIT_LIST_HEAD(&ioc->sas_expander_list);  in _scsih_probe()
12275 	INIT_LIST_HEAD(&ioc->enclosure_list);  in _scsih_probe()
12276 	INIT_LIST_HEAD(&ioc->pcie_device_list);  in _scsih_probe()
12277 	INIT_LIST_HEAD(&ioc->pcie_device_init_list);  in _scsih_probe()
12278 	INIT_LIST_HEAD(&ioc->fw_event_list);  in _scsih_probe()
12279 	INIT_LIST_HEAD(&ioc->raid_device_list);  in _scsih_probe()
12280 	INIT_LIST_HEAD(&ioc->sas_hba.sas_port_list);  in _scsih_probe()
12281 	INIT_LIST_HEAD(&ioc->delayed_tr_list);  in _scsih_probe()
12282 	INIT_LIST_HEAD(&ioc->delayed_sc_list);  in _scsih_probe()
12283 	INIT_LIST_HEAD(&ioc->delayed_event_ack_list);  in _scsih_probe()
12284 	INIT_LIST_HEAD(&ioc->delayed_tr_volume_list);  in _scsih_probe()
12285 	INIT_LIST_HEAD(&ioc->reply_queue_list);  in _scsih_probe()
12286 	INIT_LIST_HEAD(&ioc->port_table_list);  in _scsih_probe()
12288 	sprintf(ioc->name, "%s_cm%d", ioc->driver_name, ioc->id);  in _scsih_probe()
12294 	shost->unique_id = ioc->id;  in _scsih_probe()
12296 	if (ioc->is_mcpu_endpoint) {  in _scsih_probe()
12299 		ioc_info(ioc, "The max_sectors value is set to %d\n",  in _scsih_probe()
12305 …ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767. Assigning value of 6…  in _scsih_probe()
12309 …ioc_warn(ioc, "Invalid value %d passed for max_sectors, range is 64 to 32767.Assigning default val…  in _scsih_probe()
12313 				ioc_info(ioc, "The max_sectors value is set to %d\n",  in _scsih_probe()
12329 	snprintf(ioc->firmware_event_name, sizeof(ioc->firmware_event_name),  in _scsih_probe()
12330 	    "fw_event_%s%d", ioc->driver_name, ioc->id);  in _scsih_probe()
12331 	ioc->firmware_event_thread = alloc_ordered_workqueue(  in _scsih_probe()
12332 	    ioc->firmware_event_name, 0);  in _scsih_probe()
12333 	if (!ioc->firmware_event_thread) {  in _scsih_probe()
12334 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_probe()
12342 	if (ioc->is_gen35_ioc && host_tagset_enable)  in _scsih_probe()
12345 	ioc->is_driver_loading = 1;  in _scsih_probe()
12346 	if ((mpt3sas_base_attach(ioc))) {  in _scsih_probe()
12347 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_probe()
12353 	if (ioc->is_warpdrive) {  in _scsih_probe()
12354 		if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_EXPOSE_ALL_DISKS)  in _scsih_probe()
12355 			ioc->hide_drives = 0;  in _scsih_probe()
12356 		else if (ioc->mfg_pg10_hide_flag ==  MFG_PAGE10_HIDE_ALL_DISKS)  in _scsih_probe()
12357 			ioc->hide_drives = 1;  in _scsih_probe()
12359 			if (mpt3sas_get_num_volumes(ioc))  in _scsih_probe()
12360 				ioc->hide_drives = 1;  in _scsih_probe()
12362 				ioc->hide_drives = 0;  in _scsih_probe()
12365 		ioc->hide_drives = 0;  in _scsih_probe()
12371 		    ioc->reply_queue_count - ioc->high_iops_queues;  in _scsih_probe()
12374 		    ioc->reply_queue_count - ioc->iopoll_q_start_index;  in _scsih_probe()
12378 		dev_info(&ioc->pdev->dev,  in _scsih_probe()
12385 		ioc_err(ioc, "failure at %s:%d/%s()!\n",  in _scsih_probe()
12391 	mpt3sas_setup_debugfs(ioc);  in _scsih_probe()
12394 	mpt3sas_base_detach(ioc);  in _scsih_probe()
12396 	destroy_workqueue(ioc->firmware_event_thread);  in _scsih_probe()
12399 	list_del(&ioc->list);  in _scsih_probe()
12416 	struct MPT3SAS_ADAPTER *ioc;  in scsih_suspend()  local
12419 	rc = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);  in scsih_suspend()
12423 	mpt3sas_base_stop_watchdog(ioc);  in scsih_suspend()
12425 	_scsih_nvme_shutdown(ioc);  in scsih_suspend()
12426 	ioc_info(ioc, "pdev=0x%p, slot=%s, entering operating state\n",  in scsih_suspend()
12429 	mpt3sas_base_free_resources(ioc);  in scsih_suspend()
12444 	struct MPT3SAS_ADAPTER *ioc;  in scsih_resume()  local
12448 	r = _scsih_get_shost_and_ioc(pdev, &shost, &ioc);  in scsih_resume()
12452 	ioc_info(ioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",  in scsih_resume()
12455 	ioc->pdev = pdev;  in scsih_resume()
12456 	r = mpt3sas_base_map_resources(ioc);  in scsih_resume()
12459 	ioc_info(ioc, "Issuing Hard Reset as part of OS Resume\n");  in scsih_resume()
12460 	mpt3sas_base_hard_reset_handler(ioc, SOFT_RESET);  in scsih_resume()
12462 	mpt3sas_base_start_watchdog(ioc);  in scsih_resume()
12479 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_error_detected()  local
12481 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_error_detected()
12484 	ioc_info(ioc, "PCI error: detected callback, state(%d)!!\n", state);  in scsih_pci_error_detected()
12491 		ioc->pci_error_recovery = 1;  in scsih_pci_error_detected()
12492 		scsi_block_requests(ioc->shost);  in scsih_pci_error_detected()
12493 		mpt3sas_base_stop_watchdog(ioc);  in scsih_pci_error_detected()
12494 		mpt3sas_base_free_resources(ioc);  in scsih_pci_error_detected()
12498 		ioc->pci_error_recovery = 1;  in scsih_pci_error_detected()
12499 		mpt3sas_base_stop_watchdog(ioc);  in scsih_pci_error_detected()
12500 		mpt3sas_base_pause_mq_polling(ioc);  in scsih_pci_error_detected()
12501 		_scsih_flush_running_cmds(ioc);  in scsih_pci_error_detected()
12519 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_slot_reset()  local
12522 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_slot_reset()
12525 	ioc_info(ioc, "PCI error: slot reset callback!!\n");  in scsih_pci_slot_reset()
12527 	ioc->pci_error_recovery = 0;  in scsih_pci_slot_reset()
12528 	ioc->pdev = pdev;  in scsih_pci_slot_reset()
12530 	rc = mpt3sas_base_map_resources(ioc);  in scsih_pci_slot_reset()
12534 	ioc_info(ioc, "Issuing Hard Reset as part of PCI Slot Reset\n");  in scsih_pci_slot_reset()
12535 	rc = mpt3sas_base_hard_reset_handler(ioc, FORCE_BIG_HAMMER);  in scsih_pci_slot_reset()
12537 	ioc_warn(ioc, "hard reset: %s\n",  in scsih_pci_slot_reset()
12558 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_resume()  local
12560 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_resume()
12563 	ioc_info(ioc, "PCI error: resume callback!!\n");  in scsih_pci_resume()
12565 	mpt3sas_base_start_watchdog(ioc);  in scsih_pci_resume()
12566 	scsi_unblock_requests(ioc->shost);  in scsih_pci_resume()
12577 	struct MPT3SAS_ADAPTER *ioc;  in scsih_pci_mmio_enabled()  local
12579 	if (_scsih_get_shost_and_ioc(pdev, &shost, &ioc))  in scsih_pci_mmio_enabled()
12582 	ioc_info(ioc, "PCI error: mmio enabled callback!!\n");  in scsih_pci_mmio_enabled()