Lines Matching refs:dd
108 static int mtip_block_initialize(struct driver_data *dd);
133 static bool mtip_check_surprise_removal(struct driver_data *dd) in mtip_check_surprise_removal() argument
137 if (dd->sr) in mtip_check_surprise_removal()
141 pci_read_config_word(dd->pdev, 0x00, &vendor_id); in mtip_check_surprise_removal()
143 dd->sr = true; in mtip_check_surprise_removal()
144 if (dd->disk) in mtip_check_surprise_removal()
145 blk_mark_disk_dead(dd->disk); in mtip_check_surprise_removal()
152 static struct mtip_cmd *mtip_cmd_from_tag(struct driver_data *dd, in mtip_cmd_from_tag() argument
155 return blk_mq_rq_to_pdu(blk_mq_tag_to_rq(dd->tags.tags[0], tag)); in mtip_cmd_from_tag()
167 static int mtip_hba_reset(struct driver_data *dd) in mtip_hba_reset() argument
172 writel(HOST_RESET, dd->mmio + HOST_CTL); in mtip_hba_reset()
175 readl(dd->mmio + HOST_CTL); in mtip_hba_reset()
184 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in mtip_hba_reset()
187 } while ((readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
190 if (readl(dd->mmio + HOST_CTL) & HOST_RESET) in mtip_hba_reset()
326 if (readl(port->dd->mmio + HOST_CAP) & HOST_CAP_64) { in mtip_init_port()
342 for (i = 0; i < port->dd->slot_groups; i++) in mtip_init_port()
349 writel(readl(port->dd->mmio + HOST_IRQ_STAT), in mtip_init_port()
350 port->dd->mmio + HOST_IRQ_STAT); in mtip_init_port()
377 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
385 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
388 if (mtip_hba_reset(port->dd)) in mtip_restart_port()
389 dev_err(&port->dd->pdev->dev, in mtip_restart_port()
396 dev_warn(&port->dd->pdev->dev, "Issuing COM reset\n"); in mtip_restart_port()
408 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
422 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_restart_port()
426 dev_warn(&port->dd->pdev->dev, in mtip_restart_port()
434 static int mtip_device_reset(struct driver_data *dd) in mtip_device_reset() argument
438 if (mtip_check_surprise_removal(dd)) in mtip_device_reset()
441 if (mtip_hba_reset(dd) < 0) in mtip_device_reset()
445 mtip_init_port(dd->port); in mtip_device_reset()
446 mtip_start_port(dd->port); in mtip_device_reset()
449 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_device_reset()
450 dd->mmio + HOST_CTL); in mtip_device_reset()
457 static void print_tags(struct driver_data *dd, in print_tags() argument
469 dev_warn(&dd->pdev->dev, in print_tags()
495 static void mtip_handle_tfe(struct driver_data *dd) in mtip_handle_tfe() argument
508 dev_warn(&dd->pdev->dev, "Taskfile error\n"); in mtip_handle_tfe()
510 port = dd->port; in mtip_handle_tfe()
513 cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); in mtip_handle_tfe()
523 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
526 dev_warn(&dd->pdev->dev, "g=%u, comp=%x\n", group, completed); in mtip_handle_tfe()
541 cmd = mtip_cmd_from_tag(dd, tag); in mtip_handle_tfe()
548 print_tags(dd, "completed (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
555 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_handle_tfe()
556 dd->port->log_buf, in mtip_handle_tfe()
557 dd->port->log_buf_dma, 1); in mtip_handle_tfe()
559 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
563 buf = (unsigned char *)dd->port->log_buf; in mtip_handle_tfe()
565 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
567 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_handle_tfe()
572 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
574 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_handle_tfe()
579 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_handle_tfe()
580 dev_info(&dd->pdev->dev, in mtip_handle_tfe()
591 for (group = 0; group < dd->slot_groups; group++) { in mtip_handle_tfe()
595 cmd = mtip_cmd_from_tag(dd, tag); in mtip_handle_tfe()
607 dev_warn(&dd->pdev->dev, in mtip_handle_tfe()
634 dev_warn(&port->dd->pdev->dev, in mtip_handle_tfe()
640 print_tags(dd, "reissued (TFE)", tagaccum, cmd_cnt); in mtip_handle_tfe()
649 struct driver_data *dd = port->dd; in mtip_workq_sdbfx() local
669 command = mtip_cmd_from_tag(dd, tag); in mtip_workq_sdbfx()
676 if (atomic_dec_return(&dd->irq_workers_active) == 0) in mtip_workq_sdbfx()
677 writel(0xffffffff, dd->mmio + HOST_IRQ_STAT); in mtip_workq_sdbfx()
683 static inline void mtip_process_legacy(struct driver_data *dd, u32 port_stat) in mtip_process_legacy() argument
685 struct mtip_port *port = dd->port; in mtip_process_legacy()
686 struct mtip_cmd *cmd = mtip_cmd_from_tag(dd, MTIP_TAG_INTERNAL); in mtip_process_legacy()
700 static inline void mtip_process_errors(struct driver_data *dd, u32 port_stat) in mtip_process_errors() argument
703 dev_warn(&dd->pdev->dev, in mtip_process_errors()
705 writel((1 << 26), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
709 dev_warn(&dd->pdev->dev, in mtip_process_errors()
711 writel((1 << 16), dd->port->mmio + PORT_SCR_ERR); in mtip_process_errors()
715 dev_warn(&dd->pdev->dev, in mtip_process_errors()
718 if (mtip_check_surprise_removal(dd)) in mtip_process_errors()
722 set_bit(MTIP_PF_EH_ACTIVE_BIT, &dd->port->flags); in mtip_process_errors()
723 wake_up_interruptible(&dd->port->svc_wait); in mtip_process_errors()
729 struct driver_data *dd = (struct driver_data *) data; in mtip_handle_irq() local
730 struct mtip_port *port = dd->port; in mtip_handle_irq()
736 hba_stat = readl(dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
743 mtip_check_surprise_removal(dd); in mtip_handle_irq()
751 WARN_ON_ONCE(atomic_read(&dd->irq_workers_active) != 0); in mtip_handle_irq()
756 twork = &dd->work[i]; in mtip_handle_irq()
762 atomic_set(&dd->irq_workers_active, workers); in mtip_handle_irq()
765 twork = &dd->work[i]; in mtip_handle_irq()
769 dd->isr_workq, in mtip_handle_irq()
773 if (likely(dd->work[0].completed)) in mtip_handle_irq()
775 dd->work[0].completed); in mtip_handle_irq()
787 if (unlikely(mtip_check_surprise_removal(dd))) { in mtip_handle_irq()
792 &dd->dd_flag)) in mtip_handle_irq()
795 mtip_process_errors(dd, port_stat & PORT_IRQ_ERR); in mtip_handle_irq()
799 mtip_process_legacy(dd, port_stat & PORT_IRQ_LEGACY); in mtip_handle_irq()
804 writel(hba_stat, dd->mmio + HOST_IRQ_STAT); in mtip_handle_irq()
821 struct driver_data *dd = instance; in mtip_irq_handler() local
823 return mtip_handle_irq(dd); in mtip_irq_handler()
852 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
853 clear_bit(MTIP_DDF_REBUILD_FAILED_BIT, &port->dd->dd_flag); in mtip_pause_ncq()
873 for (n = 1; n < port->dd->slot_groups; n++) in mtip_commands_active()
894 blk_mq_quiesce_queue(port->dd->queue); in mtip_quiesce_io()
906 if (mtip_check_surprise_removal(port->dd)) in mtip_quiesce_io()
914 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
917 blk_mq_unquiesce_queue(port->dd->queue); in mtip_quiesce_io()
955 struct driver_data *dd = port->dd; in mtip_exec_internal_command() local
967 dev_err(&dd->pdev->dev, "SG buffer is not 8 byte aligned\n"); in mtip_exec_internal_command()
971 if (mtip_check_surprise_removal(dd)) in mtip_exec_internal_command()
974 rq = blk_mq_alloc_request(dd->queue, REQ_OP_DRV_IN, BLK_MQ_REQ_RESERVED); in mtip_exec_internal_command()
990 dev_warn(&dd->pdev->dev, "Failed to quiesce IO\n"); in mtip_exec_internal_command()
1009 dev_err(&dd->pdev->dev, "Internal command [%02X] failed %d\n", in mtip_exec_internal_command()
1013 if (mtip_check_surprise_removal(dd) || in mtip_exec_internal_command()
1015 &dd->dd_flag)) { in mtip_exec_internal_command()
1016 dev_err(&dd->pdev->dev, in mtip_exec_internal_command()
1022 mtip_device_reset(dd); /* recover from timeout issue */ in mtip_exec_internal_command()
1030 if (!test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_exec_internal_command()
1031 mtip_device_reset(dd); in mtip_exec_internal_command()
1068 static void mtip_set_timeout(struct driver_data *dd, in mtip_set_timeout() argument
1079 *timeout = ((*(dd->port->identify + 90) * 2) * 60000); in mtip_set_timeout()
1081 *timeout = ((*(dd->port->identify + 89) * 2) * 60000); in mtip_set_timeout()
1123 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &port->dd->dd_flag)) in mtip_get_identify()
1170 set_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1172 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &port->dd->dd_flag); in mtip_get_identify()
1213 mtip_set_timeout(port->dd, &fis, &timeout, 0); in mtip_standby_immediate()
1226 dev_warn(&port->dd->pdev->dev, in mtip_standby_immediate()
1325 dev_warn(&port->dd->pdev->dev, "IDENTIFY DATA not valid\n"); in mtip_get_smart_attr()
1329 dev_warn(&port->dd->pdev->dev, "SMART not supported\n"); in mtip_get_smart_attr()
1333 dev_warn(&port->dd->pdev->dev, "SMART not enabled\n"); in mtip_get_smart_attr()
1340 dev_warn(&port->dd->pdev->dev, "Failed to ge SMART data\n"); in mtip_get_smart_attr()
1352 dev_warn(&port->dd->pdev->dev, in mtip_get_smart_attr()
1370 static bool mtip_hw_get_capacity(struct driver_data *dd, sector_t *sectors) in mtip_hw_get_capacity() argument
1372 struct mtip_port *port = dd->port; in mtip_hw_get_capacity()
1401 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1405 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1409 dev_info(&port->dd->pdev->dev, "Model: %s\n", cbuf); in mtip_dump_identify()
1411 dev_info(&port->dd->pdev->dev, "Security: %04x %s\n", in mtip_dump_identify()
1415 if (mtip_hw_get_capacity(port->dd, §ors)) in mtip_dump_identify()
1416 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1421 pci_read_config_word(port->dd->pdev, PCI_REVISION_ID, &revid); in mtip_dump_identify()
1433 dev_info(&port->dd->pdev->dev, in mtip_dump_identify()
1446 static inline void fill_command_sg(struct driver_data *dd, in fill_command_sg() argument
1460 dev_err(&dd->pdev->dev, in fill_command_sg()
1494 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_task()
1559 buf = dma_alloc_coherent(&port->dd->pdev->dev, in exec_drive_command()
1564 dev_err(&port->dd->pdev->dev, in exec_drive_command()
1584 mtip_set_timeout(port->dd, &fis, &to, 0); in exec_drive_command()
1636 dma_free_coherent(&port->dd->pdev->dev, in exec_drive_command()
1692 static int exec_drive_taskfile(struct driver_data *dd, in exec_drive_taskfile() argument
1726 outbuf_dma = dma_map_single(&dd->pdev->dev, outbuf, in exec_drive_taskfile()
1728 if (dma_mapping_error(&dd->pdev->dev, outbuf_dma)) { in exec_drive_taskfile()
1742 inbuf_dma = dma_map_single(&dd->pdev->dev, inbuf, in exec_drive_taskfile()
1744 if (dma_mapping_error(&dd->pdev->dev, inbuf_dma)) { in exec_drive_taskfile()
1755 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1758 reply = (dd->port->rxfis + RX_FIS_PIO_SETUP); in exec_drive_taskfile()
1761 reply = (dd->port->rxfis + RX_FIS_D2H_REG); in exec_drive_taskfile()
1803 dev_warn(&dd->pdev->dev, in exec_drive_taskfile()
1831 mtip_set_timeout(dd, &fis, &timeout, erasemode); in exec_drive_taskfile()
1840 if (mtip_exec_internal_command(dd->port, in exec_drive_taskfile()
1851 task_file_data = readl(dd->port->mmio+PORT_TFDATA); in exec_drive_taskfile()
1854 reply = dd->port->rxfis + RX_FIS_PIO_SETUP; in exec_drive_taskfile()
1857 reply = dd->port->rxfis + RX_FIS_D2H_REG; in exec_drive_taskfile()
1863 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1866 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1914 dma_unmap_single(&dd->pdev->dev, inbuf_dma, taskin, in exec_drive_taskfile()
1917 dma_unmap_single(&dd->pdev->dev, outbuf_dma, taskout, in exec_drive_taskfile()
1942 static int mtip_hw_ioctl(struct driver_data *dd, unsigned int cmd, in mtip_hw_ioctl() argument
1948 if (copy_to_user((void __user *)arg, dd->port->identify, in mtip_hw_ioctl()
1964 if (exec_drive_command(dd->port, in mtip_hw_ioctl()
1988 if (exec_drive_task(dd->port, drive_command)) in mtip_hw_ioctl()
2009 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_hw_ioctl()
2045 static void mtip_hw_submit_io(struct driver_data *dd, struct request *rq, in mtip_hw_submit_io() argument
2050 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_hw_submit_io()
2052 struct mtip_port *port = dd->port; in mtip_hw_submit_io()
2060 nents = dma_map_sg(&dd->pdev->dev, command->sg, nents, dma_dir); in mtip_hw_submit_io()
2094 fill_command_sg(dd, command, nents); in mtip_hw_submit_io()
2101 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_hw_submit_io()
2136 struct driver_data *dd = dev_to_disk(dev)->private_data; in mtip_hw_show_status() local
2139 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2141 else if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag)) in mtip_hw_show_status()
2168 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_registers() local
2183 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2185 readl(dd->port->s_active[n])); in mtip_hw_read_registers()
2190 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2192 readl(dd->port->cmd_issue[n])); in mtip_hw_read_registers()
2197 for (n = dd->slot_groups-1; n >= 0; n--) in mtip_hw_read_registers()
2199 readl(dd->port->completed[n])); in mtip_hw_read_registers()
2203 readl(dd->port->mmio + PORT_IRQ_STAT)); in mtip_hw_read_registers()
2205 readl(dd->mmio + HOST_IRQ_STAT)); in mtip_hw_read_registers()
2210 for (n = dd->slot_groups-1; n >= 0; n--) { in mtip_hw_read_registers()
2213 dd->port->cmds_to_issue[n/2] >> (32*(n&1)); in mtip_hw_read_registers()
2215 group_allocated = dd->port->cmds_to_issue[n]; in mtip_hw_read_registers()
2232 struct driver_data *dd = (struct driver_data *)f->private_data; in mtip_hw_read_flags() local
2245 dd->port->flags); in mtip_hw_read_flags()
2247 dd->dd_flag); in mtip_hw_read_flags()
2272 static int mtip_hw_debugfs_init(struct driver_data *dd) in mtip_hw_debugfs_init() argument
2277 dd->dfs_node = debugfs_create_dir(dd->disk->disk_name, dfs_parent); in mtip_hw_debugfs_init()
2278 if (IS_ERR_OR_NULL(dd->dfs_node)) { in mtip_hw_debugfs_init()
2279 dev_warn(&dd->pdev->dev, in mtip_hw_debugfs_init()
2281 dd->disk->disk_name); in mtip_hw_debugfs_init()
2282 dd->dfs_node = NULL; in mtip_hw_debugfs_init()
2286 debugfs_create_file("flags", 0444, dd->dfs_node, dd, &mtip_flags_fops); in mtip_hw_debugfs_init()
2287 debugfs_create_file("registers", 0444, dd->dfs_node, dd, in mtip_hw_debugfs_init()
2293 static void mtip_hw_debugfs_exit(struct driver_data *dd) in mtip_hw_debugfs_exit() argument
2295 debugfs_remove_recursive(dd->dfs_node); in mtip_hw_debugfs_exit()
2306 static inline void hba_setup(struct driver_data *dd) in hba_setup() argument
2309 hwdata = readl(dd->mmio + HOST_HSORG); in hba_setup()
2315 dd->mmio + HOST_HSORG); in hba_setup()
2318 static int mtip_device_unaligned_constrained(struct driver_data *dd) in mtip_device_unaligned_constrained() argument
2320 return (dd->pdev->device == P420M_DEVICE_ID ? 1 : 0); in mtip_device_unaligned_constrained()
2333 static void mtip_detect_product(struct driver_data *dd) in mtip_detect_product() argument
2345 hwdata = readl(dd->mmio + HOST_HSORG); in mtip_detect_product()
2347 dd->product_type = MTIP_PRODUCT_UNKNOWN; in mtip_detect_product()
2348 dd->slot_groups = 1; in mtip_detect_product()
2351 dd->product_type = MTIP_PRODUCT_ASICFPGA; in mtip_detect_product()
2354 dev_info(&dd->pdev->dev, in mtip_detect_product()
2362 dev_warn(&dd->pdev->dev, in mtip_detect_product()
2367 dd->slot_groups = slotgroups; in mtip_detect_product()
2371 dev_warn(&dd->pdev->dev, "Unrecognized product id\n"); in mtip_detect_product()
2383 static int mtip_ftl_rebuild_poll(struct driver_data *dd) in mtip_ftl_rebuild_poll() argument
2387 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2395 &dd->dd_flag))) in mtip_ftl_rebuild_poll()
2397 if (mtip_check_surprise_removal(dd)) in mtip_ftl_rebuild_poll()
2400 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_ftl_rebuild_poll()
2403 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_ftl_rebuild_poll()
2408 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2414 dev_warn(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2417 mtip_block_initialize(dd); in mtip_ftl_rebuild_poll()
2423 dev_err(&dd->pdev->dev, in mtip_ftl_rebuild_poll()
2432 struct driver_data *dd = rq->q->queuedata; in mtip_softirq_done_fn() local
2435 dma_unmap_sg(&dd->pdev->dev, cmd->sg, cmd->scatter_ents, in mtip_softirq_done_fn()
2439 atomic_inc(&dd->port->cmd_slot_unal); in mtip_softirq_done_fn()
2447 struct driver_data *dd = data; in mtip_abort_cmd() local
2451 clear_bit(req->tag, dd->port->cmds_to_issue); in mtip_abort_cmd()
2459 struct driver_data *dd = data; in mtip_queue_cmd() local
2461 set_bit(req->tag, dd->port->cmds_to_issue); in mtip_queue_cmd()
2477 struct driver_data *dd = (struct driver_data *)data; in mtip_service_thread() local
2479 unsigned int num_cmd_slots = dd->slot_groups * 32; in mtip_service_thread()
2480 struct mtip_port *port = dd->port; in mtip_service_thread()
2500 &dd->dd_flag))) in mtip_service_thread()
2508 mtip_handle_tfe(dd); in mtip_service_thread()
2520 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_service_thread()
2523 if (atomic_read(&dd->irq_workers_active) != 0) in mtip_service_thread()
2524 dev_warn(&dd->pdev->dev, in mtip_service_thread()
2527 blk_mq_quiesce_queue(dd->queue); in mtip_service_thread()
2529 blk_mq_tagset_busy_iter(&dd->tags, mtip_queue_cmd, dd); in mtip_service_thread()
2531 set_bit(MTIP_PF_ISSUE_CMDS_BIT, &dd->port->flags); in mtip_service_thread()
2533 if (mtip_device_reset(dd)) in mtip_service_thread()
2534 blk_mq_tagset_busy_iter(&dd->tags, in mtip_service_thread()
2535 mtip_abort_cmd, dd); in mtip_service_thread()
2537 clear_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags); in mtip_service_thread()
2539 blk_mq_unquiesce_queue(dd->queue); in mtip_service_thread()
2574 if (mtip_ftl_rebuild_poll(dd) == 0) in mtip_service_thread()
2591 static void mtip_dma_free(struct driver_data *dd) in mtip_dma_free() argument
2593 struct mtip_port *port = dd->port; in mtip_dma_free()
2596 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_free()
2600 dma_free_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_free()
2613 static int mtip_dma_alloc(struct driver_data *dd) in mtip_dma_alloc() argument
2615 struct mtip_port *port = dd->port; in mtip_dma_alloc()
2619 dma_alloc_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2626 dma_alloc_coherent(&dd->pdev->dev, AHCI_CMD_TBL_SZ, in mtip_dma_alloc()
2629 dma_free_coherent(&dd->pdev->dev, BLOCK_DMA_ALLOC_SZ, in mtip_dma_alloc()
2649 static int mtip_hw_get_identify(struct driver_data *dd) in mtip_hw_get_identify() argument
2655 if (mtip_get_identify(dd->port, NULL) < 0) in mtip_hw_get_identify()
2658 if (*(dd->port->identify + MTIP_FTL_REBUILD_OFFSET) == in mtip_hw_get_identify()
2660 set_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags); in mtip_hw_get_identify()
2663 mtip_dump_identify(dd->port); in mtip_hw_get_identify()
2666 rv = mtip_read_log_page(dd->port, ATA_LOG_SATA_NCQ, in mtip_hw_get_identify()
2667 dd->port->log_buf, in mtip_hw_get_identify()
2668 dd->port->log_buf_dma, 1); in mtip_hw_get_identify()
2670 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2674 buf = (unsigned char *)dd->port->log_buf; in mtip_hw_get_identify()
2676 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2678 set_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2681 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2683 set_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2686 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2688 set_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag); in mtip_hw_get_identify()
2694 if (mtip_get_smart_attr(dd->port, 242, &attr242)) in mtip_hw_get_identify()
2695 dev_warn(&dd->pdev->dev, in mtip_hw_get_identify()
2698 dev_info(&dd->pdev->dev, in mtip_hw_get_identify()
2713 static int mtip_hw_init(struct driver_data *dd) in mtip_hw_init() argument
2719 dd->mmio = pcim_iomap_table(dd->pdev)[MTIP_ABAR]; in mtip_hw_init()
2721 mtip_detect_product(dd); in mtip_hw_init()
2722 if (dd->product_type == MTIP_PRODUCT_UNKNOWN) { in mtip_hw_init()
2727 hba_setup(dd); in mtip_hw_init()
2729 dd->port = kzalloc_node(sizeof(struct mtip_port), GFP_KERNEL, in mtip_hw_init()
2730 dd->numa_node); in mtip_hw_init()
2731 if (!dd->port) in mtip_hw_init()
2736 dd->work[i].port = dd->port; in mtip_hw_init()
2739 if (mtip_device_unaligned_constrained(dd)) in mtip_hw_init()
2740 dd->unal_qdepth = MTIP_MAX_UNALIGNED_SLOTS; in mtip_hw_init()
2742 dd->unal_qdepth = 0; in mtip_hw_init()
2744 atomic_set(&dd->port->cmd_slot_unal, dd->unal_qdepth); in mtip_hw_init()
2748 spin_lock_init(&dd->port->cmd_issue_lock[i]); in mtip_hw_init()
2751 dd->port->mmio = dd->mmio + PORT_OFFSET; in mtip_hw_init()
2752 dd->port->dd = dd; in mtip_hw_init()
2755 rv = mtip_dma_alloc(dd); in mtip_hw_init()
2760 for (i = 0; i < dd->slot_groups; i++) { in mtip_hw_init()
2761 dd->port->s_active[i] = in mtip_hw_init()
2762 dd->port->mmio + i*0x80 + PORT_SCR_ACT; in mtip_hw_init()
2763 dd->port->cmd_issue[i] = in mtip_hw_init()
2764 dd->port->mmio + i*0x80 + PORT_COMMAND_ISSUE; in mtip_hw_init()
2765 dd->port->completed[i] = in mtip_hw_init()
2766 dd->port->mmio + i*0x80 + PORT_SDBV; in mtip_hw_init()
2771 while (((readl(dd->port->mmio + PORT_SCR_STAT) & 0x0F) != 0x03) && in mtip_hw_init()
2775 if (unlikely(mtip_check_surprise_removal(dd))) { in mtip_hw_init()
2777 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2783 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) { in mtip_hw_init()
2785 dev_warn(&dd->pdev->dev, in mtip_hw_init()
2793 if (!(readl(dd->mmio + HOST_CAP) & HOST_CAP_NZDMA)) { in mtip_hw_init()
2794 if (mtip_hba_reset(dd) < 0) { in mtip_hw_init()
2795 dev_err(&dd->pdev->dev, in mtip_hw_init()
2802 writel(readl(dd->mmio + HOST_IRQ_STAT), in mtip_hw_init()
2803 dd->mmio + HOST_IRQ_STAT); in mtip_hw_init()
2806 mtip_init_port(dd->port); in mtip_hw_init()
2807 mtip_start_port(dd->port); in mtip_hw_init()
2810 rv = request_irq(dd->pdev->irq, mtip_irq_handler, IRQF_SHARED, in mtip_hw_init()
2811 dev_driver_string(&dd->pdev->dev), dd); in mtip_hw_init()
2813 dev_err(&dd->pdev->dev, in mtip_hw_init()
2814 "Unable to allocate IRQ %d\n", dd->pdev->irq); in mtip_hw_init()
2817 irq_set_affinity_hint(dd->pdev->irq, get_cpu_mask(dd->isr_binding)); in mtip_hw_init()
2820 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_init()
2821 dd->mmio + HOST_CTL); in mtip_hw_init()
2823 init_waitqueue_head(&dd->port->svc_wait); in mtip_hw_init()
2825 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) { in mtip_hw_init()
2834 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_init()
2835 dd->mmio + HOST_CTL); in mtip_hw_init()
2838 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_init()
2839 free_irq(dd->pdev->irq, dd); in mtip_hw_init()
2842 mtip_deinit_port(dd->port); in mtip_hw_init()
2843 mtip_dma_free(dd); in mtip_hw_init()
2847 kfree(dd->port); in mtip_hw_init()
2852 static int mtip_standby_drive(struct driver_data *dd) in mtip_standby_drive() argument
2856 if (dd->sr || !dd->port) in mtip_standby_drive()
2862 if (!test_bit(MTIP_PF_REBUILD_BIT, &dd->port->flags) && in mtip_standby_drive()
2863 !test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag) && in mtip_standby_drive()
2864 !test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) { in mtip_standby_drive()
2865 rv = mtip_standby_immediate(dd->port); in mtip_standby_drive()
2867 dev_warn(&dd->pdev->dev, in mtip_standby_drive()
2881 static int mtip_hw_exit(struct driver_data *dd) in mtip_hw_exit() argument
2883 if (!dd->sr) { in mtip_hw_exit()
2885 mtip_deinit_port(dd->port); in mtip_hw_exit()
2888 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_exit()
2889 dd->mmio + HOST_CTL); in mtip_hw_exit()
2893 irq_set_affinity_hint(dd->pdev->irq, NULL); in mtip_hw_exit()
2894 free_irq(dd->pdev->irq, dd); in mtip_hw_exit()
2898 mtip_dma_free(dd); in mtip_hw_exit()
2901 kfree(dd->port); in mtip_hw_exit()
2902 dd->port = NULL; in mtip_hw_exit()
2918 static int mtip_hw_shutdown(struct driver_data *dd) in mtip_hw_shutdown() argument
2924 mtip_standby_drive(dd); in mtip_hw_shutdown()
2941 static int mtip_hw_suspend(struct driver_data *dd) in mtip_hw_suspend() argument
2947 if (mtip_standby_drive(dd) != 0) { in mtip_hw_suspend()
2948 dev_err(&dd->pdev->dev, in mtip_hw_suspend()
2954 writel(readl(dd->mmio + HOST_CTL) & ~HOST_IRQ_EN, in mtip_hw_suspend()
2955 dd->mmio + HOST_CTL); in mtip_hw_suspend()
2956 mtip_deinit_port(dd->port); in mtip_hw_suspend()
2973 static int mtip_hw_resume(struct driver_data *dd) in mtip_hw_resume() argument
2976 hba_setup(dd); in mtip_hw_resume()
2979 if (mtip_hba_reset(dd) != 0) { in mtip_hw_resume()
2980 dev_err(&dd->pdev->dev, in mtip_hw_resume()
2989 mtip_init_port(dd->port); in mtip_hw_resume()
2990 mtip_start_port(dd->port); in mtip_hw_resume()
2993 writel(readl(dd->mmio + HOST_CTL) | HOST_IRQ_EN, in mtip_hw_resume()
2994 dd->mmio + HOST_CTL); in mtip_hw_resume()
3048 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_ioctl() local
3053 if (!dd) in mtip_block_ioctl()
3056 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_ioctl()
3063 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_ioctl()
3086 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_compat_ioctl() local
3091 if (!dd) in mtip_block_compat_ioctl()
3094 if (unlikely(test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag))) in mtip_block_compat_ioctl()
3123 ret = exec_drive_taskfile(dd, (void __user *) arg, in mtip_block_compat_ioctl()
3140 return mtip_hw_ioctl(dd, cmd, arg); in mtip_block_compat_ioctl()
3166 struct driver_data *dd = dev->bd_disk->private_data; in mtip_block_getgeo() local
3169 if (!dd) in mtip_block_getgeo()
3172 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_getgeo()
3173 dev_warn(&dd->pdev->dev, in mtip_block_getgeo()
3187 struct driver_data *dd = disk->private_data; in mtip_block_free_disk() local
3189 ida_free(&rssd_index_ida, dd->index); in mtip_block_free_disk()
3190 kfree(dd); in mtip_block_free_disk()
3209 static inline bool is_se_active(struct driver_data *dd) in is_se_active() argument
3211 if (unlikely(test_bit(MTIP_PF_SE_ACTIVE_BIT, &dd->port->flags))) { in is_se_active()
3212 if (dd->port->ic_pause_timer) { in is_se_active()
3213 unsigned long to = dd->port->ic_pause_timer + in is_se_active()
3217 &dd->port->flags); in is_se_active()
3218 clear_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag); in is_se_active()
3219 dd->port->ic_pause_timer = 0; in is_se_active()
3220 wake_up_interruptible(&dd->port->svc_wait); in is_se_active()
3229 static inline bool is_stopped(struct driver_data *dd, struct request *rq) in is_stopped() argument
3231 if (likely(!(dd->dd_flag & MTIP_DDF_STOP_IO))) in is_stopped()
3234 if (test_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag)) in is_stopped()
3236 if (test_bit(MTIP_DDF_OVER_TEMP_BIT, &dd->dd_flag)) in is_stopped()
3238 if (test_bit(MTIP_DDF_WRITE_PROTECT_BIT, &dd->dd_flag) && in is_stopped()
3241 if (test_bit(MTIP_DDF_SEC_LOCK_BIT, &dd->dd_flag)) in is_stopped()
3243 if (test_bit(MTIP_DDF_REBUILD_FAILED_BIT, &dd->dd_flag)) in is_stopped()
3252 struct driver_data *dd = hctx->queue->queuedata; in mtip_check_unal_depth() local
3255 if (rq_data_dir(rq) == READ || !dd->unal_qdepth) in mtip_check_unal_depth()
3267 if (cmd->unaligned && atomic_dec_if_positive(&dd->port->cmd_slot_unal) >= 0) in mtip_check_unal_depth()
3276 struct driver_data *dd = hctx->queue->queuedata; in mtip_issue_reserved_cmd() local
3280 dd->port->command_list + sizeof(struct mtip_cmd_hdr) * rq->tag; in mtip_issue_reserved_cmd()
3283 if (mtip_commands_active(dd->port)) in mtip_issue_reserved_cmd()
3287 if (test_bit(MTIP_PF_HOST_CAP_64, &dd->port->flags)) in mtip_issue_reserved_cmd()
3306 mtip_issue_non_ncq_command(dd->port, rq->tag); in mtip_issue_reserved_cmd()
3313 struct driver_data *dd = hctx->queue->queuedata; in mtip_queue_rq() local
3323 if (is_se_active(dd) || is_stopped(dd, rq)) in mtip_queue_rq()
3328 mtip_hw_submit_io(dd, rq, cmd, hctx); in mtip_queue_rq()
3335 struct driver_data *dd = set->driver_data; in mtip_free_cmd() local
3341 dma_free_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, cmd->command, in mtip_free_cmd()
3348 struct driver_data *dd = set->driver_data; in mtip_init_cmd() local
3351 cmd->command = dma_alloc_coherent(&dd->pdev->dev, CMD_DMA_ALLOC_SZ, in mtip_init_cmd()
3362 struct driver_data *dd = req->q->queuedata; in mtip_cmd_timeout() local
3372 if (test_bit(req->tag, dd->port->cmds_to_issue)) in mtip_cmd_timeout()
3375 if (test_and_set_bit(MTIP_PF_TO_ACTIVE_BIT, &dd->port->flags)) in mtip_cmd_timeout()
3378 wake_up_interruptible(&dd->port->svc_wait); in mtip_cmd_timeout()
3402 static int mtip_block_initialize(struct driver_data *dd) in mtip_block_initialize() argument
3408 if (dd->disk) in mtip_block_initialize()
3411 if (mtip_hw_init(dd)) { in mtip_block_initialize()
3416 memset(&dd->tags, 0, sizeof(dd->tags)); in mtip_block_initialize()
3417 dd->tags.ops = &mtip_mq_ops; in mtip_block_initialize()
3418 dd->tags.nr_hw_queues = 1; in mtip_block_initialize()
3419 dd->tags.queue_depth = MTIP_MAX_COMMAND_SLOTS; in mtip_block_initialize()
3420 dd->tags.reserved_tags = 1; in mtip_block_initialize()
3421 dd->tags.cmd_size = sizeof(struct mtip_cmd); in mtip_block_initialize()
3422 dd->tags.numa_node = dd->numa_node; in mtip_block_initialize()
3423 dd->tags.flags = BLK_MQ_F_SHOULD_MERGE; in mtip_block_initialize()
3424 dd->tags.driver_data = dd; in mtip_block_initialize()
3425 dd->tags.timeout = MTIP_NCQ_CMD_TIMEOUT_MS; in mtip_block_initialize()
3427 rv = blk_mq_alloc_tag_set(&dd->tags); in mtip_block_initialize()
3429 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3434 dd->disk = blk_mq_alloc_disk(&dd->tags, dd); in mtip_block_initialize()
3435 if (IS_ERR(dd->disk)) { in mtip_block_initialize()
3436 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3441 dd->queue = dd->disk->queue; in mtip_block_initialize()
3450 dd->disk->disk_name, in mtip_block_initialize()
3455 dd->disk->major = dd->major; in mtip_block_initialize()
3456 dd->disk->first_minor = index * MTIP_MAX_MINORS; in mtip_block_initialize()
3457 dd->disk->minors = MTIP_MAX_MINORS; in mtip_block_initialize()
3458 dd->disk->fops = &mtip_block_ops; in mtip_block_initialize()
3459 dd->disk->private_data = dd; in mtip_block_initialize()
3460 dd->index = index; in mtip_block_initialize()
3462 mtip_hw_debugfs_init(dd); in mtip_block_initialize()
3466 wait_for_rebuild = mtip_hw_get_identify(dd); in mtip_block_initialize()
3468 dev_err(&dd->pdev->dev, in mtip_block_initialize()
3482 blk_queue_flag_set(QUEUE_FLAG_NONROT, dd->queue); in mtip_block_initialize()
3483 blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, dd->queue); in mtip_block_initialize()
3484 blk_queue_max_segments(dd->queue, MTIP_MAX_SG); in mtip_block_initialize()
3485 blk_queue_physical_block_size(dd->queue, 4096); in mtip_block_initialize()
3486 blk_queue_max_hw_sectors(dd->queue, 0xffff); in mtip_block_initialize()
3487 blk_queue_max_segment_size(dd->queue, 0x400000); in mtip_block_initialize()
3488 dma_set_max_seg_size(&dd->pdev->dev, 0x400000); in mtip_block_initialize()
3489 blk_queue_io_min(dd->queue, 4096); in mtip_block_initialize()
3492 if (!(mtip_hw_get_capacity(dd, &capacity))) { in mtip_block_initialize()
3493 dev_warn(&dd->pdev->dev, in mtip_block_initialize()
3498 set_capacity(dd->disk, capacity); in mtip_block_initialize()
3501 rv = device_add_disk(&dd->pdev->dev, dd->disk, mtip_disk_attr_groups); in mtip_block_initialize()
3505 if (dd->mtip_svc_handler) { in mtip_block_initialize()
3506 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_block_initialize()
3511 dd->mtip_svc_handler = kthread_create_on_node(mtip_service_thread, in mtip_block_initialize()
3512 dd, dd->numa_node, in mtip_block_initialize()
3515 if (IS_ERR(dd->mtip_svc_handler)) { in mtip_block_initialize()
3516 dev_err(&dd->pdev->dev, "service thread failed to start\n"); in mtip_block_initialize()
3517 dd->mtip_svc_handler = NULL; in mtip_block_initialize()
3521 wake_up_process(dd->mtip_svc_handler); in mtip_block_initialize()
3529 del_gendisk(dd->disk); in mtip_block_initialize()
3532 mtip_hw_debugfs_exit(dd); in mtip_block_initialize()
3536 put_disk(dd->disk); in mtip_block_initialize()
3538 blk_mq_free_tag_set(&dd->tags); in mtip_block_initialize()
3540 mtip_hw_exit(dd); /* De-initialize the protocol layer. */ in mtip_block_initialize()
3557 static int mtip_block_shutdown(struct driver_data *dd) in mtip_block_shutdown() argument
3559 mtip_hw_shutdown(dd); in mtip_block_shutdown()
3561 dev_info(&dd->pdev->dev, in mtip_block_shutdown()
3562 "Shutting down %s ...\n", dd->disk->disk_name); in mtip_block_shutdown()
3564 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_block_shutdown()
3565 del_gendisk(dd->disk); in mtip_block_shutdown()
3567 blk_mq_free_tag_set(&dd->tags); in mtip_block_shutdown()
3568 put_disk(dd->disk); in mtip_block_shutdown()
3572 static int mtip_block_suspend(struct driver_data *dd) in mtip_block_suspend() argument
3574 dev_info(&dd->pdev->dev, in mtip_block_suspend()
3575 "Suspending %s ...\n", dd->disk->disk_name); in mtip_block_suspend()
3576 mtip_hw_suspend(dd); in mtip_block_suspend()
3580 static int mtip_block_resume(struct driver_data *dd) in mtip_block_resume() argument
3582 dev_info(&dd->pdev->dev, "Resuming %s ...\n", in mtip_block_resume()
3583 dd->disk->disk_name); in mtip_block_resume()
3584 mtip_hw_resume(dd); in mtip_block_resume()
3638 static void mtip_disable_link_opts(struct driver_data *dd, struct pci_dev *pdev) in mtip_disable_link_opts() argument
3646 dev_info(&dd->pdev->dev, in mtip_disable_link_opts()
3657 static void mtip_fix_ero_nosnoop(struct driver_data *dd, struct pci_dev *pdev) in mtip_fix_ero_nosnoop() argument
3666 mtip_disable_link_opts(dd, pdev->bus->self); in mtip_fix_ero_nosnoop()
3677 mtip_disable_link_opts(dd, in mtip_fix_ero_nosnoop()
3697 struct driver_data *dd = NULL; in mtip_pci_probe() local
3716 dd = kzalloc_node(sizeof(struct driver_data), GFP_KERNEL, my_node); in mtip_pci_probe()
3717 if (!dd) in mtip_pci_probe()
3721 pci_set_drvdata(pdev, dd); in mtip_pci_probe()
3743 dd->major = mtip_major; in mtip_pci_probe()
3744 dd->instance = instance; in mtip_pci_probe()
3745 dd->pdev = pdev; in mtip_pci_probe()
3746 dd->numa_node = my_node; in mtip_pci_probe()
3748 memset(dd->workq_name, 0, 32); in mtip_pci_probe()
3749 snprintf(dd->workq_name, 31, "mtipq%d", dd->instance); in mtip_pci_probe()
3751 dd->isr_workq = create_workqueue(dd->workq_name); in mtip_pci_probe()
3752 if (!dd->isr_workq) { in mtip_pci_probe()
3753 dev_warn(&pdev->dev, "Can't create wq %d\n", dd->instance); in mtip_pci_probe()
3760 node_mask = cpumask_of_node(dd->numa_node); in mtip_pci_probe()
3769 dd->numa_node, in mtip_pci_probe()
3771 nr_cpus_node(dd->numa_node), in mtip_pci_probe()
3776 dd->isr_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3778 cpu_to_node(dd->isr_binding), dd->isr_binding); in mtip_pci_probe()
3781 dd->work[0].cpu_binding = dd->isr_binding; in mtip_pci_probe()
3782 dd->work[1].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3783 dd->work[2].cpu_binding = get_least_used_cpu_on_node(dd->numa_node); in mtip_pci_probe()
3784 dd->work[3].cpu_binding = dd->work[0].cpu_binding; in mtip_pci_probe()
3785 dd->work[4].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3786 dd->work[5].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3787 dd->work[6].cpu_binding = dd->work[2].cpu_binding; in mtip_pci_probe()
3788 dd->work[7].cpu_binding = dd->work[1].cpu_binding; in mtip_pci_probe()
3794 if (dd->work[i].cpu_binding == cpu) { in mtip_pci_probe()
3803 INIT_WORK(&dd->work[0].work, mtip_workq_sdbf0); in mtip_pci_probe()
3804 INIT_WORK(&dd->work[1].work, mtip_workq_sdbf1); in mtip_pci_probe()
3805 INIT_WORK(&dd->work[2].work, mtip_workq_sdbf2); in mtip_pci_probe()
3806 INIT_WORK(&dd->work[3].work, mtip_workq_sdbf3); in mtip_pci_probe()
3807 INIT_WORK(&dd->work[4].work, mtip_workq_sdbf4); in mtip_pci_probe()
3808 INIT_WORK(&dd->work[5].work, mtip_workq_sdbf5); in mtip_pci_probe()
3809 INIT_WORK(&dd->work[6].work, mtip_workq_sdbf6); in mtip_pci_probe()
3810 INIT_WORK(&dd->work[7].work, mtip_workq_sdbf7); in mtip_pci_probe()
3820 mtip_fix_ero_nosnoop(dd, pdev); in mtip_pci_probe()
3823 rv = mtip_block_initialize(dd); in mtip_pci_probe()
3836 set_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag); in mtip_pci_probe()
3846 if (dd->isr_workq) { in mtip_pci_probe()
3847 destroy_workqueue(dd->isr_workq); in mtip_pci_probe()
3848 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_probe()
3849 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_probe()
3850 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_probe()
3856 kfree(dd); in mtip_pci_probe()
3872 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_remove() local
3875 mtip_check_surprise_removal(dd); in mtip_pci_remove()
3876 synchronize_irq(dd->pdev->irq); in mtip_pci_remove()
3882 } while (atomic_read(&dd->irq_workers_active) != 0 && in mtip_pci_remove()
3885 if (atomic_read(&dd->irq_workers_active) != 0) { in mtip_pci_remove()
3886 dev_warn(&dd->pdev->dev, in mtip_pci_remove()
3890 set_bit(MTIP_DDF_REMOVE_PENDING_BIT, &dd->dd_flag); in mtip_pci_remove()
3892 if (test_bit(MTIP_DDF_INIT_DONE_BIT, &dd->dd_flag)) in mtip_pci_remove()
3893 del_gendisk(dd->disk); in mtip_pci_remove()
3895 mtip_hw_debugfs_exit(dd); in mtip_pci_remove()
3897 if (dd->mtip_svc_handler) { in mtip_pci_remove()
3898 set_bit(MTIP_PF_SVC_THD_STOP_BIT, &dd->port->flags); in mtip_pci_remove()
3899 wake_up_interruptible(&dd->port->svc_wait); in mtip_pci_remove()
3900 kthread_stop(dd->mtip_svc_handler); in mtip_pci_remove()
3903 if (!dd->sr) { in mtip_pci_remove()
3908 if (!mtip_quiesce_io(dd->port, MTIP_QUIESCE_IO_TIMEOUT_MS)) in mtip_pci_remove()
3909 mtip_standby_drive(dd); in mtip_pci_remove()
3912 dev_info(&dd->pdev->dev, "device %s surprise removal\n", in mtip_pci_remove()
3913 dd->disk->disk_name); in mtip_pci_remove()
3915 blk_mq_free_tag_set(&dd->tags); in mtip_pci_remove()
3918 mtip_hw_exit(dd); in mtip_pci_remove()
3920 if (dd->isr_workq) { in mtip_pci_remove()
3921 destroy_workqueue(dd->isr_workq); in mtip_pci_remove()
3922 drop_cpu(dd->work[0].cpu_binding); in mtip_pci_remove()
3923 drop_cpu(dd->work[1].cpu_binding); in mtip_pci_remove()
3924 drop_cpu(dd->work[2].cpu_binding); in mtip_pci_remove()
3932 put_disk(dd->disk); in mtip_pci_remove()
3945 struct driver_data *dd = dev_get_drvdata(dev); in mtip_pci_suspend() local
3947 set_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_suspend()
3950 rv = mtip_block_suspend(dd); in mtip_pci_suspend()
3967 struct driver_data *dd = dev_get_drvdata(dev); in mtip_pci_resume() local
3973 rv = mtip_block_resume(dd); in mtip_pci_resume()
3977 clear_bit(MTIP_DDF_RESUME_BIT, &dd->dd_flag); in mtip_pci_resume()
3990 struct driver_data *dd = pci_get_drvdata(pdev); in mtip_pci_shutdown() local
3991 if (dd) in mtip_pci_shutdown()
3992 mtip_block_shutdown(dd); in mtip_pci_shutdown()