1a4b16dadSTom Zanussi // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2a4b16dadSTom Zanussi /* Copyright(c) 2014 - 2020 Intel Corporation */ 3a4b16dadSTom Zanussi #include <linux/kernel.h> 4a4b16dadSTom Zanussi #include <linux/pci.h> 5a4b16dadSTom Zanussi #include <linux/completion.h> 6a4b16dadSTom Zanussi #include <linux/workqueue.h> 7a4b16dadSTom Zanussi #include <linux/delay.h> 8a4b16dadSTom Zanussi #include "adf_accel_devices.h" 9a4b16dadSTom Zanussi #include "adf_common_drv.h" 10a4b16dadSTom Zanussi 11a4b16dadSTom Zanussi static struct workqueue_struct *device_reset_wq; 12a4b16dadSTom Zanussi 13a4b16dadSTom Zanussi static pci_ers_result_t adf_error_detected(struct pci_dev *pdev, 14a4b16dadSTom Zanussi pci_channel_state_t state) 15a4b16dadSTom Zanussi { 16a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 17a4b16dadSTom Zanussi 18a4b16dadSTom Zanussi dev_info(&pdev->dev, "Acceleration driver hardware error detected.\n"); 19a4b16dadSTom Zanussi if (!accel_dev) { 20a4b16dadSTom Zanussi dev_err(&pdev->dev, "Can't find acceleration device\n"); 21a4b16dadSTom Zanussi return PCI_ERS_RESULT_DISCONNECT; 22a4b16dadSTom Zanussi } 23a4b16dadSTom Zanussi 24a4b16dadSTom Zanussi if (state == pci_channel_io_perm_failure) { 25a4b16dadSTom Zanussi dev_err(&pdev->dev, "Can't recover from device error\n"); 26a4b16dadSTom Zanussi return PCI_ERS_RESULT_DISCONNECT; 27a4b16dadSTom Zanussi } 28a4b16dadSTom Zanussi 29a4b16dadSTom Zanussi return PCI_ERS_RESULT_NEED_RESET; 30a4b16dadSTom Zanussi } 31a4b16dadSTom Zanussi 32a4b16dadSTom Zanussi /* reset dev data */ 33a4b16dadSTom Zanussi struct adf_reset_dev_data { 34a4b16dadSTom Zanussi int mode; 35a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev; 36a4b16dadSTom Zanussi struct completion compl; 37a4b16dadSTom Zanussi struct work_struct reset_work; 38a4b16dadSTom Zanussi }; 39a4b16dadSTom Zanussi 40a4b16dadSTom Zanussi void adf_reset_sbr(struct adf_accel_dev *accel_dev) 41a4b16dadSTom Zanussi { 42a4b16dadSTom Zanussi struct pci_dev *pdev = accel_to_pci_dev(accel_dev); 43a4b16dadSTom Zanussi struct pci_dev *parent = pdev->bus->self; 44a4b16dadSTom Zanussi u16 bridge_ctl = 0; 45a4b16dadSTom Zanussi 46a4b16dadSTom Zanussi if (!parent) 47a4b16dadSTom Zanussi parent = pdev; 48a4b16dadSTom Zanussi 49a4b16dadSTom Zanussi if (!pci_wait_for_pending_transaction(pdev)) 50a4b16dadSTom Zanussi dev_info(&GET_DEV(accel_dev), 51a4b16dadSTom Zanussi "Transaction still in progress. Proceeding\n"); 52a4b16dadSTom Zanussi 53a4b16dadSTom Zanussi dev_info(&GET_DEV(accel_dev), "Secondary bus reset\n"); 54a4b16dadSTom Zanussi 55a4b16dadSTom Zanussi pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); 56a4b16dadSTom Zanussi bridge_ctl |= PCI_BRIDGE_CTL_BUS_RESET; 57a4b16dadSTom Zanussi pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); 58a4b16dadSTom Zanussi msleep(100); 59a4b16dadSTom Zanussi bridge_ctl &= ~PCI_BRIDGE_CTL_BUS_RESET; 60a4b16dadSTom Zanussi pci_write_config_word(parent, PCI_BRIDGE_CONTROL, bridge_ctl); 61a4b16dadSTom Zanussi msleep(100); 62a4b16dadSTom Zanussi } 63a4b16dadSTom Zanussi EXPORT_SYMBOL_GPL(adf_reset_sbr); 64a4b16dadSTom Zanussi 65a4b16dadSTom Zanussi void adf_reset_flr(struct adf_accel_dev *accel_dev) 66a4b16dadSTom Zanussi { 67a4b16dadSTom Zanussi pcie_flr(accel_to_pci_dev(accel_dev)); 68a4b16dadSTom Zanussi } 69a4b16dadSTom Zanussi EXPORT_SYMBOL_GPL(adf_reset_flr); 70a4b16dadSTom Zanussi 71a4b16dadSTom Zanussi void adf_dev_restore(struct adf_accel_dev *accel_dev) 72a4b16dadSTom Zanussi { 73a4b16dadSTom Zanussi struct adf_hw_device_data *hw_device = accel_dev->hw_device; 74a4b16dadSTom Zanussi struct pci_dev *pdev = accel_to_pci_dev(accel_dev); 75a4b16dadSTom Zanussi 76a4b16dadSTom Zanussi if (hw_device->reset_device) { 77a4b16dadSTom Zanussi dev_info(&GET_DEV(accel_dev), "Resetting device qat_dev%d\n", 78a4b16dadSTom Zanussi accel_dev->accel_id); 79a4b16dadSTom Zanussi hw_device->reset_device(accel_dev); 80a4b16dadSTom Zanussi pci_restore_state(pdev); 81a4b16dadSTom Zanussi pci_save_state(pdev); 82a4b16dadSTom Zanussi } 83a4b16dadSTom Zanussi } 84a4b16dadSTom Zanussi 85a4b16dadSTom Zanussi static void adf_device_reset_worker(struct work_struct *work) 86a4b16dadSTom Zanussi { 87a4b16dadSTom Zanussi struct adf_reset_dev_data *reset_data = 88a4b16dadSTom Zanussi container_of(work, struct adf_reset_dev_data, reset_work); 89a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = reset_data->accel_dev; 90a4b16dadSTom Zanussi 91a4b16dadSTom Zanussi adf_dev_restarting_notify(accel_dev); 92a4b16dadSTom Zanussi if (adf_dev_restart(accel_dev)) { 93a4b16dadSTom Zanussi /* The device hanged and we can't restart it so stop here */ 94a4b16dadSTom Zanussi dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); 958a5a7611SDamian Muszynski if (reset_data->mode == ADF_DEV_RESET_ASYNC || 968a5a7611SDamian Muszynski completion_done(&reset_data->compl)) 97a4b16dadSTom Zanussi kfree(reset_data); 98a4b16dadSTom Zanussi WARN(1, "QAT: device restart failed. Device is unusable\n"); 99a4b16dadSTom Zanussi return; 100a4b16dadSTom Zanussi } 101a4b16dadSTom Zanussi adf_dev_restarted_notify(accel_dev); 102a4b16dadSTom Zanussi clear_bit(ADF_STATUS_RESTARTING, &accel_dev->status); 103a4b16dadSTom Zanussi 1048a5a7611SDamian Muszynski /* 1058a5a7611SDamian Muszynski * The dev is back alive. Notify the caller if in sync mode 1068a5a7611SDamian Muszynski * 1078a5a7611SDamian Muszynski * If device restart will take a more time than expected, 1088a5a7611SDamian Muszynski * the schedule_reset() function can timeout and exit. This can be 1098a5a7611SDamian Muszynski * detected by calling the completion_done() function. In this case 1108a5a7611SDamian Muszynski * the reset_data structure needs to be freed here. 1118a5a7611SDamian Muszynski */ 1128a5a7611SDamian Muszynski if (reset_data->mode == ADF_DEV_RESET_ASYNC || 1138a5a7611SDamian Muszynski completion_done(&reset_data->compl)) 114a4b16dadSTom Zanussi kfree(reset_data); 1158a5a7611SDamian Muszynski else 1168a5a7611SDamian Muszynski complete(&reset_data->compl); 117a4b16dadSTom Zanussi } 118a4b16dadSTom Zanussi 119a4b16dadSTom Zanussi static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, 120a4b16dadSTom Zanussi enum adf_dev_reset_mode mode) 121a4b16dadSTom Zanussi { 122a4b16dadSTom Zanussi struct adf_reset_dev_data *reset_data; 123a4b16dadSTom Zanussi 124a4b16dadSTom Zanussi if (!adf_dev_started(accel_dev) || 125a4b16dadSTom Zanussi test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) 126a4b16dadSTom Zanussi return 0; 127a4b16dadSTom Zanussi 128a4b16dadSTom Zanussi set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); 129a4b16dadSTom Zanussi reset_data = kzalloc(sizeof(*reset_data), GFP_KERNEL); 130a4b16dadSTom Zanussi if (!reset_data) 131a4b16dadSTom Zanussi return -ENOMEM; 132a4b16dadSTom Zanussi reset_data->accel_dev = accel_dev; 133a4b16dadSTom Zanussi init_completion(&reset_data->compl); 134a4b16dadSTom Zanussi reset_data->mode = mode; 135a4b16dadSTom Zanussi INIT_WORK(&reset_data->reset_work, adf_device_reset_worker); 136a4b16dadSTom Zanussi queue_work(device_reset_wq, &reset_data->reset_work); 137a4b16dadSTom Zanussi 138a4b16dadSTom Zanussi /* If in sync mode wait for the result */ 139a4b16dadSTom Zanussi if (mode == ADF_DEV_RESET_SYNC) { 140a4b16dadSTom Zanussi int ret = 0; 141a4b16dadSTom Zanussi /* Maximum device reset time is 10 seconds */ 142a4b16dadSTom Zanussi unsigned long wait_jiffies = msecs_to_jiffies(10000); 143a4b16dadSTom Zanussi unsigned long timeout = wait_for_completion_timeout( 144a4b16dadSTom Zanussi &reset_data->compl, wait_jiffies); 145a4b16dadSTom Zanussi if (!timeout) { 146a4b16dadSTom Zanussi dev_err(&GET_DEV(accel_dev), 147a4b16dadSTom Zanussi "Reset device timeout expired\n"); 148a4b16dadSTom Zanussi ret = -EFAULT; 1498a5a7611SDamian Muszynski } else { 150a4b16dadSTom Zanussi kfree(reset_data); 1518a5a7611SDamian Muszynski } 152a4b16dadSTom Zanussi return ret; 153a4b16dadSTom Zanussi } 154a4b16dadSTom Zanussi return 0; 155a4b16dadSTom Zanussi } 156a4b16dadSTom Zanussi 157a4b16dadSTom Zanussi static pci_ers_result_t adf_slot_reset(struct pci_dev *pdev) 158a4b16dadSTom Zanussi { 159a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 160a4b16dadSTom Zanussi 161a4b16dadSTom Zanussi if (!accel_dev) { 162a4b16dadSTom Zanussi pr_err("QAT: Can't find acceleration device\n"); 163a4b16dadSTom Zanussi return PCI_ERS_RESULT_DISCONNECT; 164a4b16dadSTom Zanussi } 165a4b16dadSTom Zanussi if (adf_dev_aer_schedule_reset(accel_dev, ADF_DEV_RESET_SYNC)) 166a4b16dadSTom Zanussi return PCI_ERS_RESULT_DISCONNECT; 167a4b16dadSTom Zanussi 168a4b16dadSTom Zanussi return PCI_ERS_RESULT_RECOVERED; 169a4b16dadSTom Zanussi } 170a4b16dadSTom Zanussi 171a4b16dadSTom Zanussi static void adf_resume(struct pci_dev *pdev) 172a4b16dadSTom Zanussi { 173a4b16dadSTom Zanussi dev_info(&pdev->dev, "Acceleration driver reset completed\n"); 174a4b16dadSTom Zanussi dev_info(&pdev->dev, "Device is up and running\n"); 175a4b16dadSTom Zanussi } 176a4b16dadSTom Zanussi 177a4b16dadSTom Zanussi const struct pci_error_handlers adf_err_handler = { 178a4b16dadSTom Zanussi .error_detected = adf_error_detected, 179a4b16dadSTom Zanussi .slot_reset = adf_slot_reset, 180a4b16dadSTom Zanussi .resume = adf_resume, 181a4b16dadSTom Zanussi }; 182a4b16dadSTom Zanussi EXPORT_SYMBOL_GPL(adf_err_handler); 183a4b16dadSTom Zanussi 184a4b16dadSTom Zanussi int adf_init_aer(void) 185a4b16dadSTom Zanussi { 186a4b16dadSTom Zanussi device_reset_wq = alloc_workqueue("qat_device_reset_wq", 187a4b16dadSTom Zanussi WQ_MEM_RECLAIM, 0); 188a4b16dadSTom Zanussi return !device_reset_wq ? -EFAULT : 0; 189a4b16dadSTom Zanussi } 190a4b16dadSTom Zanussi 191a4b16dadSTom Zanussi void adf_exit_aer(void) 192a4b16dadSTom Zanussi { 193a4b16dadSTom Zanussi if (device_reset_wq) 194a4b16dadSTom Zanussi destroy_workqueue(device_reset_wq); 195a4b16dadSTom Zanussi device_reset_wq = NULL; 196a4b16dadSTom Zanussi } 197