155435ea7SShannon Nelson // SPDX-License-Identifier: GPL-2.0 255435ea7SShannon Nelson /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 355435ea7SShannon Nelson 455435ea7SShannon Nelson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 555435ea7SShannon Nelson 655435ea7SShannon Nelson #include <linux/pci.h> 755435ea7SShannon Nelson 855435ea7SShannon Nelson #include <linux/pds/pds_common.h> 955435ea7SShannon Nelson 1055435ea7SShannon Nelson #include "core.h" 1155435ea7SShannon Nelson 1255435ea7SShannon Nelson MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION); 1355435ea7SShannon Nelson MODULE_AUTHOR("Advanced Micro Devices, Inc"); 1455435ea7SShannon Nelson MODULE_LICENSE("GPL"); 1555435ea7SShannon Nelson 1655435ea7SShannon Nelson /* Supported devices */ 1755435ea7SShannon Nelson static const struct pci_device_id pdsc_id_table[] = { 1855435ea7SShannon Nelson { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) }, 19f53d9311SShannon Nelson { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) }, 2055435ea7SShannon Nelson { 0, } /* end of table */ 2155435ea7SShannon Nelson }; 2255435ea7SShannon Nelson MODULE_DEVICE_TABLE(pci, pdsc_id_table); 2355435ea7SShannon Nelson 24c2dbb090SShannon Nelson static void pdsc_wdtimer_cb(struct timer_list *t) 25c2dbb090SShannon Nelson { 26c2dbb090SShannon Nelson struct pdsc *pdsc = from_timer(pdsc, t, wdtimer); 27c2dbb090SShannon Nelson 28c2dbb090SShannon Nelson dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies); 29c2dbb090SShannon Nelson mod_timer(&pdsc->wdtimer, 30c2dbb090SShannon Nelson round_jiffies(jiffies + pdsc->wdtimer_period)); 31c2dbb090SShannon Nelson 32c2dbb090SShannon Nelson queue_work(pdsc->wq, &pdsc->health_work); 33c2dbb090SShannon Nelson } 34c2dbb090SShannon Nelson 3555435ea7SShannon Nelson static void pdsc_unmap_bars(struct pdsc *pdsc) 3655435ea7SShannon Nelson { 3755435ea7SShannon Nelson struct pdsc_dev_bar *bars = pdsc->bars; 3855435ea7SShannon Nelson unsigned int i; 3955435ea7SShannon Nelson 4055435ea7SShannon Nelson for (i = 0; i < PDS_CORE_BARS_MAX; i++) { 4155435ea7SShannon Nelson if (bars[i].vaddr) 4255435ea7SShannon Nelson pci_iounmap(pdsc->pdev, bars[i].vaddr); 4355435ea7SShannon Nelson } 4455435ea7SShannon Nelson } 4555435ea7SShannon Nelson 4655435ea7SShannon Nelson static int pdsc_map_bars(struct pdsc *pdsc) 4755435ea7SShannon Nelson { 4855435ea7SShannon Nelson struct pdsc_dev_bar *bar = pdsc->bars; 4955435ea7SShannon Nelson struct pci_dev *pdev = pdsc->pdev; 5055435ea7SShannon Nelson struct device *dev = pdsc->dev; 5155435ea7SShannon Nelson struct pdsc_dev_bar *bars; 5255435ea7SShannon Nelson unsigned int i, j; 5355435ea7SShannon Nelson int num_bars = 0; 5455435ea7SShannon Nelson int err; 5555435ea7SShannon Nelson u32 sig; 5655435ea7SShannon Nelson 5755435ea7SShannon Nelson bars = pdsc->bars; 5855435ea7SShannon Nelson 5955435ea7SShannon Nelson /* Since the PCI interface in the hardware is configurable, 6055435ea7SShannon Nelson * we need to poke into all the bars to find the set we're 6155435ea7SShannon Nelson * expecting. 6255435ea7SShannon Nelson */ 6355435ea7SShannon Nelson for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) { 6455435ea7SShannon Nelson if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 6555435ea7SShannon Nelson continue; 6655435ea7SShannon Nelson 6755435ea7SShannon Nelson bars[j].len = pci_resource_len(pdev, i); 6855435ea7SShannon Nelson bars[j].bus_addr = pci_resource_start(pdev, i); 6955435ea7SShannon Nelson bars[j].res_index = i; 7055435ea7SShannon Nelson 7155435ea7SShannon Nelson /* only map the whole bar 0 */ 7255435ea7SShannon Nelson if (j > 0) { 7355435ea7SShannon Nelson bars[j].vaddr = NULL; 7455435ea7SShannon Nelson } else { 7555435ea7SShannon Nelson bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 7655435ea7SShannon Nelson if (!bars[j].vaddr) { 7755435ea7SShannon Nelson dev_err(dev, "Cannot map BAR %d, aborting\n", i); 7855435ea7SShannon Nelson return -ENODEV; 7955435ea7SShannon Nelson } 8055435ea7SShannon Nelson } 8155435ea7SShannon Nelson 8255435ea7SShannon Nelson j++; 8355435ea7SShannon Nelson } 8455435ea7SShannon Nelson num_bars = j; 8555435ea7SShannon Nelson 8655435ea7SShannon Nelson /* BAR0: dev_cmd and interrupts */ 8755435ea7SShannon Nelson if (num_bars < 1) { 8855435ea7SShannon Nelson dev_err(dev, "No bars found\n"); 8955435ea7SShannon Nelson err = -EFAULT; 9055435ea7SShannon Nelson goto err_out; 9155435ea7SShannon Nelson } 9255435ea7SShannon Nelson 9355435ea7SShannon Nelson if (bar->len < PDS_CORE_BAR0_SIZE) { 9455435ea7SShannon Nelson dev_err(dev, "Resource bar size %lu too small\n", bar->len); 9555435ea7SShannon Nelson err = -EFAULT; 9655435ea7SShannon Nelson goto err_out; 9755435ea7SShannon Nelson } 9855435ea7SShannon Nelson 9955435ea7SShannon Nelson pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET; 10055435ea7SShannon Nelson pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET; 10155435ea7SShannon Nelson pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET; 10255435ea7SShannon Nelson pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET; 10355435ea7SShannon Nelson 10455435ea7SShannon Nelson sig = ioread32(&pdsc->info_regs->signature); 10555435ea7SShannon Nelson if (sig != PDS_CORE_DEV_INFO_SIGNATURE) { 10655435ea7SShannon Nelson dev_err(dev, "Incompatible firmware signature %x", sig); 10755435ea7SShannon Nelson err = -EFAULT; 10855435ea7SShannon Nelson goto err_out; 10955435ea7SShannon Nelson } 11055435ea7SShannon Nelson 11155435ea7SShannon Nelson /* BAR1: doorbells */ 11255435ea7SShannon Nelson bar++; 11355435ea7SShannon Nelson if (num_bars < 2) { 11455435ea7SShannon Nelson dev_err(dev, "Doorbell bar missing\n"); 11555435ea7SShannon Nelson err = -EFAULT; 11655435ea7SShannon Nelson goto err_out; 11755435ea7SShannon Nelson } 11855435ea7SShannon Nelson 11955435ea7SShannon Nelson pdsc->db_pages = bar->vaddr; 12055435ea7SShannon Nelson pdsc->phy_db_pages = bar->bus_addr; 12155435ea7SShannon Nelson 12255435ea7SShannon Nelson return 0; 12355435ea7SShannon Nelson 12455435ea7SShannon Nelson err_out: 12555435ea7SShannon Nelson pdsc_unmap_bars(pdsc); 12655435ea7SShannon Nelson return err; 12755435ea7SShannon Nelson } 12855435ea7SShannon Nelson 12945d76f49SShannon Nelson void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num) 13045d76f49SShannon Nelson { 13145d76f49SShannon Nelson return pci_iomap_range(pdsc->pdev, 13245d76f49SShannon Nelson pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index, 13345d76f49SShannon Nelson (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 13445d76f49SShannon Nelson } 13545d76f49SShannon Nelson 136f53d9311SShannon Nelson static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs) 137f53d9311SShannon Nelson { 138f53d9311SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev); 139f53d9311SShannon Nelson struct device *dev = pdsc->dev; 140f53d9311SShannon Nelson int ret = 0; 141f53d9311SShannon Nelson 142f53d9311SShannon Nelson if (num_vfs > 0) { 143f53d9311SShannon Nelson pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf), 144f53d9311SShannon Nelson GFP_KERNEL); 145f53d9311SShannon Nelson if (!pdsc->vfs) 146f53d9311SShannon Nelson return -ENOMEM; 147f53d9311SShannon Nelson pdsc->num_vfs = num_vfs; 148f53d9311SShannon Nelson 149f53d9311SShannon Nelson ret = pci_enable_sriov(pdev, num_vfs); 150f53d9311SShannon Nelson if (ret) { 151f53d9311SShannon Nelson dev_err(dev, "Cannot enable SRIOV: %pe\n", 152f53d9311SShannon Nelson ERR_PTR(ret)); 153f53d9311SShannon Nelson goto no_vfs; 154f53d9311SShannon Nelson } 155f53d9311SShannon Nelson 156f53d9311SShannon Nelson return num_vfs; 157f53d9311SShannon Nelson } 158f53d9311SShannon Nelson 159f53d9311SShannon Nelson no_vfs: 160f53d9311SShannon Nelson pci_disable_sriov(pdev); 161f53d9311SShannon Nelson 162f53d9311SShannon Nelson kfree(pdsc->vfs); 163f53d9311SShannon Nelson pdsc->vfs = NULL; 164f53d9311SShannon Nelson pdsc->num_vfs = 0; 165f53d9311SShannon Nelson 166f53d9311SShannon Nelson return ret; 167f53d9311SShannon Nelson } 168f53d9311SShannon Nelson 16955435ea7SShannon Nelson static int pdsc_init_vf(struct pdsc *vf) 17055435ea7SShannon Nelson { 171f53d9311SShannon Nelson struct devlink *dl; 1724569cce4SShannon Nelson struct pdsc *pf; 1734569cce4SShannon Nelson int err; 1744569cce4SShannon Nelson 1754569cce4SShannon Nelson pf = pdsc_get_pf_struct(vf->pdev); 1764569cce4SShannon Nelson if (IS_ERR_OR_NULL(pf)) 1774569cce4SShannon Nelson return PTR_ERR(pf) ?: -1; 178f53d9311SShannon Nelson 179f53d9311SShannon Nelson vf->vf_id = pci_iov_vf_id(vf->pdev); 180f53d9311SShannon Nelson 181f53d9311SShannon Nelson dl = priv_to_devlink(vf); 182f53d9311SShannon Nelson devl_lock(dl); 183f53d9311SShannon Nelson devl_register(dl); 184f53d9311SShannon Nelson devl_unlock(dl); 185f53d9311SShannon Nelson 1864569cce4SShannon Nelson pf->vfs[vf->vf_id].vf = vf; 1874569cce4SShannon Nelson err = pdsc_auxbus_dev_add(vf, pf); 1884569cce4SShannon Nelson if (err) { 1894569cce4SShannon Nelson devl_lock(dl); 1904569cce4SShannon Nelson devl_unregister(dl); 1914569cce4SShannon Nelson devl_unlock(dl); 1924569cce4SShannon Nelson } 1934569cce4SShannon Nelson 1944569cce4SShannon Nelson return err; 19555435ea7SShannon Nelson } 19655435ea7SShannon Nelson 19725b450c0SShannon Nelson static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = { 19825b450c0SShannon Nelson .name = "fw", 19925b450c0SShannon Nelson .diagnose = pdsc_fw_reporter_diagnose, 20025b450c0SShannon Nelson }; 20125b450c0SShannon Nelson 20240ced894SShannon Nelson static const struct devlink_param pdsc_dl_params[] = { 20340ced894SShannon Nelson DEVLINK_PARAM_GENERIC(ENABLE_VNET, 20440ced894SShannon Nelson BIT(DEVLINK_PARAM_CMODE_RUNTIME), 20540ced894SShannon Nelson pdsc_dl_enable_get, 20640ced894SShannon Nelson pdsc_dl_enable_set, 20740ced894SShannon Nelson pdsc_dl_enable_validate), 20840ced894SShannon Nelson }; 20940ced894SShannon Nelson 210c2dbb090SShannon Nelson #define PDSC_WQ_NAME_LEN 24 211c2dbb090SShannon Nelson 21255435ea7SShannon Nelson static int pdsc_init_pf(struct pdsc *pdsc) 21355435ea7SShannon Nelson { 21425b450c0SShannon Nelson struct devlink_health_reporter *hr; 215c2dbb090SShannon Nelson char wq_name[PDSC_WQ_NAME_LEN]; 21655435ea7SShannon Nelson struct devlink *dl; 21755435ea7SShannon Nelson int err; 21855435ea7SShannon Nelson 21955435ea7SShannon Nelson pcie_print_link_status(pdsc->pdev); 22055435ea7SShannon Nelson 22155435ea7SShannon Nelson err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 22255435ea7SShannon Nelson if (err) { 22355435ea7SShannon Nelson dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 22455435ea7SShannon Nelson ERR_PTR(err)); 22555435ea7SShannon Nelson return err; 22655435ea7SShannon Nelson } 22755435ea7SShannon Nelson 22855435ea7SShannon Nelson err = pdsc_map_bars(pdsc); 22955435ea7SShannon Nelson if (err) 23055435ea7SShannon Nelson goto err_out_release_regions; 23155435ea7SShannon Nelson 232c2dbb090SShannon Nelson /* General workqueue and timer, but don't start timer yet */ 233c2dbb090SShannon Nelson snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); 234c2dbb090SShannon Nelson pdsc->wq = create_singlethread_workqueue(wq_name); 235c2dbb090SShannon Nelson INIT_WORK(&pdsc->health_work, pdsc_health_thread); 236c2dbb090SShannon Nelson timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 237c2dbb090SShannon Nelson pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ; 238c2dbb090SShannon Nelson 239523847dfSShannon Nelson mutex_init(&pdsc->devcmd_lock); 240523847dfSShannon Nelson mutex_init(&pdsc->config_lock); 24145d76f49SShannon Nelson spin_lock_init(&pdsc->adminq_lock); 242523847dfSShannon Nelson 243523847dfSShannon Nelson mutex_lock(&pdsc->config_lock); 244523847dfSShannon Nelson set_bit(PDSC_S_FW_DEAD, &pdsc->state); 245523847dfSShannon Nelson 246523847dfSShannon Nelson err = pdsc_setup(pdsc, PDSC_SETUP_INIT); 2471e76f427SShannon Nelson if (err) { 2481e76f427SShannon Nelson mutex_unlock(&pdsc->config_lock); 249523847dfSShannon Nelson goto err_out_unmap_bars; 2501e76f427SShannon Nelson } 2511e76f427SShannon Nelson 25245d76f49SShannon Nelson err = pdsc_start(pdsc); 2531e76f427SShannon Nelson if (err) { 2541e76f427SShannon Nelson mutex_unlock(&pdsc->config_lock); 25545d76f49SShannon Nelson goto err_out_teardown; 2561e76f427SShannon Nelson } 257523847dfSShannon Nelson 258523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock); 259523847dfSShannon Nelson 26055435ea7SShannon Nelson dl = priv_to_devlink(pdsc); 26155435ea7SShannon Nelson devl_lock(dl); 26240ced894SShannon Nelson err = devl_params_register(dl, pdsc_dl_params, 26340ced894SShannon Nelson ARRAY_SIZE(pdsc_dl_params)); 26440ced894SShannon Nelson if (err) { 2651e76f427SShannon Nelson devl_unlock(dl); 26640ced894SShannon Nelson dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n", 26740ced894SShannon Nelson ERR_PTR(err)); 2681e76f427SShannon Nelson goto err_out_stop; 26940ced894SShannon Nelson } 27025b450c0SShannon Nelson 27125b450c0SShannon Nelson hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc); 27225b450c0SShannon Nelson if (IS_ERR(hr)) { 2731e76f427SShannon Nelson devl_unlock(dl); 27425b450c0SShannon Nelson dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr); 27525b450c0SShannon Nelson err = PTR_ERR(hr); 27640ced894SShannon Nelson goto err_out_unreg_params; 27725b450c0SShannon Nelson } 27825b450c0SShannon Nelson pdsc->fw_reporter = hr; 27925b450c0SShannon Nelson 28055435ea7SShannon Nelson devl_register(dl); 28155435ea7SShannon Nelson devl_unlock(dl); 28255435ea7SShannon Nelson 283c2dbb090SShannon Nelson /* Lastly, start the health check timer */ 284c2dbb090SShannon Nelson mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period)); 285c2dbb090SShannon Nelson 28655435ea7SShannon Nelson return 0; 28755435ea7SShannon Nelson 28840ced894SShannon Nelson err_out_unreg_params: 2891e76f427SShannon Nelson devlink_params_unregister(dl, pdsc_dl_params, 29040ced894SShannon Nelson ARRAY_SIZE(pdsc_dl_params)); 2911e76f427SShannon Nelson err_out_stop: 29245d76f49SShannon Nelson pdsc_stop(pdsc); 29325b450c0SShannon Nelson err_out_teardown: 29425b450c0SShannon Nelson pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 295523847dfSShannon Nelson err_out_unmap_bars: 296c2dbb090SShannon Nelson del_timer_sync(&pdsc->wdtimer); 297c2dbb090SShannon Nelson if (pdsc->wq) 298c2dbb090SShannon Nelson destroy_workqueue(pdsc->wq); 299523847dfSShannon Nelson mutex_destroy(&pdsc->config_lock); 300523847dfSShannon Nelson mutex_destroy(&pdsc->devcmd_lock); 301523847dfSShannon Nelson pci_free_irq_vectors(pdsc->pdev); 302523847dfSShannon Nelson pdsc_unmap_bars(pdsc); 30355435ea7SShannon Nelson err_out_release_regions: 30455435ea7SShannon Nelson pci_release_regions(pdsc->pdev); 30555435ea7SShannon Nelson 30655435ea7SShannon Nelson return err; 30755435ea7SShannon Nelson } 30855435ea7SShannon Nelson 30955435ea7SShannon Nelson static const struct devlink_ops pdsc_dl_ops = { 31045d76f49SShannon Nelson .info_get = pdsc_dl_info_get, 31149ce92fbSShannon Nelson .flash_update = pdsc_dl_flash_update, 31255435ea7SShannon Nelson }; 31355435ea7SShannon Nelson 31455435ea7SShannon Nelson static const struct devlink_ops pdsc_dl_vf_ops = { 31555435ea7SShannon Nelson }; 31655435ea7SShannon Nelson 31755435ea7SShannon Nelson static DEFINE_IDA(pdsc_ida); 31855435ea7SShannon Nelson 31955435ea7SShannon Nelson static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 32055435ea7SShannon Nelson { 32155435ea7SShannon Nelson struct device *dev = &pdev->dev; 32255435ea7SShannon Nelson const struct devlink_ops *ops; 32355435ea7SShannon Nelson struct devlink *dl; 32455435ea7SShannon Nelson struct pdsc *pdsc; 32555435ea7SShannon Nelson bool is_pf; 32655435ea7SShannon Nelson int err; 32755435ea7SShannon Nelson 32855435ea7SShannon Nelson is_pf = !pdev->is_virtfn; 32955435ea7SShannon Nelson ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops; 33055435ea7SShannon Nelson dl = devlink_alloc(ops, sizeof(struct pdsc), dev); 33155435ea7SShannon Nelson if (!dl) 33255435ea7SShannon Nelson return -ENOMEM; 33355435ea7SShannon Nelson pdsc = devlink_priv(dl); 33455435ea7SShannon Nelson 33555435ea7SShannon Nelson pdsc->pdev = pdev; 33655435ea7SShannon Nelson pdsc->dev = &pdev->dev; 33755435ea7SShannon Nelson set_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 33855435ea7SShannon Nelson pci_set_drvdata(pdev, pdsc); 33955435ea7SShannon Nelson pdsc_debugfs_add_dev(pdsc); 34055435ea7SShannon Nelson 34155435ea7SShannon Nelson err = ida_alloc(&pdsc_ida, GFP_KERNEL); 34255435ea7SShannon Nelson if (err < 0) { 34355435ea7SShannon Nelson dev_err(pdsc->dev, "%s: id alloc failed: %pe\n", 34455435ea7SShannon Nelson __func__, ERR_PTR(err)); 34555435ea7SShannon Nelson goto err_out_free_devlink; 34655435ea7SShannon Nelson } 34755435ea7SShannon Nelson pdsc->uid = err; 34855435ea7SShannon Nelson 34955435ea7SShannon Nelson /* Query system for DMA addressing limitation for the device. */ 35055435ea7SShannon Nelson err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN)); 35155435ea7SShannon Nelson if (err) { 35255435ea7SShannon Nelson dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n", 35355435ea7SShannon Nelson ERR_PTR(err)); 35455435ea7SShannon Nelson goto err_out_free_ida; 35555435ea7SShannon Nelson } 35655435ea7SShannon Nelson 35755435ea7SShannon Nelson err = pci_enable_device(pdev); 35855435ea7SShannon Nelson if (err) { 35955435ea7SShannon Nelson dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 36055435ea7SShannon Nelson goto err_out_free_ida; 36155435ea7SShannon Nelson } 36255435ea7SShannon Nelson pci_set_master(pdev); 36355435ea7SShannon Nelson 36455435ea7SShannon Nelson if (is_pf) 36555435ea7SShannon Nelson err = pdsc_init_pf(pdsc); 36655435ea7SShannon Nelson else 36755435ea7SShannon Nelson err = pdsc_init_vf(pdsc); 36855435ea7SShannon Nelson if (err) { 36955435ea7SShannon Nelson dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err)); 3702f48b1d8SYu Liao goto err_out_disable_device; 37155435ea7SShannon Nelson } 37255435ea7SShannon Nelson 37355435ea7SShannon Nelson clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 37455435ea7SShannon Nelson return 0; 37555435ea7SShannon Nelson 3762f48b1d8SYu Liao err_out_disable_device: 37755435ea7SShannon Nelson pci_disable_device(pdev); 37855435ea7SShannon Nelson err_out_free_ida: 37955435ea7SShannon Nelson ida_free(&pdsc_ida, pdsc->uid); 38055435ea7SShannon Nelson err_out_free_devlink: 38155435ea7SShannon Nelson pdsc_debugfs_del_dev(pdsc); 38255435ea7SShannon Nelson devlink_free(dl); 38355435ea7SShannon Nelson 38455435ea7SShannon Nelson return err; 38555435ea7SShannon Nelson } 38655435ea7SShannon Nelson 38755435ea7SShannon Nelson static void pdsc_remove(struct pci_dev *pdev) 38855435ea7SShannon Nelson { 38955435ea7SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev); 39055435ea7SShannon Nelson struct devlink *dl; 39155435ea7SShannon Nelson 39255435ea7SShannon Nelson /* Unhook the registrations first to be sure there 39355435ea7SShannon Nelson * are no requests while we're stopping. 39455435ea7SShannon Nelson */ 39555435ea7SShannon Nelson dl = priv_to_devlink(pdsc); 39655435ea7SShannon Nelson devl_lock(dl); 39755435ea7SShannon Nelson devl_unregister(dl); 39840ced894SShannon Nelson if (!pdev->is_virtfn) { 39925b450c0SShannon Nelson if (pdsc->fw_reporter) { 40025b450c0SShannon Nelson devl_health_reporter_destroy(pdsc->fw_reporter); 40125b450c0SShannon Nelson pdsc->fw_reporter = NULL; 40225b450c0SShannon Nelson } 40340ced894SShannon Nelson devl_params_unregister(dl, pdsc_dl_params, 40440ced894SShannon Nelson ARRAY_SIZE(pdsc_dl_params)); 40540ced894SShannon Nelson } 40655435ea7SShannon Nelson devl_unlock(dl); 40755435ea7SShannon Nelson 4084569cce4SShannon Nelson if (pdev->is_virtfn) { 4094569cce4SShannon Nelson struct pdsc *pf; 4104569cce4SShannon Nelson 4114569cce4SShannon Nelson pf = pdsc_get_pf_struct(pdsc->pdev); 4124569cce4SShannon Nelson if (!IS_ERR(pf)) { 4134569cce4SShannon Nelson pdsc_auxbus_dev_del(pdsc, pf); 4144569cce4SShannon Nelson pf->vfs[pdsc->vf_id].vf = NULL; 4154569cce4SShannon Nelson } 4164569cce4SShannon Nelson } else { 4174569cce4SShannon Nelson /* Remove the VFs and their aux_bus connections before other 4184569cce4SShannon Nelson * cleanup so that the clients can use the AdminQ to cleanly 4194569cce4SShannon Nelson * shut themselves down. 4204569cce4SShannon Nelson */ 421f53d9311SShannon Nelson pdsc_sriov_configure(pdev, 0); 422f53d9311SShannon Nelson 423c2dbb090SShannon Nelson del_timer_sync(&pdsc->wdtimer); 424c2dbb090SShannon Nelson if (pdsc->wq) 425c2dbb090SShannon Nelson destroy_workqueue(pdsc->wq); 426c2dbb090SShannon Nelson 427523847dfSShannon Nelson mutex_lock(&pdsc->config_lock); 428523847dfSShannon Nelson set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); 429523847dfSShannon Nelson 43045d76f49SShannon Nelson pdsc_stop(pdsc); 431523847dfSShannon Nelson pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 432523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock); 433523847dfSShannon Nelson mutex_destroy(&pdsc->config_lock); 434523847dfSShannon Nelson mutex_destroy(&pdsc->devcmd_lock); 435523847dfSShannon Nelson 436523847dfSShannon Nelson pci_free_irq_vectors(pdev); 43755435ea7SShannon Nelson pdsc_unmap_bars(pdsc); 43855435ea7SShannon Nelson pci_release_regions(pdev); 439523847dfSShannon Nelson } 44055435ea7SShannon Nelson 44155435ea7SShannon Nelson pci_disable_device(pdev); 44255435ea7SShannon Nelson 44355435ea7SShannon Nelson ida_free(&pdsc_ida, pdsc->uid); 44455435ea7SShannon Nelson pdsc_debugfs_del_dev(pdsc); 44555435ea7SShannon Nelson devlink_free(dl); 44655435ea7SShannon Nelson } 44755435ea7SShannon Nelson 448*699f5416SShannon Nelson static void pdsc_reset_prepare(struct pci_dev *pdev) 449*699f5416SShannon Nelson { 450*699f5416SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev); 451*699f5416SShannon Nelson 452*699f5416SShannon Nelson pdsc_fw_down(pdsc); 453*699f5416SShannon Nelson 454*699f5416SShannon Nelson pci_free_irq_vectors(pdev); 455*699f5416SShannon Nelson pdsc_unmap_bars(pdsc); 456*699f5416SShannon Nelson pci_release_regions(pdev); 457*699f5416SShannon Nelson pci_disable_device(pdev); 458*699f5416SShannon Nelson } 459*699f5416SShannon Nelson 460*699f5416SShannon Nelson static void pdsc_reset_done(struct pci_dev *pdev) 461*699f5416SShannon Nelson { 462*699f5416SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev); 463*699f5416SShannon Nelson struct device *dev = pdsc->dev; 464*699f5416SShannon Nelson int err; 465*699f5416SShannon Nelson 466*699f5416SShannon Nelson err = pci_enable_device(pdev); 467*699f5416SShannon Nelson if (err) { 468*699f5416SShannon Nelson dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 469*699f5416SShannon Nelson return; 470*699f5416SShannon Nelson } 471*699f5416SShannon Nelson pci_set_master(pdev); 472*699f5416SShannon Nelson 473*699f5416SShannon Nelson if (!pdev->is_virtfn) { 474*699f5416SShannon Nelson pcie_print_link_status(pdsc->pdev); 475*699f5416SShannon Nelson 476*699f5416SShannon Nelson err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 477*699f5416SShannon Nelson if (err) { 478*699f5416SShannon Nelson dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 479*699f5416SShannon Nelson ERR_PTR(err)); 480*699f5416SShannon Nelson return; 481*699f5416SShannon Nelson } 482*699f5416SShannon Nelson 483*699f5416SShannon Nelson err = pdsc_map_bars(pdsc); 484*699f5416SShannon Nelson if (err) 485*699f5416SShannon Nelson return; 486*699f5416SShannon Nelson } 487*699f5416SShannon Nelson 488*699f5416SShannon Nelson pdsc_fw_up(pdsc); 489*699f5416SShannon Nelson } 490*699f5416SShannon Nelson 491*699f5416SShannon Nelson static const struct pci_error_handlers pdsc_err_handler = { 492*699f5416SShannon Nelson /* FLR handling */ 493*699f5416SShannon Nelson .reset_prepare = pdsc_reset_prepare, 494*699f5416SShannon Nelson .reset_done = pdsc_reset_done, 495*699f5416SShannon Nelson }; 496*699f5416SShannon Nelson 49755435ea7SShannon Nelson static struct pci_driver pdsc_driver = { 49855435ea7SShannon Nelson .name = PDS_CORE_DRV_NAME, 49955435ea7SShannon Nelson .id_table = pdsc_id_table, 50055435ea7SShannon Nelson .probe = pdsc_probe, 50155435ea7SShannon Nelson .remove = pdsc_remove, 502f53d9311SShannon Nelson .sriov_configure = pdsc_sriov_configure, 503*699f5416SShannon Nelson .err_handler = &pdsc_err_handler, 50455435ea7SShannon Nelson }; 50555435ea7SShannon Nelson 5064569cce4SShannon Nelson void *pdsc_get_pf_struct(struct pci_dev *vf_pdev) 5074569cce4SShannon Nelson { 5084569cce4SShannon Nelson return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver); 5094569cce4SShannon Nelson } 5104569cce4SShannon Nelson EXPORT_SYMBOL_GPL(pdsc_get_pf_struct); 5114569cce4SShannon Nelson 51255435ea7SShannon Nelson static int __init pdsc_init_module(void) 51355435ea7SShannon Nelson { 51455435ea7SShannon Nelson if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME)) 51555435ea7SShannon Nelson return -EINVAL; 51655435ea7SShannon Nelson 51755435ea7SShannon Nelson pdsc_debugfs_create(); 51855435ea7SShannon Nelson return pci_register_driver(&pdsc_driver); 51955435ea7SShannon Nelson } 52055435ea7SShannon Nelson 52155435ea7SShannon Nelson static void __exit pdsc_cleanup_module(void) 52255435ea7SShannon Nelson { 52355435ea7SShannon Nelson pci_unregister_driver(&pdsc_driver); 52455435ea7SShannon Nelson pdsc_debugfs_destroy(); 52555435ea7SShannon Nelson } 52655435ea7SShannon Nelson 52755435ea7SShannon Nelson module_init(pdsc_init_module); 52855435ea7SShannon Nelson module_exit(pdsc_cleanup_module); 529