155435ea7SShannon Nelson // SPDX-License-Identifier: GPL-2.0
255435ea7SShannon Nelson /* Copyright(c) 2023 Advanced Micro Devices, Inc */
355435ea7SShannon Nelson
455435ea7SShannon Nelson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
555435ea7SShannon Nelson
655435ea7SShannon Nelson #include <linux/pci.h>
755435ea7SShannon Nelson
855435ea7SShannon Nelson #include <linux/pds/pds_common.h>
955435ea7SShannon Nelson
1055435ea7SShannon Nelson #include "core.h"
1155435ea7SShannon Nelson
1255435ea7SShannon Nelson MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION);
1355435ea7SShannon Nelson MODULE_AUTHOR("Advanced Micro Devices, Inc");
1455435ea7SShannon Nelson MODULE_LICENSE("GPL");
1555435ea7SShannon Nelson
1655435ea7SShannon Nelson /* Supported devices */
1755435ea7SShannon Nelson static const struct pci_device_id pdsc_id_table[] = {
1855435ea7SShannon Nelson { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) },
19f53d9311SShannon Nelson { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) },
2055435ea7SShannon Nelson { 0, } /* end of table */
2155435ea7SShannon Nelson };
2255435ea7SShannon Nelson MODULE_DEVICE_TABLE(pci, pdsc_id_table);
2355435ea7SShannon Nelson
pdsc_wdtimer_cb(struct timer_list * t)24c2dbb090SShannon Nelson static void pdsc_wdtimer_cb(struct timer_list *t)
25c2dbb090SShannon Nelson {
26c2dbb090SShannon Nelson struct pdsc *pdsc = from_timer(pdsc, t, wdtimer);
27c2dbb090SShannon Nelson
28c2dbb090SShannon Nelson dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies);
29c2dbb090SShannon Nelson mod_timer(&pdsc->wdtimer,
30c2dbb090SShannon Nelson round_jiffies(jiffies + pdsc->wdtimer_period));
31c2dbb090SShannon Nelson
32c2dbb090SShannon Nelson queue_work(pdsc->wq, &pdsc->health_work);
33c2dbb090SShannon Nelson }
34c2dbb090SShannon Nelson
pdsc_unmap_bars(struct pdsc * pdsc)3555435ea7SShannon Nelson static void pdsc_unmap_bars(struct pdsc *pdsc)
3655435ea7SShannon Nelson {
3755435ea7SShannon Nelson struct pdsc_dev_bar *bars = pdsc->bars;
3855435ea7SShannon Nelson unsigned int i;
3955435ea7SShannon Nelson
40f6ec6ac9SBrett Creeley pdsc->info_regs = NULL;
41f6ec6ac9SBrett Creeley pdsc->cmd_regs = NULL;
42f6ec6ac9SBrett Creeley pdsc->intr_status = NULL;
43f6ec6ac9SBrett Creeley pdsc->intr_ctrl = NULL;
44f6ec6ac9SBrett Creeley
4555435ea7SShannon Nelson for (i = 0; i < PDS_CORE_BARS_MAX; i++) {
4655435ea7SShannon Nelson if (bars[i].vaddr)
4755435ea7SShannon Nelson pci_iounmap(pdsc->pdev, bars[i].vaddr);
4855435ea7SShannon Nelson }
4955435ea7SShannon Nelson }
5055435ea7SShannon Nelson
pdsc_map_bars(struct pdsc * pdsc)5155435ea7SShannon Nelson static int pdsc_map_bars(struct pdsc *pdsc)
5255435ea7SShannon Nelson {
5355435ea7SShannon Nelson struct pdsc_dev_bar *bar = pdsc->bars;
5455435ea7SShannon Nelson struct pci_dev *pdev = pdsc->pdev;
5555435ea7SShannon Nelson struct device *dev = pdsc->dev;
5655435ea7SShannon Nelson struct pdsc_dev_bar *bars;
5755435ea7SShannon Nelson unsigned int i, j;
5855435ea7SShannon Nelson int num_bars = 0;
5955435ea7SShannon Nelson int err;
6055435ea7SShannon Nelson u32 sig;
6155435ea7SShannon Nelson
6255435ea7SShannon Nelson bars = pdsc->bars;
6355435ea7SShannon Nelson
6455435ea7SShannon Nelson /* Since the PCI interface in the hardware is configurable,
6555435ea7SShannon Nelson * we need to poke into all the bars to find the set we're
6655435ea7SShannon Nelson * expecting.
6755435ea7SShannon Nelson */
6855435ea7SShannon Nelson for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) {
6955435ea7SShannon Nelson if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
7055435ea7SShannon Nelson continue;
7155435ea7SShannon Nelson
7255435ea7SShannon Nelson bars[j].len = pci_resource_len(pdev, i);
7355435ea7SShannon Nelson bars[j].bus_addr = pci_resource_start(pdev, i);
7455435ea7SShannon Nelson bars[j].res_index = i;
7555435ea7SShannon Nelson
7655435ea7SShannon Nelson /* only map the whole bar 0 */
7755435ea7SShannon Nelson if (j > 0) {
7855435ea7SShannon Nelson bars[j].vaddr = NULL;
7955435ea7SShannon Nelson } else {
8055435ea7SShannon Nelson bars[j].vaddr = pci_iomap(pdev, i, bars[j].len);
8155435ea7SShannon Nelson if (!bars[j].vaddr) {
8255435ea7SShannon Nelson dev_err(dev, "Cannot map BAR %d, aborting\n", i);
8355435ea7SShannon Nelson return -ENODEV;
8455435ea7SShannon Nelson }
8555435ea7SShannon Nelson }
8655435ea7SShannon Nelson
8755435ea7SShannon Nelson j++;
8855435ea7SShannon Nelson }
8955435ea7SShannon Nelson num_bars = j;
9055435ea7SShannon Nelson
9155435ea7SShannon Nelson /* BAR0: dev_cmd and interrupts */
9255435ea7SShannon Nelson if (num_bars < 1) {
9355435ea7SShannon Nelson dev_err(dev, "No bars found\n");
9455435ea7SShannon Nelson err = -EFAULT;
9555435ea7SShannon Nelson goto err_out;
9655435ea7SShannon Nelson }
9755435ea7SShannon Nelson
9855435ea7SShannon Nelson if (bar->len < PDS_CORE_BAR0_SIZE) {
9955435ea7SShannon Nelson dev_err(dev, "Resource bar size %lu too small\n", bar->len);
10055435ea7SShannon Nelson err = -EFAULT;
10155435ea7SShannon Nelson goto err_out;
10255435ea7SShannon Nelson }
10355435ea7SShannon Nelson
10455435ea7SShannon Nelson pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET;
10555435ea7SShannon Nelson pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET;
10655435ea7SShannon Nelson pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET;
10755435ea7SShannon Nelson pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET;
10855435ea7SShannon Nelson
10955435ea7SShannon Nelson sig = ioread32(&pdsc->info_regs->signature);
11055435ea7SShannon Nelson if (sig != PDS_CORE_DEV_INFO_SIGNATURE) {
11155435ea7SShannon Nelson dev_err(dev, "Incompatible firmware signature %x", sig);
11255435ea7SShannon Nelson err = -EFAULT;
11355435ea7SShannon Nelson goto err_out;
11455435ea7SShannon Nelson }
11555435ea7SShannon Nelson
11655435ea7SShannon Nelson /* BAR1: doorbells */
11755435ea7SShannon Nelson bar++;
11855435ea7SShannon Nelson if (num_bars < 2) {
11955435ea7SShannon Nelson dev_err(dev, "Doorbell bar missing\n");
12055435ea7SShannon Nelson err = -EFAULT;
12155435ea7SShannon Nelson goto err_out;
12255435ea7SShannon Nelson }
12355435ea7SShannon Nelson
12455435ea7SShannon Nelson pdsc->db_pages = bar->vaddr;
12555435ea7SShannon Nelson pdsc->phy_db_pages = bar->bus_addr;
12655435ea7SShannon Nelson
12755435ea7SShannon Nelson return 0;
12855435ea7SShannon Nelson
12955435ea7SShannon Nelson err_out:
13055435ea7SShannon Nelson pdsc_unmap_bars(pdsc);
13155435ea7SShannon Nelson return err;
13255435ea7SShannon Nelson }
13355435ea7SShannon Nelson
pdsc_map_dbpage(struct pdsc * pdsc,int page_num)13445d76f49SShannon Nelson void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num)
13545d76f49SShannon Nelson {
13645d76f49SShannon Nelson return pci_iomap_range(pdsc->pdev,
13745d76f49SShannon Nelson pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index,
13845d76f49SShannon Nelson (u64)page_num << PAGE_SHIFT, PAGE_SIZE);
13945d76f49SShannon Nelson }
14045d76f49SShannon Nelson
pdsc_sriov_configure(struct pci_dev * pdev,int num_vfs)141f53d9311SShannon Nelson static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs)
142f53d9311SShannon Nelson {
143f53d9311SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev);
144f53d9311SShannon Nelson struct device *dev = pdsc->dev;
145f53d9311SShannon Nelson int ret = 0;
146f53d9311SShannon Nelson
147f53d9311SShannon Nelson if (num_vfs > 0) {
148f53d9311SShannon Nelson pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf),
149f53d9311SShannon Nelson GFP_KERNEL);
150f53d9311SShannon Nelson if (!pdsc->vfs)
151f53d9311SShannon Nelson return -ENOMEM;
152f53d9311SShannon Nelson pdsc->num_vfs = num_vfs;
153f53d9311SShannon Nelson
154f53d9311SShannon Nelson ret = pci_enable_sriov(pdev, num_vfs);
155f53d9311SShannon Nelson if (ret) {
156f53d9311SShannon Nelson dev_err(dev, "Cannot enable SRIOV: %pe\n",
157f53d9311SShannon Nelson ERR_PTR(ret));
158f53d9311SShannon Nelson goto no_vfs;
159f53d9311SShannon Nelson }
160f53d9311SShannon Nelson
161f53d9311SShannon Nelson return num_vfs;
162f53d9311SShannon Nelson }
163f53d9311SShannon Nelson
164f53d9311SShannon Nelson no_vfs:
165f53d9311SShannon Nelson pci_disable_sriov(pdev);
166f53d9311SShannon Nelson
167f53d9311SShannon Nelson kfree(pdsc->vfs);
168f53d9311SShannon Nelson pdsc->vfs = NULL;
169f53d9311SShannon Nelson pdsc->num_vfs = 0;
170f53d9311SShannon Nelson
171f53d9311SShannon Nelson return ret;
172f53d9311SShannon Nelson }
173f53d9311SShannon Nelson
pdsc_init_vf(struct pdsc * vf)17455435ea7SShannon Nelson static int pdsc_init_vf(struct pdsc *vf)
17555435ea7SShannon Nelson {
176f53d9311SShannon Nelson struct devlink *dl;
1774569cce4SShannon Nelson struct pdsc *pf;
1784569cce4SShannon Nelson int err;
1794569cce4SShannon Nelson
1804569cce4SShannon Nelson pf = pdsc_get_pf_struct(vf->pdev);
1814569cce4SShannon Nelson if (IS_ERR_OR_NULL(pf))
1824569cce4SShannon Nelson return PTR_ERR(pf) ?: -1;
183f53d9311SShannon Nelson
184f53d9311SShannon Nelson vf->vf_id = pci_iov_vf_id(vf->pdev);
185f53d9311SShannon Nelson
186f53d9311SShannon Nelson dl = priv_to_devlink(vf);
187f53d9311SShannon Nelson devl_lock(dl);
188f53d9311SShannon Nelson devl_register(dl);
189f53d9311SShannon Nelson devl_unlock(dl);
190f53d9311SShannon Nelson
1914569cce4SShannon Nelson pf->vfs[vf->vf_id].vf = vf;
1924569cce4SShannon Nelson err = pdsc_auxbus_dev_add(vf, pf);
1934569cce4SShannon Nelson if (err) {
1944569cce4SShannon Nelson devl_lock(dl);
1954569cce4SShannon Nelson devl_unregister(dl);
1964569cce4SShannon Nelson devl_unlock(dl);
1974569cce4SShannon Nelson }
1984569cce4SShannon Nelson
1994569cce4SShannon Nelson return err;
20055435ea7SShannon Nelson }
20155435ea7SShannon Nelson
20225b450c0SShannon Nelson static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = {
20325b450c0SShannon Nelson .name = "fw",
20425b450c0SShannon Nelson .diagnose = pdsc_fw_reporter_diagnose,
20525b450c0SShannon Nelson };
20625b450c0SShannon Nelson
20740ced894SShannon Nelson static const struct devlink_param pdsc_dl_params[] = {
20840ced894SShannon Nelson DEVLINK_PARAM_GENERIC(ENABLE_VNET,
20940ced894SShannon Nelson BIT(DEVLINK_PARAM_CMODE_RUNTIME),
21040ced894SShannon Nelson pdsc_dl_enable_get,
21140ced894SShannon Nelson pdsc_dl_enable_set,
21240ced894SShannon Nelson pdsc_dl_enable_validate),
21340ced894SShannon Nelson };
21440ced894SShannon Nelson
215c2dbb090SShannon Nelson #define PDSC_WQ_NAME_LEN 24
216c2dbb090SShannon Nelson
pdsc_init_pf(struct pdsc * pdsc)21755435ea7SShannon Nelson static int pdsc_init_pf(struct pdsc *pdsc)
21855435ea7SShannon Nelson {
21925b450c0SShannon Nelson struct devlink_health_reporter *hr;
220c2dbb090SShannon Nelson char wq_name[PDSC_WQ_NAME_LEN];
22155435ea7SShannon Nelson struct devlink *dl;
22255435ea7SShannon Nelson int err;
22355435ea7SShannon Nelson
22455435ea7SShannon Nelson pcie_print_link_status(pdsc->pdev);
22555435ea7SShannon Nelson
22655435ea7SShannon Nelson err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
22755435ea7SShannon Nelson if (err) {
22855435ea7SShannon Nelson dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
22955435ea7SShannon Nelson ERR_PTR(err));
23055435ea7SShannon Nelson return err;
23155435ea7SShannon Nelson }
23255435ea7SShannon Nelson
23355435ea7SShannon Nelson err = pdsc_map_bars(pdsc);
23455435ea7SShannon Nelson if (err)
23555435ea7SShannon Nelson goto err_out_release_regions;
23655435ea7SShannon Nelson
237c2dbb090SShannon Nelson /* General workqueue and timer, but don't start timer yet */
238c2dbb090SShannon Nelson snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid);
239c2dbb090SShannon Nelson pdsc->wq = create_singlethread_workqueue(wq_name);
240c2dbb090SShannon Nelson INIT_WORK(&pdsc->health_work, pdsc_health_thread);
241c2dbb090SShannon Nelson timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
242c2dbb090SShannon Nelson pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ;
243c2dbb090SShannon Nelson
244523847dfSShannon Nelson mutex_init(&pdsc->devcmd_lock);
245523847dfSShannon Nelson mutex_init(&pdsc->config_lock);
24645d76f49SShannon Nelson spin_lock_init(&pdsc->adminq_lock);
247523847dfSShannon Nelson
248523847dfSShannon Nelson mutex_lock(&pdsc->config_lock);
249523847dfSShannon Nelson set_bit(PDSC_S_FW_DEAD, &pdsc->state);
250523847dfSShannon Nelson
251523847dfSShannon Nelson err = pdsc_setup(pdsc, PDSC_SETUP_INIT);
2521e76f427SShannon Nelson if (err) {
2531e76f427SShannon Nelson mutex_unlock(&pdsc->config_lock);
254523847dfSShannon Nelson goto err_out_unmap_bars;
2551e76f427SShannon Nelson }
2561e76f427SShannon Nelson
25745d76f49SShannon Nelson err = pdsc_start(pdsc);
2581e76f427SShannon Nelson if (err) {
2591e76f427SShannon Nelson mutex_unlock(&pdsc->config_lock);
26045d76f49SShannon Nelson goto err_out_teardown;
2611e76f427SShannon Nelson }
262523847dfSShannon Nelson
263523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock);
264523847dfSShannon Nelson
26555435ea7SShannon Nelson dl = priv_to_devlink(pdsc);
26655435ea7SShannon Nelson devl_lock(dl);
26740ced894SShannon Nelson err = devl_params_register(dl, pdsc_dl_params,
26840ced894SShannon Nelson ARRAY_SIZE(pdsc_dl_params));
26940ced894SShannon Nelson if (err) {
2701e76f427SShannon Nelson devl_unlock(dl);
27140ced894SShannon Nelson dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n",
27240ced894SShannon Nelson ERR_PTR(err));
2731e76f427SShannon Nelson goto err_out_stop;
27440ced894SShannon Nelson }
27525b450c0SShannon Nelson
27625b450c0SShannon Nelson hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc);
27725b450c0SShannon Nelson if (IS_ERR(hr)) {
2781e76f427SShannon Nelson devl_unlock(dl);
27925b450c0SShannon Nelson dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr);
28025b450c0SShannon Nelson err = PTR_ERR(hr);
28140ced894SShannon Nelson goto err_out_unreg_params;
28225b450c0SShannon Nelson }
28325b450c0SShannon Nelson pdsc->fw_reporter = hr;
28425b450c0SShannon Nelson
28555435ea7SShannon Nelson devl_register(dl);
28655435ea7SShannon Nelson devl_unlock(dl);
28755435ea7SShannon Nelson
288c2dbb090SShannon Nelson /* Lastly, start the health check timer */
289c2dbb090SShannon Nelson mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period));
290c2dbb090SShannon Nelson
29155435ea7SShannon Nelson return 0;
29255435ea7SShannon Nelson
29340ced894SShannon Nelson err_out_unreg_params:
2941e76f427SShannon Nelson devlink_params_unregister(dl, pdsc_dl_params,
29540ced894SShannon Nelson ARRAY_SIZE(pdsc_dl_params));
2961e76f427SShannon Nelson err_out_stop:
29745d76f49SShannon Nelson pdsc_stop(pdsc);
29825b450c0SShannon Nelson err_out_teardown:
29925b450c0SShannon Nelson pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
300523847dfSShannon Nelson err_out_unmap_bars:
301*bd874092SBrett Creeley timer_shutdown_sync(&pdsc->wdtimer);
302c2dbb090SShannon Nelson if (pdsc->wq)
303c2dbb090SShannon Nelson destroy_workqueue(pdsc->wq);
304523847dfSShannon Nelson mutex_destroy(&pdsc->config_lock);
305523847dfSShannon Nelson mutex_destroy(&pdsc->devcmd_lock);
306523847dfSShannon Nelson pci_free_irq_vectors(pdsc->pdev);
307523847dfSShannon Nelson pdsc_unmap_bars(pdsc);
30855435ea7SShannon Nelson err_out_release_regions:
30955435ea7SShannon Nelson pci_release_regions(pdsc->pdev);
31055435ea7SShannon Nelson
31155435ea7SShannon Nelson return err;
31255435ea7SShannon Nelson }
31355435ea7SShannon Nelson
31455435ea7SShannon Nelson static const struct devlink_ops pdsc_dl_ops = {
31545d76f49SShannon Nelson .info_get = pdsc_dl_info_get,
31649ce92fbSShannon Nelson .flash_update = pdsc_dl_flash_update,
31755435ea7SShannon Nelson };
31855435ea7SShannon Nelson
31955435ea7SShannon Nelson static const struct devlink_ops pdsc_dl_vf_ops = {
32055435ea7SShannon Nelson };
32155435ea7SShannon Nelson
32255435ea7SShannon Nelson static DEFINE_IDA(pdsc_ida);
32355435ea7SShannon Nelson
pdsc_probe(struct pci_dev * pdev,const struct pci_device_id * ent)32455435ea7SShannon Nelson static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
32555435ea7SShannon Nelson {
32655435ea7SShannon Nelson struct device *dev = &pdev->dev;
32755435ea7SShannon Nelson const struct devlink_ops *ops;
32855435ea7SShannon Nelson struct devlink *dl;
32955435ea7SShannon Nelson struct pdsc *pdsc;
33055435ea7SShannon Nelson bool is_pf;
33155435ea7SShannon Nelson int err;
33255435ea7SShannon Nelson
33355435ea7SShannon Nelson is_pf = !pdev->is_virtfn;
33455435ea7SShannon Nelson ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops;
33555435ea7SShannon Nelson dl = devlink_alloc(ops, sizeof(struct pdsc), dev);
33655435ea7SShannon Nelson if (!dl)
33755435ea7SShannon Nelson return -ENOMEM;
33855435ea7SShannon Nelson pdsc = devlink_priv(dl);
33955435ea7SShannon Nelson
34055435ea7SShannon Nelson pdsc->pdev = pdev;
34155435ea7SShannon Nelson pdsc->dev = &pdev->dev;
34255435ea7SShannon Nelson set_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
34355435ea7SShannon Nelson pci_set_drvdata(pdev, pdsc);
34455435ea7SShannon Nelson pdsc_debugfs_add_dev(pdsc);
34555435ea7SShannon Nelson
34655435ea7SShannon Nelson err = ida_alloc(&pdsc_ida, GFP_KERNEL);
34755435ea7SShannon Nelson if (err < 0) {
34855435ea7SShannon Nelson dev_err(pdsc->dev, "%s: id alloc failed: %pe\n",
34955435ea7SShannon Nelson __func__, ERR_PTR(err));
35055435ea7SShannon Nelson goto err_out_free_devlink;
35155435ea7SShannon Nelson }
35255435ea7SShannon Nelson pdsc->uid = err;
35355435ea7SShannon Nelson
35455435ea7SShannon Nelson /* Query system for DMA addressing limitation for the device. */
35555435ea7SShannon Nelson err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN));
35655435ea7SShannon Nelson if (err) {
35755435ea7SShannon Nelson dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n",
35855435ea7SShannon Nelson ERR_PTR(err));
35955435ea7SShannon Nelson goto err_out_free_ida;
36055435ea7SShannon Nelson }
36155435ea7SShannon Nelson
36255435ea7SShannon Nelson err = pci_enable_device(pdev);
36355435ea7SShannon Nelson if (err) {
36455435ea7SShannon Nelson dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
36555435ea7SShannon Nelson goto err_out_free_ida;
36655435ea7SShannon Nelson }
36755435ea7SShannon Nelson pci_set_master(pdev);
36855435ea7SShannon Nelson
36955435ea7SShannon Nelson if (is_pf)
37055435ea7SShannon Nelson err = pdsc_init_pf(pdsc);
37155435ea7SShannon Nelson else
37255435ea7SShannon Nelson err = pdsc_init_vf(pdsc);
37355435ea7SShannon Nelson if (err) {
37455435ea7SShannon Nelson dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err));
3752f48b1d8SYu Liao goto err_out_disable_device;
37655435ea7SShannon Nelson }
37755435ea7SShannon Nelson
37855435ea7SShannon Nelson clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state);
37955435ea7SShannon Nelson return 0;
38055435ea7SShannon Nelson
3812f48b1d8SYu Liao err_out_disable_device:
38255435ea7SShannon Nelson pci_disable_device(pdev);
38355435ea7SShannon Nelson err_out_free_ida:
38455435ea7SShannon Nelson ida_free(&pdsc_ida, pdsc->uid);
38555435ea7SShannon Nelson err_out_free_devlink:
38655435ea7SShannon Nelson pdsc_debugfs_del_dev(pdsc);
38755435ea7SShannon Nelson devlink_free(dl);
38855435ea7SShannon Nelson
38955435ea7SShannon Nelson return err;
39055435ea7SShannon Nelson }
39155435ea7SShannon Nelson
pdsc_remove(struct pci_dev * pdev)39255435ea7SShannon Nelson static void pdsc_remove(struct pci_dev *pdev)
39355435ea7SShannon Nelson {
39455435ea7SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev);
39555435ea7SShannon Nelson struct devlink *dl;
39655435ea7SShannon Nelson
39755435ea7SShannon Nelson /* Unhook the registrations first to be sure there
39855435ea7SShannon Nelson * are no requests while we're stopping.
39955435ea7SShannon Nelson */
40055435ea7SShannon Nelson dl = priv_to_devlink(pdsc);
40155435ea7SShannon Nelson devl_lock(dl);
40255435ea7SShannon Nelson devl_unregister(dl);
40340ced894SShannon Nelson if (!pdev->is_virtfn) {
40425b450c0SShannon Nelson if (pdsc->fw_reporter) {
40525b450c0SShannon Nelson devl_health_reporter_destroy(pdsc->fw_reporter);
40625b450c0SShannon Nelson pdsc->fw_reporter = NULL;
40725b450c0SShannon Nelson }
40840ced894SShannon Nelson devl_params_unregister(dl, pdsc_dl_params,
40940ced894SShannon Nelson ARRAY_SIZE(pdsc_dl_params));
41040ced894SShannon Nelson }
41155435ea7SShannon Nelson devl_unlock(dl);
41255435ea7SShannon Nelson
4134569cce4SShannon Nelson if (pdev->is_virtfn) {
4144569cce4SShannon Nelson struct pdsc *pf;
4154569cce4SShannon Nelson
4164569cce4SShannon Nelson pf = pdsc_get_pf_struct(pdsc->pdev);
4174569cce4SShannon Nelson if (!IS_ERR(pf)) {
4184569cce4SShannon Nelson pdsc_auxbus_dev_del(pdsc, pf);
4194569cce4SShannon Nelson pf->vfs[pdsc->vf_id].vf = NULL;
4204569cce4SShannon Nelson }
4214569cce4SShannon Nelson } else {
4224569cce4SShannon Nelson /* Remove the VFs and their aux_bus connections before other
4234569cce4SShannon Nelson * cleanup so that the clients can use the AdminQ to cleanly
4244569cce4SShannon Nelson * shut themselves down.
4254569cce4SShannon Nelson */
426f53d9311SShannon Nelson pdsc_sriov_configure(pdev, 0);
427f53d9311SShannon Nelson
428*bd874092SBrett Creeley timer_shutdown_sync(&pdsc->wdtimer);
429c2dbb090SShannon Nelson if (pdsc->wq)
430c2dbb090SShannon Nelson destroy_workqueue(pdsc->wq);
431c2dbb090SShannon Nelson
432523847dfSShannon Nelson mutex_lock(&pdsc->config_lock);
433523847dfSShannon Nelson set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state);
434523847dfSShannon Nelson
43545d76f49SShannon Nelson pdsc_stop(pdsc);
436523847dfSShannon Nelson pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING);
437523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock);
438523847dfSShannon Nelson mutex_destroy(&pdsc->config_lock);
439523847dfSShannon Nelson mutex_destroy(&pdsc->devcmd_lock);
440523847dfSShannon Nelson
44155435ea7SShannon Nelson pdsc_unmap_bars(pdsc);
44255435ea7SShannon Nelson pci_release_regions(pdev);
443523847dfSShannon Nelson }
44455435ea7SShannon Nelson
44555435ea7SShannon Nelson pci_disable_device(pdev);
44655435ea7SShannon Nelson
44755435ea7SShannon Nelson ida_free(&pdsc_ida, pdsc->uid);
44855435ea7SShannon Nelson pdsc_debugfs_del_dev(pdsc);
44955435ea7SShannon Nelson devlink_free(dl);
45055435ea7SShannon Nelson }
45155435ea7SShannon Nelson
pdsc_stop_health_thread(struct pdsc * pdsc)452*bd874092SBrett Creeley static void pdsc_stop_health_thread(struct pdsc *pdsc)
453*bd874092SBrett Creeley {
454*bd874092SBrett Creeley timer_shutdown_sync(&pdsc->wdtimer);
455*bd874092SBrett Creeley if (pdsc->health_work.func)
456*bd874092SBrett Creeley cancel_work_sync(&pdsc->health_work);
457*bd874092SBrett Creeley }
458*bd874092SBrett Creeley
pdsc_restart_health_thread(struct pdsc * pdsc)459*bd874092SBrett Creeley static void pdsc_restart_health_thread(struct pdsc *pdsc)
460*bd874092SBrett Creeley {
461*bd874092SBrett Creeley timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0);
462*bd874092SBrett Creeley mod_timer(&pdsc->wdtimer, jiffies + 1);
463*bd874092SBrett Creeley }
464*bd874092SBrett Creeley
pdsc_reset_prepare(struct pci_dev * pdev)465699f5416SShannon Nelson static void pdsc_reset_prepare(struct pci_dev *pdev)
466699f5416SShannon Nelson {
467699f5416SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev);
468699f5416SShannon Nelson
469*bd874092SBrett Creeley pdsc_stop_health_thread(pdsc);
470699f5416SShannon Nelson pdsc_fw_down(pdsc);
471699f5416SShannon Nelson
472699f5416SShannon Nelson pdsc_unmap_bars(pdsc);
473699f5416SShannon Nelson pci_release_regions(pdev);
474699f5416SShannon Nelson pci_disable_device(pdev);
475699f5416SShannon Nelson }
476699f5416SShannon Nelson
pdsc_reset_done(struct pci_dev * pdev)477699f5416SShannon Nelson static void pdsc_reset_done(struct pci_dev *pdev)
478699f5416SShannon Nelson {
479699f5416SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev);
480699f5416SShannon Nelson struct device *dev = pdsc->dev;
481699f5416SShannon Nelson int err;
482699f5416SShannon Nelson
483699f5416SShannon Nelson err = pci_enable_device(pdev);
484699f5416SShannon Nelson if (err) {
485699f5416SShannon Nelson dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err));
486699f5416SShannon Nelson return;
487699f5416SShannon Nelson }
488699f5416SShannon Nelson pci_set_master(pdev);
489699f5416SShannon Nelson
490699f5416SShannon Nelson if (!pdev->is_virtfn) {
491699f5416SShannon Nelson pcie_print_link_status(pdsc->pdev);
492699f5416SShannon Nelson
493699f5416SShannon Nelson err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME);
494699f5416SShannon Nelson if (err) {
495699f5416SShannon Nelson dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n",
496699f5416SShannon Nelson ERR_PTR(err));
497699f5416SShannon Nelson return;
498699f5416SShannon Nelson }
499699f5416SShannon Nelson
500699f5416SShannon Nelson err = pdsc_map_bars(pdsc);
501699f5416SShannon Nelson if (err)
502699f5416SShannon Nelson return;
503699f5416SShannon Nelson }
504699f5416SShannon Nelson
505699f5416SShannon Nelson pdsc_fw_up(pdsc);
506*bd874092SBrett Creeley pdsc_restart_health_thread(pdsc);
507699f5416SShannon Nelson }
508699f5416SShannon Nelson
509699f5416SShannon Nelson static const struct pci_error_handlers pdsc_err_handler = {
510699f5416SShannon Nelson /* FLR handling */
511699f5416SShannon Nelson .reset_prepare = pdsc_reset_prepare,
512699f5416SShannon Nelson .reset_done = pdsc_reset_done,
513699f5416SShannon Nelson };
514699f5416SShannon Nelson
51555435ea7SShannon Nelson static struct pci_driver pdsc_driver = {
51655435ea7SShannon Nelson .name = PDS_CORE_DRV_NAME,
51755435ea7SShannon Nelson .id_table = pdsc_id_table,
51855435ea7SShannon Nelson .probe = pdsc_probe,
51955435ea7SShannon Nelson .remove = pdsc_remove,
520f53d9311SShannon Nelson .sriov_configure = pdsc_sriov_configure,
521699f5416SShannon Nelson .err_handler = &pdsc_err_handler,
52255435ea7SShannon Nelson };
52355435ea7SShannon Nelson
pdsc_get_pf_struct(struct pci_dev * vf_pdev)5244569cce4SShannon Nelson void *pdsc_get_pf_struct(struct pci_dev *vf_pdev)
5254569cce4SShannon Nelson {
5264569cce4SShannon Nelson return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver);
5274569cce4SShannon Nelson }
5284569cce4SShannon Nelson EXPORT_SYMBOL_GPL(pdsc_get_pf_struct);
5294569cce4SShannon Nelson
pdsc_init_module(void)53055435ea7SShannon Nelson static int __init pdsc_init_module(void)
53155435ea7SShannon Nelson {
53255435ea7SShannon Nelson if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME))
53355435ea7SShannon Nelson return -EINVAL;
53455435ea7SShannon Nelson
53555435ea7SShannon Nelson pdsc_debugfs_create();
53655435ea7SShannon Nelson return pci_register_driver(&pdsc_driver);
53755435ea7SShannon Nelson }
53855435ea7SShannon Nelson
pdsc_cleanup_module(void)53955435ea7SShannon Nelson static void __exit pdsc_cleanup_module(void)
54055435ea7SShannon Nelson {
54155435ea7SShannon Nelson pci_unregister_driver(&pdsc_driver);
54255435ea7SShannon Nelson pdsc_debugfs_destroy();
54355435ea7SShannon Nelson }
54455435ea7SShannon Nelson
54555435ea7SShannon Nelson module_init(pdsc_init_module);
54655435ea7SShannon Nelson module_exit(pdsc_cleanup_module);
547