155435ea7SShannon Nelson // SPDX-License-Identifier: GPL-2.0 255435ea7SShannon Nelson /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 355435ea7SShannon Nelson 455435ea7SShannon Nelson #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 555435ea7SShannon Nelson 655435ea7SShannon Nelson #include <linux/pci.h> 755435ea7SShannon Nelson 855435ea7SShannon Nelson #include <linux/pds/pds_common.h> 955435ea7SShannon Nelson 1055435ea7SShannon Nelson #include "core.h" 1155435ea7SShannon Nelson 1255435ea7SShannon Nelson MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION); 1355435ea7SShannon Nelson MODULE_AUTHOR("Advanced Micro Devices, Inc"); 1455435ea7SShannon Nelson MODULE_LICENSE("GPL"); 1555435ea7SShannon Nelson 1655435ea7SShannon Nelson /* Supported devices */ 1755435ea7SShannon Nelson static const struct pci_device_id pdsc_id_table[] = { 1855435ea7SShannon Nelson { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) }, 1955435ea7SShannon Nelson { 0, } /* end of table */ 2055435ea7SShannon Nelson }; 2155435ea7SShannon Nelson MODULE_DEVICE_TABLE(pci, pdsc_id_table); 2255435ea7SShannon Nelson 23c2dbb090SShannon Nelson static void pdsc_wdtimer_cb(struct timer_list *t) 24c2dbb090SShannon Nelson { 25c2dbb090SShannon Nelson struct pdsc *pdsc = from_timer(pdsc, t, wdtimer); 26c2dbb090SShannon Nelson 27c2dbb090SShannon Nelson dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies); 28c2dbb090SShannon Nelson mod_timer(&pdsc->wdtimer, 29c2dbb090SShannon Nelson round_jiffies(jiffies + pdsc->wdtimer_period)); 30c2dbb090SShannon Nelson 31c2dbb090SShannon Nelson queue_work(pdsc->wq, &pdsc->health_work); 32c2dbb090SShannon Nelson } 33c2dbb090SShannon Nelson 3455435ea7SShannon Nelson static void pdsc_unmap_bars(struct pdsc *pdsc) 3555435ea7SShannon Nelson { 3655435ea7SShannon Nelson struct pdsc_dev_bar *bars = pdsc->bars; 3755435ea7SShannon Nelson unsigned int i; 3855435ea7SShannon Nelson 3955435ea7SShannon Nelson for (i = 0; i < PDS_CORE_BARS_MAX; i++) { 4055435ea7SShannon Nelson if (bars[i].vaddr) 4155435ea7SShannon Nelson pci_iounmap(pdsc->pdev, bars[i].vaddr); 4255435ea7SShannon Nelson } 4355435ea7SShannon Nelson } 4455435ea7SShannon Nelson 4555435ea7SShannon Nelson static int pdsc_map_bars(struct pdsc *pdsc) 4655435ea7SShannon Nelson { 4755435ea7SShannon Nelson struct pdsc_dev_bar *bar = pdsc->bars; 4855435ea7SShannon Nelson struct pci_dev *pdev = pdsc->pdev; 4955435ea7SShannon Nelson struct device *dev = pdsc->dev; 5055435ea7SShannon Nelson struct pdsc_dev_bar *bars; 5155435ea7SShannon Nelson unsigned int i, j; 5255435ea7SShannon Nelson int num_bars = 0; 5355435ea7SShannon Nelson int err; 5455435ea7SShannon Nelson u32 sig; 5555435ea7SShannon Nelson 5655435ea7SShannon Nelson bars = pdsc->bars; 5755435ea7SShannon Nelson 5855435ea7SShannon Nelson /* Since the PCI interface in the hardware is configurable, 5955435ea7SShannon Nelson * we need to poke into all the bars to find the set we're 6055435ea7SShannon Nelson * expecting. 6155435ea7SShannon Nelson */ 6255435ea7SShannon Nelson for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) { 6355435ea7SShannon Nelson if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 6455435ea7SShannon Nelson continue; 6555435ea7SShannon Nelson 6655435ea7SShannon Nelson bars[j].len = pci_resource_len(pdev, i); 6755435ea7SShannon Nelson bars[j].bus_addr = pci_resource_start(pdev, i); 6855435ea7SShannon Nelson bars[j].res_index = i; 6955435ea7SShannon Nelson 7055435ea7SShannon Nelson /* only map the whole bar 0 */ 7155435ea7SShannon Nelson if (j > 0) { 7255435ea7SShannon Nelson bars[j].vaddr = NULL; 7355435ea7SShannon Nelson } else { 7455435ea7SShannon Nelson bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 7555435ea7SShannon Nelson if (!bars[j].vaddr) { 7655435ea7SShannon Nelson dev_err(dev, "Cannot map BAR %d, aborting\n", i); 7755435ea7SShannon Nelson return -ENODEV; 7855435ea7SShannon Nelson } 7955435ea7SShannon Nelson } 8055435ea7SShannon Nelson 8155435ea7SShannon Nelson j++; 8255435ea7SShannon Nelson } 8355435ea7SShannon Nelson num_bars = j; 8455435ea7SShannon Nelson 8555435ea7SShannon Nelson /* BAR0: dev_cmd and interrupts */ 8655435ea7SShannon Nelson if (num_bars < 1) { 8755435ea7SShannon Nelson dev_err(dev, "No bars found\n"); 8855435ea7SShannon Nelson err = -EFAULT; 8955435ea7SShannon Nelson goto err_out; 9055435ea7SShannon Nelson } 9155435ea7SShannon Nelson 9255435ea7SShannon Nelson if (bar->len < PDS_CORE_BAR0_SIZE) { 9355435ea7SShannon Nelson dev_err(dev, "Resource bar size %lu too small\n", bar->len); 9455435ea7SShannon Nelson err = -EFAULT; 9555435ea7SShannon Nelson goto err_out; 9655435ea7SShannon Nelson } 9755435ea7SShannon Nelson 9855435ea7SShannon Nelson pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET; 9955435ea7SShannon Nelson pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET; 10055435ea7SShannon Nelson pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET; 10155435ea7SShannon Nelson pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET; 10255435ea7SShannon Nelson 10355435ea7SShannon Nelson sig = ioread32(&pdsc->info_regs->signature); 10455435ea7SShannon Nelson if (sig != PDS_CORE_DEV_INFO_SIGNATURE) { 10555435ea7SShannon Nelson dev_err(dev, "Incompatible firmware signature %x", sig); 10655435ea7SShannon Nelson err = -EFAULT; 10755435ea7SShannon Nelson goto err_out; 10855435ea7SShannon Nelson } 10955435ea7SShannon Nelson 11055435ea7SShannon Nelson /* BAR1: doorbells */ 11155435ea7SShannon Nelson bar++; 11255435ea7SShannon Nelson if (num_bars < 2) { 11355435ea7SShannon Nelson dev_err(dev, "Doorbell bar missing\n"); 11455435ea7SShannon Nelson err = -EFAULT; 11555435ea7SShannon Nelson goto err_out; 11655435ea7SShannon Nelson } 11755435ea7SShannon Nelson 11855435ea7SShannon Nelson pdsc->db_pages = bar->vaddr; 11955435ea7SShannon Nelson pdsc->phy_db_pages = bar->bus_addr; 12055435ea7SShannon Nelson 12155435ea7SShannon Nelson return 0; 12255435ea7SShannon Nelson 12355435ea7SShannon Nelson err_out: 12455435ea7SShannon Nelson pdsc_unmap_bars(pdsc); 12555435ea7SShannon Nelson return err; 12655435ea7SShannon Nelson } 12755435ea7SShannon Nelson 128*45d76f49SShannon Nelson void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num) 129*45d76f49SShannon Nelson { 130*45d76f49SShannon Nelson return pci_iomap_range(pdsc->pdev, 131*45d76f49SShannon Nelson pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index, 132*45d76f49SShannon Nelson (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 133*45d76f49SShannon Nelson } 134*45d76f49SShannon Nelson 13555435ea7SShannon Nelson static int pdsc_init_vf(struct pdsc *vf) 13655435ea7SShannon Nelson { 13755435ea7SShannon Nelson return -1; 13855435ea7SShannon Nelson } 13955435ea7SShannon Nelson 14025b450c0SShannon Nelson static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = { 14125b450c0SShannon Nelson .name = "fw", 14225b450c0SShannon Nelson .diagnose = pdsc_fw_reporter_diagnose, 14325b450c0SShannon Nelson }; 14425b450c0SShannon Nelson 145c2dbb090SShannon Nelson #define PDSC_WQ_NAME_LEN 24 146c2dbb090SShannon Nelson 14755435ea7SShannon Nelson static int pdsc_init_pf(struct pdsc *pdsc) 14855435ea7SShannon Nelson { 14925b450c0SShannon Nelson struct devlink_health_reporter *hr; 150c2dbb090SShannon Nelson char wq_name[PDSC_WQ_NAME_LEN]; 15155435ea7SShannon Nelson struct devlink *dl; 15255435ea7SShannon Nelson int err; 15355435ea7SShannon Nelson 15455435ea7SShannon Nelson pcie_print_link_status(pdsc->pdev); 15555435ea7SShannon Nelson 15655435ea7SShannon Nelson err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 15755435ea7SShannon Nelson if (err) { 15855435ea7SShannon Nelson dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 15955435ea7SShannon Nelson ERR_PTR(err)); 16055435ea7SShannon Nelson return err; 16155435ea7SShannon Nelson } 16255435ea7SShannon Nelson 16355435ea7SShannon Nelson err = pdsc_map_bars(pdsc); 16455435ea7SShannon Nelson if (err) 16555435ea7SShannon Nelson goto err_out_release_regions; 16655435ea7SShannon Nelson 167c2dbb090SShannon Nelson /* General workqueue and timer, but don't start timer yet */ 168c2dbb090SShannon Nelson snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); 169c2dbb090SShannon Nelson pdsc->wq = create_singlethread_workqueue(wq_name); 170c2dbb090SShannon Nelson INIT_WORK(&pdsc->health_work, pdsc_health_thread); 171c2dbb090SShannon Nelson timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 172c2dbb090SShannon Nelson pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ; 173c2dbb090SShannon Nelson 174523847dfSShannon Nelson mutex_init(&pdsc->devcmd_lock); 175523847dfSShannon Nelson mutex_init(&pdsc->config_lock); 176*45d76f49SShannon Nelson spin_lock_init(&pdsc->adminq_lock); 177523847dfSShannon Nelson 178523847dfSShannon Nelson mutex_lock(&pdsc->config_lock); 179523847dfSShannon Nelson set_bit(PDSC_S_FW_DEAD, &pdsc->state); 180523847dfSShannon Nelson 181523847dfSShannon Nelson err = pdsc_setup(pdsc, PDSC_SETUP_INIT); 182523847dfSShannon Nelson if (err) 183523847dfSShannon Nelson goto err_out_unmap_bars; 184*45d76f49SShannon Nelson err = pdsc_start(pdsc); 185*45d76f49SShannon Nelson if (err) 186*45d76f49SShannon Nelson goto err_out_teardown; 187523847dfSShannon Nelson 188523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock); 189523847dfSShannon Nelson 19055435ea7SShannon Nelson dl = priv_to_devlink(pdsc); 19155435ea7SShannon Nelson devl_lock(dl); 19225b450c0SShannon Nelson 19325b450c0SShannon Nelson hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc); 19425b450c0SShannon Nelson if (IS_ERR(hr)) { 19525b450c0SShannon Nelson dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr); 19625b450c0SShannon Nelson err = PTR_ERR(hr); 19725b450c0SShannon Nelson devl_unlock(dl); 198*45d76f49SShannon Nelson goto err_out_stop; 19925b450c0SShannon Nelson } 20025b450c0SShannon Nelson pdsc->fw_reporter = hr; 20125b450c0SShannon Nelson 20255435ea7SShannon Nelson devl_register(dl); 20355435ea7SShannon Nelson devl_unlock(dl); 20455435ea7SShannon Nelson 205c2dbb090SShannon Nelson /* Lastly, start the health check timer */ 206c2dbb090SShannon Nelson mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period)); 207c2dbb090SShannon Nelson 20855435ea7SShannon Nelson return 0; 20955435ea7SShannon Nelson 210*45d76f49SShannon Nelson err_out_stop: 211*45d76f49SShannon Nelson pdsc_stop(pdsc); 21225b450c0SShannon Nelson err_out_teardown: 21325b450c0SShannon Nelson pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 214523847dfSShannon Nelson err_out_unmap_bars: 215523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock); 216c2dbb090SShannon Nelson del_timer_sync(&pdsc->wdtimer); 217c2dbb090SShannon Nelson if (pdsc->wq) 218c2dbb090SShannon Nelson destroy_workqueue(pdsc->wq); 219523847dfSShannon Nelson mutex_destroy(&pdsc->config_lock); 220523847dfSShannon Nelson mutex_destroy(&pdsc->devcmd_lock); 221523847dfSShannon Nelson pci_free_irq_vectors(pdsc->pdev); 222523847dfSShannon Nelson pdsc_unmap_bars(pdsc); 22355435ea7SShannon Nelson err_out_release_regions: 22455435ea7SShannon Nelson pci_release_regions(pdsc->pdev); 22555435ea7SShannon Nelson 22655435ea7SShannon Nelson return err; 22755435ea7SShannon Nelson } 22855435ea7SShannon Nelson 22955435ea7SShannon Nelson static const struct devlink_ops pdsc_dl_ops = { 230*45d76f49SShannon Nelson .info_get = pdsc_dl_info_get, 23155435ea7SShannon Nelson }; 23255435ea7SShannon Nelson 23355435ea7SShannon Nelson static const struct devlink_ops pdsc_dl_vf_ops = { 23455435ea7SShannon Nelson }; 23555435ea7SShannon Nelson 23655435ea7SShannon Nelson static DEFINE_IDA(pdsc_ida); 23755435ea7SShannon Nelson 23855435ea7SShannon Nelson static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 23955435ea7SShannon Nelson { 24055435ea7SShannon Nelson struct device *dev = &pdev->dev; 24155435ea7SShannon Nelson const struct devlink_ops *ops; 24255435ea7SShannon Nelson struct devlink *dl; 24355435ea7SShannon Nelson struct pdsc *pdsc; 24455435ea7SShannon Nelson bool is_pf; 24555435ea7SShannon Nelson int err; 24655435ea7SShannon Nelson 24755435ea7SShannon Nelson is_pf = !pdev->is_virtfn; 24855435ea7SShannon Nelson ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops; 24955435ea7SShannon Nelson dl = devlink_alloc(ops, sizeof(struct pdsc), dev); 25055435ea7SShannon Nelson if (!dl) 25155435ea7SShannon Nelson return -ENOMEM; 25255435ea7SShannon Nelson pdsc = devlink_priv(dl); 25355435ea7SShannon Nelson 25455435ea7SShannon Nelson pdsc->pdev = pdev; 25555435ea7SShannon Nelson pdsc->dev = &pdev->dev; 25655435ea7SShannon Nelson set_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 25755435ea7SShannon Nelson pci_set_drvdata(pdev, pdsc); 25855435ea7SShannon Nelson pdsc_debugfs_add_dev(pdsc); 25955435ea7SShannon Nelson 26055435ea7SShannon Nelson err = ida_alloc(&pdsc_ida, GFP_KERNEL); 26155435ea7SShannon Nelson if (err < 0) { 26255435ea7SShannon Nelson dev_err(pdsc->dev, "%s: id alloc failed: %pe\n", 26355435ea7SShannon Nelson __func__, ERR_PTR(err)); 26455435ea7SShannon Nelson goto err_out_free_devlink; 26555435ea7SShannon Nelson } 26655435ea7SShannon Nelson pdsc->uid = err; 26755435ea7SShannon Nelson 26855435ea7SShannon Nelson /* Query system for DMA addressing limitation for the device. */ 26955435ea7SShannon Nelson err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN)); 27055435ea7SShannon Nelson if (err) { 27155435ea7SShannon Nelson dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n", 27255435ea7SShannon Nelson ERR_PTR(err)); 27355435ea7SShannon Nelson goto err_out_free_ida; 27455435ea7SShannon Nelson } 27555435ea7SShannon Nelson 27655435ea7SShannon Nelson err = pci_enable_device(pdev); 27755435ea7SShannon Nelson if (err) { 27855435ea7SShannon Nelson dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 27955435ea7SShannon Nelson goto err_out_free_ida; 28055435ea7SShannon Nelson } 28155435ea7SShannon Nelson pci_set_master(pdev); 28255435ea7SShannon Nelson 28355435ea7SShannon Nelson if (is_pf) 28455435ea7SShannon Nelson err = pdsc_init_pf(pdsc); 28555435ea7SShannon Nelson else 28655435ea7SShannon Nelson err = pdsc_init_vf(pdsc); 28755435ea7SShannon Nelson if (err) { 28855435ea7SShannon Nelson dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err)); 28955435ea7SShannon Nelson goto err_out_clear_master; 29055435ea7SShannon Nelson } 29155435ea7SShannon Nelson 29255435ea7SShannon Nelson clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 29355435ea7SShannon Nelson return 0; 29455435ea7SShannon Nelson 29555435ea7SShannon Nelson err_out_clear_master: 29655435ea7SShannon Nelson pci_clear_master(pdev); 29755435ea7SShannon Nelson pci_disable_device(pdev); 29855435ea7SShannon Nelson err_out_free_ida: 29955435ea7SShannon Nelson ida_free(&pdsc_ida, pdsc->uid); 30055435ea7SShannon Nelson err_out_free_devlink: 30155435ea7SShannon Nelson pdsc_debugfs_del_dev(pdsc); 30255435ea7SShannon Nelson devlink_free(dl); 30355435ea7SShannon Nelson 30455435ea7SShannon Nelson return err; 30555435ea7SShannon Nelson } 30655435ea7SShannon Nelson 30755435ea7SShannon Nelson static void pdsc_remove(struct pci_dev *pdev) 30855435ea7SShannon Nelson { 30955435ea7SShannon Nelson struct pdsc *pdsc = pci_get_drvdata(pdev); 31055435ea7SShannon Nelson struct devlink *dl; 31155435ea7SShannon Nelson 31255435ea7SShannon Nelson /* Unhook the registrations first to be sure there 31355435ea7SShannon Nelson * are no requests while we're stopping. 31455435ea7SShannon Nelson */ 31555435ea7SShannon Nelson dl = priv_to_devlink(pdsc); 31655435ea7SShannon Nelson devl_lock(dl); 31755435ea7SShannon Nelson devl_unregister(dl); 31825b450c0SShannon Nelson if (pdsc->fw_reporter) { 31925b450c0SShannon Nelson devl_health_reporter_destroy(pdsc->fw_reporter); 32025b450c0SShannon Nelson pdsc->fw_reporter = NULL; 32125b450c0SShannon Nelson } 32255435ea7SShannon Nelson devl_unlock(dl); 32355435ea7SShannon Nelson 324523847dfSShannon Nelson if (!pdev->is_virtfn) { 325c2dbb090SShannon Nelson del_timer_sync(&pdsc->wdtimer); 326c2dbb090SShannon Nelson if (pdsc->wq) 327c2dbb090SShannon Nelson destroy_workqueue(pdsc->wq); 328c2dbb090SShannon Nelson 329523847dfSShannon Nelson mutex_lock(&pdsc->config_lock); 330523847dfSShannon Nelson set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); 331523847dfSShannon Nelson 332*45d76f49SShannon Nelson pdsc_stop(pdsc); 333523847dfSShannon Nelson pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 334523847dfSShannon Nelson mutex_unlock(&pdsc->config_lock); 335523847dfSShannon Nelson mutex_destroy(&pdsc->config_lock); 336523847dfSShannon Nelson mutex_destroy(&pdsc->devcmd_lock); 337523847dfSShannon Nelson 338523847dfSShannon Nelson pci_free_irq_vectors(pdev); 33955435ea7SShannon Nelson pdsc_unmap_bars(pdsc); 34055435ea7SShannon Nelson pci_release_regions(pdev); 341523847dfSShannon Nelson } 34255435ea7SShannon Nelson 34355435ea7SShannon Nelson pci_clear_master(pdev); 34455435ea7SShannon Nelson pci_disable_device(pdev); 34555435ea7SShannon Nelson 34655435ea7SShannon Nelson ida_free(&pdsc_ida, pdsc->uid); 34755435ea7SShannon Nelson pdsc_debugfs_del_dev(pdsc); 34855435ea7SShannon Nelson devlink_free(dl); 34955435ea7SShannon Nelson } 35055435ea7SShannon Nelson 35155435ea7SShannon Nelson static struct pci_driver pdsc_driver = { 35255435ea7SShannon Nelson .name = PDS_CORE_DRV_NAME, 35355435ea7SShannon Nelson .id_table = pdsc_id_table, 35455435ea7SShannon Nelson .probe = pdsc_probe, 35555435ea7SShannon Nelson .remove = pdsc_remove, 35655435ea7SShannon Nelson }; 35755435ea7SShannon Nelson 35855435ea7SShannon Nelson static int __init pdsc_init_module(void) 35955435ea7SShannon Nelson { 36055435ea7SShannon Nelson if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME)) 36155435ea7SShannon Nelson return -EINVAL; 36255435ea7SShannon Nelson 36355435ea7SShannon Nelson pdsc_debugfs_create(); 36455435ea7SShannon Nelson return pci_register_driver(&pdsc_driver); 36555435ea7SShannon Nelson } 36655435ea7SShannon Nelson 36755435ea7SShannon Nelson static void __exit pdsc_cleanup_module(void) 36855435ea7SShannon Nelson { 36955435ea7SShannon Nelson pci_unregister_driver(&pdsc_driver); 37055435ea7SShannon Nelson pdsc_debugfs_destroy(); 37155435ea7SShannon Nelson } 37255435ea7SShannon Nelson 37355435ea7SShannon Nelson module_init(pdsc_init_module); 37455435ea7SShannon Nelson module_exit(pdsc_cleanup_module); 375