1*a4b16dadSTom Zanussi // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2*a4b16dadSTom Zanussi /* Copyright(c) 2014 - 2020 Intel Corporation */ 3*a4b16dadSTom Zanussi #include <linux/kernel.h> 4*a4b16dadSTom Zanussi #include <linux/module.h> 5*a4b16dadSTom Zanussi #include <linux/pci.h> 6*a4b16dadSTom Zanussi #include <linux/init.h> 7*a4b16dadSTom Zanussi #include <linux/types.h> 8*a4b16dadSTom Zanussi #include <linux/fs.h> 9*a4b16dadSTom Zanussi #include <linux/slab.h> 10*a4b16dadSTom Zanussi #include <linux/errno.h> 11*a4b16dadSTom Zanussi #include <linux/device.h> 12*a4b16dadSTom Zanussi #include <linux/dma-mapping.h> 13*a4b16dadSTom Zanussi #include <linux/platform_device.h> 14*a4b16dadSTom Zanussi #include <linux/workqueue.h> 15*a4b16dadSTom Zanussi #include <linux/io.h> 16*a4b16dadSTom Zanussi #include <adf_accel_devices.h> 17*a4b16dadSTom Zanussi #include <adf_common_drv.h> 18*a4b16dadSTom Zanussi #include <adf_cfg.h> 19*a4b16dadSTom Zanussi #include "adf_c62xvf_hw_data.h" 20*a4b16dadSTom Zanussi 21*a4b16dadSTom Zanussi static const struct pci_device_id adf_pci_tbl[] = { 22*a4b16dadSTom Zanussi { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_QAT_C62X_VF), }, 23*a4b16dadSTom Zanussi { } 24*a4b16dadSTom Zanussi }; 25*a4b16dadSTom Zanussi MODULE_DEVICE_TABLE(pci, adf_pci_tbl); 26*a4b16dadSTom Zanussi 27*a4b16dadSTom Zanussi static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); 28*a4b16dadSTom Zanussi static void adf_remove(struct pci_dev *dev); 29*a4b16dadSTom Zanussi 30*a4b16dadSTom Zanussi static struct pci_driver adf_driver = { 31*a4b16dadSTom Zanussi .id_table = adf_pci_tbl, 32*a4b16dadSTom Zanussi .name = ADF_C62XVF_DEVICE_NAME, 33*a4b16dadSTom Zanussi .probe = adf_probe, 34*a4b16dadSTom Zanussi .remove = adf_remove, 35*a4b16dadSTom Zanussi }; 36*a4b16dadSTom Zanussi 37*a4b16dadSTom Zanussi static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) 38*a4b16dadSTom Zanussi { 39*a4b16dadSTom Zanussi pci_release_regions(accel_dev->accel_pci_dev.pci_dev); 40*a4b16dadSTom Zanussi pci_disable_device(accel_dev->accel_pci_dev.pci_dev); 41*a4b16dadSTom Zanussi } 42*a4b16dadSTom Zanussi 43*a4b16dadSTom Zanussi static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) 44*a4b16dadSTom Zanussi { 45*a4b16dadSTom Zanussi struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; 46*a4b16dadSTom Zanussi struct adf_accel_dev *pf; 47*a4b16dadSTom Zanussi int i; 48*a4b16dadSTom Zanussi 49*a4b16dadSTom Zanussi for (i = 0; i < ADF_PCI_MAX_BARS; i++) { 50*a4b16dadSTom Zanussi struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; 51*a4b16dadSTom Zanussi 52*a4b16dadSTom Zanussi if (bar->virt_addr) 53*a4b16dadSTom Zanussi pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); 54*a4b16dadSTom Zanussi } 55*a4b16dadSTom Zanussi 56*a4b16dadSTom Zanussi if (accel_dev->hw_device) { 57*a4b16dadSTom Zanussi switch (accel_pci_dev->pci_dev->device) { 58*a4b16dadSTom Zanussi case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: 59*a4b16dadSTom Zanussi adf_clean_hw_data_c62xiov(accel_dev->hw_device); 60*a4b16dadSTom Zanussi break; 61*a4b16dadSTom Zanussi default: 62*a4b16dadSTom Zanussi break; 63*a4b16dadSTom Zanussi } 64*a4b16dadSTom Zanussi kfree(accel_dev->hw_device); 65*a4b16dadSTom Zanussi accel_dev->hw_device = NULL; 66*a4b16dadSTom Zanussi } 67*a4b16dadSTom Zanussi adf_cfg_dev_remove(accel_dev); 68*a4b16dadSTom Zanussi debugfs_remove(accel_dev->debugfs_dir); 69*a4b16dadSTom Zanussi pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); 70*a4b16dadSTom Zanussi adf_devmgr_rm_dev(accel_dev, pf); 71*a4b16dadSTom Zanussi } 72*a4b16dadSTom Zanussi 73*a4b16dadSTom Zanussi static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 74*a4b16dadSTom Zanussi { 75*a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev; 76*a4b16dadSTom Zanussi struct adf_accel_dev *pf; 77*a4b16dadSTom Zanussi struct adf_accel_pci *accel_pci_dev; 78*a4b16dadSTom Zanussi struct adf_hw_device_data *hw_data; 79*a4b16dadSTom Zanussi char name[ADF_DEVICE_NAME_LENGTH]; 80*a4b16dadSTom Zanussi unsigned int i, bar_nr; 81*a4b16dadSTom Zanussi unsigned long bar_mask; 82*a4b16dadSTom Zanussi int ret; 83*a4b16dadSTom Zanussi 84*a4b16dadSTom Zanussi switch (ent->device) { 85*a4b16dadSTom Zanussi case PCI_DEVICE_ID_INTEL_QAT_C62X_VF: 86*a4b16dadSTom Zanussi break; 87*a4b16dadSTom Zanussi default: 88*a4b16dadSTom Zanussi dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); 89*a4b16dadSTom Zanussi return -ENODEV; 90*a4b16dadSTom Zanussi } 91*a4b16dadSTom Zanussi 92*a4b16dadSTom Zanussi accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, 93*a4b16dadSTom Zanussi dev_to_node(&pdev->dev)); 94*a4b16dadSTom Zanussi if (!accel_dev) 95*a4b16dadSTom Zanussi return -ENOMEM; 96*a4b16dadSTom Zanussi 97*a4b16dadSTom Zanussi accel_dev->is_vf = true; 98*a4b16dadSTom Zanussi pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); 99*a4b16dadSTom Zanussi accel_pci_dev = &accel_dev->accel_pci_dev; 100*a4b16dadSTom Zanussi accel_pci_dev->pci_dev = pdev; 101*a4b16dadSTom Zanussi 102*a4b16dadSTom Zanussi /* Add accel device to accel table */ 103*a4b16dadSTom Zanussi if (adf_devmgr_add_dev(accel_dev, pf)) { 104*a4b16dadSTom Zanussi dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); 105*a4b16dadSTom Zanussi kfree(accel_dev); 106*a4b16dadSTom Zanussi return -EFAULT; 107*a4b16dadSTom Zanussi } 108*a4b16dadSTom Zanussi INIT_LIST_HEAD(&accel_dev->crypto_list); 109*a4b16dadSTom Zanussi 110*a4b16dadSTom Zanussi accel_dev->owner = THIS_MODULE; 111*a4b16dadSTom Zanussi /* Allocate and configure device configuration structure */ 112*a4b16dadSTom Zanussi hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, 113*a4b16dadSTom Zanussi dev_to_node(&pdev->dev)); 114*a4b16dadSTom Zanussi if (!hw_data) { 115*a4b16dadSTom Zanussi ret = -ENOMEM; 116*a4b16dadSTom Zanussi goto out_err; 117*a4b16dadSTom Zanussi } 118*a4b16dadSTom Zanussi accel_dev->hw_device = hw_data; 119*a4b16dadSTom Zanussi adf_init_hw_data_c62xiov(accel_dev->hw_device); 120*a4b16dadSTom Zanussi 121*a4b16dadSTom Zanussi /* Get Accelerators and Accelerators Engines masks */ 122*a4b16dadSTom Zanussi hw_data->accel_mask = hw_data->get_accel_mask(hw_data); 123*a4b16dadSTom Zanussi hw_data->ae_mask = hw_data->get_ae_mask(hw_data); 124*a4b16dadSTom Zanussi accel_pci_dev->sku = hw_data->get_sku(hw_data); 125*a4b16dadSTom Zanussi 126*a4b16dadSTom Zanussi /* Create dev top level debugfs entry */ 127*a4b16dadSTom Zanussi snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, 128*a4b16dadSTom Zanussi hw_data->dev_class->name, pci_name(pdev)); 129*a4b16dadSTom Zanussi 130*a4b16dadSTom Zanussi accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); 131*a4b16dadSTom Zanussi 132*a4b16dadSTom Zanussi /* Create device configuration table */ 133*a4b16dadSTom Zanussi ret = adf_cfg_dev_add(accel_dev); 134*a4b16dadSTom Zanussi if (ret) 135*a4b16dadSTom Zanussi goto out_err; 136*a4b16dadSTom Zanussi 137*a4b16dadSTom Zanussi /* enable PCI device */ 138*a4b16dadSTom Zanussi if (pci_enable_device(pdev)) { 139*a4b16dadSTom Zanussi ret = -EFAULT; 140*a4b16dadSTom Zanussi goto out_err; 141*a4b16dadSTom Zanussi } 142*a4b16dadSTom Zanussi 143*a4b16dadSTom Zanussi /* set dma identifier */ 144*a4b16dadSTom Zanussi ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); 145*a4b16dadSTom Zanussi if (ret) { 146*a4b16dadSTom Zanussi dev_err(&pdev->dev, "No usable DMA configuration\n"); 147*a4b16dadSTom Zanussi goto out_err_disable; 148*a4b16dadSTom Zanussi } 149*a4b16dadSTom Zanussi 150*a4b16dadSTom Zanussi if (pci_request_regions(pdev, ADF_C62XVF_DEVICE_NAME)) { 151*a4b16dadSTom Zanussi ret = -EFAULT; 152*a4b16dadSTom Zanussi goto out_err_disable; 153*a4b16dadSTom Zanussi } 154*a4b16dadSTom Zanussi 155*a4b16dadSTom Zanussi /* Find and map all the device's BARS */ 156*a4b16dadSTom Zanussi i = 0; 157*a4b16dadSTom Zanussi bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); 158*a4b16dadSTom Zanussi for_each_set_bit(bar_nr, &bar_mask, ADF_PCI_MAX_BARS * 2) { 159*a4b16dadSTom Zanussi struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; 160*a4b16dadSTom Zanussi 161*a4b16dadSTom Zanussi bar->base_addr = pci_resource_start(pdev, bar_nr); 162*a4b16dadSTom Zanussi if (!bar->base_addr) 163*a4b16dadSTom Zanussi break; 164*a4b16dadSTom Zanussi bar->size = pci_resource_len(pdev, bar_nr); 165*a4b16dadSTom Zanussi bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); 166*a4b16dadSTom Zanussi if (!bar->virt_addr) { 167*a4b16dadSTom Zanussi dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); 168*a4b16dadSTom Zanussi ret = -EFAULT; 169*a4b16dadSTom Zanussi goto out_err_free_reg; 170*a4b16dadSTom Zanussi } 171*a4b16dadSTom Zanussi } 172*a4b16dadSTom Zanussi pci_set_master(pdev); 173*a4b16dadSTom Zanussi /* Completion for VF2PF request/response message exchange */ 174*a4b16dadSTom Zanussi init_completion(&accel_dev->vf.msg_received); 175*a4b16dadSTom Zanussi 176*a4b16dadSTom Zanussi ret = adf_dev_up(accel_dev, false); 177*a4b16dadSTom Zanussi if (ret) 178*a4b16dadSTom Zanussi goto out_err_dev_stop; 179*a4b16dadSTom Zanussi 180*a4b16dadSTom Zanussi return ret; 181*a4b16dadSTom Zanussi 182*a4b16dadSTom Zanussi out_err_dev_stop: 183*a4b16dadSTom Zanussi adf_dev_down(accel_dev, false); 184*a4b16dadSTom Zanussi out_err_free_reg: 185*a4b16dadSTom Zanussi pci_release_regions(accel_pci_dev->pci_dev); 186*a4b16dadSTom Zanussi out_err_disable: 187*a4b16dadSTom Zanussi pci_disable_device(accel_pci_dev->pci_dev); 188*a4b16dadSTom Zanussi out_err: 189*a4b16dadSTom Zanussi adf_cleanup_accel(accel_dev); 190*a4b16dadSTom Zanussi kfree(accel_dev); 191*a4b16dadSTom Zanussi return ret; 192*a4b16dadSTom Zanussi } 193*a4b16dadSTom Zanussi 194*a4b16dadSTom Zanussi static void adf_remove(struct pci_dev *pdev) 195*a4b16dadSTom Zanussi { 196*a4b16dadSTom Zanussi struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); 197*a4b16dadSTom Zanussi 198*a4b16dadSTom Zanussi if (!accel_dev) { 199*a4b16dadSTom Zanussi pr_err("QAT: Driver removal failed\n"); 200*a4b16dadSTom Zanussi return; 201*a4b16dadSTom Zanussi } 202*a4b16dadSTom Zanussi adf_flush_vf_wq(accel_dev); 203*a4b16dadSTom Zanussi adf_dev_down(accel_dev, false); 204*a4b16dadSTom Zanussi adf_cleanup_accel(accel_dev); 205*a4b16dadSTom Zanussi adf_cleanup_pci_dev(accel_dev); 206*a4b16dadSTom Zanussi kfree(accel_dev); 207*a4b16dadSTom Zanussi } 208*a4b16dadSTom Zanussi 209*a4b16dadSTom Zanussi static int __init adfdrv_init(void) 210*a4b16dadSTom Zanussi { 211*a4b16dadSTom Zanussi request_module("intel_qat"); 212*a4b16dadSTom Zanussi 213*a4b16dadSTom Zanussi if (pci_register_driver(&adf_driver)) { 214*a4b16dadSTom Zanussi pr_err("QAT: Driver initialization failed\n"); 215*a4b16dadSTom Zanussi return -EFAULT; 216*a4b16dadSTom Zanussi } 217*a4b16dadSTom Zanussi return 0; 218*a4b16dadSTom Zanussi } 219*a4b16dadSTom Zanussi 220*a4b16dadSTom Zanussi static void __exit adfdrv_release(void) 221*a4b16dadSTom Zanussi { 222*a4b16dadSTom Zanussi pci_unregister_driver(&adf_driver); 223*a4b16dadSTom Zanussi adf_clean_vf_map(true); 224*a4b16dadSTom Zanussi } 225*a4b16dadSTom Zanussi 226*a4b16dadSTom Zanussi module_init(adfdrv_init); 227*a4b16dadSTom Zanussi module_exit(adfdrv_release); 228*a4b16dadSTom Zanussi 229*a4b16dadSTom Zanussi MODULE_LICENSE("Dual BSD/GPL"); 230*a4b16dadSTom Zanussi MODULE_AUTHOR("Intel"); 231*a4b16dadSTom Zanussi MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); 232*a4b16dadSTom Zanussi MODULE_VERSION(ADF_DRV_VERSION); 233