1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/pci.h> 7 8 #include <linux/pds/pds_common.h> 9 10 #include "core.h" 11 12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION); 13 MODULE_AUTHOR("Advanced Micro Devices, Inc"); 14 MODULE_LICENSE("GPL"); 15 16 /* Supported devices */ 17 static const struct pci_device_id pdsc_id_table[] = { 18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) }, 19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) }, 20 { 0, } /* end of table */ 21 }; 22 MODULE_DEVICE_TABLE(pci, pdsc_id_table); 23 24 static void pdsc_wdtimer_cb(struct timer_list *t) 25 { 26 struct pdsc *pdsc = from_timer(pdsc, t, wdtimer); 27 28 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies); 29 mod_timer(&pdsc->wdtimer, 30 round_jiffies(jiffies + pdsc->wdtimer_period)); 31 32 queue_work(pdsc->wq, &pdsc->health_work); 33 } 34 35 static void pdsc_unmap_bars(struct pdsc *pdsc) 36 { 37 struct pdsc_dev_bar *bars = pdsc->bars; 38 unsigned int i; 39 40 for (i = 0; i < PDS_CORE_BARS_MAX; i++) { 41 if (bars[i].vaddr) 42 pci_iounmap(pdsc->pdev, bars[i].vaddr); 43 } 44 } 45 46 static int pdsc_map_bars(struct pdsc *pdsc) 47 { 48 struct pdsc_dev_bar *bar = pdsc->bars; 49 struct pci_dev *pdev = pdsc->pdev; 50 struct device *dev = pdsc->dev; 51 struct pdsc_dev_bar *bars; 52 unsigned int i, j; 53 int num_bars = 0; 54 int err; 55 u32 sig; 56 57 bars = pdsc->bars; 58 59 /* Since the PCI interface in the hardware is configurable, 60 * we need to poke into all the bars to find the set we're 61 * expecting. 62 */ 63 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) { 64 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 65 continue; 66 67 bars[j].len = pci_resource_len(pdev, i); 68 bars[j].bus_addr = pci_resource_start(pdev, i); 69 bars[j].res_index = i; 70 71 /* only map the whole bar 0 */ 72 if (j > 0) { 73 bars[j].vaddr = NULL; 74 } else { 75 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 76 if (!bars[j].vaddr) { 77 dev_err(dev, "Cannot map BAR %d, aborting\n", i); 78 return -ENODEV; 79 } 80 } 81 82 j++; 83 } 84 num_bars = j; 85 86 /* BAR0: dev_cmd and interrupts */ 87 if (num_bars < 1) { 88 dev_err(dev, "No bars found\n"); 89 err = -EFAULT; 90 goto err_out; 91 } 92 93 if (bar->len < PDS_CORE_BAR0_SIZE) { 94 dev_err(dev, "Resource bar size %lu too small\n", bar->len); 95 err = -EFAULT; 96 goto err_out; 97 } 98 99 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET; 100 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET; 101 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET; 102 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET; 103 104 sig = ioread32(&pdsc->info_regs->signature); 105 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) { 106 dev_err(dev, "Incompatible firmware signature %x", sig); 107 err = -EFAULT; 108 goto err_out; 109 } 110 111 /* BAR1: doorbells */ 112 bar++; 113 if (num_bars < 2) { 114 dev_err(dev, "Doorbell bar missing\n"); 115 err = -EFAULT; 116 goto err_out; 117 } 118 119 pdsc->db_pages = bar->vaddr; 120 pdsc->phy_db_pages = bar->bus_addr; 121 122 return 0; 123 124 err_out: 125 pdsc_unmap_bars(pdsc); 126 return err; 127 } 128 129 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num) 130 { 131 return pci_iomap_range(pdsc->pdev, 132 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index, 133 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 134 } 135 136 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs) 137 { 138 struct pdsc *pdsc = pci_get_drvdata(pdev); 139 struct device *dev = pdsc->dev; 140 int ret = 0; 141 142 if (num_vfs > 0) { 143 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf), 144 GFP_KERNEL); 145 if (!pdsc->vfs) 146 return -ENOMEM; 147 pdsc->num_vfs = num_vfs; 148 149 ret = pci_enable_sriov(pdev, num_vfs); 150 if (ret) { 151 dev_err(dev, "Cannot enable SRIOV: %pe\n", 152 ERR_PTR(ret)); 153 goto no_vfs; 154 } 155 156 return num_vfs; 157 } 158 159 no_vfs: 160 pci_disable_sriov(pdev); 161 162 kfree(pdsc->vfs); 163 pdsc->vfs = NULL; 164 pdsc->num_vfs = 0; 165 166 return ret; 167 } 168 169 static int pdsc_init_vf(struct pdsc *vf) 170 { 171 struct devlink *dl; 172 struct pdsc *pf; 173 int err; 174 175 pf = pdsc_get_pf_struct(vf->pdev); 176 if (IS_ERR_OR_NULL(pf)) 177 return PTR_ERR(pf) ?: -1; 178 179 vf->vf_id = pci_iov_vf_id(vf->pdev); 180 181 dl = priv_to_devlink(vf); 182 devl_lock(dl); 183 devl_register(dl); 184 devl_unlock(dl); 185 186 pf->vfs[vf->vf_id].vf = vf; 187 err = pdsc_auxbus_dev_add(vf, pf); 188 if (err) { 189 devl_lock(dl); 190 devl_unregister(dl); 191 devl_unlock(dl); 192 } 193 194 return err; 195 } 196 197 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = { 198 .name = "fw", 199 .diagnose = pdsc_fw_reporter_diagnose, 200 }; 201 202 #define PDSC_WQ_NAME_LEN 24 203 204 static int pdsc_init_pf(struct pdsc *pdsc) 205 { 206 struct devlink_health_reporter *hr; 207 char wq_name[PDSC_WQ_NAME_LEN]; 208 struct devlink *dl; 209 int err; 210 211 pcie_print_link_status(pdsc->pdev); 212 213 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 214 if (err) { 215 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 216 ERR_PTR(err)); 217 return err; 218 } 219 220 err = pdsc_map_bars(pdsc); 221 if (err) 222 goto err_out_release_regions; 223 224 /* General workqueue and timer, but don't start timer yet */ 225 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); 226 pdsc->wq = create_singlethread_workqueue(wq_name); 227 INIT_WORK(&pdsc->health_work, pdsc_health_thread); 228 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 229 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ; 230 231 mutex_init(&pdsc->devcmd_lock); 232 mutex_init(&pdsc->config_lock); 233 spin_lock_init(&pdsc->adminq_lock); 234 235 mutex_lock(&pdsc->config_lock); 236 set_bit(PDSC_S_FW_DEAD, &pdsc->state); 237 238 err = pdsc_setup(pdsc, PDSC_SETUP_INIT); 239 if (err) 240 goto err_out_unmap_bars; 241 err = pdsc_start(pdsc); 242 if (err) 243 goto err_out_teardown; 244 245 mutex_unlock(&pdsc->config_lock); 246 247 dl = priv_to_devlink(pdsc); 248 devl_lock(dl); 249 250 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc); 251 if (IS_ERR(hr)) { 252 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr); 253 err = PTR_ERR(hr); 254 devl_unlock(dl); 255 goto err_out_stop; 256 } 257 pdsc->fw_reporter = hr; 258 259 devl_register(dl); 260 devl_unlock(dl); 261 262 /* Lastly, start the health check timer */ 263 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period)); 264 265 return 0; 266 267 err_out_stop: 268 pdsc_stop(pdsc); 269 err_out_teardown: 270 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 271 err_out_unmap_bars: 272 mutex_unlock(&pdsc->config_lock); 273 del_timer_sync(&pdsc->wdtimer); 274 if (pdsc->wq) 275 destroy_workqueue(pdsc->wq); 276 mutex_destroy(&pdsc->config_lock); 277 mutex_destroy(&pdsc->devcmd_lock); 278 pci_free_irq_vectors(pdsc->pdev); 279 pdsc_unmap_bars(pdsc); 280 err_out_release_regions: 281 pci_release_regions(pdsc->pdev); 282 283 return err; 284 } 285 286 static const struct devlink_ops pdsc_dl_ops = { 287 .info_get = pdsc_dl_info_get, 288 .flash_update = pdsc_dl_flash_update, 289 }; 290 291 static const struct devlink_ops pdsc_dl_vf_ops = { 292 }; 293 294 static DEFINE_IDA(pdsc_ida); 295 296 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 297 { 298 struct device *dev = &pdev->dev; 299 const struct devlink_ops *ops; 300 struct devlink *dl; 301 struct pdsc *pdsc; 302 bool is_pf; 303 int err; 304 305 is_pf = !pdev->is_virtfn; 306 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops; 307 dl = devlink_alloc(ops, sizeof(struct pdsc), dev); 308 if (!dl) 309 return -ENOMEM; 310 pdsc = devlink_priv(dl); 311 312 pdsc->pdev = pdev; 313 pdsc->dev = &pdev->dev; 314 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 315 pci_set_drvdata(pdev, pdsc); 316 pdsc_debugfs_add_dev(pdsc); 317 318 err = ida_alloc(&pdsc_ida, GFP_KERNEL); 319 if (err < 0) { 320 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n", 321 __func__, ERR_PTR(err)); 322 goto err_out_free_devlink; 323 } 324 pdsc->uid = err; 325 326 /* Query system for DMA addressing limitation for the device. */ 327 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN)); 328 if (err) { 329 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n", 330 ERR_PTR(err)); 331 goto err_out_free_ida; 332 } 333 334 err = pci_enable_device(pdev); 335 if (err) { 336 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 337 goto err_out_free_ida; 338 } 339 pci_set_master(pdev); 340 341 if (is_pf) 342 err = pdsc_init_pf(pdsc); 343 else 344 err = pdsc_init_vf(pdsc); 345 if (err) { 346 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err)); 347 goto err_out_clear_master; 348 } 349 350 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 351 return 0; 352 353 err_out_clear_master: 354 pci_clear_master(pdev); 355 pci_disable_device(pdev); 356 err_out_free_ida: 357 ida_free(&pdsc_ida, pdsc->uid); 358 err_out_free_devlink: 359 pdsc_debugfs_del_dev(pdsc); 360 devlink_free(dl); 361 362 return err; 363 } 364 365 static void pdsc_remove(struct pci_dev *pdev) 366 { 367 struct pdsc *pdsc = pci_get_drvdata(pdev); 368 struct devlink *dl; 369 370 /* Unhook the registrations first to be sure there 371 * are no requests while we're stopping. 372 */ 373 dl = priv_to_devlink(pdsc); 374 devl_lock(dl); 375 devl_unregister(dl); 376 if (pdsc->fw_reporter) { 377 devl_health_reporter_destroy(pdsc->fw_reporter); 378 pdsc->fw_reporter = NULL; 379 } 380 devl_unlock(dl); 381 382 if (pdev->is_virtfn) { 383 struct pdsc *pf; 384 385 pf = pdsc_get_pf_struct(pdsc->pdev); 386 if (!IS_ERR(pf)) { 387 pdsc_auxbus_dev_del(pdsc, pf); 388 pf->vfs[pdsc->vf_id].vf = NULL; 389 } 390 } else { 391 /* Remove the VFs and their aux_bus connections before other 392 * cleanup so that the clients can use the AdminQ to cleanly 393 * shut themselves down. 394 */ 395 pdsc_sriov_configure(pdev, 0); 396 397 del_timer_sync(&pdsc->wdtimer); 398 if (pdsc->wq) 399 destroy_workqueue(pdsc->wq); 400 401 mutex_lock(&pdsc->config_lock); 402 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); 403 404 pdsc_stop(pdsc); 405 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 406 mutex_unlock(&pdsc->config_lock); 407 mutex_destroy(&pdsc->config_lock); 408 mutex_destroy(&pdsc->devcmd_lock); 409 410 pci_free_irq_vectors(pdev); 411 pdsc_unmap_bars(pdsc); 412 pci_release_regions(pdev); 413 } 414 415 pci_clear_master(pdev); 416 pci_disable_device(pdev); 417 418 ida_free(&pdsc_ida, pdsc->uid); 419 pdsc_debugfs_del_dev(pdsc); 420 devlink_free(dl); 421 } 422 423 static struct pci_driver pdsc_driver = { 424 .name = PDS_CORE_DRV_NAME, 425 .id_table = pdsc_id_table, 426 .probe = pdsc_probe, 427 .remove = pdsc_remove, 428 .sriov_configure = pdsc_sriov_configure, 429 }; 430 431 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev) 432 { 433 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver); 434 } 435 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct); 436 437 static int __init pdsc_init_module(void) 438 { 439 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME)) 440 return -EINVAL; 441 442 pdsc_debugfs_create(); 443 return pci_register_driver(&pdsc_driver); 444 } 445 446 static void __exit pdsc_cleanup_module(void) 447 { 448 pci_unregister_driver(&pdsc_driver); 449 pdsc_debugfs_destroy(); 450 } 451 452 module_init(pdsc_init_module); 453 module_exit(pdsc_cleanup_module); 454