1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2023 Advanced Micro Devices, Inc */ 3 4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 5 6 #include <linux/pci.h> 7 8 #include <linux/pds/pds_common.h> 9 10 #include "core.h" 11 12 MODULE_DESCRIPTION(PDSC_DRV_DESCRIPTION); 13 MODULE_AUTHOR("Advanced Micro Devices, Inc"); 14 MODULE_LICENSE("GPL"); 15 16 /* Supported devices */ 17 static const struct pci_device_id pdsc_id_table[] = { 18 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_CORE_PF) }, 19 { PCI_VDEVICE(PENSANDO, PCI_DEVICE_ID_PENSANDO_VDPA_VF) }, 20 { 0, } /* end of table */ 21 }; 22 MODULE_DEVICE_TABLE(pci, pdsc_id_table); 23 24 static void pdsc_wdtimer_cb(struct timer_list *t) 25 { 26 struct pdsc *pdsc = from_timer(pdsc, t, wdtimer); 27 28 dev_dbg(pdsc->dev, "%s: jiffies %ld\n", __func__, jiffies); 29 mod_timer(&pdsc->wdtimer, 30 round_jiffies(jiffies + pdsc->wdtimer_period)); 31 32 queue_work(pdsc->wq, &pdsc->health_work); 33 } 34 35 static void pdsc_unmap_bars(struct pdsc *pdsc) 36 { 37 struct pdsc_dev_bar *bars = pdsc->bars; 38 unsigned int i; 39 40 for (i = 0; i < PDS_CORE_BARS_MAX; i++) { 41 if (bars[i].vaddr) 42 pci_iounmap(pdsc->pdev, bars[i].vaddr); 43 } 44 } 45 46 static int pdsc_map_bars(struct pdsc *pdsc) 47 { 48 struct pdsc_dev_bar *bar = pdsc->bars; 49 struct pci_dev *pdev = pdsc->pdev; 50 struct device *dev = pdsc->dev; 51 struct pdsc_dev_bar *bars; 52 unsigned int i, j; 53 int num_bars = 0; 54 int err; 55 u32 sig; 56 57 bars = pdsc->bars; 58 59 /* Since the PCI interface in the hardware is configurable, 60 * we need to poke into all the bars to find the set we're 61 * expecting. 62 */ 63 for (i = 0, j = 0; i < PDS_CORE_BARS_MAX; i++) { 64 if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM)) 65 continue; 66 67 bars[j].len = pci_resource_len(pdev, i); 68 bars[j].bus_addr = pci_resource_start(pdev, i); 69 bars[j].res_index = i; 70 71 /* only map the whole bar 0 */ 72 if (j > 0) { 73 bars[j].vaddr = NULL; 74 } else { 75 bars[j].vaddr = pci_iomap(pdev, i, bars[j].len); 76 if (!bars[j].vaddr) { 77 dev_err(dev, "Cannot map BAR %d, aborting\n", i); 78 return -ENODEV; 79 } 80 } 81 82 j++; 83 } 84 num_bars = j; 85 86 /* BAR0: dev_cmd and interrupts */ 87 if (num_bars < 1) { 88 dev_err(dev, "No bars found\n"); 89 err = -EFAULT; 90 goto err_out; 91 } 92 93 if (bar->len < PDS_CORE_BAR0_SIZE) { 94 dev_err(dev, "Resource bar size %lu too small\n", bar->len); 95 err = -EFAULT; 96 goto err_out; 97 } 98 99 pdsc->info_regs = bar->vaddr + PDS_CORE_BAR0_DEV_INFO_REGS_OFFSET; 100 pdsc->cmd_regs = bar->vaddr + PDS_CORE_BAR0_DEV_CMD_REGS_OFFSET; 101 pdsc->intr_status = bar->vaddr + PDS_CORE_BAR0_INTR_STATUS_OFFSET; 102 pdsc->intr_ctrl = bar->vaddr + PDS_CORE_BAR0_INTR_CTRL_OFFSET; 103 104 sig = ioread32(&pdsc->info_regs->signature); 105 if (sig != PDS_CORE_DEV_INFO_SIGNATURE) { 106 dev_err(dev, "Incompatible firmware signature %x", sig); 107 err = -EFAULT; 108 goto err_out; 109 } 110 111 /* BAR1: doorbells */ 112 bar++; 113 if (num_bars < 2) { 114 dev_err(dev, "Doorbell bar missing\n"); 115 err = -EFAULT; 116 goto err_out; 117 } 118 119 pdsc->db_pages = bar->vaddr; 120 pdsc->phy_db_pages = bar->bus_addr; 121 122 return 0; 123 124 err_out: 125 pdsc_unmap_bars(pdsc); 126 return err; 127 } 128 129 void __iomem *pdsc_map_dbpage(struct pdsc *pdsc, int page_num) 130 { 131 return pci_iomap_range(pdsc->pdev, 132 pdsc->bars[PDS_CORE_PCI_BAR_DBELL].res_index, 133 (u64)page_num << PAGE_SHIFT, PAGE_SIZE); 134 } 135 136 static int pdsc_sriov_configure(struct pci_dev *pdev, int num_vfs) 137 { 138 struct pdsc *pdsc = pci_get_drvdata(pdev); 139 struct device *dev = pdsc->dev; 140 int ret = 0; 141 142 if (num_vfs > 0) { 143 pdsc->vfs = kcalloc(num_vfs, sizeof(struct pdsc_vf), 144 GFP_KERNEL); 145 if (!pdsc->vfs) 146 return -ENOMEM; 147 pdsc->num_vfs = num_vfs; 148 149 ret = pci_enable_sriov(pdev, num_vfs); 150 if (ret) { 151 dev_err(dev, "Cannot enable SRIOV: %pe\n", 152 ERR_PTR(ret)); 153 goto no_vfs; 154 } 155 156 return num_vfs; 157 } 158 159 no_vfs: 160 pci_disable_sriov(pdev); 161 162 kfree(pdsc->vfs); 163 pdsc->vfs = NULL; 164 pdsc->num_vfs = 0; 165 166 return ret; 167 } 168 169 static int pdsc_init_vf(struct pdsc *vf) 170 { 171 struct devlink *dl; 172 struct pdsc *pf; 173 int err; 174 175 pf = pdsc_get_pf_struct(vf->pdev); 176 if (IS_ERR_OR_NULL(pf)) 177 return PTR_ERR(pf) ?: -1; 178 179 vf->vf_id = pci_iov_vf_id(vf->pdev); 180 181 dl = priv_to_devlink(vf); 182 devl_lock(dl); 183 devl_register(dl); 184 devl_unlock(dl); 185 186 pf->vfs[vf->vf_id].vf = vf; 187 err = pdsc_auxbus_dev_add(vf, pf); 188 if (err) { 189 devl_lock(dl); 190 devl_unregister(dl); 191 devl_unlock(dl); 192 } 193 194 return err; 195 } 196 197 static const struct devlink_health_reporter_ops pdsc_fw_reporter_ops = { 198 .name = "fw", 199 .diagnose = pdsc_fw_reporter_diagnose, 200 }; 201 202 static const struct devlink_param pdsc_dl_params[] = { 203 DEVLINK_PARAM_GENERIC(ENABLE_VNET, 204 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 205 pdsc_dl_enable_get, 206 pdsc_dl_enable_set, 207 pdsc_dl_enable_validate), 208 }; 209 210 #define PDSC_WQ_NAME_LEN 24 211 212 static int pdsc_init_pf(struct pdsc *pdsc) 213 { 214 struct devlink_health_reporter *hr; 215 char wq_name[PDSC_WQ_NAME_LEN]; 216 struct devlink *dl; 217 int err; 218 219 pcie_print_link_status(pdsc->pdev); 220 221 err = pci_request_regions(pdsc->pdev, PDS_CORE_DRV_NAME); 222 if (err) { 223 dev_err(pdsc->dev, "Cannot request PCI regions: %pe\n", 224 ERR_PTR(err)); 225 return err; 226 } 227 228 err = pdsc_map_bars(pdsc); 229 if (err) 230 goto err_out_release_regions; 231 232 /* General workqueue and timer, but don't start timer yet */ 233 snprintf(wq_name, sizeof(wq_name), "%s.%d", PDS_CORE_DRV_NAME, pdsc->uid); 234 pdsc->wq = create_singlethread_workqueue(wq_name); 235 INIT_WORK(&pdsc->health_work, pdsc_health_thread); 236 timer_setup(&pdsc->wdtimer, pdsc_wdtimer_cb, 0); 237 pdsc->wdtimer_period = PDSC_WATCHDOG_SECS * HZ; 238 239 mutex_init(&pdsc->devcmd_lock); 240 mutex_init(&pdsc->config_lock); 241 spin_lock_init(&pdsc->adminq_lock); 242 243 mutex_lock(&pdsc->config_lock); 244 set_bit(PDSC_S_FW_DEAD, &pdsc->state); 245 246 err = pdsc_setup(pdsc, PDSC_SETUP_INIT); 247 if (err) { 248 mutex_unlock(&pdsc->config_lock); 249 goto err_out_unmap_bars; 250 } 251 252 err = pdsc_start(pdsc); 253 if (err) { 254 mutex_unlock(&pdsc->config_lock); 255 goto err_out_teardown; 256 } 257 258 mutex_unlock(&pdsc->config_lock); 259 260 dl = priv_to_devlink(pdsc); 261 devl_lock(dl); 262 err = devl_params_register(dl, pdsc_dl_params, 263 ARRAY_SIZE(pdsc_dl_params)); 264 if (err) { 265 devl_unlock(dl); 266 dev_warn(pdsc->dev, "Failed to register devlink params: %pe\n", 267 ERR_PTR(err)); 268 goto err_out_stop; 269 } 270 271 hr = devl_health_reporter_create(dl, &pdsc_fw_reporter_ops, 0, pdsc); 272 if (IS_ERR(hr)) { 273 devl_unlock(dl); 274 dev_warn(pdsc->dev, "Failed to create fw reporter: %pe\n", hr); 275 err = PTR_ERR(hr); 276 goto err_out_unreg_params; 277 } 278 pdsc->fw_reporter = hr; 279 280 devl_register(dl); 281 devl_unlock(dl); 282 283 /* Lastly, start the health check timer */ 284 mod_timer(&pdsc->wdtimer, round_jiffies(jiffies + pdsc->wdtimer_period)); 285 286 return 0; 287 288 err_out_unreg_params: 289 devlink_params_unregister(dl, pdsc_dl_params, 290 ARRAY_SIZE(pdsc_dl_params)); 291 err_out_stop: 292 pdsc_stop(pdsc); 293 err_out_teardown: 294 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 295 err_out_unmap_bars: 296 del_timer_sync(&pdsc->wdtimer); 297 if (pdsc->wq) 298 destroy_workqueue(pdsc->wq); 299 mutex_destroy(&pdsc->config_lock); 300 mutex_destroy(&pdsc->devcmd_lock); 301 pci_free_irq_vectors(pdsc->pdev); 302 pdsc_unmap_bars(pdsc); 303 err_out_release_regions: 304 pci_release_regions(pdsc->pdev); 305 306 return err; 307 } 308 309 static const struct devlink_ops pdsc_dl_ops = { 310 .info_get = pdsc_dl_info_get, 311 .flash_update = pdsc_dl_flash_update, 312 }; 313 314 static const struct devlink_ops pdsc_dl_vf_ops = { 315 }; 316 317 static DEFINE_IDA(pdsc_ida); 318 319 static int pdsc_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 320 { 321 struct device *dev = &pdev->dev; 322 const struct devlink_ops *ops; 323 struct devlink *dl; 324 struct pdsc *pdsc; 325 bool is_pf; 326 int err; 327 328 is_pf = !pdev->is_virtfn; 329 ops = is_pf ? &pdsc_dl_ops : &pdsc_dl_vf_ops; 330 dl = devlink_alloc(ops, sizeof(struct pdsc), dev); 331 if (!dl) 332 return -ENOMEM; 333 pdsc = devlink_priv(dl); 334 335 pdsc->pdev = pdev; 336 pdsc->dev = &pdev->dev; 337 set_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 338 pci_set_drvdata(pdev, pdsc); 339 pdsc_debugfs_add_dev(pdsc); 340 341 err = ida_alloc(&pdsc_ida, GFP_KERNEL); 342 if (err < 0) { 343 dev_err(pdsc->dev, "%s: id alloc failed: %pe\n", 344 __func__, ERR_PTR(err)); 345 goto err_out_free_devlink; 346 } 347 pdsc->uid = err; 348 349 /* Query system for DMA addressing limitation for the device. */ 350 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(PDS_CORE_ADDR_LEN)); 351 if (err) { 352 dev_err(dev, "Unable to obtain 64-bit DMA for consistent allocations, aborting: %pe\n", 353 ERR_PTR(err)); 354 goto err_out_free_ida; 355 } 356 357 err = pci_enable_device(pdev); 358 if (err) { 359 dev_err(dev, "Cannot enable PCI device: %pe\n", ERR_PTR(err)); 360 goto err_out_free_ida; 361 } 362 pci_set_master(pdev); 363 364 if (is_pf) 365 err = pdsc_init_pf(pdsc); 366 else 367 err = pdsc_init_vf(pdsc); 368 if (err) { 369 dev_err(dev, "Cannot init device: %pe\n", ERR_PTR(err)); 370 goto err_out_disable_device; 371 } 372 373 clear_bit(PDSC_S_INITING_DRIVER, &pdsc->state); 374 return 0; 375 376 err_out_disable_device: 377 pci_disable_device(pdev); 378 err_out_free_ida: 379 ida_free(&pdsc_ida, pdsc->uid); 380 err_out_free_devlink: 381 pdsc_debugfs_del_dev(pdsc); 382 devlink_free(dl); 383 384 return err; 385 } 386 387 static void pdsc_remove(struct pci_dev *pdev) 388 { 389 struct pdsc *pdsc = pci_get_drvdata(pdev); 390 struct devlink *dl; 391 392 /* Unhook the registrations first to be sure there 393 * are no requests while we're stopping. 394 */ 395 dl = priv_to_devlink(pdsc); 396 devl_lock(dl); 397 devl_unregister(dl); 398 if (!pdev->is_virtfn) { 399 if (pdsc->fw_reporter) { 400 devl_health_reporter_destroy(pdsc->fw_reporter); 401 pdsc->fw_reporter = NULL; 402 } 403 devl_params_unregister(dl, pdsc_dl_params, 404 ARRAY_SIZE(pdsc_dl_params)); 405 } 406 devl_unlock(dl); 407 408 if (pdev->is_virtfn) { 409 struct pdsc *pf; 410 411 pf = pdsc_get_pf_struct(pdsc->pdev); 412 if (!IS_ERR(pf)) { 413 pdsc_auxbus_dev_del(pdsc, pf); 414 pf->vfs[pdsc->vf_id].vf = NULL; 415 } 416 } else { 417 /* Remove the VFs and their aux_bus connections before other 418 * cleanup so that the clients can use the AdminQ to cleanly 419 * shut themselves down. 420 */ 421 pdsc_sriov_configure(pdev, 0); 422 423 del_timer_sync(&pdsc->wdtimer); 424 if (pdsc->wq) 425 destroy_workqueue(pdsc->wq); 426 427 mutex_lock(&pdsc->config_lock); 428 set_bit(PDSC_S_STOPPING_DRIVER, &pdsc->state); 429 430 pdsc_stop(pdsc); 431 pdsc_teardown(pdsc, PDSC_TEARDOWN_REMOVING); 432 mutex_unlock(&pdsc->config_lock); 433 mutex_destroy(&pdsc->config_lock); 434 mutex_destroy(&pdsc->devcmd_lock); 435 436 pci_free_irq_vectors(pdev); 437 pdsc_unmap_bars(pdsc); 438 pci_release_regions(pdev); 439 } 440 441 pci_disable_device(pdev); 442 443 ida_free(&pdsc_ida, pdsc->uid); 444 pdsc_debugfs_del_dev(pdsc); 445 devlink_free(dl); 446 } 447 448 static struct pci_driver pdsc_driver = { 449 .name = PDS_CORE_DRV_NAME, 450 .id_table = pdsc_id_table, 451 .probe = pdsc_probe, 452 .remove = pdsc_remove, 453 .sriov_configure = pdsc_sriov_configure, 454 }; 455 456 void *pdsc_get_pf_struct(struct pci_dev *vf_pdev) 457 { 458 return pci_iov_get_pf_drvdata(vf_pdev, &pdsc_driver); 459 } 460 EXPORT_SYMBOL_GPL(pdsc_get_pf_struct); 461 462 static int __init pdsc_init_module(void) 463 { 464 if (strcmp(KBUILD_MODNAME, PDS_CORE_DRV_NAME)) 465 return -EINVAL; 466 467 pdsc_debugfs_create(); 468 return pci_register_driver(&pdsc_driver); 469 } 470 471 static void __exit pdsc_cleanup_module(void) 472 { 473 pci_unregister_driver(&pdsc_driver); 474 pdsc_debugfs_destroy(); 475 } 476 477 module_init(pdsc_init_module); 478 module_exit(pdsc_cleanup_module); 479