1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/aer.h> 13 #include <linux/fs.h> 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/device.h> 16 #include <linux/idr.h> 17 #include <linux/intel-svm.h> 18 #include <linux/iommu.h> 19 #include <uapi/linux/idxd.h> 20 #include <linux/dmaengine.h> 21 #include "../dmaengine.h" 22 #include "registers.h" 23 #include "idxd.h" 24 25 MODULE_VERSION(IDXD_DRIVER_VERSION); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 29 #define DRV_NAME "idxd" 30 31 bool support_enqcmd; 32 33 static struct idr idxd_idrs[IDXD_TYPE_MAX]; 34 static struct mutex idxd_idr_lock; 35 36 static struct pci_device_id idxd_pci_tbl[] = { 37 /* DSA ver 1.0 platforms */ 38 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) }, 39 40 /* IAX ver 1.0 platforms */ 41 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) }, 42 { 0, } 43 }; 44 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 45 46 static char *idxd_name[] = { 47 "dsa", 48 "iax" 49 }; 50 51 const char *idxd_get_dev_name(struct idxd_device *idxd) 52 { 53 return idxd_name[idxd->type]; 54 } 55 56 static int idxd_setup_interrupts(struct idxd_device *idxd) 57 { 58 struct pci_dev *pdev = idxd->pdev; 59 struct device *dev = &pdev->dev; 60 struct msix_entry *msix; 61 struct idxd_irq_entry *irq_entry; 62 int i, msixcnt; 63 int rc = 0; 64 union msix_perm mperm; 65 66 msixcnt = pci_msix_vec_count(pdev); 67 if (msixcnt < 0) { 68 dev_err(dev, "Not MSI-X interrupt capable.\n"); 69 goto err_no_irq; 70 } 71 72 idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) * 73 msixcnt, GFP_KERNEL); 74 if (!idxd->msix_entries) { 75 rc = -ENOMEM; 76 goto err_no_irq; 77 } 78 79 for (i = 0; i < msixcnt; i++) 80 idxd->msix_entries[i].entry = i; 81 82 rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt); 83 if (rc) { 84 dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt); 85 goto err_no_irq; 86 } 87 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 88 89 /* 90 * We implement 1 completion list per MSI-X entry except for 91 * entry 0, which is for errors and others. 92 */ 93 idxd->irq_entries = devm_kcalloc(dev, msixcnt, 94 sizeof(struct idxd_irq_entry), 95 GFP_KERNEL); 96 if (!idxd->irq_entries) { 97 rc = -ENOMEM; 98 goto err_no_irq; 99 } 100 101 for (i = 0; i < msixcnt; i++) { 102 idxd->irq_entries[i].id = i; 103 idxd->irq_entries[i].idxd = idxd; 104 spin_lock_init(&idxd->irq_entries[i].list_lock); 105 } 106 107 msix = &idxd->msix_entries[0]; 108 irq_entry = &idxd->irq_entries[0]; 109 rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler, 110 idxd_misc_thread, 0, "idxd-misc", 111 irq_entry); 112 if (rc < 0) { 113 dev_err(dev, "Failed to allocate misc interrupt.\n"); 114 goto err_no_irq; 115 } 116 117 dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", 118 msix->vector); 119 120 /* first MSI-X entry is not for wq interrupts */ 121 idxd->num_wq_irqs = msixcnt - 1; 122 123 for (i = 1; i < msixcnt; i++) { 124 msix = &idxd->msix_entries[i]; 125 irq_entry = &idxd->irq_entries[i]; 126 127 init_llist_head(&idxd->irq_entries[i].pending_llist); 128 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); 129 rc = devm_request_threaded_irq(dev, msix->vector, 130 idxd_irq_handler, 131 idxd_wq_thread, 0, 132 "idxd-portal", irq_entry); 133 if (rc < 0) { 134 dev_err(dev, "Failed to allocate irq %d.\n", 135 msix->vector); 136 goto err_no_irq; 137 } 138 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", 139 i, msix->vector); 140 } 141 142 idxd_unmask_error_interrupts(idxd); 143 144 /* Setup MSIX permission table */ 145 mperm.bits = 0; 146 mperm.pasid = idxd->pasid; 147 mperm.pasid_en = device_pasid_enabled(idxd); 148 for (i = 1; i < msixcnt; i++) 149 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); 150 151 return 0; 152 153 err_no_irq: 154 /* Disable error interrupt generation */ 155 idxd_mask_error_interrupts(idxd); 156 pci_disable_msix(pdev); 157 dev_err(dev, "No usable interrupts\n"); 158 return rc; 159 } 160 161 static int idxd_setup_internals(struct idxd_device *idxd) 162 { 163 struct device *dev = &idxd->pdev->dev; 164 int i; 165 166 init_waitqueue_head(&idxd->cmd_waitq); 167 idxd->groups = devm_kcalloc(dev, idxd->max_groups, 168 sizeof(struct idxd_group), GFP_KERNEL); 169 if (!idxd->groups) 170 return -ENOMEM; 171 172 for (i = 0; i < idxd->max_groups; i++) { 173 idxd->groups[i].idxd = idxd; 174 idxd->groups[i].id = i; 175 idxd->groups[i].tc_a = -1; 176 idxd->groups[i].tc_b = -1; 177 } 178 179 idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq), 180 GFP_KERNEL); 181 if (!idxd->wqs) 182 return -ENOMEM; 183 184 idxd->engines = devm_kcalloc(dev, idxd->max_engines, 185 sizeof(struct idxd_engine), GFP_KERNEL); 186 if (!idxd->engines) 187 return -ENOMEM; 188 189 for (i = 0; i < idxd->max_wqs; i++) { 190 struct idxd_wq *wq = &idxd->wqs[i]; 191 192 wq->id = i; 193 wq->idxd = idxd; 194 mutex_init(&wq->wq_lock); 195 wq->idxd_cdev.minor = -1; 196 wq->max_xfer_bytes = idxd->max_xfer_bytes; 197 wq->max_batch_size = idxd->max_batch_size; 198 wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL); 199 if (!wq->wqcfg) 200 return -ENOMEM; 201 } 202 203 for (i = 0; i < idxd->max_engines; i++) { 204 idxd->engines[i].idxd = idxd; 205 idxd->engines[i].id = i; 206 } 207 208 idxd->wq = create_workqueue(dev_name(dev)); 209 if (!idxd->wq) 210 return -ENOMEM; 211 212 return 0; 213 } 214 215 static void idxd_read_table_offsets(struct idxd_device *idxd) 216 { 217 union offsets_reg offsets; 218 struct device *dev = &idxd->pdev->dev; 219 220 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 221 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 222 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 223 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 224 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 225 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 226 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 227 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 228 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 229 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 230 } 231 232 static void idxd_read_caps(struct idxd_device *idxd) 233 { 234 struct device *dev = &idxd->pdev->dev; 235 int i; 236 237 /* reading generic capabilities */ 238 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 239 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 240 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 241 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 242 idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; 243 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 244 if (idxd->hw.gen_cap.config_en) 245 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 246 247 /* reading group capabilities */ 248 idxd->hw.group_cap.bits = 249 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 250 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 251 idxd->max_groups = idxd->hw.group_cap.num_groups; 252 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 253 idxd->max_tokens = idxd->hw.group_cap.total_tokens; 254 dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens); 255 idxd->nr_tokens = idxd->max_tokens; 256 257 /* read engine capabilities */ 258 idxd->hw.engine_cap.bits = 259 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 260 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 261 idxd->max_engines = idxd->hw.engine_cap.num_engines; 262 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 263 264 /* read workqueue capabilities */ 265 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 266 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 267 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 268 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 269 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 270 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 271 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 272 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 273 274 /* reading operation capabilities */ 275 for (i = 0; i < 4; i++) { 276 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 277 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 278 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 279 } 280 } 281 282 static struct idxd_device *idxd_alloc(struct pci_dev *pdev) 283 { 284 struct device *dev = &pdev->dev; 285 struct idxd_device *idxd; 286 287 idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL); 288 if (!idxd) 289 return NULL; 290 291 idxd->pdev = pdev; 292 spin_lock_init(&idxd->dev_lock); 293 294 return idxd; 295 } 296 297 static int idxd_enable_system_pasid(struct idxd_device *idxd) 298 { 299 int flags; 300 unsigned int pasid; 301 struct iommu_sva *sva; 302 303 flags = SVM_FLAG_SUPERVISOR_MODE; 304 305 sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); 306 if (IS_ERR(sva)) { 307 dev_warn(&idxd->pdev->dev, 308 "iommu sva bind failed: %ld\n", PTR_ERR(sva)); 309 return PTR_ERR(sva); 310 } 311 312 pasid = iommu_sva_get_pasid(sva); 313 if (pasid == IOMMU_PASID_INVALID) { 314 iommu_sva_unbind_device(sva); 315 return -ENODEV; 316 } 317 318 idxd->sva = sva; 319 idxd->pasid = pasid; 320 dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); 321 return 0; 322 } 323 324 static void idxd_disable_system_pasid(struct idxd_device *idxd) 325 { 326 327 iommu_sva_unbind_device(idxd->sva); 328 idxd->sva = NULL; 329 } 330 331 static int idxd_probe(struct idxd_device *idxd) 332 { 333 struct pci_dev *pdev = idxd->pdev; 334 struct device *dev = &pdev->dev; 335 int rc; 336 337 dev_dbg(dev, "%s entered and resetting device\n", __func__); 338 rc = idxd_device_init_reset(idxd); 339 if (rc < 0) 340 return rc; 341 342 dev_dbg(dev, "IDXD reset complete\n"); 343 344 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) { 345 rc = idxd_enable_system_pasid(idxd); 346 if (rc < 0) 347 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); 348 else 349 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 350 } 351 352 idxd_read_caps(idxd); 353 idxd_read_table_offsets(idxd); 354 355 rc = idxd_setup_internals(idxd); 356 if (rc) 357 goto err_setup; 358 359 rc = idxd_setup_interrupts(idxd); 360 if (rc) 361 goto err_setup; 362 363 dev_dbg(dev, "IDXD interrupt setup complete.\n"); 364 365 mutex_lock(&idxd_idr_lock); 366 idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL); 367 mutex_unlock(&idxd_idr_lock); 368 if (idxd->id < 0) { 369 rc = -ENOMEM; 370 goto err_idr_fail; 371 } 372 373 idxd->major = idxd_cdev_get_major(idxd); 374 375 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 376 return 0; 377 378 err_idr_fail: 379 idxd_mask_error_interrupts(idxd); 380 idxd_mask_msix_vectors(idxd); 381 err_setup: 382 if (device_pasid_enabled(idxd)) 383 idxd_disable_system_pasid(idxd); 384 return rc; 385 } 386 387 static void idxd_type_init(struct idxd_device *idxd) 388 { 389 if (idxd->type == IDXD_TYPE_DSA) 390 idxd->compl_size = sizeof(struct dsa_completion_record); 391 else if (idxd->type == IDXD_TYPE_IAX) 392 idxd->compl_size = sizeof(struct iax_completion_record); 393 } 394 395 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 396 { 397 struct device *dev = &pdev->dev; 398 struct idxd_device *idxd; 399 int rc; 400 401 rc = pcim_enable_device(pdev); 402 if (rc) 403 return rc; 404 405 dev_dbg(dev, "Alloc IDXD context\n"); 406 idxd = idxd_alloc(pdev); 407 if (!idxd) 408 return -ENOMEM; 409 410 dev_dbg(dev, "Mapping BARs\n"); 411 idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0); 412 if (!idxd->reg_base) 413 return -ENOMEM; 414 415 dev_dbg(dev, "Set DMA masks\n"); 416 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 417 if (rc) 418 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 419 if (rc) 420 return rc; 421 422 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 423 if (rc) 424 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 425 if (rc) 426 return rc; 427 428 idxd_set_type(idxd); 429 430 idxd_type_init(idxd); 431 432 dev_dbg(dev, "Set PCI master\n"); 433 pci_set_master(pdev); 434 pci_set_drvdata(pdev, idxd); 435 436 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 437 rc = idxd_probe(idxd); 438 if (rc) { 439 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 440 return -ENODEV; 441 } 442 443 rc = idxd_setup_sysfs(idxd); 444 if (rc) { 445 dev_err(dev, "IDXD sysfs setup failed\n"); 446 return -ENODEV; 447 } 448 449 idxd->state = IDXD_DEV_CONF_READY; 450 451 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 452 idxd->hw.version); 453 454 return 0; 455 } 456 457 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) 458 { 459 struct idxd_desc *desc, *itr; 460 struct llist_node *head; 461 462 head = llist_del_all(&ie->pending_llist); 463 if (!head) 464 return; 465 466 llist_for_each_entry_safe(desc, itr, head, llnode) { 467 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 468 idxd_free_desc(desc->wq, desc); 469 } 470 } 471 472 static void idxd_flush_work_list(struct idxd_irq_entry *ie) 473 { 474 struct idxd_desc *desc, *iter; 475 476 list_for_each_entry_safe(desc, iter, &ie->work_list, list) { 477 list_del(&desc->list); 478 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 479 idxd_free_desc(desc->wq, desc); 480 } 481 } 482 483 static void idxd_shutdown(struct pci_dev *pdev) 484 { 485 struct idxd_device *idxd = pci_get_drvdata(pdev); 486 int rc, i; 487 struct idxd_irq_entry *irq_entry; 488 int msixcnt = pci_msix_vec_count(pdev); 489 490 rc = idxd_device_disable(idxd); 491 if (rc) 492 dev_err(&pdev->dev, "Disabling device failed\n"); 493 494 dev_dbg(&pdev->dev, "%s called\n", __func__); 495 idxd_mask_msix_vectors(idxd); 496 idxd_mask_error_interrupts(idxd); 497 498 for (i = 0; i < msixcnt; i++) { 499 irq_entry = &idxd->irq_entries[i]; 500 synchronize_irq(idxd->msix_entries[i].vector); 501 if (i == 0) 502 continue; 503 idxd_flush_pending_llist(irq_entry); 504 idxd_flush_work_list(irq_entry); 505 } 506 507 destroy_workqueue(idxd->wq); 508 } 509 510 static void idxd_remove(struct pci_dev *pdev) 511 { 512 struct idxd_device *idxd = pci_get_drvdata(pdev); 513 514 dev_dbg(&pdev->dev, "%s called\n", __func__); 515 idxd_cleanup_sysfs(idxd); 516 idxd_shutdown(pdev); 517 if (device_pasid_enabled(idxd)) 518 idxd_disable_system_pasid(idxd); 519 mutex_lock(&idxd_idr_lock); 520 idr_remove(&idxd_idrs[idxd->type], idxd->id); 521 mutex_unlock(&idxd_idr_lock); 522 } 523 524 static struct pci_driver idxd_pci_driver = { 525 .name = DRV_NAME, 526 .id_table = idxd_pci_tbl, 527 .probe = idxd_pci_probe, 528 .remove = idxd_remove, 529 .shutdown = idxd_shutdown, 530 }; 531 532 static int __init idxd_init_module(void) 533 { 534 int err, i; 535 536 /* 537 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 538 * enumerating the device. We can not utilize it. 539 */ 540 if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) { 541 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 542 return -ENODEV; 543 } 544 545 if (!boot_cpu_has(X86_FEATURE_ENQCMD)) 546 pr_warn("Platform does not have ENQCMD(S) support.\n"); 547 else 548 support_enqcmd = true; 549 550 mutex_init(&idxd_idr_lock); 551 for (i = 0; i < IDXD_TYPE_MAX; i++) 552 idr_init(&idxd_idrs[i]); 553 554 err = idxd_register_bus_type(); 555 if (err < 0) 556 return err; 557 558 err = idxd_register_driver(); 559 if (err < 0) 560 goto err_idxd_driver_register; 561 562 err = idxd_cdev_register(); 563 if (err) 564 goto err_cdev_register; 565 566 err = pci_register_driver(&idxd_pci_driver); 567 if (err) 568 goto err_pci_register; 569 570 return 0; 571 572 err_pci_register: 573 idxd_cdev_remove(); 574 err_cdev_register: 575 idxd_unregister_driver(); 576 err_idxd_driver_register: 577 idxd_unregister_bus_type(); 578 return err; 579 } 580 module_init(idxd_init_module); 581 582 static void __exit idxd_exit_module(void) 583 { 584 pci_unregister_driver(&idxd_pci_driver); 585 idxd_cdev_remove(); 586 idxd_unregister_bus_type(); 587 } 588 module_exit(idxd_exit_module); 589