1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/aer.h> 13 #include <linux/fs.h> 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/device.h> 16 #include <linux/idr.h> 17 #include <linux/intel-svm.h> 18 #include <linux/iommu.h> 19 #include <uapi/linux/idxd.h> 20 #include <linux/dmaengine.h> 21 #include "../dmaengine.h" 22 #include "registers.h" 23 #include "idxd.h" 24 25 MODULE_VERSION(IDXD_DRIVER_VERSION); 26 MODULE_LICENSE("GPL v2"); 27 MODULE_AUTHOR("Intel Corporation"); 28 29 #define DRV_NAME "idxd" 30 31 bool support_enqcmd; 32 33 static struct idr idxd_idrs[IDXD_TYPE_MAX]; 34 static struct mutex idxd_idr_lock; 35 36 static struct pci_device_id idxd_pci_tbl[] = { 37 /* DSA ver 1.0 platforms */ 38 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_DSA_SPR0) }, 39 40 /* IAX ver 1.0 platforms */ 41 { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_IAX_SPR0) }, 42 { 0, } 43 }; 44 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 45 46 static char *idxd_name[] = { 47 "dsa", 48 "iax" 49 }; 50 51 const char *idxd_get_dev_name(struct idxd_device *idxd) 52 { 53 return idxd_name[idxd->type]; 54 } 55 56 static int idxd_setup_interrupts(struct idxd_device *idxd) 57 { 58 struct pci_dev *pdev = idxd->pdev; 59 struct device *dev = &pdev->dev; 60 struct msix_entry *msix; 61 struct idxd_irq_entry *irq_entry; 62 int i, msixcnt; 63 int rc = 0; 64 union msix_perm mperm; 65 66 msixcnt = pci_msix_vec_count(pdev); 67 if (msixcnt < 0) { 68 dev_err(dev, "Not MSI-X interrupt capable.\n"); 69 goto err_no_irq; 70 } 71 72 idxd->msix_entries = devm_kzalloc(dev, sizeof(struct msix_entry) * 73 msixcnt, GFP_KERNEL); 74 if (!idxd->msix_entries) { 75 rc = -ENOMEM; 76 goto err_no_irq; 77 } 78 79 for (i = 0; i < msixcnt; i++) 80 idxd->msix_entries[i].entry = i; 81 82 rc = pci_enable_msix_exact(pdev, idxd->msix_entries, msixcnt); 83 if (rc) { 84 dev_err(dev, "Failed enabling %d MSIX entries.\n", msixcnt); 85 goto err_no_irq; 86 } 87 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 88 89 /* 90 * We implement 1 completion list per MSI-X entry except for 91 * entry 0, which is for errors and others. 92 */ 93 idxd->irq_entries = devm_kcalloc(dev, msixcnt, 94 sizeof(struct idxd_irq_entry), 95 GFP_KERNEL); 96 if (!idxd->irq_entries) { 97 rc = -ENOMEM; 98 goto err_no_irq; 99 } 100 101 for (i = 0; i < msixcnt; i++) { 102 idxd->irq_entries[i].id = i; 103 idxd->irq_entries[i].idxd = idxd; 104 spin_lock_init(&idxd->irq_entries[i].list_lock); 105 } 106 107 msix = &idxd->msix_entries[0]; 108 irq_entry = &idxd->irq_entries[0]; 109 rc = devm_request_threaded_irq(dev, msix->vector, idxd_irq_handler, 110 idxd_misc_thread, 0, "idxd-misc", 111 irq_entry); 112 if (rc < 0) { 113 dev_err(dev, "Failed to allocate misc interrupt.\n"); 114 goto err_no_irq; 115 } 116 117 dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", 118 msix->vector); 119 120 /* first MSI-X entry is not for wq interrupts */ 121 idxd->num_wq_irqs = msixcnt - 1; 122 123 for (i = 1; i < msixcnt; i++) { 124 msix = &idxd->msix_entries[i]; 125 irq_entry = &idxd->irq_entries[i]; 126 127 init_llist_head(&idxd->irq_entries[i].pending_llist); 128 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); 129 rc = devm_request_threaded_irq(dev, msix->vector, 130 idxd_irq_handler, 131 idxd_wq_thread, 0, 132 "idxd-portal", irq_entry); 133 if (rc < 0) { 134 dev_err(dev, "Failed to allocate irq %d.\n", 135 msix->vector); 136 goto err_no_irq; 137 } 138 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", 139 i, msix->vector); 140 } 141 142 idxd_unmask_error_interrupts(idxd); 143 144 /* Setup MSIX permission table */ 145 mperm.bits = 0; 146 mperm.pasid = idxd->pasid; 147 mperm.pasid_en = device_pasid_enabled(idxd); 148 for (i = 1; i < msixcnt; i++) 149 iowrite32(mperm.bits, idxd->reg_base + idxd->msix_perm_offset + i * 8); 150 151 return 0; 152 153 err_no_irq: 154 /* Disable error interrupt generation */ 155 idxd_mask_error_interrupts(idxd); 156 pci_disable_msix(pdev); 157 dev_err(dev, "No usable interrupts\n"); 158 return rc; 159 } 160 161 static int idxd_setup_internals(struct idxd_device *idxd) 162 { 163 struct device *dev = &idxd->pdev->dev; 164 int i; 165 166 init_waitqueue_head(&idxd->cmd_waitq); 167 idxd->groups = devm_kcalloc(dev, idxd->max_groups, 168 sizeof(struct idxd_group), GFP_KERNEL); 169 if (!idxd->groups) 170 return -ENOMEM; 171 172 for (i = 0; i < idxd->max_groups; i++) { 173 idxd->groups[i].idxd = idxd; 174 idxd->groups[i].id = i; 175 idxd->groups[i].tc_a = -1; 176 idxd->groups[i].tc_b = -1; 177 } 178 179 idxd->wqs = devm_kcalloc(dev, idxd->max_wqs, sizeof(struct idxd_wq), 180 GFP_KERNEL); 181 if (!idxd->wqs) 182 return -ENOMEM; 183 184 idxd->engines = devm_kcalloc(dev, idxd->max_engines, 185 sizeof(struct idxd_engine), GFP_KERNEL); 186 if (!idxd->engines) 187 return -ENOMEM; 188 189 for (i = 0; i < idxd->max_wqs; i++) { 190 struct idxd_wq *wq = &idxd->wqs[i]; 191 192 wq->id = i; 193 wq->idxd = idxd; 194 mutex_init(&wq->wq_lock); 195 wq->idxd_cdev.minor = -1; 196 wq->max_xfer_bytes = idxd->max_xfer_bytes; 197 wq->max_batch_size = idxd->max_batch_size; 198 wq->wqcfg = devm_kzalloc(dev, idxd->wqcfg_size, GFP_KERNEL); 199 if (!wq->wqcfg) 200 return -ENOMEM; 201 } 202 203 for (i = 0; i < idxd->max_engines; i++) { 204 idxd->engines[i].idxd = idxd; 205 idxd->engines[i].id = i; 206 } 207 208 idxd->wq = create_workqueue(dev_name(dev)); 209 if (!idxd->wq) 210 return -ENOMEM; 211 212 return 0; 213 } 214 215 static void idxd_read_table_offsets(struct idxd_device *idxd) 216 { 217 union offsets_reg offsets; 218 struct device *dev = &idxd->pdev->dev; 219 220 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 221 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 222 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 223 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 224 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 225 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 226 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 227 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 228 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 229 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 230 } 231 232 static void idxd_read_caps(struct idxd_device *idxd) 233 { 234 struct device *dev = &idxd->pdev->dev; 235 int i; 236 237 /* reading generic capabilities */ 238 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 239 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 240 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 241 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 242 idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; 243 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 244 if (idxd->hw.gen_cap.config_en) 245 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 246 247 /* reading group capabilities */ 248 idxd->hw.group_cap.bits = 249 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 250 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 251 idxd->max_groups = idxd->hw.group_cap.num_groups; 252 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 253 idxd->max_tokens = idxd->hw.group_cap.total_tokens; 254 dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens); 255 idxd->nr_tokens = idxd->max_tokens; 256 257 /* read engine capabilities */ 258 idxd->hw.engine_cap.bits = 259 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 260 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 261 idxd->max_engines = idxd->hw.engine_cap.num_engines; 262 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 263 264 /* read workqueue capabilities */ 265 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 266 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 267 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 268 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 269 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 270 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 271 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 272 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 273 274 /* reading operation capabilities */ 275 for (i = 0; i < 4; i++) { 276 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 277 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 278 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 279 } 280 } 281 282 static struct idxd_device *idxd_alloc(struct pci_dev *pdev) 283 { 284 struct device *dev = &pdev->dev; 285 struct idxd_device *idxd; 286 287 idxd = devm_kzalloc(dev, sizeof(struct idxd_device), GFP_KERNEL); 288 if (!idxd) 289 return NULL; 290 291 idxd->pdev = pdev; 292 spin_lock_init(&idxd->dev_lock); 293 294 return idxd; 295 } 296 297 static int idxd_enable_system_pasid(struct idxd_device *idxd) 298 { 299 int flags; 300 unsigned int pasid; 301 struct iommu_sva *sva; 302 303 flags = SVM_FLAG_SUPERVISOR_MODE; 304 305 sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); 306 if (IS_ERR(sva)) { 307 dev_warn(&idxd->pdev->dev, 308 "iommu sva bind failed: %ld\n", PTR_ERR(sva)); 309 return PTR_ERR(sva); 310 } 311 312 pasid = iommu_sva_get_pasid(sva); 313 if (pasid == IOMMU_PASID_INVALID) { 314 iommu_sva_unbind_device(sva); 315 return -ENODEV; 316 } 317 318 idxd->sva = sva; 319 idxd->pasid = pasid; 320 dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); 321 return 0; 322 } 323 324 static void idxd_disable_system_pasid(struct idxd_device *idxd) 325 { 326 327 iommu_sva_unbind_device(idxd->sva); 328 idxd->sva = NULL; 329 } 330 331 static int idxd_probe(struct idxd_device *idxd) 332 { 333 struct pci_dev *pdev = idxd->pdev; 334 struct device *dev = &pdev->dev; 335 int rc; 336 337 dev_dbg(dev, "%s entered and resetting device\n", __func__); 338 idxd_device_init_reset(idxd); 339 dev_dbg(dev, "IDXD reset complete\n"); 340 341 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM)) { 342 rc = idxd_enable_system_pasid(idxd); 343 if (rc < 0) 344 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); 345 else 346 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 347 } 348 349 idxd_read_caps(idxd); 350 idxd_read_table_offsets(idxd); 351 352 rc = idxd_setup_internals(idxd); 353 if (rc) 354 goto err_setup; 355 356 rc = idxd_setup_interrupts(idxd); 357 if (rc) 358 goto err_setup; 359 360 dev_dbg(dev, "IDXD interrupt setup complete.\n"); 361 362 mutex_lock(&idxd_idr_lock); 363 idxd->id = idr_alloc(&idxd_idrs[idxd->type], idxd, 0, 0, GFP_KERNEL); 364 mutex_unlock(&idxd_idr_lock); 365 if (idxd->id < 0) { 366 rc = -ENOMEM; 367 goto err_idr_fail; 368 } 369 370 idxd->major = idxd_cdev_get_major(idxd); 371 372 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 373 return 0; 374 375 err_idr_fail: 376 idxd_mask_error_interrupts(idxd); 377 idxd_mask_msix_vectors(idxd); 378 err_setup: 379 if (device_pasid_enabled(idxd)) 380 idxd_disable_system_pasid(idxd); 381 return rc; 382 } 383 384 static void idxd_type_init(struct idxd_device *idxd) 385 { 386 if (idxd->type == IDXD_TYPE_DSA) 387 idxd->compl_size = sizeof(struct dsa_completion_record); 388 else if (idxd->type == IDXD_TYPE_IAX) 389 idxd->compl_size = sizeof(struct iax_completion_record); 390 } 391 392 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 393 { 394 struct device *dev = &pdev->dev; 395 struct idxd_device *idxd; 396 int rc; 397 398 rc = pcim_enable_device(pdev); 399 if (rc) 400 return rc; 401 402 dev_dbg(dev, "Alloc IDXD context\n"); 403 idxd = idxd_alloc(pdev); 404 if (!idxd) 405 return -ENOMEM; 406 407 dev_dbg(dev, "Mapping BARs\n"); 408 idxd->reg_base = pcim_iomap(pdev, IDXD_MMIO_BAR, 0); 409 if (!idxd->reg_base) 410 return -ENOMEM; 411 412 dev_dbg(dev, "Set DMA masks\n"); 413 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64)); 414 if (rc) 415 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 416 if (rc) 417 return rc; 418 419 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 420 if (rc) 421 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); 422 if (rc) 423 return rc; 424 425 idxd_set_type(idxd); 426 427 idxd_type_init(idxd); 428 429 dev_dbg(dev, "Set PCI master\n"); 430 pci_set_master(pdev); 431 pci_set_drvdata(pdev, idxd); 432 433 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 434 rc = idxd_probe(idxd); 435 if (rc) { 436 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 437 return -ENODEV; 438 } 439 440 rc = idxd_setup_sysfs(idxd); 441 if (rc) { 442 dev_err(dev, "IDXD sysfs setup failed\n"); 443 return -ENODEV; 444 } 445 446 idxd->state = IDXD_DEV_CONF_READY; 447 448 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 449 idxd->hw.version); 450 451 return 0; 452 } 453 454 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) 455 { 456 struct idxd_desc *desc, *itr; 457 struct llist_node *head; 458 459 head = llist_del_all(&ie->pending_llist); 460 if (!head) 461 return; 462 463 llist_for_each_entry_safe(desc, itr, head, llnode) { 464 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 465 idxd_free_desc(desc->wq, desc); 466 } 467 } 468 469 static void idxd_flush_work_list(struct idxd_irq_entry *ie) 470 { 471 struct idxd_desc *desc, *iter; 472 473 list_for_each_entry_safe(desc, iter, &ie->work_list, list) { 474 list_del(&desc->list); 475 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 476 idxd_free_desc(desc->wq, desc); 477 } 478 } 479 480 static void idxd_shutdown(struct pci_dev *pdev) 481 { 482 struct idxd_device *idxd = pci_get_drvdata(pdev); 483 int rc, i; 484 struct idxd_irq_entry *irq_entry; 485 int msixcnt = pci_msix_vec_count(pdev); 486 487 rc = idxd_device_disable(idxd); 488 if (rc) 489 dev_err(&pdev->dev, "Disabling device failed\n"); 490 491 dev_dbg(&pdev->dev, "%s called\n", __func__); 492 idxd_mask_msix_vectors(idxd); 493 idxd_mask_error_interrupts(idxd); 494 495 for (i = 0; i < msixcnt; i++) { 496 irq_entry = &idxd->irq_entries[i]; 497 synchronize_irq(idxd->msix_entries[i].vector); 498 if (i == 0) 499 continue; 500 idxd_flush_pending_llist(irq_entry); 501 idxd_flush_work_list(irq_entry); 502 } 503 504 destroy_workqueue(idxd->wq); 505 } 506 507 static void idxd_remove(struct pci_dev *pdev) 508 { 509 struct idxd_device *idxd = pci_get_drvdata(pdev); 510 511 dev_dbg(&pdev->dev, "%s called\n", __func__); 512 idxd_cleanup_sysfs(idxd); 513 idxd_shutdown(pdev); 514 if (device_pasid_enabled(idxd)) 515 idxd_disable_system_pasid(idxd); 516 mutex_lock(&idxd_idr_lock); 517 idr_remove(&idxd_idrs[idxd->type], idxd->id); 518 mutex_unlock(&idxd_idr_lock); 519 } 520 521 static struct pci_driver idxd_pci_driver = { 522 .name = DRV_NAME, 523 .id_table = idxd_pci_tbl, 524 .probe = idxd_pci_probe, 525 .remove = idxd_remove, 526 .shutdown = idxd_shutdown, 527 }; 528 529 static int __init idxd_init_module(void) 530 { 531 int err, i; 532 533 /* 534 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 535 * enumerating the device. We can not utilize it. 536 */ 537 if (!boot_cpu_has(X86_FEATURE_MOVDIR64B)) { 538 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 539 return -ENODEV; 540 } 541 542 if (!boot_cpu_has(X86_FEATURE_ENQCMD)) 543 pr_warn("Platform does not have ENQCMD(S) support.\n"); 544 else 545 support_enqcmd = true; 546 547 mutex_init(&idxd_idr_lock); 548 for (i = 0; i < IDXD_TYPE_MAX; i++) 549 idr_init(&idxd_idrs[i]); 550 551 err = idxd_register_bus_type(); 552 if (err < 0) 553 return err; 554 555 err = idxd_register_driver(); 556 if (err < 0) 557 goto err_idxd_driver_register; 558 559 err = idxd_cdev_register(); 560 if (err) 561 goto err_cdev_register; 562 563 err = pci_register_driver(&idxd_pci_driver); 564 if (err) 565 goto err_pci_register; 566 567 return 0; 568 569 err_pci_register: 570 idxd_cdev_remove(); 571 err_cdev_register: 572 idxd_unregister_driver(); 573 err_idxd_driver_register: 574 idxd_unregister_bus_type(); 575 return err; 576 } 577 module_init(idxd_init_module); 578 579 static void __exit idxd_exit_module(void) 580 { 581 pci_unregister_driver(&idxd_pci_driver); 582 idxd_cdev_remove(); 583 idxd_unregister_bus_type(); 584 } 585 module_exit(idxd_exit_module); 586