1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/aer.h> 13 #include <linux/fs.h> 14 #include <linux/io-64-nonatomic-lo-hi.h> 15 #include <linux/device.h> 16 #include <linux/idr.h> 17 #include <linux/intel-svm.h> 18 #include <linux/iommu.h> 19 #include <uapi/linux/idxd.h> 20 #include <linux/dmaengine.h> 21 #include "../dmaengine.h" 22 #include "registers.h" 23 #include "idxd.h" 24 #include "perfmon.h" 25 26 MODULE_VERSION(IDXD_DRIVER_VERSION); 27 MODULE_LICENSE("GPL v2"); 28 MODULE_AUTHOR("Intel Corporation"); 29 MODULE_IMPORT_NS(IDXD); 30 31 static bool sva = true; 32 module_param(sva, bool, 0644); 33 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 34 35 bool tc_override; 36 module_param(tc_override, bool, 0644); 37 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 38 39 #define DRV_NAME "idxd" 40 41 bool support_enqcmd; 42 DEFINE_IDA(idxd_ida); 43 44 static struct idxd_driver_data idxd_driver_data[] = { 45 [IDXD_TYPE_DSA] = { 46 .name_prefix = "dsa", 47 .type = IDXD_TYPE_DSA, 48 .compl_size = sizeof(struct dsa_completion_record), 49 .align = 32, 50 .dev_type = &dsa_device_type, 51 }, 52 [IDXD_TYPE_IAX] = { 53 .name_prefix = "iax", 54 .type = IDXD_TYPE_IAX, 55 .compl_size = sizeof(struct iax_completion_record), 56 .align = 64, 57 .dev_type = &iax_device_type, 58 }, 59 }; 60 61 static struct pci_device_id idxd_pci_tbl[] = { 62 /* DSA ver 1.0 platforms */ 63 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 64 65 /* IAX ver 1.0 platforms */ 66 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 67 { 0, } 68 }; 69 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 70 71 static int idxd_setup_interrupts(struct idxd_device *idxd) 72 { 73 struct pci_dev *pdev = idxd->pdev; 74 struct device *dev = &pdev->dev; 75 struct idxd_irq_entry *irq_entry; 76 int i, msixcnt; 77 int rc = 0; 78 79 msixcnt = pci_msix_vec_count(pdev); 80 if (msixcnt < 0) { 81 dev_err(dev, "Not MSI-X interrupt capable.\n"); 82 return -ENOSPC; 83 } 84 85 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 86 if (rc != msixcnt) { 87 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 88 return -ENOSPC; 89 } 90 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 91 92 /* 93 * We implement 1 completion list per MSI-X entry except for 94 * entry 0, which is for errors and others. 95 */ 96 idxd->irq_entries = kcalloc_node(msixcnt, sizeof(struct idxd_irq_entry), 97 GFP_KERNEL, dev_to_node(dev)); 98 if (!idxd->irq_entries) { 99 rc = -ENOMEM; 100 goto err_irq_entries; 101 } 102 103 for (i = 0; i < msixcnt; i++) { 104 idxd->irq_entries[i].id = i; 105 idxd->irq_entries[i].idxd = idxd; 106 idxd->irq_entries[i].vector = pci_irq_vector(pdev, i); 107 spin_lock_init(&idxd->irq_entries[i].list_lock); 108 } 109 110 idxd_msix_perm_setup(idxd); 111 112 irq_entry = &idxd->irq_entries[0]; 113 rc = request_threaded_irq(irq_entry->vector, NULL, idxd_misc_thread, 114 0, "idxd-misc", irq_entry); 115 if (rc < 0) { 116 dev_err(dev, "Failed to allocate misc interrupt.\n"); 117 goto err_misc_irq; 118 } 119 120 dev_dbg(dev, "Allocated idxd-misc handler on msix vector %d\n", irq_entry->vector); 121 122 /* first MSI-X entry is not for wq interrupts */ 123 idxd->num_wq_irqs = msixcnt - 1; 124 125 for (i = 1; i < msixcnt; i++) { 126 irq_entry = &idxd->irq_entries[i]; 127 128 init_llist_head(&idxd->irq_entries[i].pending_llist); 129 INIT_LIST_HEAD(&idxd->irq_entries[i].work_list); 130 rc = request_threaded_irq(irq_entry->vector, NULL, 131 idxd_wq_thread, 0, "idxd-portal", irq_entry); 132 if (rc < 0) { 133 dev_err(dev, "Failed to allocate irq %d.\n", irq_entry->vector); 134 goto err_wq_irqs; 135 } 136 137 dev_dbg(dev, "Allocated idxd-msix %d for vector %d\n", i, irq_entry->vector); 138 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { 139 /* 140 * The MSIX vector enumeration starts at 1 with vector 0 being the 141 * misc interrupt that handles non I/O completion events. The 142 * interrupt handles are for IMS enumeration on guest. The misc 143 * interrupt vector does not require a handle and therefore we start 144 * the int_handles at index 0. Since 'i' starts at 1, the first 145 * int_handles index will be 0. 146 */ 147 rc = idxd_device_request_int_handle(idxd, i, &idxd->int_handles[i - 1], 148 IDXD_IRQ_MSIX); 149 if (rc < 0) { 150 free_irq(irq_entry->vector, irq_entry); 151 goto err_wq_irqs; 152 } 153 dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i - 1]); 154 } 155 } 156 157 idxd_unmask_error_interrupts(idxd); 158 return 0; 159 160 err_wq_irqs: 161 while (--i >= 0) { 162 irq_entry = &idxd->irq_entries[i]; 163 free_irq(irq_entry->vector, irq_entry); 164 if (i != 0) 165 idxd_device_release_int_handle(idxd, 166 idxd->int_handles[i], IDXD_IRQ_MSIX); 167 } 168 err_misc_irq: 169 /* Disable error interrupt generation */ 170 idxd_mask_error_interrupts(idxd); 171 idxd_msix_perm_clear(idxd); 172 err_irq_entries: 173 pci_free_irq_vectors(pdev); 174 dev_err(dev, "No usable interrupts\n"); 175 return rc; 176 } 177 178 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 179 { 180 struct pci_dev *pdev = idxd->pdev; 181 struct idxd_irq_entry *irq_entry; 182 int i, msixcnt; 183 184 msixcnt = pci_msix_vec_count(pdev); 185 if (msixcnt <= 0) 186 return; 187 188 irq_entry = &idxd->irq_entries[0]; 189 free_irq(irq_entry->vector, irq_entry); 190 191 for (i = 1; i < msixcnt; i++) { 192 193 irq_entry = &idxd->irq_entries[i]; 194 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) 195 idxd_device_release_int_handle(idxd, idxd->int_handles[i], 196 IDXD_IRQ_MSIX); 197 free_irq(irq_entry->vector, irq_entry); 198 } 199 200 idxd_mask_error_interrupts(idxd); 201 pci_free_irq_vectors(pdev); 202 } 203 204 static int idxd_setup_wqs(struct idxd_device *idxd) 205 { 206 struct device *dev = &idxd->pdev->dev; 207 struct idxd_wq *wq; 208 struct device *conf_dev; 209 int i, rc; 210 211 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 212 GFP_KERNEL, dev_to_node(dev)); 213 if (!idxd->wqs) 214 return -ENOMEM; 215 216 for (i = 0; i < idxd->max_wqs; i++) { 217 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 218 if (!wq) { 219 rc = -ENOMEM; 220 goto err; 221 } 222 223 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 224 conf_dev = wq_confdev(wq); 225 wq->id = i; 226 wq->idxd = idxd; 227 device_initialize(wq_confdev(wq)); 228 conf_dev->parent = idxd_confdev(idxd); 229 conf_dev->bus = &dsa_bus_type; 230 conf_dev->type = &idxd_wq_device_type; 231 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 232 if (rc < 0) { 233 put_device(conf_dev); 234 goto err; 235 } 236 237 mutex_init(&wq->wq_lock); 238 init_waitqueue_head(&wq->err_queue); 239 init_completion(&wq->wq_dead); 240 wq->max_xfer_bytes = idxd->max_xfer_bytes; 241 wq->max_batch_size = idxd->max_batch_size; 242 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 243 if (!wq->wqcfg) { 244 put_device(conf_dev); 245 rc = -ENOMEM; 246 goto err; 247 } 248 idxd->wqs[i] = wq; 249 } 250 251 return 0; 252 253 err: 254 while (--i >= 0) { 255 wq = idxd->wqs[i]; 256 conf_dev = wq_confdev(wq); 257 put_device(conf_dev); 258 } 259 return rc; 260 } 261 262 static int idxd_setup_engines(struct idxd_device *idxd) 263 { 264 struct idxd_engine *engine; 265 struct device *dev = &idxd->pdev->dev; 266 struct device *conf_dev; 267 int i, rc; 268 269 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 270 GFP_KERNEL, dev_to_node(dev)); 271 if (!idxd->engines) 272 return -ENOMEM; 273 274 for (i = 0; i < idxd->max_engines; i++) { 275 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 276 if (!engine) { 277 rc = -ENOMEM; 278 goto err; 279 } 280 281 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 282 conf_dev = engine_confdev(engine); 283 engine->id = i; 284 engine->idxd = idxd; 285 device_initialize(conf_dev); 286 conf_dev->parent = idxd_confdev(idxd); 287 conf_dev->bus = &dsa_bus_type; 288 conf_dev->type = &idxd_engine_device_type; 289 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 290 if (rc < 0) { 291 put_device(conf_dev); 292 goto err; 293 } 294 295 idxd->engines[i] = engine; 296 } 297 298 return 0; 299 300 err: 301 while (--i >= 0) { 302 engine = idxd->engines[i]; 303 conf_dev = engine_confdev(engine); 304 put_device(conf_dev); 305 } 306 return rc; 307 } 308 309 static int idxd_setup_groups(struct idxd_device *idxd) 310 { 311 struct device *dev = &idxd->pdev->dev; 312 struct device *conf_dev; 313 struct idxd_group *group; 314 int i, rc; 315 316 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 317 GFP_KERNEL, dev_to_node(dev)); 318 if (!idxd->groups) 319 return -ENOMEM; 320 321 for (i = 0; i < idxd->max_groups; i++) { 322 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 323 if (!group) { 324 rc = -ENOMEM; 325 goto err; 326 } 327 328 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 329 conf_dev = group_confdev(group); 330 group->id = i; 331 group->idxd = idxd; 332 device_initialize(conf_dev); 333 conf_dev->parent = idxd_confdev(idxd); 334 conf_dev->bus = &dsa_bus_type; 335 conf_dev->type = &idxd_group_device_type; 336 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 337 if (rc < 0) { 338 put_device(conf_dev); 339 goto err; 340 } 341 342 idxd->groups[i] = group; 343 if (idxd->hw.version < DEVICE_VERSION_2 && !tc_override) { 344 group->tc_a = 1; 345 group->tc_b = 1; 346 } else { 347 group->tc_a = -1; 348 group->tc_b = -1; 349 } 350 } 351 352 return 0; 353 354 err: 355 while (--i >= 0) { 356 group = idxd->groups[i]; 357 put_device(group_confdev(group)); 358 } 359 return rc; 360 } 361 362 static void idxd_cleanup_internals(struct idxd_device *idxd) 363 { 364 int i; 365 366 for (i = 0; i < idxd->max_groups; i++) 367 put_device(group_confdev(idxd->groups[i])); 368 for (i = 0; i < idxd->max_engines; i++) 369 put_device(engine_confdev(idxd->engines[i])); 370 for (i = 0; i < idxd->max_wqs; i++) 371 put_device(wq_confdev(idxd->wqs[i])); 372 destroy_workqueue(idxd->wq); 373 } 374 375 static int idxd_setup_internals(struct idxd_device *idxd) 376 { 377 struct device *dev = &idxd->pdev->dev; 378 int rc, i; 379 380 init_waitqueue_head(&idxd->cmd_waitq); 381 382 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) { 383 idxd->int_handles = kcalloc_node(idxd->max_wqs, sizeof(int), GFP_KERNEL, 384 dev_to_node(dev)); 385 if (!idxd->int_handles) 386 return -ENOMEM; 387 } 388 389 rc = idxd_setup_wqs(idxd); 390 if (rc < 0) 391 goto err_wqs; 392 393 rc = idxd_setup_engines(idxd); 394 if (rc < 0) 395 goto err_engine; 396 397 rc = idxd_setup_groups(idxd); 398 if (rc < 0) 399 goto err_group; 400 401 idxd->wq = create_workqueue(dev_name(dev)); 402 if (!idxd->wq) { 403 rc = -ENOMEM; 404 goto err_wkq_create; 405 } 406 407 return 0; 408 409 err_wkq_create: 410 for (i = 0; i < idxd->max_groups; i++) 411 put_device(group_confdev(idxd->groups[i])); 412 err_group: 413 for (i = 0; i < idxd->max_engines; i++) 414 put_device(engine_confdev(idxd->engines[i])); 415 err_engine: 416 for (i = 0; i < idxd->max_wqs; i++) 417 put_device(wq_confdev(idxd->wqs[i])); 418 err_wqs: 419 kfree(idxd->int_handles); 420 return rc; 421 } 422 423 static void idxd_read_table_offsets(struct idxd_device *idxd) 424 { 425 union offsets_reg offsets; 426 struct device *dev = &idxd->pdev->dev; 427 428 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 429 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 430 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 431 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 432 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 433 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 434 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 435 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 436 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 437 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 438 } 439 440 static void idxd_read_caps(struct idxd_device *idxd) 441 { 442 struct device *dev = &idxd->pdev->dev; 443 int i; 444 445 /* reading generic capabilities */ 446 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 447 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 448 449 if (idxd->hw.gen_cap.cmd_cap) { 450 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 451 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 452 } 453 454 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 455 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 456 idxd->max_batch_size = 1U << idxd->hw.gen_cap.max_batch_shift; 457 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 458 if (idxd->hw.gen_cap.config_en) 459 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 460 461 /* reading group capabilities */ 462 idxd->hw.group_cap.bits = 463 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 464 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 465 idxd->max_groups = idxd->hw.group_cap.num_groups; 466 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 467 idxd->max_tokens = idxd->hw.group_cap.total_tokens; 468 dev_dbg(dev, "max tokens: %u\n", idxd->max_tokens); 469 idxd->nr_tokens = idxd->max_tokens; 470 471 /* read engine capabilities */ 472 idxd->hw.engine_cap.bits = 473 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 474 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 475 idxd->max_engines = idxd->hw.engine_cap.num_engines; 476 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 477 478 /* read workqueue capabilities */ 479 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 480 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 481 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 482 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 483 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 484 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 485 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 486 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 487 488 /* reading operation capabilities */ 489 for (i = 0; i < 4; i++) { 490 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 491 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 492 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 493 } 494 } 495 496 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 497 { 498 struct device *dev = &pdev->dev; 499 struct device *conf_dev; 500 struct idxd_device *idxd; 501 int rc; 502 503 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 504 if (!idxd) 505 return NULL; 506 507 conf_dev = idxd_confdev(idxd); 508 idxd->pdev = pdev; 509 idxd->data = data; 510 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 511 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 512 if (idxd->id < 0) 513 return NULL; 514 515 device_initialize(conf_dev); 516 conf_dev->parent = dev; 517 conf_dev->bus = &dsa_bus_type; 518 conf_dev->type = idxd->data->dev_type; 519 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 520 if (rc < 0) { 521 put_device(conf_dev); 522 return NULL; 523 } 524 525 spin_lock_init(&idxd->dev_lock); 526 spin_lock_init(&idxd->cmd_lock); 527 528 return idxd; 529 } 530 531 static int idxd_enable_system_pasid(struct idxd_device *idxd) 532 { 533 int flags; 534 unsigned int pasid; 535 struct iommu_sva *sva; 536 537 flags = SVM_FLAG_SUPERVISOR_MODE; 538 539 sva = iommu_sva_bind_device(&idxd->pdev->dev, NULL, &flags); 540 if (IS_ERR(sva)) { 541 dev_warn(&idxd->pdev->dev, 542 "iommu sva bind failed: %ld\n", PTR_ERR(sva)); 543 return PTR_ERR(sva); 544 } 545 546 pasid = iommu_sva_get_pasid(sva); 547 if (pasid == IOMMU_PASID_INVALID) { 548 iommu_sva_unbind_device(sva); 549 return -ENODEV; 550 } 551 552 idxd->sva = sva; 553 idxd->pasid = pasid; 554 dev_dbg(&idxd->pdev->dev, "system pasid: %u\n", pasid); 555 return 0; 556 } 557 558 static void idxd_disable_system_pasid(struct idxd_device *idxd) 559 { 560 561 iommu_sva_unbind_device(idxd->sva); 562 idxd->sva = NULL; 563 } 564 565 static int idxd_probe(struct idxd_device *idxd) 566 { 567 struct pci_dev *pdev = idxd->pdev; 568 struct device *dev = &pdev->dev; 569 int rc; 570 571 dev_dbg(dev, "%s entered and resetting device\n", __func__); 572 rc = idxd_device_init_reset(idxd); 573 if (rc < 0) 574 return rc; 575 576 dev_dbg(dev, "IDXD reset complete\n"); 577 578 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 579 rc = iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA); 580 if (rc == 0) { 581 rc = idxd_enable_system_pasid(idxd); 582 if (rc < 0) { 583 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 584 dev_warn(dev, "Failed to enable PASID. No SVA support: %d\n", rc); 585 } else { 586 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 587 } 588 } else { 589 dev_warn(dev, "Unable to turn on SVA feature.\n"); 590 } 591 } else if (!sva) { 592 dev_warn(dev, "User forced SVA off via module param.\n"); 593 } 594 595 idxd_read_caps(idxd); 596 idxd_read_table_offsets(idxd); 597 598 rc = idxd_setup_internals(idxd); 599 if (rc) 600 goto err; 601 602 /* If the configs are readonly, then load them from device */ 603 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 604 dev_dbg(dev, "Loading RO device config\n"); 605 rc = idxd_device_load_config(idxd); 606 if (rc < 0) 607 goto err_config; 608 } 609 610 rc = idxd_setup_interrupts(idxd); 611 if (rc) 612 goto err_config; 613 614 dev_dbg(dev, "IDXD interrupt setup complete.\n"); 615 616 idxd->major = idxd_cdev_get_major(idxd); 617 618 rc = perfmon_pmu_init(idxd); 619 if (rc < 0) 620 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 621 622 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 623 return 0; 624 625 err_config: 626 idxd_cleanup_internals(idxd); 627 err: 628 if (device_pasid_enabled(idxd)) 629 idxd_disable_system_pasid(idxd); 630 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 631 return rc; 632 } 633 634 static void idxd_cleanup(struct idxd_device *idxd) 635 { 636 struct device *dev = &idxd->pdev->dev; 637 638 perfmon_pmu_remove(idxd); 639 idxd_cleanup_interrupts(idxd); 640 idxd_cleanup_internals(idxd); 641 if (device_pasid_enabled(idxd)) 642 idxd_disable_system_pasid(idxd); 643 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 644 } 645 646 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 647 { 648 struct device *dev = &pdev->dev; 649 struct idxd_device *idxd; 650 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 651 int rc; 652 653 rc = pci_enable_device(pdev); 654 if (rc) 655 return rc; 656 657 dev_dbg(dev, "Alloc IDXD context\n"); 658 idxd = idxd_alloc(pdev, data); 659 if (!idxd) { 660 rc = -ENOMEM; 661 goto err_idxd_alloc; 662 } 663 664 dev_dbg(dev, "Mapping BARs\n"); 665 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 666 if (!idxd->reg_base) { 667 rc = -ENOMEM; 668 goto err_iomap; 669 } 670 671 dev_dbg(dev, "Set DMA masks\n"); 672 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 673 if (rc) 674 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 675 if (rc) 676 goto err; 677 678 dev_dbg(dev, "Set PCI master\n"); 679 pci_set_master(pdev); 680 pci_set_drvdata(pdev, idxd); 681 682 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 683 rc = idxd_probe(idxd); 684 if (rc) { 685 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 686 goto err; 687 } 688 689 rc = idxd_register_devices(idxd); 690 if (rc) { 691 dev_err(dev, "IDXD sysfs setup failed\n"); 692 goto err_dev_register; 693 } 694 695 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 696 idxd->hw.version); 697 698 return 0; 699 700 err_dev_register: 701 idxd_cleanup(idxd); 702 err: 703 pci_iounmap(pdev, idxd->reg_base); 704 err_iomap: 705 put_device(idxd_confdev(idxd)); 706 err_idxd_alloc: 707 pci_disable_device(pdev); 708 return rc; 709 } 710 711 static void idxd_flush_pending_llist(struct idxd_irq_entry *ie) 712 { 713 struct idxd_desc *desc, *itr; 714 struct llist_node *head; 715 716 head = llist_del_all(&ie->pending_llist); 717 if (!head) 718 return; 719 720 llist_for_each_entry_safe(desc, itr, head, llnode) { 721 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 722 idxd_free_desc(desc->wq, desc); 723 } 724 } 725 726 static void idxd_flush_work_list(struct idxd_irq_entry *ie) 727 { 728 struct idxd_desc *desc, *iter; 729 730 list_for_each_entry_safe(desc, iter, &ie->work_list, list) { 731 list_del(&desc->list); 732 idxd_dma_complete_txd(desc, IDXD_COMPLETE_ABORT); 733 idxd_free_desc(desc->wq, desc); 734 } 735 } 736 737 void idxd_wqs_quiesce(struct idxd_device *idxd) 738 { 739 struct idxd_wq *wq; 740 int i; 741 742 for (i = 0; i < idxd->max_wqs; i++) { 743 wq = idxd->wqs[i]; 744 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 745 idxd_wq_quiesce(wq); 746 } 747 } 748 749 static void idxd_release_int_handles(struct idxd_device *idxd) 750 { 751 struct device *dev = &idxd->pdev->dev; 752 int i, rc; 753 754 for (i = 0; i < idxd->num_wq_irqs; i++) { 755 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_RELEASE_INT_HANDLE)) { 756 rc = idxd_device_release_int_handle(idxd, idxd->int_handles[i], 757 IDXD_IRQ_MSIX); 758 if (rc < 0) 759 dev_warn(dev, "irq handle %d release failed\n", 760 idxd->int_handles[i]); 761 else 762 dev_dbg(dev, "int handle requested: %u\n", idxd->int_handles[i]); 763 } 764 } 765 } 766 767 static void idxd_shutdown(struct pci_dev *pdev) 768 { 769 struct idxd_device *idxd = pci_get_drvdata(pdev); 770 int rc, i; 771 struct idxd_irq_entry *irq_entry; 772 int msixcnt = pci_msix_vec_count(pdev); 773 774 rc = idxd_device_disable(idxd); 775 if (rc) 776 dev_err(&pdev->dev, "Disabling device failed\n"); 777 778 dev_dbg(&pdev->dev, "%s called\n", __func__); 779 idxd_mask_msix_vectors(idxd); 780 idxd_mask_error_interrupts(idxd); 781 782 for (i = 0; i < msixcnt; i++) { 783 irq_entry = &idxd->irq_entries[i]; 784 synchronize_irq(irq_entry->vector); 785 if (i == 0) 786 continue; 787 idxd_flush_pending_llist(irq_entry); 788 idxd_flush_work_list(irq_entry); 789 } 790 flush_workqueue(idxd->wq); 791 } 792 793 static void idxd_remove(struct pci_dev *pdev) 794 { 795 struct idxd_device *idxd = pci_get_drvdata(pdev); 796 struct idxd_irq_entry *irq_entry; 797 int msixcnt = pci_msix_vec_count(pdev); 798 int i; 799 800 dev_dbg(&pdev->dev, "%s called\n", __func__); 801 idxd_shutdown(pdev); 802 if (device_pasid_enabled(idxd)) 803 idxd_disable_system_pasid(idxd); 804 idxd_unregister_devices(idxd); 805 806 for (i = 0; i < msixcnt; i++) { 807 irq_entry = &idxd->irq_entries[i]; 808 free_irq(irq_entry->vector, irq_entry); 809 } 810 idxd_msix_perm_clear(idxd); 811 idxd_release_int_handles(idxd); 812 pci_free_irq_vectors(pdev); 813 pci_iounmap(pdev, idxd->reg_base); 814 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 815 pci_disable_device(pdev); 816 destroy_workqueue(idxd->wq); 817 perfmon_pmu_remove(idxd); 818 device_unregister(idxd_confdev(idxd)); 819 } 820 821 static struct pci_driver idxd_pci_driver = { 822 .name = DRV_NAME, 823 .id_table = idxd_pci_tbl, 824 .probe = idxd_pci_probe, 825 .remove = idxd_remove, 826 .shutdown = idxd_shutdown, 827 }; 828 829 static int __init idxd_init_module(void) 830 { 831 int err; 832 833 /* 834 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 835 * enumerating the device. We can not utilize it. 836 */ 837 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 838 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 839 return -ENODEV; 840 } 841 842 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 843 pr_warn("Platform does not have ENQCMD(S) support.\n"); 844 else 845 support_enqcmd = true; 846 847 perfmon_init(); 848 849 err = idxd_driver_register(&idxd_drv); 850 if (err < 0) 851 goto err_idxd_driver_register; 852 853 err = idxd_driver_register(&idxd_dmaengine_drv); 854 if (err < 0) 855 goto err_idxd_dmaengine_driver_register; 856 857 err = idxd_driver_register(&idxd_user_drv); 858 if (err < 0) 859 goto err_idxd_user_driver_register; 860 861 err = idxd_cdev_register(); 862 if (err) 863 goto err_cdev_register; 864 865 err = pci_register_driver(&idxd_pci_driver); 866 if (err) 867 goto err_pci_register; 868 869 return 0; 870 871 err_pci_register: 872 idxd_cdev_remove(); 873 err_cdev_register: 874 idxd_driver_unregister(&idxd_user_drv); 875 err_idxd_user_driver_register: 876 idxd_driver_unregister(&idxd_dmaengine_drv); 877 err_idxd_dmaengine_driver_register: 878 idxd_driver_unregister(&idxd_drv); 879 err_idxd_driver_register: 880 return err; 881 } 882 module_init(idxd_init_module); 883 884 static void __exit idxd_exit_module(void) 885 { 886 idxd_driver_unregister(&idxd_user_drv); 887 idxd_driver_unregister(&idxd_dmaengine_drv); 888 idxd_driver_unregister(&idxd_drv); 889 pci_unregister_driver(&idxd_pci_driver); 890 idxd_cdev_remove(); 891 perfmon_exit(); 892 } 893 module_exit(idxd_exit_module); 894