1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_LICENSE("GPL v2"); 26 MODULE_AUTHOR("Intel Corporation"); 27 MODULE_IMPORT_NS(IDXD); 28 29 static bool sva = true; 30 module_param(sva, bool, 0644); 31 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 32 33 bool tc_override; 34 module_param(tc_override, bool, 0644); 35 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 36 37 #define DRV_NAME "idxd" 38 39 bool support_enqcmd; 40 DEFINE_IDA(idxd_ida); 41 42 static struct idxd_driver_data idxd_driver_data[] = { 43 [IDXD_TYPE_DSA] = { 44 .name_prefix = "dsa", 45 .type = IDXD_TYPE_DSA, 46 .compl_size = sizeof(struct dsa_completion_record), 47 .align = 32, 48 .dev_type = &dsa_device_type, 49 }, 50 [IDXD_TYPE_IAX] = { 51 .name_prefix = "iax", 52 .type = IDXD_TYPE_IAX, 53 .compl_size = sizeof(struct iax_completion_record), 54 .align = 64, 55 .dev_type = &iax_device_type, 56 }, 57 }; 58 59 static struct pci_device_id idxd_pci_tbl[] = { 60 /* DSA ver 1.0 platforms */ 61 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 62 63 /* IAX ver 1.0 platforms */ 64 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 65 { 0, } 66 }; 67 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 68 69 static int idxd_setup_interrupts(struct idxd_device *idxd) 70 { 71 struct pci_dev *pdev = idxd->pdev; 72 struct device *dev = &pdev->dev; 73 struct idxd_irq_entry *ie; 74 int i, msixcnt; 75 int rc = 0; 76 77 msixcnt = pci_msix_vec_count(pdev); 78 if (msixcnt < 0) { 79 dev_err(dev, "Not MSI-X interrupt capable.\n"); 80 return -ENOSPC; 81 } 82 idxd->irq_cnt = msixcnt; 83 84 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 85 if (rc != msixcnt) { 86 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 87 return -ENOSPC; 88 } 89 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 90 91 92 ie = idxd_get_ie(idxd, 0); 93 ie->vector = pci_irq_vector(pdev, 0); 94 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 95 if (rc < 0) { 96 dev_err(dev, "Failed to allocate misc interrupt.\n"); 97 goto err_misc_irq; 98 } 99 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 100 101 for (i = 0; i < idxd->max_wqs; i++) { 102 int msix_idx = i + 1; 103 104 ie = idxd_get_ie(idxd, msix_idx); 105 ie->id = msix_idx; 106 ie->int_handle = INVALID_INT_HANDLE; 107 ie->pasid = INVALID_IOASID; 108 109 spin_lock_init(&ie->list_lock); 110 init_llist_head(&ie->pending_llist); 111 INIT_LIST_HEAD(&ie->work_list); 112 } 113 114 idxd_unmask_error_interrupts(idxd); 115 return 0; 116 117 err_misc_irq: 118 idxd_mask_error_interrupts(idxd); 119 pci_free_irq_vectors(pdev); 120 dev_err(dev, "No usable interrupts\n"); 121 return rc; 122 } 123 124 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 125 { 126 struct pci_dev *pdev = idxd->pdev; 127 struct idxd_irq_entry *ie; 128 int msixcnt; 129 130 msixcnt = pci_msix_vec_count(pdev); 131 if (msixcnt <= 0) 132 return; 133 134 ie = idxd_get_ie(idxd, 0); 135 idxd_mask_error_interrupts(idxd); 136 free_irq(ie->vector, ie); 137 pci_free_irq_vectors(pdev); 138 } 139 140 static int idxd_setup_wqs(struct idxd_device *idxd) 141 { 142 struct device *dev = &idxd->pdev->dev; 143 struct idxd_wq *wq; 144 struct device *conf_dev; 145 int i, rc; 146 147 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 148 GFP_KERNEL, dev_to_node(dev)); 149 if (!idxd->wqs) 150 return -ENOMEM; 151 152 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 153 if (!idxd->wq_enable_map) { 154 kfree(idxd->wqs); 155 return -ENOMEM; 156 } 157 158 for (i = 0; i < idxd->max_wqs; i++) { 159 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 160 if (!wq) { 161 rc = -ENOMEM; 162 goto err; 163 } 164 165 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 166 conf_dev = wq_confdev(wq); 167 wq->id = i; 168 wq->idxd = idxd; 169 device_initialize(wq_confdev(wq)); 170 conf_dev->parent = idxd_confdev(idxd); 171 conf_dev->bus = &dsa_bus_type; 172 conf_dev->type = &idxd_wq_device_type; 173 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 174 if (rc < 0) { 175 put_device(conf_dev); 176 goto err; 177 } 178 179 mutex_init(&wq->wq_lock); 180 init_waitqueue_head(&wq->err_queue); 181 init_completion(&wq->wq_dead); 182 init_completion(&wq->wq_resurrect); 183 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 184 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 185 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 186 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 187 if (!wq->wqcfg) { 188 put_device(conf_dev); 189 rc = -ENOMEM; 190 goto err; 191 } 192 193 if (idxd->hw.wq_cap.op_config) { 194 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 195 if (!wq->opcap_bmap) { 196 put_device(conf_dev); 197 rc = -ENOMEM; 198 goto err; 199 } 200 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 201 } 202 mutex_init(&wq->uc_lock); 203 xa_init(&wq->upasid_xa); 204 idxd->wqs[i] = wq; 205 } 206 207 return 0; 208 209 err: 210 while (--i >= 0) { 211 wq = idxd->wqs[i]; 212 conf_dev = wq_confdev(wq); 213 put_device(conf_dev); 214 } 215 return rc; 216 } 217 218 static int idxd_setup_engines(struct idxd_device *idxd) 219 { 220 struct idxd_engine *engine; 221 struct device *dev = &idxd->pdev->dev; 222 struct device *conf_dev; 223 int i, rc; 224 225 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 226 GFP_KERNEL, dev_to_node(dev)); 227 if (!idxd->engines) 228 return -ENOMEM; 229 230 for (i = 0; i < idxd->max_engines; i++) { 231 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 232 if (!engine) { 233 rc = -ENOMEM; 234 goto err; 235 } 236 237 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 238 conf_dev = engine_confdev(engine); 239 engine->id = i; 240 engine->idxd = idxd; 241 device_initialize(conf_dev); 242 conf_dev->parent = idxd_confdev(idxd); 243 conf_dev->bus = &dsa_bus_type; 244 conf_dev->type = &idxd_engine_device_type; 245 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 246 if (rc < 0) { 247 put_device(conf_dev); 248 goto err; 249 } 250 251 idxd->engines[i] = engine; 252 } 253 254 return 0; 255 256 err: 257 while (--i >= 0) { 258 engine = idxd->engines[i]; 259 conf_dev = engine_confdev(engine); 260 put_device(conf_dev); 261 } 262 return rc; 263 } 264 265 static int idxd_setup_groups(struct idxd_device *idxd) 266 { 267 struct device *dev = &idxd->pdev->dev; 268 struct device *conf_dev; 269 struct idxd_group *group; 270 int i, rc; 271 272 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 273 GFP_KERNEL, dev_to_node(dev)); 274 if (!idxd->groups) 275 return -ENOMEM; 276 277 for (i = 0; i < idxd->max_groups; i++) { 278 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 279 if (!group) { 280 rc = -ENOMEM; 281 goto err; 282 } 283 284 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 285 conf_dev = group_confdev(group); 286 group->id = i; 287 group->idxd = idxd; 288 device_initialize(conf_dev); 289 conf_dev->parent = idxd_confdev(idxd); 290 conf_dev->bus = &dsa_bus_type; 291 conf_dev->type = &idxd_group_device_type; 292 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 293 if (rc < 0) { 294 put_device(conf_dev); 295 goto err; 296 } 297 298 idxd->groups[i] = group; 299 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 300 group->tc_a = 1; 301 group->tc_b = 1; 302 } else { 303 group->tc_a = -1; 304 group->tc_b = -1; 305 } 306 /* 307 * The default value is the same as the value of 308 * total read buffers in GRPCAP. 309 */ 310 group->rdbufs_allowed = idxd->max_rdbufs; 311 } 312 313 return 0; 314 315 err: 316 while (--i >= 0) { 317 group = idxd->groups[i]; 318 put_device(group_confdev(group)); 319 } 320 return rc; 321 } 322 323 static void idxd_cleanup_internals(struct idxd_device *idxd) 324 { 325 int i; 326 327 for (i = 0; i < idxd->max_groups; i++) 328 put_device(group_confdev(idxd->groups[i])); 329 for (i = 0; i < idxd->max_engines; i++) 330 put_device(engine_confdev(idxd->engines[i])); 331 for (i = 0; i < idxd->max_wqs; i++) 332 put_device(wq_confdev(idxd->wqs[i])); 333 destroy_workqueue(idxd->wq); 334 } 335 336 static int idxd_init_evl(struct idxd_device *idxd) 337 { 338 struct device *dev = &idxd->pdev->dev; 339 struct idxd_evl *evl; 340 341 if (idxd->hw.gen_cap.evl_support == 0) 342 return 0; 343 344 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 345 if (!evl) 346 return -ENOMEM; 347 348 spin_lock_init(&evl->lock); 349 evl->size = IDXD_EVL_SIZE_MIN; 350 351 idxd->evl_cache = kmem_cache_create(dev_name(idxd_confdev(idxd)), 352 sizeof(struct idxd_evl_fault) + evl_ent_size(idxd), 353 0, 0, NULL); 354 if (!idxd->evl_cache) { 355 kfree(evl); 356 return -ENOMEM; 357 } 358 359 idxd->evl = evl; 360 return 0; 361 } 362 363 static int idxd_setup_internals(struct idxd_device *idxd) 364 { 365 struct device *dev = &idxd->pdev->dev; 366 int rc, i; 367 368 init_waitqueue_head(&idxd->cmd_waitq); 369 370 rc = idxd_setup_wqs(idxd); 371 if (rc < 0) 372 goto err_wqs; 373 374 rc = idxd_setup_engines(idxd); 375 if (rc < 0) 376 goto err_engine; 377 378 rc = idxd_setup_groups(idxd); 379 if (rc < 0) 380 goto err_group; 381 382 idxd->wq = create_workqueue(dev_name(dev)); 383 if (!idxd->wq) { 384 rc = -ENOMEM; 385 goto err_wkq_create; 386 } 387 388 rc = idxd_init_evl(idxd); 389 if (rc < 0) 390 goto err_evl; 391 392 return 0; 393 394 err_evl: 395 destroy_workqueue(idxd->wq); 396 err_wkq_create: 397 for (i = 0; i < idxd->max_groups; i++) 398 put_device(group_confdev(idxd->groups[i])); 399 err_group: 400 for (i = 0; i < idxd->max_engines; i++) 401 put_device(engine_confdev(idxd->engines[i])); 402 err_engine: 403 for (i = 0; i < idxd->max_wqs; i++) 404 put_device(wq_confdev(idxd->wqs[i])); 405 err_wqs: 406 return rc; 407 } 408 409 static void idxd_read_table_offsets(struct idxd_device *idxd) 410 { 411 union offsets_reg offsets; 412 struct device *dev = &idxd->pdev->dev; 413 414 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 415 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 416 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 417 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 418 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 419 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 420 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 421 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 422 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 423 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 424 } 425 426 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 427 { 428 int i, j, nr; 429 430 for (i = 0, nr = 0; i < count; i++) { 431 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 432 if (val[i] & BIT(j)) 433 set_bit(nr, bmap); 434 nr++; 435 } 436 } 437 } 438 439 static void idxd_read_caps(struct idxd_device *idxd) 440 { 441 struct device *dev = &idxd->pdev->dev; 442 int i; 443 444 /* reading generic capabilities */ 445 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 446 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 447 448 if (idxd->hw.gen_cap.cmd_cap) { 449 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 450 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 451 } 452 453 /* reading command capabilities */ 454 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 455 idxd->request_int_handles = true; 456 457 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 458 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 459 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 460 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 461 if (idxd->hw.gen_cap.config_en) 462 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 463 464 /* reading group capabilities */ 465 idxd->hw.group_cap.bits = 466 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 467 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 468 idxd->max_groups = idxd->hw.group_cap.num_groups; 469 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 470 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 471 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 472 idxd->nr_rdbufs = idxd->max_rdbufs; 473 474 /* read engine capabilities */ 475 idxd->hw.engine_cap.bits = 476 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 477 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 478 idxd->max_engines = idxd->hw.engine_cap.num_engines; 479 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 480 481 /* read workqueue capabilities */ 482 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 483 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 484 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 485 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 486 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 487 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 488 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 489 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 490 491 /* reading operation capabilities */ 492 for (i = 0; i < 4; i++) { 493 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 494 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 495 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 496 } 497 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 498 499 /* read iaa cap */ 500 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 501 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 502 } 503 504 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 505 { 506 struct device *dev = &pdev->dev; 507 struct device *conf_dev; 508 struct idxd_device *idxd; 509 int rc; 510 511 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 512 if (!idxd) 513 return NULL; 514 515 conf_dev = idxd_confdev(idxd); 516 idxd->pdev = pdev; 517 idxd->data = data; 518 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 519 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 520 if (idxd->id < 0) 521 return NULL; 522 523 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 524 if (!idxd->opcap_bmap) { 525 ida_free(&idxd_ida, idxd->id); 526 return NULL; 527 } 528 529 device_initialize(conf_dev); 530 conf_dev->parent = dev; 531 conf_dev->bus = &dsa_bus_type; 532 conf_dev->type = idxd->data->dev_type; 533 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 534 if (rc < 0) { 535 put_device(conf_dev); 536 return NULL; 537 } 538 539 spin_lock_init(&idxd->dev_lock); 540 spin_lock_init(&idxd->cmd_lock); 541 542 return idxd; 543 } 544 545 static int idxd_enable_system_pasid(struct idxd_device *idxd) 546 { 547 return -EOPNOTSUPP; 548 } 549 550 static void idxd_disable_system_pasid(struct idxd_device *idxd) 551 { 552 553 iommu_sva_unbind_device(idxd->sva); 554 idxd->sva = NULL; 555 } 556 557 static int idxd_probe(struct idxd_device *idxd) 558 { 559 struct pci_dev *pdev = idxd->pdev; 560 struct device *dev = &pdev->dev; 561 int rc; 562 563 dev_dbg(dev, "%s entered and resetting device\n", __func__); 564 rc = idxd_device_init_reset(idxd); 565 if (rc < 0) 566 return rc; 567 568 dev_dbg(dev, "IDXD reset complete\n"); 569 570 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 571 if (iommu_dev_enable_feature(dev, IOMMU_DEV_FEAT_SVA)) { 572 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 573 } else { 574 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 575 576 if (idxd_enable_system_pasid(idxd)) 577 dev_warn(dev, "No in-kernel DMA with PASID.\n"); 578 else 579 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 580 } 581 } else if (!sva) { 582 dev_warn(dev, "User forced SVA off via module param.\n"); 583 } 584 585 idxd_read_caps(idxd); 586 idxd_read_table_offsets(idxd); 587 588 rc = idxd_setup_internals(idxd); 589 if (rc) 590 goto err; 591 592 /* If the configs are readonly, then load them from device */ 593 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 594 dev_dbg(dev, "Loading RO device config\n"); 595 rc = idxd_device_load_config(idxd); 596 if (rc < 0) 597 goto err_config; 598 } 599 600 rc = idxd_setup_interrupts(idxd); 601 if (rc) 602 goto err_config; 603 604 idxd->major = idxd_cdev_get_major(idxd); 605 606 rc = perfmon_pmu_init(idxd); 607 if (rc < 0) 608 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 609 610 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 611 return 0; 612 613 err_config: 614 idxd_cleanup_internals(idxd); 615 err: 616 if (device_pasid_enabled(idxd)) 617 idxd_disable_system_pasid(idxd); 618 if (device_user_pasid_enabled(idxd)) 619 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 620 return rc; 621 } 622 623 static void idxd_cleanup(struct idxd_device *idxd) 624 { 625 struct device *dev = &idxd->pdev->dev; 626 627 perfmon_pmu_remove(idxd); 628 idxd_cleanup_interrupts(idxd); 629 idxd_cleanup_internals(idxd); 630 if (device_pasid_enabled(idxd)) 631 idxd_disable_system_pasid(idxd); 632 if (device_user_pasid_enabled(idxd)) 633 iommu_dev_disable_feature(dev, IOMMU_DEV_FEAT_SVA); 634 } 635 636 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 637 { 638 struct device *dev = &pdev->dev; 639 struct idxd_device *idxd; 640 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 641 int rc; 642 643 rc = pci_enable_device(pdev); 644 if (rc) 645 return rc; 646 647 dev_dbg(dev, "Alloc IDXD context\n"); 648 idxd = idxd_alloc(pdev, data); 649 if (!idxd) { 650 rc = -ENOMEM; 651 goto err_idxd_alloc; 652 } 653 654 dev_dbg(dev, "Mapping BARs\n"); 655 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 656 if (!idxd->reg_base) { 657 rc = -ENOMEM; 658 goto err_iomap; 659 } 660 661 dev_dbg(dev, "Set DMA masks\n"); 662 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 663 if (rc) 664 goto err; 665 666 dev_dbg(dev, "Set PCI master\n"); 667 pci_set_master(pdev); 668 pci_set_drvdata(pdev, idxd); 669 670 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 671 rc = idxd_probe(idxd); 672 if (rc) { 673 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 674 goto err; 675 } 676 677 rc = idxd_register_devices(idxd); 678 if (rc) { 679 dev_err(dev, "IDXD sysfs setup failed\n"); 680 goto err_dev_register; 681 } 682 683 rc = idxd_device_init_debugfs(idxd); 684 if (rc) 685 dev_warn(dev, "IDXD debugfs failed to setup\n"); 686 687 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 688 idxd->hw.version); 689 690 return 0; 691 692 err_dev_register: 693 idxd_cleanup(idxd); 694 err: 695 pci_iounmap(pdev, idxd->reg_base); 696 err_iomap: 697 put_device(idxd_confdev(idxd)); 698 err_idxd_alloc: 699 pci_disable_device(pdev); 700 return rc; 701 } 702 703 void idxd_wqs_quiesce(struct idxd_device *idxd) 704 { 705 struct idxd_wq *wq; 706 int i; 707 708 for (i = 0; i < idxd->max_wqs; i++) { 709 wq = idxd->wqs[i]; 710 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 711 idxd_wq_quiesce(wq); 712 } 713 } 714 715 static void idxd_shutdown(struct pci_dev *pdev) 716 { 717 struct idxd_device *idxd = pci_get_drvdata(pdev); 718 struct idxd_irq_entry *irq_entry; 719 int rc; 720 721 rc = idxd_device_disable(idxd); 722 if (rc) 723 dev_err(&pdev->dev, "Disabling device failed\n"); 724 725 irq_entry = &idxd->ie; 726 synchronize_irq(irq_entry->vector); 727 idxd_mask_error_interrupts(idxd); 728 flush_workqueue(idxd->wq); 729 } 730 731 static void idxd_remove(struct pci_dev *pdev) 732 { 733 struct idxd_device *idxd = pci_get_drvdata(pdev); 734 struct idxd_irq_entry *irq_entry; 735 736 idxd_unregister_devices(idxd); 737 /* 738 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 739 * to the idxd context. The driver still needs those bits in order to do the rest of 740 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 741 * on the device here to hold off the freeing while allowing the idxd sub-driver 742 * to unbind. 743 */ 744 get_device(idxd_confdev(idxd)); 745 device_unregister(idxd_confdev(idxd)); 746 idxd_shutdown(pdev); 747 if (device_pasid_enabled(idxd)) 748 idxd_disable_system_pasid(idxd); 749 idxd_device_remove_debugfs(idxd); 750 751 irq_entry = idxd_get_ie(idxd, 0); 752 free_irq(irq_entry->vector, irq_entry); 753 pci_free_irq_vectors(pdev); 754 pci_iounmap(pdev, idxd->reg_base); 755 if (device_user_pasid_enabled(idxd)) 756 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 757 pci_disable_device(pdev); 758 destroy_workqueue(idxd->wq); 759 perfmon_pmu_remove(idxd); 760 put_device(idxd_confdev(idxd)); 761 } 762 763 static struct pci_driver idxd_pci_driver = { 764 .name = DRV_NAME, 765 .id_table = idxd_pci_tbl, 766 .probe = idxd_pci_probe, 767 .remove = idxd_remove, 768 .shutdown = idxd_shutdown, 769 }; 770 771 static int __init idxd_init_module(void) 772 { 773 int err; 774 775 /* 776 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 777 * enumerating the device. We can not utilize it. 778 */ 779 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 780 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 781 return -ENODEV; 782 } 783 784 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 785 pr_warn("Platform does not have ENQCMD(S) support.\n"); 786 else 787 support_enqcmd = true; 788 789 perfmon_init(); 790 791 err = idxd_driver_register(&idxd_drv); 792 if (err < 0) 793 goto err_idxd_driver_register; 794 795 err = idxd_driver_register(&idxd_dmaengine_drv); 796 if (err < 0) 797 goto err_idxd_dmaengine_driver_register; 798 799 err = idxd_driver_register(&idxd_user_drv); 800 if (err < 0) 801 goto err_idxd_user_driver_register; 802 803 err = idxd_cdev_register(); 804 if (err) 805 goto err_cdev_register; 806 807 err = idxd_init_debugfs(); 808 if (err) 809 goto err_debugfs; 810 811 err = pci_register_driver(&idxd_pci_driver); 812 if (err) 813 goto err_pci_register; 814 815 return 0; 816 817 err_pci_register: 818 idxd_remove_debugfs(); 819 err_debugfs: 820 idxd_cdev_remove(); 821 err_cdev_register: 822 idxd_driver_unregister(&idxd_user_drv); 823 err_idxd_user_driver_register: 824 idxd_driver_unregister(&idxd_dmaengine_drv); 825 err_idxd_dmaengine_driver_register: 826 idxd_driver_unregister(&idxd_drv); 827 err_idxd_driver_register: 828 return err; 829 } 830 module_init(idxd_init_module); 831 832 static void __exit idxd_exit_module(void) 833 { 834 idxd_driver_unregister(&idxd_user_drv); 835 idxd_driver_unregister(&idxd_dmaengine_drv); 836 idxd_driver_unregister(&idxd_drv); 837 pci_unregister_driver(&idxd_pci_driver); 838 idxd_cdev_remove(); 839 perfmon_exit(); 840 idxd_remove_debugfs(); 841 } 842 module_exit(idxd_exit_module); 843