1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2019 Intel Corporation. All rights rsvd. */ 3 #include <linux/init.h> 4 #include <linux/kernel.h> 5 #include <linux/module.h> 6 #include <linux/slab.h> 7 #include <linux/pci.h> 8 #include <linux/interrupt.h> 9 #include <linux/delay.h> 10 #include <linux/dma-mapping.h> 11 #include <linux/workqueue.h> 12 #include <linux/fs.h> 13 #include <linux/io-64-nonatomic-lo-hi.h> 14 #include <linux/device.h> 15 #include <linux/idr.h> 16 #include <linux/iommu.h> 17 #include <uapi/linux/idxd.h> 18 #include <linux/dmaengine.h> 19 #include "../dmaengine.h" 20 #include "registers.h" 21 #include "idxd.h" 22 #include "perfmon.h" 23 24 MODULE_VERSION(IDXD_DRIVER_VERSION); 25 MODULE_LICENSE("GPL v2"); 26 MODULE_AUTHOR("Intel Corporation"); 27 MODULE_IMPORT_NS(IDXD); 28 29 static bool sva = true; 30 module_param(sva, bool, 0644); 31 MODULE_PARM_DESC(sva, "Toggle SVA support on/off"); 32 33 bool tc_override; 34 module_param(tc_override, bool, 0644); 35 MODULE_PARM_DESC(tc_override, "Override traffic class defaults"); 36 37 #define DRV_NAME "idxd" 38 39 bool support_enqcmd; 40 DEFINE_IDA(idxd_ida); 41 42 static struct idxd_driver_data idxd_driver_data[] = { 43 [IDXD_TYPE_DSA] = { 44 .name_prefix = "dsa", 45 .type = IDXD_TYPE_DSA, 46 .compl_size = sizeof(struct dsa_completion_record), 47 .align = 32, 48 .dev_type = &dsa_device_type, 49 .evl_cr_off = offsetof(struct dsa_evl_entry, cr), 50 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 51 .cr_status_off = offsetof(struct dsa_completion_record, status), 52 .cr_result_off = offsetof(struct dsa_completion_record, result), 53 }, 54 [IDXD_TYPE_IAX] = { 55 .name_prefix = "iax", 56 .type = IDXD_TYPE_IAX, 57 .compl_size = sizeof(struct iax_completion_record), 58 .align = 64, 59 .dev_type = &iax_device_type, 60 .evl_cr_off = offsetof(struct iax_evl_entry, cr), 61 .user_submission_safe = false, /* See INTEL-SA-01084 security advisory */ 62 .cr_status_off = offsetof(struct iax_completion_record, status), 63 .cr_result_off = offsetof(struct iax_completion_record, error_code), 64 }, 65 }; 66 67 static struct pci_device_id idxd_pci_tbl[] = { 68 /* DSA ver 1.0 platforms */ 69 { PCI_DEVICE_DATA(INTEL, DSA_SPR0, &idxd_driver_data[IDXD_TYPE_DSA]) }, 70 71 /* IAX ver 1.0 platforms */ 72 { PCI_DEVICE_DATA(INTEL, IAX_SPR0, &idxd_driver_data[IDXD_TYPE_IAX]) }, 73 { 0, } 74 }; 75 MODULE_DEVICE_TABLE(pci, idxd_pci_tbl); 76 77 static int idxd_setup_interrupts(struct idxd_device *idxd) 78 { 79 struct pci_dev *pdev = idxd->pdev; 80 struct device *dev = &pdev->dev; 81 struct idxd_irq_entry *ie; 82 int i, msixcnt; 83 int rc = 0; 84 85 msixcnt = pci_msix_vec_count(pdev); 86 if (msixcnt < 0) { 87 dev_err(dev, "Not MSI-X interrupt capable.\n"); 88 return -ENOSPC; 89 } 90 idxd->irq_cnt = msixcnt; 91 92 rc = pci_alloc_irq_vectors(pdev, msixcnt, msixcnt, PCI_IRQ_MSIX); 93 if (rc != msixcnt) { 94 dev_err(dev, "Failed enabling %d MSIX entries: %d\n", msixcnt, rc); 95 return -ENOSPC; 96 } 97 dev_dbg(dev, "Enabled %d msix vectors\n", msixcnt); 98 99 100 ie = idxd_get_ie(idxd, 0); 101 ie->vector = pci_irq_vector(pdev, 0); 102 rc = request_threaded_irq(ie->vector, NULL, idxd_misc_thread, 0, "idxd-misc", ie); 103 if (rc < 0) { 104 dev_err(dev, "Failed to allocate misc interrupt.\n"); 105 goto err_misc_irq; 106 } 107 dev_dbg(dev, "Requested idxd-misc handler on msix vector %d\n", ie->vector); 108 109 for (i = 0; i < idxd->max_wqs; i++) { 110 int msix_idx = i + 1; 111 112 ie = idxd_get_ie(idxd, msix_idx); 113 ie->id = msix_idx; 114 ie->int_handle = INVALID_INT_HANDLE; 115 ie->pasid = IOMMU_PASID_INVALID; 116 117 spin_lock_init(&ie->list_lock); 118 init_llist_head(&ie->pending_llist); 119 INIT_LIST_HEAD(&ie->work_list); 120 } 121 122 idxd_unmask_error_interrupts(idxd); 123 return 0; 124 125 err_misc_irq: 126 idxd_mask_error_interrupts(idxd); 127 pci_free_irq_vectors(pdev); 128 dev_err(dev, "No usable interrupts\n"); 129 return rc; 130 } 131 132 static void idxd_cleanup_interrupts(struct idxd_device *idxd) 133 { 134 struct pci_dev *pdev = idxd->pdev; 135 struct idxd_irq_entry *ie; 136 int msixcnt; 137 138 msixcnt = pci_msix_vec_count(pdev); 139 if (msixcnt <= 0) 140 return; 141 142 ie = idxd_get_ie(idxd, 0); 143 idxd_mask_error_interrupts(idxd); 144 free_irq(ie->vector, ie); 145 pci_free_irq_vectors(pdev); 146 } 147 148 static int idxd_setup_wqs(struct idxd_device *idxd) 149 { 150 struct device *dev = &idxd->pdev->dev; 151 struct idxd_wq *wq; 152 struct device *conf_dev; 153 int i, rc; 154 155 idxd->wqs = kcalloc_node(idxd->max_wqs, sizeof(struct idxd_wq *), 156 GFP_KERNEL, dev_to_node(dev)); 157 if (!idxd->wqs) 158 return -ENOMEM; 159 160 idxd->wq_enable_map = bitmap_zalloc_node(idxd->max_wqs, GFP_KERNEL, dev_to_node(dev)); 161 if (!idxd->wq_enable_map) { 162 kfree(idxd->wqs); 163 return -ENOMEM; 164 } 165 166 for (i = 0; i < idxd->max_wqs; i++) { 167 wq = kzalloc_node(sizeof(*wq), GFP_KERNEL, dev_to_node(dev)); 168 if (!wq) { 169 rc = -ENOMEM; 170 goto err; 171 } 172 173 idxd_dev_set_type(&wq->idxd_dev, IDXD_DEV_WQ); 174 conf_dev = wq_confdev(wq); 175 wq->id = i; 176 wq->idxd = idxd; 177 device_initialize(wq_confdev(wq)); 178 conf_dev->parent = idxd_confdev(idxd); 179 conf_dev->bus = &dsa_bus_type; 180 conf_dev->type = &idxd_wq_device_type; 181 rc = dev_set_name(conf_dev, "wq%d.%d", idxd->id, wq->id); 182 if (rc < 0) { 183 put_device(conf_dev); 184 goto err; 185 } 186 187 mutex_init(&wq->wq_lock); 188 init_waitqueue_head(&wq->err_queue); 189 init_completion(&wq->wq_dead); 190 init_completion(&wq->wq_resurrect); 191 wq->max_xfer_bytes = WQ_DEFAULT_MAX_XFER; 192 idxd_wq_set_max_batch_size(idxd->data->type, wq, WQ_DEFAULT_MAX_BATCH); 193 wq->enqcmds_retries = IDXD_ENQCMDS_RETRIES; 194 wq->wqcfg = kzalloc_node(idxd->wqcfg_size, GFP_KERNEL, dev_to_node(dev)); 195 if (!wq->wqcfg) { 196 put_device(conf_dev); 197 rc = -ENOMEM; 198 goto err; 199 } 200 201 if (idxd->hw.wq_cap.op_config) { 202 wq->opcap_bmap = bitmap_zalloc(IDXD_MAX_OPCAP_BITS, GFP_KERNEL); 203 if (!wq->opcap_bmap) { 204 put_device(conf_dev); 205 rc = -ENOMEM; 206 goto err; 207 } 208 bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS); 209 } 210 mutex_init(&wq->uc_lock); 211 xa_init(&wq->upasid_xa); 212 idxd->wqs[i] = wq; 213 } 214 215 return 0; 216 217 err: 218 while (--i >= 0) { 219 wq = idxd->wqs[i]; 220 conf_dev = wq_confdev(wq); 221 put_device(conf_dev); 222 } 223 return rc; 224 } 225 226 static int idxd_setup_engines(struct idxd_device *idxd) 227 { 228 struct idxd_engine *engine; 229 struct device *dev = &idxd->pdev->dev; 230 struct device *conf_dev; 231 int i, rc; 232 233 idxd->engines = kcalloc_node(idxd->max_engines, sizeof(struct idxd_engine *), 234 GFP_KERNEL, dev_to_node(dev)); 235 if (!idxd->engines) 236 return -ENOMEM; 237 238 for (i = 0; i < idxd->max_engines; i++) { 239 engine = kzalloc_node(sizeof(*engine), GFP_KERNEL, dev_to_node(dev)); 240 if (!engine) { 241 rc = -ENOMEM; 242 goto err; 243 } 244 245 idxd_dev_set_type(&engine->idxd_dev, IDXD_DEV_ENGINE); 246 conf_dev = engine_confdev(engine); 247 engine->id = i; 248 engine->idxd = idxd; 249 device_initialize(conf_dev); 250 conf_dev->parent = idxd_confdev(idxd); 251 conf_dev->bus = &dsa_bus_type; 252 conf_dev->type = &idxd_engine_device_type; 253 rc = dev_set_name(conf_dev, "engine%d.%d", idxd->id, engine->id); 254 if (rc < 0) { 255 put_device(conf_dev); 256 goto err; 257 } 258 259 idxd->engines[i] = engine; 260 } 261 262 return 0; 263 264 err: 265 while (--i >= 0) { 266 engine = idxd->engines[i]; 267 conf_dev = engine_confdev(engine); 268 put_device(conf_dev); 269 } 270 return rc; 271 } 272 273 static int idxd_setup_groups(struct idxd_device *idxd) 274 { 275 struct device *dev = &idxd->pdev->dev; 276 struct device *conf_dev; 277 struct idxd_group *group; 278 int i, rc; 279 280 idxd->groups = kcalloc_node(idxd->max_groups, sizeof(struct idxd_group *), 281 GFP_KERNEL, dev_to_node(dev)); 282 if (!idxd->groups) 283 return -ENOMEM; 284 285 for (i = 0; i < idxd->max_groups; i++) { 286 group = kzalloc_node(sizeof(*group), GFP_KERNEL, dev_to_node(dev)); 287 if (!group) { 288 rc = -ENOMEM; 289 goto err; 290 } 291 292 idxd_dev_set_type(&group->idxd_dev, IDXD_DEV_GROUP); 293 conf_dev = group_confdev(group); 294 group->id = i; 295 group->idxd = idxd; 296 device_initialize(conf_dev); 297 conf_dev->parent = idxd_confdev(idxd); 298 conf_dev->bus = &dsa_bus_type; 299 conf_dev->type = &idxd_group_device_type; 300 rc = dev_set_name(conf_dev, "group%d.%d", idxd->id, group->id); 301 if (rc < 0) { 302 put_device(conf_dev); 303 goto err; 304 } 305 306 idxd->groups[i] = group; 307 if (idxd->hw.version <= DEVICE_VERSION_2 && !tc_override) { 308 group->tc_a = 1; 309 group->tc_b = 1; 310 } else { 311 group->tc_a = -1; 312 group->tc_b = -1; 313 } 314 /* 315 * The default value is the same as the value of 316 * total read buffers in GRPCAP. 317 */ 318 group->rdbufs_allowed = idxd->max_rdbufs; 319 } 320 321 return 0; 322 323 err: 324 while (--i >= 0) { 325 group = idxd->groups[i]; 326 put_device(group_confdev(group)); 327 } 328 return rc; 329 } 330 331 static void idxd_cleanup_internals(struct idxd_device *idxd) 332 { 333 int i; 334 335 for (i = 0; i < idxd->max_groups; i++) 336 put_device(group_confdev(idxd->groups[i])); 337 for (i = 0; i < idxd->max_engines; i++) 338 put_device(engine_confdev(idxd->engines[i])); 339 for (i = 0; i < idxd->max_wqs; i++) 340 put_device(wq_confdev(idxd->wqs[i])); 341 destroy_workqueue(idxd->wq); 342 } 343 344 static int idxd_init_evl(struct idxd_device *idxd) 345 { 346 struct device *dev = &idxd->pdev->dev; 347 unsigned int evl_cache_size; 348 struct idxd_evl *evl; 349 const char *idxd_name; 350 351 if (idxd->hw.gen_cap.evl_support == 0) 352 return 0; 353 354 evl = kzalloc_node(sizeof(*evl), GFP_KERNEL, dev_to_node(dev)); 355 if (!evl) 356 return -ENOMEM; 357 358 mutex_init(&evl->lock); 359 evl->size = IDXD_EVL_SIZE_MIN; 360 361 idxd_name = dev_name(idxd_confdev(idxd)); 362 evl_cache_size = sizeof(struct idxd_evl_fault) + evl_ent_size(idxd); 363 /* 364 * Since completion record in evl_cache will be copied to user 365 * when handling completion record page fault, need to create 366 * the cache suitable for user copy. 367 */ 368 idxd->evl_cache = kmem_cache_create_usercopy(idxd_name, evl_cache_size, 369 0, 0, 0, evl_cache_size, 370 NULL); 371 if (!idxd->evl_cache) { 372 kfree(evl); 373 return -ENOMEM; 374 } 375 376 idxd->evl = evl; 377 return 0; 378 } 379 380 static int idxd_setup_internals(struct idxd_device *idxd) 381 { 382 struct device *dev = &idxd->pdev->dev; 383 int rc, i; 384 385 init_waitqueue_head(&idxd->cmd_waitq); 386 387 rc = idxd_setup_wqs(idxd); 388 if (rc < 0) 389 goto err_wqs; 390 391 rc = idxd_setup_engines(idxd); 392 if (rc < 0) 393 goto err_engine; 394 395 rc = idxd_setup_groups(idxd); 396 if (rc < 0) 397 goto err_group; 398 399 idxd->wq = create_workqueue(dev_name(dev)); 400 if (!idxd->wq) { 401 rc = -ENOMEM; 402 goto err_wkq_create; 403 } 404 405 rc = idxd_init_evl(idxd); 406 if (rc < 0) 407 goto err_evl; 408 409 return 0; 410 411 err_evl: 412 destroy_workqueue(idxd->wq); 413 err_wkq_create: 414 for (i = 0; i < idxd->max_groups; i++) 415 put_device(group_confdev(idxd->groups[i])); 416 err_group: 417 for (i = 0; i < idxd->max_engines; i++) 418 put_device(engine_confdev(idxd->engines[i])); 419 err_engine: 420 for (i = 0; i < idxd->max_wqs; i++) 421 put_device(wq_confdev(idxd->wqs[i])); 422 err_wqs: 423 return rc; 424 } 425 426 static void idxd_read_table_offsets(struct idxd_device *idxd) 427 { 428 union offsets_reg offsets; 429 struct device *dev = &idxd->pdev->dev; 430 431 offsets.bits[0] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET); 432 offsets.bits[1] = ioread64(idxd->reg_base + IDXD_TABLE_OFFSET + sizeof(u64)); 433 idxd->grpcfg_offset = offsets.grpcfg * IDXD_TABLE_MULT; 434 dev_dbg(dev, "IDXD Group Config Offset: %#x\n", idxd->grpcfg_offset); 435 idxd->wqcfg_offset = offsets.wqcfg * IDXD_TABLE_MULT; 436 dev_dbg(dev, "IDXD Work Queue Config Offset: %#x\n", idxd->wqcfg_offset); 437 idxd->msix_perm_offset = offsets.msix_perm * IDXD_TABLE_MULT; 438 dev_dbg(dev, "IDXD MSIX Permission Offset: %#x\n", idxd->msix_perm_offset); 439 idxd->perfmon_offset = offsets.perfmon * IDXD_TABLE_MULT; 440 dev_dbg(dev, "IDXD Perfmon Offset: %#x\n", idxd->perfmon_offset); 441 } 442 443 void multi_u64_to_bmap(unsigned long *bmap, u64 *val, int count) 444 { 445 int i, j, nr; 446 447 for (i = 0, nr = 0; i < count; i++) { 448 for (j = 0; j < BITS_PER_LONG_LONG; j++) { 449 if (val[i] & BIT(j)) 450 set_bit(nr, bmap); 451 nr++; 452 } 453 } 454 } 455 456 static void idxd_read_caps(struct idxd_device *idxd) 457 { 458 struct device *dev = &idxd->pdev->dev; 459 int i; 460 461 /* reading generic capabilities */ 462 idxd->hw.gen_cap.bits = ioread64(idxd->reg_base + IDXD_GENCAP_OFFSET); 463 dev_dbg(dev, "gen_cap: %#llx\n", idxd->hw.gen_cap.bits); 464 465 if (idxd->hw.gen_cap.cmd_cap) { 466 idxd->hw.cmd_cap = ioread32(idxd->reg_base + IDXD_CMDCAP_OFFSET); 467 dev_dbg(dev, "cmd_cap: %#x\n", idxd->hw.cmd_cap); 468 } 469 470 /* reading command capabilities */ 471 if (idxd->hw.cmd_cap & BIT(IDXD_CMD_REQUEST_INT_HANDLE)) 472 idxd->request_int_handles = true; 473 474 idxd->max_xfer_bytes = 1ULL << idxd->hw.gen_cap.max_xfer_shift; 475 dev_dbg(dev, "max xfer size: %llu bytes\n", idxd->max_xfer_bytes); 476 idxd_set_max_batch_size(idxd->data->type, idxd, 1U << idxd->hw.gen_cap.max_batch_shift); 477 dev_dbg(dev, "max batch size: %u\n", idxd->max_batch_size); 478 if (idxd->hw.gen_cap.config_en) 479 set_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags); 480 481 /* reading group capabilities */ 482 idxd->hw.group_cap.bits = 483 ioread64(idxd->reg_base + IDXD_GRPCAP_OFFSET); 484 dev_dbg(dev, "group_cap: %#llx\n", idxd->hw.group_cap.bits); 485 idxd->max_groups = idxd->hw.group_cap.num_groups; 486 dev_dbg(dev, "max groups: %u\n", idxd->max_groups); 487 idxd->max_rdbufs = idxd->hw.group_cap.total_rdbufs; 488 dev_dbg(dev, "max read buffers: %u\n", idxd->max_rdbufs); 489 idxd->nr_rdbufs = idxd->max_rdbufs; 490 491 /* read engine capabilities */ 492 idxd->hw.engine_cap.bits = 493 ioread64(idxd->reg_base + IDXD_ENGCAP_OFFSET); 494 dev_dbg(dev, "engine_cap: %#llx\n", idxd->hw.engine_cap.bits); 495 idxd->max_engines = idxd->hw.engine_cap.num_engines; 496 dev_dbg(dev, "max engines: %u\n", idxd->max_engines); 497 498 /* read workqueue capabilities */ 499 idxd->hw.wq_cap.bits = ioread64(idxd->reg_base + IDXD_WQCAP_OFFSET); 500 dev_dbg(dev, "wq_cap: %#llx\n", idxd->hw.wq_cap.bits); 501 idxd->max_wq_size = idxd->hw.wq_cap.total_wq_size; 502 dev_dbg(dev, "total workqueue size: %u\n", idxd->max_wq_size); 503 idxd->max_wqs = idxd->hw.wq_cap.num_wqs; 504 dev_dbg(dev, "max workqueues: %u\n", idxd->max_wqs); 505 idxd->wqcfg_size = 1 << (idxd->hw.wq_cap.wqcfg_size + IDXD_WQCFG_MIN); 506 dev_dbg(dev, "wqcfg size: %u\n", idxd->wqcfg_size); 507 508 /* reading operation capabilities */ 509 for (i = 0; i < 4; i++) { 510 idxd->hw.opcap.bits[i] = ioread64(idxd->reg_base + 511 IDXD_OPCAP_OFFSET + i * sizeof(u64)); 512 dev_dbg(dev, "opcap[%d]: %#llx\n", i, idxd->hw.opcap.bits[i]); 513 } 514 multi_u64_to_bmap(idxd->opcap_bmap, &idxd->hw.opcap.bits[0], 4); 515 516 /* read iaa cap */ 517 if (idxd->data->type == IDXD_TYPE_IAX && idxd->hw.version >= DEVICE_VERSION_2) 518 idxd->hw.iaa_cap.bits = ioread64(idxd->reg_base + IDXD_IAACAP_OFFSET); 519 } 520 521 static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_data *data) 522 { 523 struct device *dev = &pdev->dev; 524 struct device *conf_dev; 525 struct idxd_device *idxd; 526 int rc; 527 528 idxd = kzalloc_node(sizeof(*idxd), GFP_KERNEL, dev_to_node(dev)); 529 if (!idxd) 530 return NULL; 531 532 conf_dev = idxd_confdev(idxd); 533 idxd->pdev = pdev; 534 idxd->data = data; 535 idxd_dev_set_type(&idxd->idxd_dev, idxd->data->type); 536 idxd->id = ida_alloc(&idxd_ida, GFP_KERNEL); 537 if (idxd->id < 0) 538 return NULL; 539 540 idxd->opcap_bmap = bitmap_zalloc_node(IDXD_MAX_OPCAP_BITS, GFP_KERNEL, dev_to_node(dev)); 541 if (!idxd->opcap_bmap) { 542 ida_free(&idxd_ida, idxd->id); 543 return NULL; 544 } 545 546 device_initialize(conf_dev); 547 conf_dev->parent = dev; 548 conf_dev->bus = &dsa_bus_type; 549 conf_dev->type = idxd->data->dev_type; 550 rc = dev_set_name(conf_dev, "%s%d", idxd->data->name_prefix, idxd->id); 551 if (rc < 0) { 552 put_device(conf_dev); 553 return NULL; 554 } 555 556 spin_lock_init(&idxd->dev_lock); 557 spin_lock_init(&idxd->cmd_lock); 558 559 return idxd; 560 } 561 562 static int idxd_enable_system_pasid(struct idxd_device *idxd) 563 { 564 struct pci_dev *pdev = idxd->pdev; 565 struct device *dev = &pdev->dev; 566 struct iommu_domain *domain; 567 ioasid_t pasid; 568 int ret; 569 570 /* 571 * Attach a global PASID to the DMA domain so that we can use ENQCMDS 572 * to submit work on buffers mapped by DMA API. 573 */ 574 domain = iommu_get_domain_for_dev(dev); 575 if (!domain) 576 return -EPERM; 577 578 pasid = iommu_alloc_global_pasid(dev); 579 if (pasid == IOMMU_PASID_INVALID) 580 return -ENOSPC; 581 582 /* 583 * DMA domain is owned by the driver, it should support all valid 584 * types such as DMA-FQ, identity, etc. 585 */ 586 ret = iommu_attach_device_pasid(domain, dev, pasid); 587 if (ret) { 588 dev_err(dev, "failed to attach device pasid %d, domain type %d", 589 pasid, domain->type); 590 iommu_free_global_pasid(pasid); 591 return ret; 592 } 593 594 /* Since we set user privilege for kernel DMA, enable completion IRQ */ 595 idxd_set_user_intr(idxd, 1); 596 idxd->pasid = pasid; 597 598 return ret; 599 } 600 601 static void idxd_disable_system_pasid(struct idxd_device *idxd) 602 { 603 struct pci_dev *pdev = idxd->pdev; 604 struct device *dev = &pdev->dev; 605 struct iommu_domain *domain; 606 607 domain = iommu_get_domain_for_dev(dev); 608 if (!domain) 609 return; 610 611 iommu_detach_device_pasid(domain, dev, idxd->pasid); 612 iommu_free_global_pasid(idxd->pasid); 613 614 idxd_set_user_intr(idxd, 0); 615 idxd->sva = NULL; 616 idxd->pasid = IOMMU_PASID_INVALID; 617 } 618 619 static int idxd_enable_sva(struct pci_dev *pdev) 620 { 621 int ret; 622 623 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 624 if (ret) 625 return ret; 626 627 ret = iommu_dev_enable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 628 if (ret) 629 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 630 631 return ret; 632 } 633 634 static void idxd_disable_sva(struct pci_dev *pdev) 635 { 636 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_SVA); 637 iommu_dev_disable_feature(&pdev->dev, IOMMU_DEV_FEAT_IOPF); 638 } 639 640 static int idxd_probe(struct idxd_device *idxd) 641 { 642 struct pci_dev *pdev = idxd->pdev; 643 struct device *dev = &pdev->dev; 644 int rc; 645 646 dev_dbg(dev, "%s entered and resetting device\n", __func__); 647 rc = idxd_device_init_reset(idxd); 648 if (rc < 0) 649 return rc; 650 651 dev_dbg(dev, "IDXD reset complete\n"); 652 653 if (IS_ENABLED(CONFIG_INTEL_IDXD_SVM) && sva) { 654 if (idxd_enable_sva(pdev)) { 655 dev_warn(dev, "Unable to turn on user SVA feature.\n"); 656 } else { 657 set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags); 658 659 rc = idxd_enable_system_pasid(idxd); 660 if (rc) 661 dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc); 662 else 663 set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags); 664 } 665 } else if (!sva) { 666 dev_warn(dev, "User forced SVA off via module param.\n"); 667 } 668 669 idxd_read_caps(idxd); 670 idxd_read_table_offsets(idxd); 671 672 rc = idxd_setup_internals(idxd); 673 if (rc) 674 goto err; 675 676 /* If the configs are readonly, then load them from device */ 677 if (!test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) { 678 dev_dbg(dev, "Loading RO device config\n"); 679 rc = idxd_device_load_config(idxd); 680 if (rc < 0) 681 goto err_config; 682 } 683 684 rc = idxd_setup_interrupts(idxd); 685 if (rc) 686 goto err_config; 687 688 idxd->major = idxd_cdev_get_major(idxd); 689 690 rc = perfmon_pmu_init(idxd); 691 if (rc < 0) 692 dev_warn(dev, "Failed to initialize perfmon. No PMU support: %d\n", rc); 693 694 dev_dbg(dev, "IDXD device %d probed successfully\n", idxd->id); 695 return 0; 696 697 err_config: 698 idxd_cleanup_internals(idxd); 699 err: 700 if (device_pasid_enabled(idxd)) 701 idxd_disable_system_pasid(idxd); 702 if (device_user_pasid_enabled(idxd)) 703 idxd_disable_sva(pdev); 704 return rc; 705 } 706 707 static void idxd_cleanup(struct idxd_device *idxd) 708 { 709 perfmon_pmu_remove(idxd); 710 idxd_cleanup_interrupts(idxd); 711 idxd_cleanup_internals(idxd); 712 if (device_pasid_enabled(idxd)) 713 idxd_disable_system_pasid(idxd); 714 if (device_user_pasid_enabled(idxd)) 715 idxd_disable_sva(idxd->pdev); 716 } 717 718 static int idxd_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) 719 { 720 struct device *dev = &pdev->dev; 721 struct idxd_device *idxd; 722 struct idxd_driver_data *data = (struct idxd_driver_data *)id->driver_data; 723 int rc; 724 725 rc = pci_enable_device(pdev); 726 if (rc) 727 return rc; 728 729 dev_dbg(dev, "Alloc IDXD context\n"); 730 idxd = idxd_alloc(pdev, data); 731 if (!idxd) { 732 rc = -ENOMEM; 733 goto err_idxd_alloc; 734 } 735 736 dev_dbg(dev, "Mapping BARs\n"); 737 idxd->reg_base = pci_iomap(pdev, IDXD_MMIO_BAR, 0); 738 if (!idxd->reg_base) { 739 rc = -ENOMEM; 740 goto err_iomap; 741 } 742 743 dev_dbg(dev, "Set DMA masks\n"); 744 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 745 if (rc) 746 goto err; 747 748 dev_dbg(dev, "Set PCI master\n"); 749 pci_set_master(pdev); 750 pci_set_drvdata(pdev, idxd); 751 752 idxd->hw.version = ioread32(idxd->reg_base + IDXD_VER_OFFSET); 753 rc = idxd_probe(idxd); 754 if (rc) { 755 dev_err(dev, "Intel(R) IDXD DMA Engine init failed\n"); 756 goto err; 757 } 758 759 rc = idxd_register_devices(idxd); 760 if (rc) { 761 dev_err(dev, "IDXD sysfs setup failed\n"); 762 goto err_dev_register; 763 } 764 765 rc = idxd_device_init_debugfs(idxd); 766 if (rc) 767 dev_warn(dev, "IDXD debugfs failed to setup\n"); 768 769 dev_info(&pdev->dev, "Intel(R) Accelerator Device (v%x)\n", 770 idxd->hw.version); 771 772 idxd->user_submission_safe = data->user_submission_safe; 773 774 return 0; 775 776 err_dev_register: 777 idxd_cleanup(idxd); 778 err: 779 pci_iounmap(pdev, idxd->reg_base); 780 err_iomap: 781 put_device(idxd_confdev(idxd)); 782 err_idxd_alloc: 783 pci_disable_device(pdev); 784 return rc; 785 } 786 787 void idxd_wqs_quiesce(struct idxd_device *idxd) 788 { 789 struct idxd_wq *wq; 790 int i; 791 792 for (i = 0; i < idxd->max_wqs; i++) { 793 wq = idxd->wqs[i]; 794 if (wq->state == IDXD_WQ_ENABLED && wq->type == IDXD_WQT_KERNEL) 795 idxd_wq_quiesce(wq); 796 } 797 } 798 799 static void idxd_shutdown(struct pci_dev *pdev) 800 { 801 struct idxd_device *idxd = pci_get_drvdata(pdev); 802 struct idxd_irq_entry *irq_entry; 803 int rc; 804 805 rc = idxd_device_disable(idxd); 806 if (rc) 807 dev_err(&pdev->dev, "Disabling device failed\n"); 808 809 irq_entry = &idxd->ie; 810 synchronize_irq(irq_entry->vector); 811 idxd_mask_error_interrupts(idxd); 812 flush_workqueue(idxd->wq); 813 } 814 815 static void idxd_remove(struct pci_dev *pdev) 816 { 817 struct idxd_device *idxd = pci_get_drvdata(pdev); 818 struct idxd_irq_entry *irq_entry; 819 820 idxd_unregister_devices(idxd); 821 /* 822 * When ->release() is called for the idxd->conf_dev, it frees all the memory related 823 * to the idxd context. The driver still needs those bits in order to do the rest of 824 * the cleanup. However, we do need to unbound the idxd sub-driver. So take a ref 825 * on the device here to hold off the freeing while allowing the idxd sub-driver 826 * to unbind. 827 */ 828 get_device(idxd_confdev(idxd)); 829 device_unregister(idxd_confdev(idxd)); 830 idxd_shutdown(pdev); 831 if (device_pasid_enabled(idxd)) 832 idxd_disable_system_pasid(idxd); 833 idxd_device_remove_debugfs(idxd); 834 835 irq_entry = idxd_get_ie(idxd, 0); 836 free_irq(irq_entry->vector, irq_entry); 837 pci_free_irq_vectors(pdev); 838 pci_iounmap(pdev, idxd->reg_base); 839 if (device_user_pasid_enabled(idxd)) 840 idxd_disable_sva(pdev); 841 pci_disable_device(pdev); 842 destroy_workqueue(idxd->wq); 843 perfmon_pmu_remove(idxd); 844 put_device(idxd_confdev(idxd)); 845 } 846 847 static struct pci_driver idxd_pci_driver = { 848 .name = DRV_NAME, 849 .id_table = idxd_pci_tbl, 850 .probe = idxd_pci_probe, 851 .remove = idxd_remove, 852 .shutdown = idxd_shutdown, 853 }; 854 855 static int __init idxd_init_module(void) 856 { 857 int err; 858 859 /* 860 * If the CPU does not support MOVDIR64B or ENQCMDS, there's no point in 861 * enumerating the device. We can not utilize it. 862 */ 863 if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { 864 pr_warn("idxd driver failed to load without MOVDIR64B.\n"); 865 return -ENODEV; 866 } 867 868 if (!cpu_feature_enabled(X86_FEATURE_ENQCMD)) 869 pr_warn("Platform does not have ENQCMD(S) support.\n"); 870 else 871 support_enqcmd = true; 872 873 perfmon_init(); 874 875 err = idxd_driver_register(&idxd_drv); 876 if (err < 0) 877 goto err_idxd_driver_register; 878 879 err = idxd_driver_register(&idxd_dmaengine_drv); 880 if (err < 0) 881 goto err_idxd_dmaengine_driver_register; 882 883 err = idxd_driver_register(&idxd_user_drv); 884 if (err < 0) 885 goto err_idxd_user_driver_register; 886 887 err = idxd_cdev_register(); 888 if (err) 889 goto err_cdev_register; 890 891 err = idxd_init_debugfs(); 892 if (err) 893 goto err_debugfs; 894 895 err = pci_register_driver(&idxd_pci_driver); 896 if (err) 897 goto err_pci_register; 898 899 return 0; 900 901 err_pci_register: 902 idxd_remove_debugfs(); 903 err_debugfs: 904 idxd_cdev_remove(); 905 err_cdev_register: 906 idxd_driver_unregister(&idxd_user_drv); 907 err_idxd_user_driver_register: 908 idxd_driver_unregister(&idxd_dmaengine_drv); 909 err_idxd_dmaengine_driver_register: 910 idxd_driver_unregister(&idxd_drv); 911 err_idxd_driver_register: 912 return err; 913 } 914 module_init(idxd_init_module); 915 916 static void __exit idxd_exit_module(void) 917 { 918 idxd_driver_unregister(&idxd_user_drv); 919 idxd_driver_unregister(&idxd_dmaengine_drv); 920 idxd_driver_unregister(&idxd_drv); 921 pci_unregister_driver(&idxd_pci_driver); 922 idxd_cdev_remove(); 923 perfmon_exit(); 924 idxd_remove_debugfs(); 925 } 926 module_exit(idxd_exit_module); 927