1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <dwmw2@infradead.org> 6 */ 7 8 #include <linux/intel-iommu.h> 9 #include <linux/mmu_notifier.h> 10 #include <linux/sched.h> 11 #include <linux/sched/mm.h> 12 #include <linux/slab.h> 13 #include <linux/intel-svm.h> 14 #include <linux/rculist.h> 15 #include <linux/pci.h> 16 #include <linux/pci-ats.h> 17 #include <linux/dmar.h> 18 #include <linux/interrupt.h> 19 #include <linux/mm_types.h> 20 #include <linux/ioasid.h> 21 #include <asm/page.h> 22 23 #include "pasid.h" 24 25 static irqreturn_t prq_event_thread(int irq, void *d); 26 static void intel_svm_drain_prq(struct device *dev, int pasid); 27 28 #define PRQ_ORDER 0 29 30 int intel_svm_enable_prq(struct intel_iommu *iommu) 31 { 32 struct page *pages; 33 int irq, ret; 34 35 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 36 if (!pages) { 37 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 38 iommu->name); 39 return -ENOMEM; 40 } 41 iommu->prq = page_address(pages); 42 43 irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu); 44 if (irq <= 0) { 45 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 46 iommu->name); 47 ret = -EINVAL; 48 err: 49 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 50 iommu->prq = NULL; 51 return ret; 52 } 53 iommu->pr_irq = irq; 54 55 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 56 57 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 58 iommu->prq_name, iommu); 59 if (ret) { 60 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 61 iommu->name); 62 dmar_free_hwirq(irq); 63 iommu->pr_irq = 0; 64 goto err; 65 } 66 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 67 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 68 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 69 70 init_completion(&iommu->prq_complete); 71 72 return 0; 73 } 74 75 int intel_svm_finish_prq(struct intel_iommu *iommu) 76 { 77 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 78 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 79 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 80 81 if (iommu->pr_irq) { 82 free_irq(iommu->pr_irq, iommu); 83 dmar_free_hwirq(iommu->pr_irq); 84 iommu->pr_irq = 0; 85 } 86 87 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 88 iommu->prq = NULL; 89 90 return 0; 91 } 92 93 static inline bool intel_svm_capable(struct intel_iommu *iommu) 94 { 95 return iommu->flags & VTD_FLAG_SVM_CAPABLE; 96 } 97 98 void intel_svm_check(struct intel_iommu *iommu) 99 { 100 if (!pasid_supported(iommu)) 101 return; 102 103 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 104 !cap_fl1gp_support(iommu->cap)) { 105 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 106 iommu->name); 107 return; 108 } 109 110 if (cpu_feature_enabled(X86_FEATURE_LA57) && 111 !cap_5lp_support(iommu->cap)) { 112 pr_err("%s SVM disabled, incompatible paging mode\n", 113 iommu->name); 114 return; 115 } 116 117 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 118 } 119 120 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev, 121 unsigned long address, unsigned long pages, int ih) 122 { 123 struct qi_desc desc; 124 125 if (pages == -1) { 126 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | 127 QI_EIOTLB_DID(sdev->did) | 128 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 129 QI_EIOTLB_TYPE; 130 desc.qw1 = 0; 131 } else { 132 int mask = ilog2(__roundup_pow_of_two(pages)); 133 134 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | 135 QI_EIOTLB_DID(sdev->did) | 136 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | 137 QI_EIOTLB_TYPE; 138 desc.qw1 = QI_EIOTLB_ADDR(address) | 139 QI_EIOTLB_IH(ih) | 140 QI_EIOTLB_AM(mask); 141 } 142 desc.qw2 = 0; 143 desc.qw3 = 0; 144 qi_submit_sync(svm->iommu, &desc, 1, 0); 145 146 if (sdev->dev_iotlb) { 147 desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) | 148 QI_DEV_EIOTLB_SID(sdev->sid) | 149 QI_DEV_EIOTLB_QDEP(sdev->qdep) | 150 QI_DEIOTLB_TYPE; 151 if (pages == -1) { 152 desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | 153 QI_DEV_EIOTLB_SIZE; 154 } else if (pages > 1) { 155 /* The least significant zero bit indicates the size. So, 156 * for example, an "address" value of 0x12345f000 will 157 * flush from 0x123440000 to 0x12347ffff (256KiB). */ 158 unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT); 159 unsigned long mask = __rounddown_pow_of_two(address ^ last); 160 161 desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) | 162 (mask - 1)) | QI_DEV_EIOTLB_SIZE; 163 } else { 164 desc.qw1 = QI_DEV_EIOTLB_ADDR(address); 165 } 166 desc.qw2 = 0; 167 desc.qw3 = 0; 168 qi_submit_sync(svm->iommu, &desc, 1, 0); 169 } 170 } 171 172 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 173 unsigned long pages, int ih) 174 { 175 struct intel_svm_dev *sdev; 176 177 rcu_read_lock(); 178 list_for_each_entry_rcu(sdev, &svm->devs, list) 179 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 180 rcu_read_unlock(); 181 } 182 183 /* Pages have been freed at this point */ 184 static void intel_invalidate_range(struct mmu_notifier *mn, 185 struct mm_struct *mm, 186 unsigned long start, unsigned long end) 187 { 188 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 189 190 intel_flush_svm_range(svm, start, 191 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 192 } 193 194 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 195 { 196 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 197 struct intel_svm_dev *sdev; 198 199 /* This might end up being called from exit_mmap(), *before* the page 200 * tables are cleared. And __mmu_notifier_release() will delete us from 201 * the list of notifiers so that our invalidate_range() callback doesn't 202 * get called when the page tables are cleared. So we need to protect 203 * against hardware accessing those page tables. 204 * 205 * We do it by clearing the entry in the PASID table and then flushing 206 * the IOTLB and the PASID table caches. This might upset hardware; 207 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 208 * page) so that we end up taking a fault that the hardware really 209 * *has* to handle gracefully without affecting other processes. 210 */ 211 rcu_read_lock(); 212 list_for_each_entry_rcu(sdev, &svm->devs, list) 213 intel_pasid_tear_down_entry(svm->iommu, sdev->dev, 214 svm->pasid, true); 215 rcu_read_unlock(); 216 217 } 218 219 static const struct mmu_notifier_ops intel_mmuops = { 220 .release = intel_mm_release, 221 .invalidate_range = intel_invalidate_range, 222 }; 223 224 static DEFINE_MUTEX(pasid_mutex); 225 static LIST_HEAD(global_svm_list); 226 227 #define for_each_svm_dev(sdev, svm, d) \ 228 list_for_each_entry((sdev), &(svm)->devs, list) \ 229 if ((d) != (sdev)->dev) {} else 230 231 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, 232 struct intel_svm **rsvm, 233 struct intel_svm_dev **rsdev) 234 { 235 struct intel_svm_dev *d, *sdev = NULL; 236 struct intel_svm *svm; 237 238 /* The caller should hold the pasid_mutex lock */ 239 if (WARN_ON(!mutex_is_locked(&pasid_mutex))) 240 return -EINVAL; 241 242 if (pasid == INVALID_IOASID || pasid >= PASID_MAX) 243 return -EINVAL; 244 245 svm = ioasid_find(NULL, pasid, NULL); 246 if (IS_ERR(svm)) 247 return PTR_ERR(svm); 248 249 if (!svm) 250 goto out; 251 252 /* 253 * If we found svm for the PASID, there must be at least one device 254 * bond. 255 */ 256 if (WARN_ON(list_empty(&svm->devs))) 257 return -EINVAL; 258 259 rcu_read_lock(); 260 list_for_each_entry_rcu(d, &svm->devs, list) { 261 if (d->dev == dev) { 262 sdev = d; 263 break; 264 } 265 } 266 rcu_read_unlock(); 267 268 out: 269 *rsvm = svm; 270 *rsdev = sdev; 271 272 return 0; 273 } 274 275 int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, 276 struct iommu_gpasid_bind_data *data) 277 { 278 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 279 struct intel_svm_dev *sdev = NULL; 280 struct dmar_domain *dmar_domain; 281 struct intel_svm *svm = NULL; 282 int ret = 0; 283 284 if (WARN_ON(!iommu) || !data) 285 return -EINVAL; 286 287 if (data->version != IOMMU_GPASID_BIND_VERSION_1 || 288 data->format != IOMMU_PASID_FORMAT_INTEL_VTD) 289 return -EINVAL; 290 291 if (!dev_is_pci(dev)) 292 return -ENOTSUPP; 293 294 /* VT-d supports devices with full 20 bit PASIDs only */ 295 if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX) 296 return -EINVAL; 297 298 /* 299 * We only check host PASID range, we have no knowledge to check 300 * guest PASID range. 301 */ 302 if (data->hpasid <= 0 || data->hpasid >= PASID_MAX) 303 return -EINVAL; 304 305 dmar_domain = to_dmar_domain(domain); 306 307 mutex_lock(&pasid_mutex); 308 ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev); 309 if (ret) 310 goto out; 311 312 if (sdev) { 313 /* 314 * Do not allow multiple bindings of the same device-PASID since 315 * there is only one SL page tables per PASID. We may revisit 316 * once sharing PGD across domains are supported. 317 */ 318 dev_warn_ratelimited(dev, "Already bound with PASID %u\n", 319 svm->pasid); 320 ret = -EBUSY; 321 goto out; 322 } 323 324 if (!svm) { 325 /* We come here when PASID has never been bond to a device. */ 326 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 327 if (!svm) { 328 ret = -ENOMEM; 329 goto out; 330 } 331 /* REVISIT: upper layer/VFIO can track host process that bind 332 * the PASID. ioasid_set = mm might be sufficient for vfio to 333 * check pasid VMM ownership. We can drop the following line 334 * once VFIO and IOASID set check is in place. 335 */ 336 svm->mm = get_task_mm(current); 337 svm->pasid = data->hpasid; 338 if (data->flags & IOMMU_SVA_GPASID_VAL) { 339 svm->gpasid = data->gpasid; 340 svm->flags |= SVM_FLAG_GUEST_PASID; 341 } 342 ioasid_set_data(data->hpasid, svm); 343 INIT_LIST_HEAD_RCU(&svm->devs); 344 mmput(svm->mm); 345 } 346 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 347 if (!sdev) { 348 ret = -ENOMEM; 349 goto out; 350 } 351 sdev->dev = dev; 352 353 /* Only count users if device has aux domains */ 354 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 355 sdev->users = 1; 356 357 /* Set up device context entry for PASID if not enabled already */ 358 ret = intel_iommu_enable_pasid(iommu, sdev->dev); 359 if (ret) { 360 dev_err_ratelimited(dev, "Failed to enable PASID capability\n"); 361 kfree(sdev); 362 goto out; 363 } 364 365 /* 366 * PASID table is per device for better security. Therefore, for 367 * each bind of a new device even with an existing PASID, we need to 368 * call the nested mode setup function here. 369 */ 370 spin_lock(&iommu->lock); 371 ret = intel_pasid_setup_nested(iommu, dev, 372 (pgd_t *)(uintptr_t)data->gpgd, 373 data->hpasid, &data->vtd, dmar_domain, 374 data->addr_width); 375 spin_unlock(&iommu->lock); 376 if (ret) { 377 dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n", 378 data->hpasid, ret); 379 /* 380 * PASID entry should be in cleared state if nested mode 381 * set up failed. So we only need to clear IOASID tracking 382 * data such that free call will succeed. 383 */ 384 kfree(sdev); 385 goto out; 386 } 387 388 svm->flags |= SVM_FLAG_GUEST_MODE; 389 390 init_rcu_head(&sdev->rcu); 391 list_add_rcu(&sdev->list, &svm->devs); 392 out: 393 if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) { 394 ioasid_set_data(data->hpasid, NULL); 395 kfree(svm); 396 } 397 398 mutex_unlock(&pasid_mutex); 399 return ret; 400 } 401 402 int intel_svm_unbind_gpasid(struct device *dev, int pasid) 403 { 404 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 405 struct intel_svm_dev *sdev; 406 struct intel_svm *svm; 407 int ret; 408 409 if (WARN_ON(!iommu)) 410 return -EINVAL; 411 412 mutex_lock(&pasid_mutex); 413 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 414 if (ret) 415 goto out; 416 417 if (sdev) { 418 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 419 sdev->users--; 420 if (!sdev->users) { 421 list_del_rcu(&sdev->list); 422 intel_pasid_tear_down_entry(iommu, dev, 423 svm->pasid, false); 424 intel_svm_drain_prq(dev, svm->pasid); 425 kfree_rcu(sdev, rcu); 426 427 if (list_empty(&svm->devs)) { 428 /* 429 * We do not free the IOASID here in that 430 * IOMMU driver did not allocate it. 431 * Unlike native SVM, IOASID for guest use was 432 * allocated prior to the bind call. 433 * In any case, if the free call comes before 434 * the unbind, IOMMU driver will get notified 435 * and perform cleanup. 436 */ 437 ioasid_set_data(pasid, NULL); 438 kfree(svm); 439 } 440 } 441 } 442 out: 443 mutex_unlock(&pasid_mutex); 444 return ret; 445 } 446 447 /* Caller must hold pasid_mutex, mm reference */ 448 static int 449 intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops, 450 struct mm_struct *mm, struct intel_svm_dev **sd) 451 { 452 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 453 struct device_domain_info *info; 454 struct intel_svm_dev *sdev; 455 struct intel_svm *svm = NULL; 456 int pasid_max; 457 int ret; 458 459 if (!iommu || dmar_disabled) 460 return -EINVAL; 461 462 if (!intel_svm_capable(iommu)) 463 return -ENOTSUPP; 464 465 if (dev_is_pci(dev)) { 466 pasid_max = pci_max_pasids(to_pci_dev(dev)); 467 if (pasid_max < 0) 468 return -EINVAL; 469 } else 470 pasid_max = 1 << 20; 471 472 /* Bind supervisor PASID shuld have mm = NULL */ 473 if (flags & SVM_FLAG_SUPERVISOR_MODE) { 474 if (!ecap_srs(iommu->ecap) || mm) { 475 pr_err("Supervisor PASID with user provided mm.\n"); 476 return -EINVAL; 477 } 478 } 479 480 if (!(flags & SVM_FLAG_PRIVATE_PASID)) { 481 struct intel_svm *t; 482 483 list_for_each_entry(t, &global_svm_list, list) { 484 if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID)) 485 continue; 486 487 svm = t; 488 if (svm->pasid >= pasid_max) { 489 dev_warn(dev, 490 "Limited PASID width. Cannot use existing PASID %d\n", 491 svm->pasid); 492 ret = -ENOSPC; 493 goto out; 494 } 495 496 /* Find the matching device in svm list */ 497 for_each_svm_dev(sdev, svm, dev) { 498 if (sdev->ops != ops) { 499 ret = -EBUSY; 500 goto out; 501 } 502 sdev->users++; 503 goto success; 504 } 505 506 break; 507 } 508 } 509 510 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 511 if (!sdev) { 512 ret = -ENOMEM; 513 goto out; 514 } 515 sdev->dev = dev; 516 517 ret = intel_iommu_enable_pasid(iommu, dev); 518 if (ret) { 519 kfree(sdev); 520 goto out; 521 } 522 523 info = get_domain_info(dev); 524 sdev->did = FLPT_DEFAULT_DID; 525 sdev->sid = PCI_DEVID(info->bus, info->devfn); 526 if (info->ats_enabled) { 527 sdev->dev_iotlb = 1; 528 sdev->qdep = info->ats_qdep; 529 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 530 sdev->qdep = 0; 531 } 532 533 /* Finish the setup now we know we're keeping it */ 534 sdev->users = 1; 535 sdev->ops = ops; 536 init_rcu_head(&sdev->rcu); 537 538 if (!svm) { 539 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 540 if (!svm) { 541 ret = -ENOMEM; 542 kfree(sdev); 543 goto out; 544 } 545 svm->iommu = iommu; 546 547 if (pasid_max > intel_pasid_max_id) 548 pasid_max = intel_pasid_max_id; 549 550 /* Do not use PASID 0, reserved for RID to PASID */ 551 svm->pasid = ioasid_alloc(NULL, PASID_MIN, 552 pasid_max - 1, svm); 553 if (svm->pasid == INVALID_IOASID) { 554 kfree(svm); 555 kfree(sdev); 556 ret = -ENOSPC; 557 goto out; 558 } 559 svm->notifier.ops = &intel_mmuops; 560 svm->mm = mm; 561 svm->flags = flags; 562 INIT_LIST_HEAD_RCU(&svm->devs); 563 INIT_LIST_HEAD(&svm->list); 564 ret = -ENOMEM; 565 if (mm) { 566 ret = mmu_notifier_register(&svm->notifier, mm); 567 if (ret) { 568 ioasid_free(svm->pasid); 569 kfree(svm); 570 kfree(sdev); 571 goto out; 572 } 573 } 574 575 spin_lock(&iommu->lock); 576 ret = intel_pasid_setup_first_level(iommu, dev, 577 mm ? mm->pgd : init_mm.pgd, 578 svm->pasid, FLPT_DEFAULT_DID, 579 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | 580 (cpu_feature_enabled(X86_FEATURE_LA57) ? 581 PASID_FLAG_FL5LP : 0)); 582 spin_unlock(&iommu->lock); 583 if (ret) { 584 if (mm) 585 mmu_notifier_unregister(&svm->notifier, mm); 586 ioasid_free(svm->pasid); 587 kfree(svm); 588 kfree(sdev); 589 goto out; 590 } 591 592 list_add_tail(&svm->list, &global_svm_list); 593 } else { 594 /* 595 * Binding a new device with existing PASID, need to setup 596 * the PASID entry. 597 */ 598 spin_lock(&iommu->lock); 599 ret = intel_pasid_setup_first_level(iommu, dev, 600 mm ? mm->pgd : init_mm.pgd, 601 svm->pasid, FLPT_DEFAULT_DID, 602 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | 603 (cpu_feature_enabled(X86_FEATURE_LA57) ? 604 PASID_FLAG_FL5LP : 0)); 605 spin_unlock(&iommu->lock); 606 if (ret) { 607 kfree(sdev); 608 goto out; 609 } 610 } 611 list_add_rcu(&sdev->list, &svm->devs); 612 success: 613 sdev->pasid = svm->pasid; 614 sdev->sva.dev = dev; 615 if (sd) 616 *sd = sdev; 617 ret = 0; 618 out: 619 return ret; 620 } 621 622 /* Caller must hold pasid_mutex */ 623 static int intel_svm_unbind_mm(struct device *dev, int pasid) 624 { 625 struct intel_svm_dev *sdev; 626 struct intel_iommu *iommu; 627 struct intel_svm *svm; 628 int ret = -EINVAL; 629 630 iommu = device_to_iommu(dev, NULL, NULL); 631 if (!iommu) 632 goto out; 633 634 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 635 if (ret) 636 goto out; 637 638 if (sdev) { 639 sdev->users--; 640 if (!sdev->users) { 641 list_del_rcu(&sdev->list); 642 /* Flush the PASID cache and IOTLB for this device. 643 * Note that we do depend on the hardware *not* using 644 * the PASID any more. Just as we depend on other 645 * devices never using PASIDs that they have no right 646 * to use. We have a *shared* PASID table, because it's 647 * large and has to be physically contiguous. So it's 648 * hard to be as defensive as we might like. */ 649 intel_pasid_tear_down_entry(iommu, dev, 650 svm->pasid, false); 651 intel_svm_drain_prq(dev, svm->pasid); 652 kfree_rcu(sdev, rcu); 653 654 if (list_empty(&svm->devs)) { 655 ioasid_free(svm->pasid); 656 if (svm->mm) 657 mmu_notifier_unregister(&svm->notifier, svm->mm); 658 list_del(&svm->list); 659 /* We mandate that no page faults may be outstanding 660 * for the PASID when intel_svm_unbind_mm() is called. 661 * If that is not obeyed, subtle errors will happen. 662 * Let's make them less subtle... */ 663 memset(svm, 0x6b, sizeof(*svm)); 664 kfree(svm); 665 } 666 } 667 } 668 out: 669 return ret; 670 } 671 672 /* Page request queue descriptor */ 673 struct page_req_dsc { 674 union { 675 struct { 676 u64 type:8; 677 u64 pasid_present:1; 678 u64 priv_data_present:1; 679 u64 rsvd:6; 680 u64 rid:16; 681 u64 pasid:20; 682 u64 exe_req:1; 683 u64 pm_req:1; 684 u64 rsvd2:10; 685 }; 686 u64 qw_0; 687 }; 688 union { 689 struct { 690 u64 rd_req:1; 691 u64 wr_req:1; 692 u64 lpig:1; 693 u64 prg_index:9; 694 u64 addr:52; 695 }; 696 u64 qw_1; 697 }; 698 u64 priv_data[2]; 699 }; 700 701 #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) 702 703 static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req) 704 { 705 unsigned long requested = 0; 706 707 if (req->exe_req) 708 requested |= VM_EXEC; 709 710 if (req->rd_req) 711 requested |= VM_READ; 712 713 if (req->wr_req) 714 requested |= VM_WRITE; 715 716 return (requested & ~vma->vm_flags) != 0; 717 } 718 719 static bool is_canonical_address(u64 addr) 720 { 721 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 722 long saddr = (long) addr; 723 724 return (((saddr << shift) >> shift) == saddr); 725 } 726 727 /** 728 * intel_svm_drain_prq - Drain page requests and responses for a pasid 729 * @dev: target device 730 * @pasid: pasid for draining 731 * 732 * Drain all pending page requests and responses related to @pasid in both 733 * software and hardware. This is supposed to be called after the device 734 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 735 * and DevTLB have been invalidated. 736 * 737 * It waits until all pending page requests for @pasid in the page fault 738 * queue are completed by the prq handling thread. Then follow the steps 739 * described in VT-d spec CH7.10 to drain all page requests and page 740 * responses pending in the hardware. 741 */ 742 static void intel_svm_drain_prq(struct device *dev, int pasid) 743 { 744 struct device_domain_info *info; 745 struct dmar_domain *domain; 746 struct intel_iommu *iommu; 747 struct qi_desc desc[3]; 748 struct pci_dev *pdev; 749 int head, tail; 750 u16 sid, did; 751 int qdep; 752 753 info = get_domain_info(dev); 754 if (WARN_ON(!info || !dev_is_pci(dev))) 755 return; 756 757 if (!info->pri_enabled) 758 return; 759 760 iommu = info->iommu; 761 domain = info->domain; 762 pdev = to_pci_dev(dev); 763 sid = PCI_DEVID(info->bus, info->devfn); 764 did = domain->iommu_did[iommu->seq_id]; 765 qdep = pci_ats_queue_depth(pdev); 766 767 /* 768 * Check and wait until all pending page requests in the queue are 769 * handled by the prq handling thread. 770 */ 771 prq_retry: 772 reinit_completion(&iommu->prq_complete); 773 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 774 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 775 while (head != tail) { 776 struct page_req_dsc *req; 777 778 req = &iommu->prq[head / sizeof(*req)]; 779 if (!req->pasid_present || req->pasid != pasid) { 780 head = (head + sizeof(*req)) & PRQ_RING_MASK; 781 continue; 782 } 783 784 wait_for_completion(&iommu->prq_complete); 785 goto prq_retry; 786 } 787 788 /* 789 * Perform steps described in VT-d spec CH7.10 to drain page 790 * requests and responses in hardware. 791 */ 792 memset(desc, 0, sizeof(desc)); 793 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 794 QI_IWD_FENCE | 795 QI_IWD_TYPE; 796 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 797 QI_EIOTLB_DID(did) | 798 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 799 QI_EIOTLB_TYPE; 800 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 801 QI_DEV_EIOTLB_SID(sid) | 802 QI_DEV_EIOTLB_QDEP(qdep) | 803 QI_DEIOTLB_TYPE | 804 QI_DEV_IOTLB_PFSID(info->pfsid); 805 qi_retry: 806 reinit_completion(&iommu->prq_complete); 807 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 808 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 809 wait_for_completion(&iommu->prq_complete); 810 goto qi_retry; 811 } 812 } 813 814 static int prq_to_iommu_prot(struct page_req_dsc *req) 815 { 816 int prot = 0; 817 818 if (req->rd_req) 819 prot |= IOMMU_FAULT_PERM_READ; 820 if (req->wr_req) 821 prot |= IOMMU_FAULT_PERM_WRITE; 822 if (req->exe_req) 823 prot |= IOMMU_FAULT_PERM_EXEC; 824 if (req->pm_req) 825 prot |= IOMMU_FAULT_PERM_PRIV; 826 827 return prot; 828 } 829 830 static int 831 intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc) 832 { 833 struct iommu_fault_event event; 834 835 if (!dev || !dev_is_pci(dev)) 836 return -ENODEV; 837 838 /* Fill in event data for device specific processing */ 839 memset(&event, 0, sizeof(struct iommu_fault_event)); 840 event.fault.type = IOMMU_FAULT_PAGE_REQ; 841 event.fault.prm.addr = desc->addr; 842 event.fault.prm.pasid = desc->pasid; 843 event.fault.prm.grpid = desc->prg_index; 844 event.fault.prm.perm = prq_to_iommu_prot(desc); 845 846 if (desc->lpig) 847 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 848 if (desc->pasid_present) { 849 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 850 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 851 } 852 if (desc->priv_data_present) { 853 /* 854 * Set last page in group bit if private data is present, 855 * page response is required as it does for LPIG. 856 * iommu_report_device_fault() doesn't understand this vendor 857 * specific requirement thus we set last_page as a workaround. 858 */ 859 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 860 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 861 memcpy(event.fault.prm.private_data, desc->priv_data, 862 sizeof(desc->priv_data)); 863 } 864 865 return iommu_report_device_fault(dev, &event); 866 } 867 868 static irqreturn_t prq_event_thread(int irq, void *d) 869 { 870 struct intel_svm_dev *sdev = NULL; 871 struct intel_iommu *iommu = d; 872 struct intel_svm *svm = NULL; 873 int head, tail, handled = 0; 874 875 /* Clear PPR bit before reading head/tail registers, to 876 * ensure that we get a new interrupt if needed. */ 877 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 878 879 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 880 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 881 while (head != tail) { 882 struct vm_area_struct *vma; 883 struct page_req_dsc *req; 884 struct qi_desc resp; 885 int result; 886 vm_fault_t ret; 887 u64 address; 888 889 handled = 1; 890 891 req = &iommu->prq[head / sizeof(*req)]; 892 893 result = QI_RESP_FAILURE; 894 address = (u64)req->addr << VTD_PAGE_SHIFT; 895 if (!req->pasid_present) { 896 pr_err("%s: Page request without PASID: %08llx %08llx\n", 897 iommu->name, ((unsigned long long *)req)[0], 898 ((unsigned long long *)req)[1]); 899 goto no_pasid; 900 } 901 902 if (!svm || svm->pasid != req->pasid) { 903 rcu_read_lock(); 904 svm = ioasid_find(NULL, req->pasid, NULL); 905 /* It *can't* go away, because the driver is not permitted 906 * to unbind the mm while any page faults are outstanding. 907 * So we only need RCU to protect the internal idr code. */ 908 rcu_read_unlock(); 909 if (IS_ERR_OR_NULL(svm)) { 910 pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n", 911 iommu->name, req->pasid, ((unsigned long long *)req)[0], 912 ((unsigned long long *)req)[1]); 913 goto no_pasid; 914 } 915 } 916 917 if (!sdev || sdev->sid != req->rid) { 918 struct intel_svm_dev *t; 919 920 sdev = NULL; 921 rcu_read_lock(); 922 list_for_each_entry_rcu(t, &svm->devs, list) { 923 if (t->sid == req->rid) { 924 sdev = t; 925 break; 926 } 927 } 928 rcu_read_unlock(); 929 } 930 931 result = QI_RESP_INVALID; 932 /* Since we're using init_mm.pgd directly, we should never take 933 * any faults on kernel addresses. */ 934 if (!svm->mm) 935 goto bad_req; 936 937 /* If address is not canonical, return invalid response */ 938 if (!is_canonical_address(address)) 939 goto bad_req; 940 941 /* 942 * If prq is to be handled outside iommu driver via receiver of 943 * the fault notifiers, we skip the page response here. 944 */ 945 if (svm->flags & SVM_FLAG_GUEST_MODE) { 946 if (sdev && !intel_svm_prq_report(sdev->dev, req)) 947 goto prq_advance; 948 else 949 goto bad_req; 950 } 951 952 /* If the mm is already defunct, don't handle faults. */ 953 if (!mmget_not_zero(svm->mm)) 954 goto bad_req; 955 956 mmap_read_lock(svm->mm); 957 vma = find_extend_vma(svm->mm, address); 958 if (!vma || address < vma->vm_start) 959 goto invalid; 960 961 if (access_error(vma, req)) 962 goto invalid; 963 964 ret = handle_mm_fault(vma, address, 965 req->wr_req ? FAULT_FLAG_WRITE : 0, 966 NULL); 967 if (ret & VM_FAULT_ERROR) 968 goto invalid; 969 970 result = QI_RESP_SUCCESS; 971 invalid: 972 mmap_read_unlock(svm->mm); 973 mmput(svm->mm); 974 bad_req: 975 WARN_ON(!sdev); 976 if (sdev && sdev->ops && sdev->ops->fault_cb) { 977 int rwxp = (req->rd_req << 3) | (req->wr_req << 2) | 978 (req->exe_req << 1) | (req->pm_req); 979 sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, 980 req->priv_data, rwxp, result); 981 } 982 /* We get here in the error case where the PASID lookup failed, 983 and these can be NULL. Do not use them below this point! */ 984 sdev = NULL; 985 svm = NULL; 986 no_pasid: 987 if (req->lpig || req->priv_data_present) { 988 /* 989 * Per VT-d spec. v3.0 ch7.7, system software must 990 * respond with page group response if private data 991 * is present (PDP) or last page in group (LPIG) bit 992 * is set. This is an additional VT-d feature beyond 993 * PCI ATS spec. 994 */ 995 resp.qw0 = QI_PGRP_PASID(req->pasid) | 996 QI_PGRP_DID(req->rid) | 997 QI_PGRP_PASID_P(req->pasid_present) | 998 QI_PGRP_PDP(req->pasid_present) | 999 QI_PGRP_RESP_CODE(result) | 1000 QI_PGRP_RESP_TYPE; 1001 resp.qw1 = QI_PGRP_IDX(req->prg_index) | 1002 QI_PGRP_LPIG(req->lpig); 1003 1004 if (req->priv_data_present) 1005 memcpy(&resp.qw2, req->priv_data, 1006 sizeof(req->priv_data)); 1007 resp.qw2 = 0; 1008 resp.qw3 = 0; 1009 qi_submit_sync(iommu, &resp, 1, 0); 1010 } 1011 prq_advance: 1012 head = (head + sizeof(*req)) & PRQ_RING_MASK; 1013 } 1014 1015 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 1016 1017 /* 1018 * Clear the page request overflow bit and wake up all threads that 1019 * are waiting for the completion of this handling. 1020 */ 1021 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) 1022 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 1023 1024 if (!completion_done(&iommu->prq_complete)) 1025 complete(&iommu->prq_complete); 1026 1027 return IRQ_RETVAL(handled); 1028 } 1029 1030 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva) 1031 struct iommu_sva * 1032 intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata) 1033 { 1034 struct iommu_sva *sva = ERR_PTR(-EINVAL); 1035 struct intel_svm_dev *sdev = NULL; 1036 int flags = 0; 1037 int ret; 1038 1039 /* 1040 * TODO: Consolidate with generic iommu-sva bind after it is merged. 1041 * It will require shared SVM data structures, i.e. combine io_mm 1042 * and intel_svm etc. 1043 */ 1044 if (drvdata) 1045 flags = *(int *)drvdata; 1046 mutex_lock(&pasid_mutex); 1047 ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev); 1048 if (ret) 1049 sva = ERR_PTR(ret); 1050 else if (sdev) 1051 sva = &sdev->sva; 1052 else 1053 WARN(!sdev, "SVM bind succeeded with no sdev!\n"); 1054 1055 mutex_unlock(&pasid_mutex); 1056 1057 return sva; 1058 } 1059 1060 void intel_svm_unbind(struct iommu_sva *sva) 1061 { 1062 struct intel_svm_dev *sdev; 1063 1064 mutex_lock(&pasid_mutex); 1065 sdev = to_intel_svm_dev(sva); 1066 intel_svm_unbind_mm(sdev->dev, sdev->pasid); 1067 mutex_unlock(&pasid_mutex); 1068 } 1069 1070 int intel_svm_get_pasid(struct iommu_sva *sva) 1071 { 1072 struct intel_svm_dev *sdev; 1073 int pasid; 1074 1075 mutex_lock(&pasid_mutex); 1076 sdev = to_intel_svm_dev(sva); 1077 pasid = sdev->pasid; 1078 mutex_unlock(&pasid_mutex); 1079 1080 return pasid; 1081 } 1082 1083 int intel_svm_page_response(struct device *dev, 1084 struct iommu_fault_event *evt, 1085 struct iommu_page_response *msg) 1086 { 1087 struct iommu_fault_page_request *prm; 1088 struct intel_svm_dev *sdev = NULL; 1089 struct intel_svm *svm = NULL; 1090 struct intel_iommu *iommu; 1091 bool private_present; 1092 bool pasid_present; 1093 bool last_page; 1094 u8 bus, devfn; 1095 int ret = 0; 1096 u16 sid; 1097 1098 if (!dev || !dev_is_pci(dev)) 1099 return -ENODEV; 1100 1101 iommu = device_to_iommu(dev, &bus, &devfn); 1102 if (!iommu) 1103 return -ENODEV; 1104 1105 if (!msg || !evt) 1106 return -EINVAL; 1107 1108 mutex_lock(&pasid_mutex); 1109 1110 prm = &evt->fault.prm; 1111 sid = PCI_DEVID(bus, devfn); 1112 pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1113 private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 1114 last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 1115 1116 if (!pasid_present) { 1117 ret = -EINVAL; 1118 goto out; 1119 } 1120 1121 if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { 1122 ret = -EINVAL; 1123 goto out; 1124 } 1125 1126 ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev); 1127 if (ret || !sdev) { 1128 ret = -ENODEV; 1129 goto out; 1130 } 1131 1132 /* 1133 * For responses from userspace, need to make sure that the 1134 * pasid has been bound to its mm. 1135 */ 1136 if (svm->flags & SVM_FLAG_GUEST_MODE) { 1137 struct mm_struct *mm; 1138 1139 mm = get_task_mm(current); 1140 if (!mm) { 1141 ret = -EINVAL; 1142 goto out; 1143 } 1144 1145 if (mm != svm->mm) { 1146 ret = -ENODEV; 1147 mmput(mm); 1148 goto out; 1149 } 1150 1151 mmput(mm); 1152 } 1153 1154 /* 1155 * Per VT-d spec. v3.0 ch7.7, system software must respond 1156 * with page group response if private data is present (PDP) 1157 * or last page in group (LPIG) bit is set. This is an 1158 * additional VT-d requirement beyond PCI ATS spec. 1159 */ 1160 if (last_page || private_present) { 1161 struct qi_desc desc; 1162 1163 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 1164 QI_PGRP_PASID_P(pasid_present) | 1165 QI_PGRP_PDP(private_present) | 1166 QI_PGRP_RESP_CODE(msg->code) | 1167 QI_PGRP_RESP_TYPE; 1168 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 1169 desc.qw2 = 0; 1170 desc.qw3 = 0; 1171 if (private_present) 1172 memcpy(&desc.qw2, prm->private_data, 1173 sizeof(prm->private_data)); 1174 1175 qi_submit_sync(iommu, &desc, 1, 0); 1176 } 1177 out: 1178 mutex_unlock(&pasid_mutex); 1179 return ret; 1180 } 1181