1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <dwmw2@infradead.org> 6 */ 7 8 #include <linux/intel-iommu.h> 9 #include <linux/mmu_notifier.h> 10 #include <linux/sched.h> 11 #include <linux/sched/mm.h> 12 #include <linux/slab.h> 13 #include <linux/intel-svm.h> 14 #include <linux/rculist.h> 15 #include <linux/pci.h> 16 #include <linux/pci-ats.h> 17 #include <linux/dmar.h> 18 #include <linux/interrupt.h> 19 #include <linux/mm_types.h> 20 #include <linux/ioasid.h> 21 #include <asm/page.h> 22 #include <asm/fpu/api.h> 23 24 #include "pasid.h" 25 26 static irqreturn_t prq_event_thread(int irq, void *d); 27 static void intel_svm_drain_prq(struct device *dev, u32 pasid); 28 29 #define PRQ_ORDER 0 30 31 int intel_svm_enable_prq(struct intel_iommu *iommu) 32 { 33 struct page *pages; 34 int irq, ret; 35 36 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 37 if (!pages) { 38 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 39 iommu->name); 40 return -ENOMEM; 41 } 42 iommu->prq = page_address(pages); 43 44 irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu); 45 if (irq <= 0) { 46 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 47 iommu->name); 48 ret = -EINVAL; 49 err: 50 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 51 iommu->prq = NULL; 52 return ret; 53 } 54 iommu->pr_irq = irq; 55 56 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 57 58 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 59 iommu->prq_name, iommu); 60 if (ret) { 61 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 62 iommu->name); 63 dmar_free_hwirq(irq); 64 iommu->pr_irq = 0; 65 goto err; 66 } 67 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 68 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 69 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 70 71 init_completion(&iommu->prq_complete); 72 73 return 0; 74 } 75 76 int intel_svm_finish_prq(struct intel_iommu *iommu) 77 { 78 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 79 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 80 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 81 82 if (iommu->pr_irq) { 83 free_irq(iommu->pr_irq, iommu); 84 dmar_free_hwirq(iommu->pr_irq); 85 iommu->pr_irq = 0; 86 } 87 88 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 89 iommu->prq = NULL; 90 91 return 0; 92 } 93 94 static inline bool intel_svm_capable(struct intel_iommu *iommu) 95 { 96 return iommu->flags & VTD_FLAG_SVM_CAPABLE; 97 } 98 99 void intel_svm_check(struct intel_iommu *iommu) 100 { 101 if (!pasid_supported(iommu)) 102 return; 103 104 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 105 !cap_fl1gp_support(iommu->cap)) { 106 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 107 iommu->name); 108 return; 109 } 110 111 if (cpu_feature_enabled(X86_FEATURE_LA57) && 112 !cap_5lp_support(iommu->cap)) { 113 pr_err("%s SVM disabled, incompatible paging mode\n", 114 iommu->name); 115 return; 116 } 117 118 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 119 } 120 121 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev, 122 unsigned long address, unsigned long pages, int ih) 123 { 124 struct qi_desc desc; 125 126 if (pages == -1) { 127 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | 128 QI_EIOTLB_DID(sdev->did) | 129 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 130 QI_EIOTLB_TYPE; 131 desc.qw1 = 0; 132 } else { 133 int mask = ilog2(__roundup_pow_of_two(pages)); 134 135 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | 136 QI_EIOTLB_DID(sdev->did) | 137 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | 138 QI_EIOTLB_TYPE; 139 desc.qw1 = QI_EIOTLB_ADDR(address) | 140 QI_EIOTLB_IH(ih) | 141 QI_EIOTLB_AM(mask); 142 } 143 desc.qw2 = 0; 144 desc.qw3 = 0; 145 qi_submit_sync(svm->iommu, &desc, 1, 0); 146 147 if (sdev->dev_iotlb) { 148 desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) | 149 QI_DEV_EIOTLB_SID(sdev->sid) | 150 QI_DEV_EIOTLB_QDEP(sdev->qdep) | 151 QI_DEIOTLB_TYPE; 152 if (pages == -1) { 153 desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | 154 QI_DEV_EIOTLB_SIZE; 155 } else if (pages > 1) { 156 /* The least significant zero bit indicates the size. So, 157 * for example, an "address" value of 0x12345f000 will 158 * flush from 0x123440000 to 0x12347ffff (256KiB). */ 159 unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT); 160 unsigned long mask = __rounddown_pow_of_two(address ^ last); 161 162 desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) | 163 (mask - 1)) | QI_DEV_EIOTLB_SIZE; 164 } else { 165 desc.qw1 = QI_DEV_EIOTLB_ADDR(address); 166 } 167 desc.qw2 = 0; 168 desc.qw3 = 0; 169 qi_submit_sync(svm->iommu, &desc, 1, 0); 170 } 171 } 172 173 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 174 unsigned long pages, int ih) 175 { 176 struct intel_svm_dev *sdev; 177 178 rcu_read_lock(); 179 list_for_each_entry_rcu(sdev, &svm->devs, list) 180 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 181 rcu_read_unlock(); 182 } 183 184 /* Pages have been freed at this point */ 185 static void intel_invalidate_range(struct mmu_notifier *mn, 186 struct mm_struct *mm, 187 unsigned long start, unsigned long end) 188 { 189 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 190 191 intel_flush_svm_range(svm, start, 192 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 193 } 194 195 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 196 { 197 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 198 struct intel_svm_dev *sdev; 199 200 /* This might end up being called from exit_mmap(), *before* the page 201 * tables are cleared. And __mmu_notifier_release() will delete us from 202 * the list of notifiers so that our invalidate_range() callback doesn't 203 * get called when the page tables are cleared. So we need to protect 204 * against hardware accessing those page tables. 205 * 206 * We do it by clearing the entry in the PASID table and then flushing 207 * the IOTLB and the PASID table caches. This might upset hardware; 208 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 209 * page) so that we end up taking a fault that the hardware really 210 * *has* to handle gracefully without affecting other processes. 211 */ 212 rcu_read_lock(); 213 list_for_each_entry_rcu(sdev, &svm->devs, list) 214 intel_pasid_tear_down_entry(svm->iommu, sdev->dev, 215 svm->pasid, true); 216 rcu_read_unlock(); 217 218 } 219 220 static const struct mmu_notifier_ops intel_mmuops = { 221 .release = intel_mm_release, 222 .invalidate_range = intel_invalidate_range, 223 }; 224 225 static DEFINE_MUTEX(pasid_mutex); 226 static LIST_HEAD(global_svm_list); 227 228 #define for_each_svm_dev(sdev, svm, d) \ 229 list_for_each_entry((sdev), &(svm)->devs, list) \ 230 if ((d) != (sdev)->dev) {} else 231 232 static int pasid_to_svm_sdev(struct device *dev, unsigned int pasid, 233 struct intel_svm **rsvm, 234 struct intel_svm_dev **rsdev) 235 { 236 struct intel_svm_dev *d, *sdev = NULL; 237 struct intel_svm *svm; 238 239 /* The caller should hold the pasid_mutex lock */ 240 if (WARN_ON(!mutex_is_locked(&pasid_mutex))) 241 return -EINVAL; 242 243 if (pasid == INVALID_IOASID || pasid >= PASID_MAX) 244 return -EINVAL; 245 246 svm = ioasid_find(NULL, pasid, NULL); 247 if (IS_ERR(svm)) 248 return PTR_ERR(svm); 249 250 if (!svm) 251 goto out; 252 253 /* 254 * If we found svm for the PASID, there must be at least one device 255 * bond. 256 */ 257 if (WARN_ON(list_empty(&svm->devs))) 258 return -EINVAL; 259 260 rcu_read_lock(); 261 list_for_each_entry_rcu(d, &svm->devs, list) { 262 if (d->dev == dev) { 263 sdev = d; 264 break; 265 } 266 } 267 rcu_read_unlock(); 268 269 out: 270 *rsvm = svm; 271 *rsdev = sdev; 272 273 return 0; 274 } 275 276 int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, 277 struct iommu_gpasid_bind_data *data) 278 { 279 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 280 struct intel_svm_dev *sdev = NULL; 281 struct dmar_domain *dmar_domain; 282 struct intel_svm *svm = NULL; 283 int ret = 0; 284 285 if (WARN_ON(!iommu) || !data) 286 return -EINVAL; 287 288 if (data->format != IOMMU_PASID_FORMAT_INTEL_VTD) 289 return -EINVAL; 290 291 /* IOMMU core ensures argsz is more than the start of the union */ 292 if (data->argsz < offsetofend(struct iommu_gpasid_bind_data, vendor.vtd)) 293 return -EINVAL; 294 295 /* Make sure no undefined flags are used in vendor data */ 296 if (data->vendor.vtd.flags & ~(IOMMU_SVA_VTD_GPASID_LAST - 1)) 297 return -EINVAL; 298 299 if (!dev_is_pci(dev)) 300 return -ENOTSUPP; 301 302 /* VT-d supports devices with full 20 bit PASIDs only */ 303 if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX) 304 return -EINVAL; 305 306 /* 307 * We only check host PASID range, we have no knowledge to check 308 * guest PASID range. 309 */ 310 if (data->hpasid <= 0 || data->hpasid >= PASID_MAX) 311 return -EINVAL; 312 313 dmar_domain = to_dmar_domain(domain); 314 315 mutex_lock(&pasid_mutex); 316 ret = pasid_to_svm_sdev(dev, data->hpasid, &svm, &sdev); 317 if (ret) 318 goto out; 319 320 if (sdev) { 321 /* 322 * Do not allow multiple bindings of the same device-PASID since 323 * there is only one SL page tables per PASID. We may revisit 324 * once sharing PGD across domains are supported. 325 */ 326 dev_warn_ratelimited(dev, "Already bound with PASID %u\n", 327 svm->pasid); 328 ret = -EBUSY; 329 goto out; 330 } 331 332 if (!svm) { 333 /* We come here when PASID has never been bond to a device. */ 334 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 335 if (!svm) { 336 ret = -ENOMEM; 337 goto out; 338 } 339 /* REVISIT: upper layer/VFIO can track host process that bind 340 * the PASID. ioasid_set = mm might be sufficient for vfio to 341 * check pasid VMM ownership. We can drop the following line 342 * once VFIO and IOASID set check is in place. 343 */ 344 svm->mm = get_task_mm(current); 345 svm->pasid = data->hpasid; 346 if (data->flags & IOMMU_SVA_GPASID_VAL) { 347 svm->gpasid = data->gpasid; 348 svm->flags |= SVM_FLAG_GUEST_PASID; 349 } 350 ioasid_set_data(data->hpasid, svm); 351 INIT_LIST_HEAD_RCU(&svm->devs); 352 mmput(svm->mm); 353 } 354 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 355 if (!sdev) { 356 ret = -ENOMEM; 357 goto out; 358 } 359 sdev->dev = dev; 360 361 /* Only count users if device has aux domains */ 362 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 363 sdev->users = 1; 364 365 /* Set up device context entry for PASID if not enabled already */ 366 ret = intel_iommu_enable_pasid(iommu, sdev->dev); 367 if (ret) { 368 dev_err_ratelimited(dev, "Failed to enable PASID capability\n"); 369 kfree(sdev); 370 goto out; 371 } 372 373 /* 374 * PASID table is per device for better security. Therefore, for 375 * each bind of a new device even with an existing PASID, we need to 376 * call the nested mode setup function here. 377 */ 378 spin_lock(&iommu->lock); 379 ret = intel_pasid_setup_nested(iommu, dev, 380 (pgd_t *)(uintptr_t)data->gpgd, 381 data->hpasid, &data->vendor.vtd, dmar_domain, 382 data->addr_width); 383 spin_unlock(&iommu->lock); 384 if (ret) { 385 dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n", 386 data->hpasid, ret); 387 /* 388 * PASID entry should be in cleared state if nested mode 389 * set up failed. So we only need to clear IOASID tracking 390 * data such that free call will succeed. 391 */ 392 kfree(sdev); 393 goto out; 394 } 395 396 svm->flags |= SVM_FLAG_GUEST_MODE; 397 398 init_rcu_head(&sdev->rcu); 399 list_add_rcu(&sdev->list, &svm->devs); 400 out: 401 if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) { 402 ioasid_set_data(data->hpasid, NULL); 403 kfree(svm); 404 } 405 406 mutex_unlock(&pasid_mutex); 407 return ret; 408 } 409 410 int intel_svm_unbind_gpasid(struct device *dev, u32 pasid) 411 { 412 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 413 struct intel_svm_dev *sdev; 414 struct intel_svm *svm; 415 int ret; 416 417 if (WARN_ON(!iommu)) 418 return -EINVAL; 419 420 mutex_lock(&pasid_mutex); 421 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 422 if (ret) 423 goto out; 424 425 if (sdev) { 426 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 427 sdev->users--; 428 if (!sdev->users) { 429 list_del_rcu(&sdev->list); 430 intel_pasid_tear_down_entry(iommu, dev, 431 svm->pasid, false); 432 intel_svm_drain_prq(dev, svm->pasid); 433 kfree_rcu(sdev, rcu); 434 435 if (list_empty(&svm->devs)) { 436 /* 437 * We do not free the IOASID here in that 438 * IOMMU driver did not allocate it. 439 * Unlike native SVM, IOASID for guest use was 440 * allocated prior to the bind call. 441 * In any case, if the free call comes before 442 * the unbind, IOMMU driver will get notified 443 * and perform cleanup. 444 */ 445 ioasid_set_data(pasid, NULL); 446 kfree(svm); 447 } 448 } 449 } 450 out: 451 mutex_unlock(&pasid_mutex); 452 return ret; 453 } 454 455 static void _load_pasid(void *unused) 456 { 457 update_pasid(); 458 } 459 460 static void load_pasid(struct mm_struct *mm, u32 pasid) 461 { 462 mutex_lock(&mm->context.lock); 463 464 /* Synchronize with READ_ONCE in update_pasid(). */ 465 smp_store_release(&mm->pasid, pasid); 466 467 /* Update PASID MSR on all CPUs running the mm's tasks. */ 468 on_each_cpu_mask(mm_cpumask(mm), _load_pasid, NULL, true); 469 470 mutex_unlock(&mm->context.lock); 471 } 472 473 /* Caller must hold pasid_mutex, mm reference */ 474 static int 475 intel_svm_bind_mm(struct device *dev, unsigned int flags, 476 struct svm_dev_ops *ops, 477 struct mm_struct *mm, struct intel_svm_dev **sd) 478 { 479 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); 480 struct device_domain_info *info; 481 struct intel_svm_dev *sdev; 482 struct intel_svm *svm = NULL; 483 int pasid_max; 484 int ret; 485 486 if (!iommu || dmar_disabled) 487 return -EINVAL; 488 489 if (!intel_svm_capable(iommu)) 490 return -ENOTSUPP; 491 492 if (dev_is_pci(dev)) { 493 pasid_max = pci_max_pasids(to_pci_dev(dev)); 494 if (pasid_max < 0) 495 return -EINVAL; 496 } else 497 pasid_max = 1 << 20; 498 499 /* Bind supervisor PASID shuld have mm = NULL */ 500 if (flags & SVM_FLAG_SUPERVISOR_MODE) { 501 if (!ecap_srs(iommu->ecap) || mm) { 502 pr_err("Supervisor PASID with user provided mm.\n"); 503 return -EINVAL; 504 } 505 } 506 507 if (!(flags & SVM_FLAG_PRIVATE_PASID)) { 508 struct intel_svm *t; 509 510 list_for_each_entry(t, &global_svm_list, list) { 511 if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID)) 512 continue; 513 514 svm = t; 515 if (svm->pasid >= pasid_max) { 516 dev_warn(dev, 517 "Limited PASID width. Cannot use existing PASID %d\n", 518 svm->pasid); 519 ret = -ENOSPC; 520 goto out; 521 } 522 523 /* Find the matching device in svm list */ 524 for_each_svm_dev(sdev, svm, dev) { 525 if (sdev->ops != ops) { 526 ret = -EBUSY; 527 goto out; 528 } 529 sdev->users++; 530 goto success; 531 } 532 533 break; 534 } 535 } 536 537 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 538 if (!sdev) { 539 ret = -ENOMEM; 540 goto out; 541 } 542 sdev->dev = dev; 543 544 ret = intel_iommu_enable_pasid(iommu, dev); 545 if (ret) { 546 kfree(sdev); 547 goto out; 548 } 549 550 info = get_domain_info(dev); 551 sdev->did = FLPT_DEFAULT_DID; 552 sdev->sid = PCI_DEVID(info->bus, info->devfn); 553 if (info->ats_enabled) { 554 sdev->dev_iotlb = 1; 555 sdev->qdep = info->ats_qdep; 556 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 557 sdev->qdep = 0; 558 } 559 560 /* Finish the setup now we know we're keeping it */ 561 sdev->users = 1; 562 sdev->ops = ops; 563 init_rcu_head(&sdev->rcu); 564 565 if (!svm) { 566 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 567 if (!svm) { 568 ret = -ENOMEM; 569 kfree(sdev); 570 goto out; 571 } 572 svm->iommu = iommu; 573 574 if (pasid_max > intel_pasid_max_id) 575 pasid_max = intel_pasid_max_id; 576 577 /* Do not use PASID 0, reserved for RID to PASID */ 578 svm->pasid = ioasid_alloc(NULL, PASID_MIN, 579 pasid_max - 1, svm); 580 if (svm->pasid == INVALID_IOASID) { 581 kfree(svm); 582 kfree(sdev); 583 ret = -ENOSPC; 584 goto out; 585 } 586 svm->notifier.ops = &intel_mmuops; 587 svm->mm = mm; 588 svm->flags = flags; 589 INIT_LIST_HEAD_RCU(&svm->devs); 590 INIT_LIST_HEAD(&svm->list); 591 ret = -ENOMEM; 592 if (mm) { 593 ret = mmu_notifier_register(&svm->notifier, mm); 594 if (ret) { 595 ioasid_free(svm->pasid); 596 kfree(svm); 597 kfree(sdev); 598 goto out; 599 } 600 } 601 602 spin_lock(&iommu->lock); 603 ret = intel_pasid_setup_first_level(iommu, dev, 604 mm ? mm->pgd : init_mm.pgd, 605 svm->pasid, FLPT_DEFAULT_DID, 606 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | 607 (cpu_feature_enabled(X86_FEATURE_LA57) ? 608 PASID_FLAG_FL5LP : 0)); 609 spin_unlock(&iommu->lock); 610 if (ret) { 611 if (mm) 612 mmu_notifier_unregister(&svm->notifier, mm); 613 ioasid_free(svm->pasid); 614 kfree(svm); 615 kfree(sdev); 616 goto out; 617 } 618 619 list_add_tail(&svm->list, &global_svm_list); 620 if (mm) { 621 /* The newly allocated pasid is loaded to the mm. */ 622 load_pasid(mm, svm->pasid); 623 } 624 } else { 625 /* 626 * Binding a new device with existing PASID, need to setup 627 * the PASID entry. 628 */ 629 spin_lock(&iommu->lock); 630 ret = intel_pasid_setup_first_level(iommu, dev, 631 mm ? mm->pgd : init_mm.pgd, 632 svm->pasid, FLPT_DEFAULT_DID, 633 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | 634 (cpu_feature_enabled(X86_FEATURE_LA57) ? 635 PASID_FLAG_FL5LP : 0)); 636 spin_unlock(&iommu->lock); 637 if (ret) { 638 kfree(sdev); 639 goto out; 640 } 641 } 642 list_add_rcu(&sdev->list, &svm->devs); 643 success: 644 sdev->pasid = svm->pasid; 645 sdev->sva.dev = dev; 646 if (sd) 647 *sd = sdev; 648 ret = 0; 649 out: 650 return ret; 651 } 652 653 /* Caller must hold pasid_mutex */ 654 static int intel_svm_unbind_mm(struct device *dev, u32 pasid) 655 { 656 struct intel_svm_dev *sdev; 657 struct intel_iommu *iommu; 658 struct intel_svm *svm; 659 int ret = -EINVAL; 660 661 iommu = device_to_iommu(dev, NULL, NULL); 662 if (!iommu) 663 goto out; 664 665 ret = pasid_to_svm_sdev(dev, pasid, &svm, &sdev); 666 if (ret) 667 goto out; 668 669 if (sdev) { 670 sdev->users--; 671 if (!sdev->users) { 672 list_del_rcu(&sdev->list); 673 /* Flush the PASID cache and IOTLB for this device. 674 * Note that we do depend on the hardware *not* using 675 * the PASID any more. Just as we depend on other 676 * devices never using PASIDs that they have no right 677 * to use. We have a *shared* PASID table, because it's 678 * large and has to be physically contiguous. So it's 679 * hard to be as defensive as we might like. */ 680 intel_pasid_tear_down_entry(iommu, dev, 681 svm->pasid, false); 682 intel_svm_drain_prq(dev, svm->pasid); 683 kfree_rcu(sdev, rcu); 684 685 if (list_empty(&svm->devs)) { 686 ioasid_free(svm->pasid); 687 if (svm->mm) { 688 mmu_notifier_unregister(&svm->notifier, svm->mm); 689 /* Clear mm's pasid. */ 690 load_pasid(svm->mm, PASID_DISABLED); 691 } 692 list_del(&svm->list); 693 /* We mandate that no page faults may be outstanding 694 * for the PASID when intel_svm_unbind_mm() is called. 695 * If that is not obeyed, subtle errors will happen. 696 * Let's make them less subtle... */ 697 memset(svm, 0x6b, sizeof(*svm)); 698 kfree(svm); 699 } 700 } 701 } 702 out: 703 return ret; 704 } 705 706 /* Page request queue descriptor */ 707 struct page_req_dsc { 708 union { 709 struct { 710 u64 type:8; 711 u64 pasid_present:1; 712 u64 priv_data_present:1; 713 u64 rsvd:6; 714 u64 rid:16; 715 u64 pasid:20; 716 u64 exe_req:1; 717 u64 pm_req:1; 718 u64 rsvd2:10; 719 }; 720 u64 qw_0; 721 }; 722 union { 723 struct { 724 u64 rd_req:1; 725 u64 wr_req:1; 726 u64 lpig:1; 727 u64 prg_index:9; 728 u64 addr:52; 729 }; 730 u64 qw_1; 731 }; 732 u64 priv_data[2]; 733 }; 734 735 #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) 736 737 static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req) 738 { 739 unsigned long requested = 0; 740 741 if (req->exe_req) 742 requested |= VM_EXEC; 743 744 if (req->rd_req) 745 requested |= VM_READ; 746 747 if (req->wr_req) 748 requested |= VM_WRITE; 749 750 return (requested & ~vma->vm_flags) != 0; 751 } 752 753 static bool is_canonical_address(u64 addr) 754 { 755 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 756 long saddr = (long) addr; 757 758 return (((saddr << shift) >> shift) == saddr); 759 } 760 761 /** 762 * intel_svm_drain_prq - Drain page requests and responses for a pasid 763 * @dev: target device 764 * @pasid: pasid for draining 765 * 766 * Drain all pending page requests and responses related to @pasid in both 767 * software and hardware. This is supposed to be called after the device 768 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 769 * and DevTLB have been invalidated. 770 * 771 * It waits until all pending page requests for @pasid in the page fault 772 * queue are completed by the prq handling thread. Then follow the steps 773 * described in VT-d spec CH7.10 to drain all page requests and page 774 * responses pending in the hardware. 775 */ 776 static void intel_svm_drain_prq(struct device *dev, u32 pasid) 777 { 778 struct device_domain_info *info; 779 struct dmar_domain *domain; 780 struct intel_iommu *iommu; 781 struct qi_desc desc[3]; 782 struct pci_dev *pdev; 783 int head, tail; 784 u16 sid, did; 785 int qdep; 786 787 info = get_domain_info(dev); 788 if (WARN_ON(!info || !dev_is_pci(dev))) 789 return; 790 791 if (!info->pri_enabled) 792 return; 793 794 iommu = info->iommu; 795 domain = info->domain; 796 pdev = to_pci_dev(dev); 797 sid = PCI_DEVID(info->bus, info->devfn); 798 did = domain->iommu_did[iommu->seq_id]; 799 qdep = pci_ats_queue_depth(pdev); 800 801 /* 802 * Check and wait until all pending page requests in the queue are 803 * handled by the prq handling thread. 804 */ 805 prq_retry: 806 reinit_completion(&iommu->prq_complete); 807 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 808 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 809 while (head != tail) { 810 struct page_req_dsc *req; 811 812 req = &iommu->prq[head / sizeof(*req)]; 813 if (!req->pasid_present || req->pasid != pasid) { 814 head = (head + sizeof(*req)) & PRQ_RING_MASK; 815 continue; 816 } 817 818 wait_for_completion(&iommu->prq_complete); 819 goto prq_retry; 820 } 821 822 /* 823 * Perform steps described in VT-d spec CH7.10 to drain page 824 * requests and responses in hardware. 825 */ 826 memset(desc, 0, sizeof(desc)); 827 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 828 QI_IWD_FENCE | 829 QI_IWD_TYPE; 830 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 831 QI_EIOTLB_DID(did) | 832 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 833 QI_EIOTLB_TYPE; 834 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 835 QI_DEV_EIOTLB_SID(sid) | 836 QI_DEV_EIOTLB_QDEP(qdep) | 837 QI_DEIOTLB_TYPE | 838 QI_DEV_IOTLB_PFSID(info->pfsid); 839 qi_retry: 840 reinit_completion(&iommu->prq_complete); 841 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 842 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 843 wait_for_completion(&iommu->prq_complete); 844 goto qi_retry; 845 } 846 } 847 848 static int prq_to_iommu_prot(struct page_req_dsc *req) 849 { 850 int prot = 0; 851 852 if (req->rd_req) 853 prot |= IOMMU_FAULT_PERM_READ; 854 if (req->wr_req) 855 prot |= IOMMU_FAULT_PERM_WRITE; 856 if (req->exe_req) 857 prot |= IOMMU_FAULT_PERM_EXEC; 858 if (req->pm_req) 859 prot |= IOMMU_FAULT_PERM_PRIV; 860 861 return prot; 862 } 863 864 static int 865 intel_svm_prq_report(struct device *dev, struct page_req_dsc *desc) 866 { 867 struct iommu_fault_event event; 868 869 if (!dev || !dev_is_pci(dev)) 870 return -ENODEV; 871 872 /* Fill in event data for device specific processing */ 873 memset(&event, 0, sizeof(struct iommu_fault_event)); 874 event.fault.type = IOMMU_FAULT_PAGE_REQ; 875 event.fault.prm.addr = desc->addr; 876 event.fault.prm.pasid = desc->pasid; 877 event.fault.prm.grpid = desc->prg_index; 878 event.fault.prm.perm = prq_to_iommu_prot(desc); 879 880 if (desc->lpig) 881 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 882 if (desc->pasid_present) { 883 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 884 event.fault.prm.flags |= IOMMU_FAULT_PAGE_RESPONSE_NEEDS_PASID; 885 } 886 if (desc->priv_data_present) { 887 /* 888 * Set last page in group bit if private data is present, 889 * page response is required as it does for LPIG. 890 * iommu_report_device_fault() doesn't understand this vendor 891 * specific requirement thus we set last_page as a workaround. 892 */ 893 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 894 event.fault.prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 895 memcpy(event.fault.prm.private_data, desc->priv_data, 896 sizeof(desc->priv_data)); 897 } 898 899 return iommu_report_device_fault(dev, &event); 900 } 901 902 static irqreturn_t prq_event_thread(int irq, void *d) 903 { 904 struct intel_svm_dev *sdev = NULL; 905 struct intel_iommu *iommu = d; 906 struct intel_svm *svm = NULL; 907 int head, tail, handled = 0; 908 909 /* Clear PPR bit before reading head/tail registers, to 910 * ensure that we get a new interrupt if needed. */ 911 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 912 913 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 914 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 915 while (head != tail) { 916 struct vm_area_struct *vma; 917 struct page_req_dsc *req; 918 struct qi_desc resp; 919 int result; 920 vm_fault_t ret; 921 u64 address; 922 923 handled = 1; 924 925 req = &iommu->prq[head / sizeof(*req)]; 926 927 result = QI_RESP_FAILURE; 928 address = (u64)req->addr << VTD_PAGE_SHIFT; 929 if (!req->pasid_present) { 930 pr_err("%s: Page request without PASID: %08llx %08llx\n", 931 iommu->name, ((unsigned long long *)req)[0], 932 ((unsigned long long *)req)[1]); 933 goto no_pasid; 934 } 935 936 if (!svm || svm->pasid != req->pasid) { 937 rcu_read_lock(); 938 svm = ioasid_find(NULL, req->pasid, NULL); 939 /* It *can't* go away, because the driver is not permitted 940 * to unbind the mm while any page faults are outstanding. 941 * So we only need RCU to protect the internal idr code. */ 942 rcu_read_unlock(); 943 if (IS_ERR_OR_NULL(svm)) { 944 pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n", 945 iommu->name, req->pasid, ((unsigned long long *)req)[0], 946 ((unsigned long long *)req)[1]); 947 goto no_pasid; 948 } 949 } 950 951 if (!sdev || sdev->sid != req->rid) { 952 struct intel_svm_dev *t; 953 954 sdev = NULL; 955 rcu_read_lock(); 956 list_for_each_entry_rcu(t, &svm->devs, list) { 957 if (t->sid == req->rid) { 958 sdev = t; 959 break; 960 } 961 } 962 rcu_read_unlock(); 963 } 964 965 result = QI_RESP_INVALID; 966 /* Since we're using init_mm.pgd directly, we should never take 967 * any faults on kernel addresses. */ 968 if (!svm->mm) 969 goto bad_req; 970 971 /* If address is not canonical, return invalid response */ 972 if (!is_canonical_address(address)) 973 goto bad_req; 974 975 /* 976 * If prq is to be handled outside iommu driver via receiver of 977 * the fault notifiers, we skip the page response here. 978 */ 979 if (svm->flags & SVM_FLAG_GUEST_MODE) { 980 if (sdev && !intel_svm_prq_report(sdev->dev, req)) 981 goto prq_advance; 982 else 983 goto bad_req; 984 } 985 986 /* If the mm is already defunct, don't handle faults. */ 987 if (!mmget_not_zero(svm->mm)) 988 goto bad_req; 989 990 mmap_read_lock(svm->mm); 991 vma = find_extend_vma(svm->mm, address); 992 if (!vma || address < vma->vm_start) 993 goto invalid; 994 995 if (access_error(vma, req)) 996 goto invalid; 997 998 ret = handle_mm_fault(vma, address, 999 req->wr_req ? FAULT_FLAG_WRITE : 0, 1000 NULL); 1001 if (ret & VM_FAULT_ERROR) 1002 goto invalid; 1003 1004 result = QI_RESP_SUCCESS; 1005 invalid: 1006 mmap_read_unlock(svm->mm); 1007 mmput(svm->mm); 1008 bad_req: 1009 WARN_ON(!sdev); 1010 if (sdev && sdev->ops && sdev->ops->fault_cb) { 1011 int rwxp = (req->rd_req << 3) | (req->wr_req << 2) | 1012 (req->exe_req << 1) | (req->pm_req); 1013 sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, 1014 req->priv_data, rwxp, result); 1015 } 1016 /* We get here in the error case where the PASID lookup failed, 1017 and these can be NULL. Do not use them below this point! */ 1018 sdev = NULL; 1019 svm = NULL; 1020 no_pasid: 1021 if (req->lpig || req->priv_data_present) { 1022 /* 1023 * Per VT-d spec. v3.0 ch7.7, system software must 1024 * respond with page group response if private data 1025 * is present (PDP) or last page in group (LPIG) bit 1026 * is set. This is an additional VT-d feature beyond 1027 * PCI ATS spec. 1028 */ 1029 resp.qw0 = QI_PGRP_PASID(req->pasid) | 1030 QI_PGRP_DID(req->rid) | 1031 QI_PGRP_PASID_P(req->pasid_present) | 1032 QI_PGRP_PDP(req->pasid_present) | 1033 QI_PGRP_RESP_CODE(result) | 1034 QI_PGRP_RESP_TYPE; 1035 resp.qw1 = QI_PGRP_IDX(req->prg_index) | 1036 QI_PGRP_LPIG(req->lpig); 1037 1038 if (req->priv_data_present) 1039 memcpy(&resp.qw2, req->priv_data, 1040 sizeof(req->priv_data)); 1041 resp.qw2 = 0; 1042 resp.qw3 = 0; 1043 qi_submit_sync(iommu, &resp, 1, 0); 1044 } 1045 prq_advance: 1046 head = (head + sizeof(*req)) & PRQ_RING_MASK; 1047 } 1048 1049 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 1050 1051 /* 1052 * Clear the page request overflow bit and wake up all threads that 1053 * are waiting for the completion of this handling. 1054 */ 1055 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) 1056 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 1057 1058 if (!completion_done(&iommu->prq_complete)) 1059 complete(&iommu->prq_complete); 1060 1061 return IRQ_RETVAL(handled); 1062 } 1063 1064 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva) 1065 struct iommu_sva * 1066 intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata) 1067 { 1068 struct iommu_sva *sva = ERR_PTR(-EINVAL); 1069 struct intel_svm_dev *sdev = NULL; 1070 unsigned int flags = 0; 1071 int ret; 1072 1073 /* 1074 * TODO: Consolidate with generic iommu-sva bind after it is merged. 1075 * It will require shared SVM data structures, i.e. combine io_mm 1076 * and intel_svm etc. 1077 */ 1078 if (drvdata) 1079 flags = *(unsigned int *)drvdata; 1080 mutex_lock(&pasid_mutex); 1081 ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev); 1082 if (ret) 1083 sva = ERR_PTR(ret); 1084 else if (sdev) 1085 sva = &sdev->sva; 1086 else 1087 WARN(!sdev, "SVM bind succeeded with no sdev!\n"); 1088 1089 mutex_unlock(&pasid_mutex); 1090 1091 return sva; 1092 } 1093 1094 void intel_svm_unbind(struct iommu_sva *sva) 1095 { 1096 struct intel_svm_dev *sdev; 1097 1098 mutex_lock(&pasid_mutex); 1099 sdev = to_intel_svm_dev(sva); 1100 intel_svm_unbind_mm(sdev->dev, sdev->pasid); 1101 mutex_unlock(&pasid_mutex); 1102 } 1103 1104 u32 intel_svm_get_pasid(struct iommu_sva *sva) 1105 { 1106 struct intel_svm_dev *sdev; 1107 u32 pasid; 1108 1109 mutex_lock(&pasid_mutex); 1110 sdev = to_intel_svm_dev(sva); 1111 pasid = sdev->pasid; 1112 mutex_unlock(&pasid_mutex); 1113 1114 return pasid; 1115 } 1116 1117 int intel_svm_page_response(struct device *dev, 1118 struct iommu_fault_event *evt, 1119 struct iommu_page_response *msg) 1120 { 1121 struct iommu_fault_page_request *prm; 1122 struct intel_svm_dev *sdev = NULL; 1123 struct intel_svm *svm = NULL; 1124 struct intel_iommu *iommu; 1125 bool private_present; 1126 bool pasid_present; 1127 bool last_page; 1128 u8 bus, devfn; 1129 int ret = 0; 1130 u16 sid; 1131 1132 if (!dev || !dev_is_pci(dev)) 1133 return -ENODEV; 1134 1135 iommu = device_to_iommu(dev, &bus, &devfn); 1136 if (!iommu) 1137 return -ENODEV; 1138 1139 if (!msg || !evt) 1140 return -EINVAL; 1141 1142 mutex_lock(&pasid_mutex); 1143 1144 prm = &evt->fault.prm; 1145 sid = PCI_DEVID(bus, devfn); 1146 pasid_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PASID_VALID; 1147 private_present = prm->flags & IOMMU_FAULT_PAGE_REQUEST_PRIV_DATA; 1148 last_page = prm->flags & IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE; 1149 1150 if (!pasid_present) { 1151 ret = -EINVAL; 1152 goto out; 1153 } 1154 1155 if (prm->pasid == 0 || prm->pasid >= PASID_MAX) { 1156 ret = -EINVAL; 1157 goto out; 1158 } 1159 1160 ret = pasid_to_svm_sdev(dev, prm->pasid, &svm, &sdev); 1161 if (ret || !sdev) { 1162 ret = -ENODEV; 1163 goto out; 1164 } 1165 1166 /* 1167 * For responses from userspace, need to make sure that the 1168 * pasid has been bound to its mm. 1169 */ 1170 if (svm->flags & SVM_FLAG_GUEST_MODE) { 1171 struct mm_struct *mm; 1172 1173 mm = get_task_mm(current); 1174 if (!mm) { 1175 ret = -EINVAL; 1176 goto out; 1177 } 1178 1179 if (mm != svm->mm) { 1180 ret = -ENODEV; 1181 mmput(mm); 1182 goto out; 1183 } 1184 1185 mmput(mm); 1186 } 1187 1188 /* 1189 * Per VT-d spec. v3.0 ch7.7, system software must respond 1190 * with page group response if private data is present (PDP) 1191 * or last page in group (LPIG) bit is set. This is an 1192 * additional VT-d requirement beyond PCI ATS spec. 1193 */ 1194 if (last_page || private_present) { 1195 struct qi_desc desc; 1196 1197 desc.qw0 = QI_PGRP_PASID(prm->pasid) | QI_PGRP_DID(sid) | 1198 QI_PGRP_PASID_P(pasid_present) | 1199 QI_PGRP_PDP(private_present) | 1200 QI_PGRP_RESP_CODE(msg->code) | 1201 QI_PGRP_RESP_TYPE; 1202 desc.qw1 = QI_PGRP_IDX(prm->grpid) | QI_PGRP_LPIG(last_page); 1203 desc.qw2 = 0; 1204 desc.qw3 = 0; 1205 if (private_present) 1206 memcpy(&desc.qw2, prm->private_data, 1207 sizeof(prm->private_data)); 1208 1209 qi_submit_sync(iommu, &desc, 1, 0); 1210 } 1211 out: 1212 mutex_unlock(&pasid_mutex); 1213 return ret; 1214 } 1215