1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright © 2015 Intel Corporation. 4 * 5 * Authors: David Woodhouse <dwmw2@infradead.org> 6 */ 7 8 #include <linux/intel-iommu.h> 9 #include <linux/mmu_notifier.h> 10 #include <linux/sched.h> 11 #include <linux/sched/mm.h> 12 #include <linux/slab.h> 13 #include <linux/intel-svm.h> 14 #include <linux/rculist.h> 15 #include <linux/pci.h> 16 #include <linux/pci-ats.h> 17 #include <linux/dmar.h> 18 #include <linux/interrupt.h> 19 #include <linux/mm_types.h> 20 #include <linux/ioasid.h> 21 #include <asm/page.h> 22 23 #include "intel-pasid.h" 24 25 static irqreturn_t prq_event_thread(int irq, void *d); 26 static void intel_svm_drain_prq(struct device *dev, int pasid); 27 28 #define PRQ_ORDER 0 29 30 int intel_svm_enable_prq(struct intel_iommu *iommu) 31 { 32 struct page *pages; 33 int irq, ret; 34 35 pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, PRQ_ORDER); 36 if (!pages) { 37 pr_warn("IOMMU: %s: Failed to allocate page request queue\n", 38 iommu->name); 39 return -ENOMEM; 40 } 41 iommu->prq = page_address(pages); 42 43 irq = dmar_alloc_hwirq(DMAR_UNITS_SUPPORTED + iommu->seq_id, iommu->node, iommu); 44 if (irq <= 0) { 45 pr_err("IOMMU: %s: Failed to create IRQ vector for page request queue\n", 46 iommu->name); 47 ret = -EINVAL; 48 err: 49 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 50 iommu->prq = NULL; 51 return ret; 52 } 53 iommu->pr_irq = irq; 54 55 snprintf(iommu->prq_name, sizeof(iommu->prq_name), "dmar%d-prq", iommu->seq_id); 56 57 ret = request_threaded_irq(irq, NULL, prq_event_thread, IRQF_ONESHOT, 58 iommu->prq_name, iommu); 59 if (ret) { 60 pr_err("IOMMU: %s: Failed to request IRQ for page request queue\n", 61 iommu->name); 62 dmar_free_hwirq(irq); 63 iommu->pr_irq = 0; 64 goto err; 65 } 66 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 67 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 68 dmar_writeq(iommu->reg + DMAR_PQA_REG, virt_to_phys(iommu->prq) | PRQ_ORDER); 69 70 init_completion(&iommu->prq_complete); 71 72 return 0; 73 } 74 75 int intel_svm_finish_prq(struct intel_iommu *iommu) 76 { 77 dmar_writeq(iommu->reg + DMAR_PQH_REG, 0ULL); 78 dmar_writeq(iommu->reg + DMAR_PQT_REG, 0ULL); 79 dmar_writeq(iommu->reg + DMAR_PQA_REG, 0ULL); 80 81 if (iommu->pr_irq) { 82 free_irq(iommu->pr_irq, iommu); 83 dmar_free_hwirq(iommu->pr_irq); 84 iommu->pr_irq = 0; 85 } 86 87 free_pages((unsigned long)iommu->prq, PRQ_ORDER); 88 iommu->prq = NULL; 89 90 return 0; 91 } 92 93 static inline bool intel_svm_capable(struct intel_iommu *iommu) 94 { 95 return iommu->flags & VTD_FLAG_SVM_CAPABLE; 96 } 97 98 void intel_svm_check(struct intel_iommu *iommu) 99 { 100 if (!pasid_supported(iommu)) 101 return; 102 103 if (cpu_feature_enabled(X86_FEATURE_GBPAGES) && 104 !cap_fl1gp_support(iommu->cap)) { 105 pr_err("%s SVM disabled, incompatible 1GB page capability\n", 106 iommu->name); 107 return; 108 } 109 110 if (cpu_feature_enabled(X86_FEATURE_LA57) && 111 !cap_5lp_support(iommu->cap)) { 112 pr_err("%s SVM disabled, incompatible paging mode\n", 113 iommu->name); 114 return; 115 } 116 117 iommu->flags |= VTD_FLAG_SVM_CAPABLE; 118 } 119 120 static void intel_flush_svm_range_dev (struct intel_svm *svm, struct intel_svm_dev *sdev, 121 unsigned long address, unsigned long pages, int ih) 122 { 123 struct qi_desc desc; 124 125 if (pages == -1) { 126 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | 127 QI_EIOTLB_DID(sdev->did) | 128 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 129 QI_EIOTLB_TYPE; 130 desc.qw1 = 0; 131 } else { 132 int mask = ilog2(__roundup_pow_of_two(pages)); 133 134 desc.qw0 = QI_EIOTLB_PASID(svm->pasid) | 135 QI_EIOTLB_DID(sdev->did) | 136 QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) | 137 QI_EIOTLB_TYPE; 138 desc.qw1 = QI_EIOTLB_ADDR(address) | 139 QI_EIOTLB_IH(ih) | 140 QI_EIOTLB_AM(mask); 141 } 142 desc.qw2 = 0; 143 desc.qw3 = 0; 144 qi_submit_sync(svm->iommu, &desc, 1, 0); 145 146 if (sdev->dev_iotlb) { 147 desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) | 148 QI_DEV_EIOTLB_SID(sdev->sid) | 149 QI_DEV_EIOTLB_QDEP(sdev->qdep) | 150 QI_DEIOTLB_TYPE; 151 if (pages == -1) { 152 desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) | 153 QI_DEV_EIOTLB_SIZE; 154 } else if (pages > 1) { 155 /* The least significant zero bit indicates the size. So, 156 * for example, an "address" value of 0x12345f000 will 157 * flush from 0x123440000 to 0x12347ffff (256KiB). */ 158 unsigned long last = address + ((unsigned long)(pages - 1) << VTD_PAGE_SHIFT); 159 unsigned long mask = __rounddown_pow_of_two(address ^ last); 160 161 desc.qw1 = QI_DEV_EIOTLB_ADDR((address & ~mask) | 162 (mask - 1)) | QI_DEV_EIOTLB_SIZE; 163 } else { 164 desc.qw1 = QI_DEV_EIOTLB_ADDR(address); 165 } 166 desc.qw2 = 0; 167 desc.qw3 = 0; 168 qi_submit_sync(svm->iommu, &desc, 1, 0); 169 } 170 } 171 172 static void intel_flush_svm_range(struct intel_svm *svm, unsigned long address, 173 unsigned long pages, int ih) 174 { 175 struct intel_svm_dev *sdev; 176 177 rcu_read_lock(); 178 list_for_each_entry_rcu(sdev, &svm->devs, list) 179 intel_flush_svm_range_dev(svm, sdev, address, pages, ih); 180 rcu_read_unlock(); 181 } 182 183 /* Pages have been freed at this point */ 184 static void intel_invalidate_range(struct mmu_notifier *mn, 185 struct mm_struct *mm, 186 unsigned long start, unsigned long end) 187 { 188 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 189 190 intel_flush_svm_range(svm, start, 191 (end - start + PAGE_SIZE - 1) >> VTD_PAGE_SHIFT, 0); 192 } 193 194 static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 195 { 196 struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); 197 struct intel_svm_dev *sdev; 198 199 /* This might end up being called from exit_mmap(), *before* the page 200 * tables are cleared. And __mmu_notifier_release() will delete us from 201 * the list of notifiers so that our invalidate_range() callback doesn't 202 * get called when the page tables are cleared. So we need to protect 203 * against hardware accessing those page tables. 204 * 205 * We do it by clearing the entry in the PASID table and then flushing 206 * the IOTLB and the PASID table caches. This might upset hardware; 207 * perhaps we'll want to point the PASID to a dummy PGD (like the zero 208 * page) so that we end up taking a fault that the hardware really 209 * *has* to handle gracefully without affecting other processes. 210 */ 211 rcu_read_lock(); 212 list_for_each_entry_rcu(sdev, &svm->devs, list) 213 intel_pasid_tear_down_entry(svm->iommu, sdev->dev, 214 svm->pasid, true); 215 rcu_read_unlock(); 216 217 } 218 219 static const struct mmu_notifier_ops intel_mmuops = { 220 .release = intel_mm_release, 221 .invalidate_range = intel_invalidate_range, 222 }; 223 224 static DEFINE_MUTEX(pasid_mutex); 225 static LIST_HEAD(global_svm_list); 226 227 #define for_each_svm_dev(sdev, svm, d) \ 228 list_for_each_entry((sdev), &(svm)->devs, list) \ 229 if ((d) != (sdev)->dev) {} else 230 231 int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, 232 struct iommu_gpasid_bind_data *data) 233 { 234 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); 235 struct dmar_domain *dmar_domain; 236 struct intel_svm_dev *sdev; 237 struct intel_svm *svm; 238 int ret = 0; 239 240 if (WARN_ON(!iommu) || !data) 241 return -EINVAL; 242 243 if (data->version != IOMMU_GPASID_BIND_VERSION_1 || 244 data->format != IOMMU_PASID_FORMAT_INTEL_VTD) 245 return -EINVAL; 246 247 if (!dev_is_pci(dev)) 248 return -ENOTSUPP; 249 250 /* VT-d supports devices with full 20 bit PASIDs only */ 251 if (pci_max_pasids(to_pci_dev(dev)) != PASID_MAX) 252 return -EINVAL; 253 254 /* 255 * We only check host PASID range, we have no knowledge to check 256 * guest PASID range. 257 */ 258 if (data->hpasid <= 0 || data->hpasid >= PASID_MAX) 259 return -EINVAL; 260 261 dmar_domain = to_dmar_domain(domain); 262 263 mutex_lock(&pasid_mutex); 264 svm = ioasid_find(NULL, data->hpasid, NULL); 265 if (IS_ERR(svm)) { 266 ret = PTR_ERR(svm); 267 goto out; 268 } 269 270 if (svm) { 271 /* 272 * If we found svm for the PASID, there must be at 273 * least one device bond, otherwise svm should be freed. 274 */ 275 if (WARN_ON(list_empty(&svm->devs))) { 276 ret = -EINVAL; 277 goto out; 278 } 279 280 for_each_svm_dev(sdev, svm, dev) { 281 /* 282 * For devices with aux domains, we should allow 283 * multiple bind calls with the same PASID and pdev. 284 */ 285 if (iommu_dev_feature_enabled(dev, 286 IOMMU_DEV_FEAT_AUX)) { 287 sdev->users++; 288 } else { 289 dev_warn_ratelimited(dev, 290 "Already bound with PASID %u\n", 291 svm->pasid); 292 ret = -EBUSY; 293 } 294 goto out; 295 } 296 } else { 297 /* We come here when PASID has never been bond to a device. */ 298 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 299 if (!svm) { 300 ret = -ENOMEM; 301 goto out; 302 } 303 /* REVISIT: upper layer/VFIO can track host process that bind 304 * the PASID. ioasid_set = mm might be sufficient for vfio to 305 * check pasid VMM ownership. We can drop the following line 306 * once VFIO and IOASID set check is in place. 307 */ 308 svm->mm = get_task_mm(current); 309 svm->pasid = data->hpasid; 310 if (data->flags & IOMMU_SVA_GPASID_VAL) { 311 svm->gpasid = data->gpasid; 312 svm->flags |= SVM_FLAG_GUEST_PASID; 313 } 314 ioasid_set_data(data->hpasid, svm); 315 INIT_LIST_HEAD_RCU(&svm->devs); 316 mmput(svm->mm); 317 } 318 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 319 if (!sdev) { 320 ret = -ENOMEM; 321 goto out; 322 } 323 sdev->dev = dev; 324 325 /* Only count users if device has aux domains */ 326 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 327 sdev->users = 1; 328 329 /* Set up device context entry for PASID if not enabled already */ 330 ret = intel_iommu_enable_pasid(iommu, sdev->dev); 331 if (ret) { 332 dev_err_ratelimited(dev, "Failed to enable PASID capability\n"); 333 kfree(sdev); 334 goto out; 335 } 336 337 /* 338 * PASID table is per device for better security. Therefore, for 339 * each bind of a new device even with an existing PASID, we need to 340 * call the nested mode setup function here. 341 */ 342 spin_lock(&iommu->lock); 343 ret = intel_pasid_setup_nested(iommu, dev, 344 (pgd_t *)(uintptr_t)data->gpgd, 345 data->hpasid, &data->vtd, dmar_domain, 346 data->addr_width); 347 spin_unlock(&iommu->lock); 348 if (ret) { 349 dev_err_ratelimited(dev, "Failed to set up PASID %llu in nested mode, Err %d\n", 350 data->hpasid, ret); 351 /* 352 * PASID entry should be in cleared state if nested mode 353 * set up failed. So we only need to clear IOASID tracking 354 * data such that free call will succeed. 355 */ 356 kfree(sdev); 357 goto out; 358 } 359 360 svm->flags |= SVM_FLAG_GUEST_MODE; 361 362 init_rcu_head(&sdev->rcu); 363 list_add_rcu(&sdev->list, &svm->devs); 364 out: 365 if (!IS_ERR_OR_NULL(svm) && list_empty(&svm->devs)) { 366 ioasid_set_data(data->hpasid, NULL); 367 kfree(svm); 368 } 369 370 mutex_unlock(&pasid_mutex); 371 return ret; 372 } 373 374 int intel_svm_unbind_gpasid(struct device *dev, int pasid) 375 { 376 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); 377 struct intel_svm_dev *sdev; 378 struct intel_svm *svm; 379 int ret = -EINVAL; 380 381 if (WARN_ON(!iommu)) 382 return -EINVAL; 383 384 mutex_lock(&pasid_mutex); 385 svm = ioasid_find(NULL, pasid, NULL); 386 if (!svm) { 387 ret = -EINVAL; 388 goto out; 389 } 390 391 if (IS_ERR(svm)) { 392 ret = PTR_ERR(svm); 393 goto out; 394 } 395 396 for_each_svm_dev(sdev, svm, dev) { 397 ret = 0; 398 if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) 399 sdev->users--; 400 if (!sdev->users) { 401 list_del_rcu(&sdev->list); 402 intel_pasid_tear_down_entry(iommu, dev, 403 svm->pasid, false); 404 intel_svm_drain_prq(dev, svm->pasid); 405 kfree_rcu(sdev, rcu); 406 407 if (list_empty(&svm->devs)) { 408 /* 409 * We do not free the IOASID here in that 410 * IOMMU driver did not allocate it. 411 * Unlike native SVM, IOASID for guest use was 412 * allocated prior to the bind call. 413 * In any case, if the free call comes before 414 * the unbind, IOMMU driver will get notified 415 * and perform cleanup. 416 */ 417 ioasid_set_data(pasid, NULL); 418 kfree(svm); 419 } 420 } 421 break; 422 } 423 out: 424 mutex_unlock(&pasid_mutex); 425 return ret; 426 } 427 428 /* Caller must hold pasid_mutex, mm reference */ 429 static int 430 intel_svm_bind_mm(struct device *dev, int flags, struct svm_dev_ops *ops, 431 struct mm_struct *mm, struct intel_svm_dev **sd) 432 { 433 struct intel_iommu *iommu = intel_svm_device_to_iommu(dev); 434 struct device_domain_info *info; 435 struct intel_svm_dev *sdev; 436 struct intel_svm *svm = NULL; 437 int pasid_max; 438 int ret; 439 440 if (!iommu || dmar_disabled) 441 return -EINVAL; 442 443 if (!intel_svm_capable(iommu)) 444 return -ENOTSUPP; 445 446 if (dev_is_pci(dev)) { 447 pasid_max = pci_max_pasids(to_pci_dev(dev)); 448 if (pasid_max < 0) 449 return -EINVAL; 450 } else 451 pasid_max = 1 << 20; 452 453 /* Bind supervisor PASID shuld have mm = NULL */ 454 if (flags & SVM_FLAG_SUPERVISOR_MODE) { 455 if (!ecap_srs(iommu->ecap) || mm) { 456 pr_err("Supervisor PASID with user provided mm.\n"); 457 return -EINVAL; 458 } 459 } 460 461 if (!(flags & SVM_FLAG_PRIVATE_PASID)) { 462 struct intel_svm *t; 463 464 list_for_each_entry(t, &global_svm_list, list) { 465 if (t->mm != mm || (t->flags & SVM_FLAG_PRIVATE_PASID)) 466 continue; 467 468 svm = t; 469 if (svm->pasid >= pasid_max) { 470 dev_warn(dev, 471 "Limited PASID width. Cannot use existing PASID %d\n", 472 svm->pasid); 473 ret = -ENOSPC; 474 goto out; 475 } 476 477 /* Find the matching device in svm list */ 478 for_each_svm_dev(sdev, svm, dev) { 479 if (sdev->ops != ops) { 480 ret = -EBUSY; 481 goto out; 482 } 483 sdev->users++; 484 goto success; 485 } 486 487 break; 488 } 489 } 490 491 sdev = kzalloc(sizeof(*sdev), GFP_KERNEL); 492 if (!sdev) { 493 ret = -ENOMEM; 494 goto out; 495 } 496 sdev->dev = dev; 497 498 ret = intel_iommu_enable_pasid(iommu, dev); 499 if (ret) { 500 kfree(sdev); 501 goto out; 502 } 503 504 info = get_domain_info(dev); 505 sdev->did = FLPT_DEFAULT_DID; 506 sdev->sid = PCI_DEVID(info->bus, info->devfn); 507 if (info->ats_enabled) { 508 sdev->dev_iotlb = 1; 509 sdev->qdep = info->ats_qdep; 510 if (sdev->qdep >= QI_DEV_EIOTLB_MAX_INVS) 511 sdev->qdep = 0; 512 } 513 514 /* Finish the setup now we know we're keeping it */ 515 sdev->users = 1; 516 sdev->ops = ops; 517 init_rcu_head(&sdev->rcu); 518 519 if (!svm) { 520 svm = kzalloc(sizeof(*svm), GFP_KERNEL); 521 if (!svm) { 522 ret = -ENOMEM; 523 kfree(sdev); 524 goto out; 525 } 526 svm->iommu = iommu; 527 528 if (pasid_max > intel_pasid_max_id) 529 pasid_max = intel_pasid_max_id; 530 531 /* Do not use PASID 0, reserved for RID to PASID */ 532 svm->pasid = ioasid_alloc(NULL, PASID_MIN, 533 pasid_max - 1, svm); 534 if (svm->pasid == INVALID_IOASID) { 535 kfree(svm); 536 kfree(sdev); 537 ret = -ENOSPC; 538 goto out; 539 } 540 svm->notifier.ops = &intel_mmuops; 541 svm->mm = mm; 542 svm->flags = flags; 543 INIT_LIST_HEAD_RCU(&svm->devs); 544 INIT_LIST_HEAD(&svm->list); 545 ret = -ENOMEM; 546 if (mm) { 547 ret = mmu_notifier_register(&svm->notifier, mm); 548 if (ret) { 549 ioasid_free(svm->pasid); 550 kfree(svm); 551 kfree(sdev); 552 goto out; 553 } 554 } 555 556 spin_lock(&iommu->lock); 557 ret = intel_pasid_setup_first_level(iommu, dev, 558 mm ? mm->pgd : init_mm.pgd, 559 svm->pasid, FLPT_DEFAULT_DID, 560 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | 561 (cpu_feature_enabled(X86_FEATURE_LA57) ? 562 PASID_FLAG_FL5LP : 0)); 563 spin_unlock(&iommu->lock); 564 if (ret) { 565 if (mm) 566 mmu_notifier_unregister(&svm->notifier, mm); 567 ioasid_free(svm->pasid); 568 kfree(svm); 569 kfree(sdev); 570 goto out; 571 } 572 573 list_add_tail(&svm->list, &global_svm_list); 574 } else { 575 /* 576 * Binding a new device with existing PASID, need to setup 577 * the PASID entry. 578 */ 579 spin_lock(&iommu->lock); 580 ret = intel_pasid_setup_first_level(iommu, dev, 581 mm ? mm->pgd : init_mm.pgd, 582 svm->pasid, FLPT_DEFAULT_DID, 583 (mm ? 0 : PASID_FLAG_SUPERVISOR_MODE) | 584 (cpu_feature_enabled(X86_FEATURE_LA57) ? 585 PASID_FLAG_FL5LP : 0)); 586 spin_unlock(&iommu->lock); 587 if (ret) { 588 kfree(sdev); 589 goto out; 590 } 591 } 592 list_add_rcu(&sdev->list, &svm->devs); 593 success: 594 sdev->pasid = svm->pasid; 595 sdev->sva.dev = dev; 596 if (sd) 597 *sd = sdev; 598 ret = 0; 599 out: 600 return ret; 601 } 602 603 /* Caller must hold pasid_mutex */ 604 static int intel_svm_unbind_mm(struct device *dev, int pasid) 605 { 606 struct intel_svm_dev *sdev; 607 struct intel_iommu *iommu; 608 struct intel_svm *svm; 609 int ret = -EINVAL; 610 611 iommu = intel_svm_device_to_iommu(dev); 612 if (!iommu) 613 goto out; 614 615 svm = ioasid_find(NULL, pasid, NULL); 616 if (!svm) 617 goto out; 618 619 if (IS_ERR(svm)) { 620 ret = PTR_ERR(svm); 621 goto out; 622 } 623 624 for_each_svm_dev(sdev, svm, dev) { 625 ret = 0; 626 sdev->users--; 627 if (!sdev->users) { 628 list_del_rcu(&sdev->list); 629 /* Flush the PASID cache and IOTLB for this device. 630 * Note that we do depend on the hardware *not* using 631 * the PASID any more. Just as we depend on other 632 * devices never using PASIDs that they have no right 633 * to use. We have a *shared* PASID table, because it's 634 * large and has to be physically contiguous. So it's 635 * hard to be as defensive as we might like. */ 636 intel_pasid_tear_down_entry(iommu, dev, 637 svm->pasid, false); 638 intel_svm_drain_prq(dev, svm->pasid); 639 kfree_rcu(sdev, rcu); 640 641 if (list_empty(&svm->devs)) { 642 ioasid_free(svm->pasid); 643 if (svm->mm) 644 mmu_notifier_unregister(&svm->notifier, svm->mm); 645 list_del(&svm->list); 646 /* We mandate that no page faults may be outstanding 647 * for the PASID when intel_svm_unbind_mm() is called. 648 * If that is not obeyed, subtle errors will happen. 649 * Let's make them less subtle... */ 650 memset(svm, 0x6b, sizeof(*svm)); 651 kfree(svm); 652 } 653 } 654 break; 655 } 656 out: 657 658 return ret; 659 } 660 661 /* Page request queue descriptor */ 662 struct page_req_dsc { 663 union { 664 struct { 665 u64 type:8; 666 u64 pasid_present:1; 667 u64 priv_data_present:1; 668 u64 rsvd:6; 669 u64 rid:16; 670 u64 pasid:20; 671 u64 exe_req:1; 672 u64 pm_req:1; 673 u64 rsvd2:10; 674 }; 675 u64 qw_0; 676 }; 677 union { 678 struct { 679 u64 rd_req:1; 680 u64 wr_req:1; 681 u64 lpig:1; 682 u64 prg_index:9; 683 u64 addr:52; 684 }; 685 u64 qw_1; 686 }; 687 u64 priv_data[2]; 688 }; 689 690 #define PRQ_RING_MASK ((0x1000 << PRQ_ORDER) - 0x20) 691 692 static bool access_error(struct vm_area_struct *vma, struct page_req_dsc *req) 693 { 694 unsigned long requested = 0; 695 696 if (req->exe_req) 697 requested |= VM_EXEC; 698 699 if (req->rd_req) 700 requested |= VM_READ; 701 702 if (req->wr_req) 703 requested |= VM_WRITE; 704 705 return (requested & ~vma->vm_flags) != 0; 706 } 707 708 static bool is_canonical_address(u64 addr) 709 { 710 int shift = 64 - (__VIRTUAL_MASK_SHIFT + 1); 711 long saddr = (long) addr; 712 713 return (((saddr << shift) >> shift) == saddr); 714 } 715 716 /** 717 * intel_svm_drain_prq - Drain page requests and responses for a pasid 718 * @dev: target device 719 * @pasid: pasid for draining 720 * 721 * Drain all pending page requests and responses related to @pasid in both 722 * software and hardware. This is supposed to be called after the device 723 * driver has stopped DMA, the pasid entry has been cleared, and both IOTLB 724 * and DevTLB have been invalidated. 725 * 726 * It waits until all pending page requests for @pasid in the page fault 727 * queue are completed by the prq handling thread. Then follow the steps 728 * described in VT-d spec CH7.10 to drain all page requests and page 729 * responses pending in the hardware. 730 */ 731 static void intel_svm_drain_prq(struct device *dev, int pasid) 732 { 733 struct device_domain_info *info; 734 struct dmar_domain *domain; 735 struct intel_iommu *iommu; 736 struct qi_desc desc[3]; 737 struct pci_dev *pdev; 738 int head, tail; 739 u16 sid, did; 740 int qdep; 741 742 info = get_domain_info(dev); 743 if (WARN_ON(!info || !dev_is_pci(dev))) 744 return; 745 746 if (!info->pri_enabled) 747 return; 748 749 iommu = info->iommu; 750 domain = info->domain; 751 pdev = to_pci_dev(dev); 752 sid = PCI_DEVID(info->bus, info->devfn); 753 did = domain->iommu_did[iommu->seq_id]; 754 qdep = pci_ats_queue_depth(pdev); 755 756 /* 757 * Check and wait until all pending page requests in the queue are 758 * handled by the prq handling thread. 759 */ 760 prq_retry: 761 reinit_completion(&iommu->prq_complete); 762 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 763 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 764 while (head != tail) { 765 struct page_req_dsc *req; 766 767 req = &iommu->prq[head / sizeof(*req)]; 768 if (!req->pasid_present || req->pasid != pasid) { 769 head = (head + sizeof(*req)) & PRQ_RING_MASK; 770 continue; 771 } 772 773 wait_for_completion(&iommu->prq_complete); 774 goto prq_retry; 775 } 776 777 /* 778 * Perform steps described in VT-d spec CH7.10 to drain page 779 * requests and responses in hardware. 780 */ 781 memset(desc, 0, sizeof(desc)); 782 desc[0].qw0 = QI_IWD_STATUS_DATA(QI_DONE) | 783 QI_IWD_FENCE | 784 QI_IWD_TYPE; 785 desc[1].qw0 = QI_EIOTLB_PASID(pasid) | 786 QI_EIOTLB_DID(did) | 787 QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) | 788 QI_EIOTLB_TYPE; 789 desc[2].qw0 = QI_DEV_EIOTLB_PASID(pasid) | 790 QI_DEV_EIOTLB_SID(sid) | 791 QI_DEV_EIOTLB_QDEP(qdep) | 792 QI_DEIOTLB_TYPE | 793 QI_DEV_IOTLB_PFSID(info->pfsid); 794 qi_retry: 795 reinit_completion(&iommu->prq_complete); 796 qi_submit_sync(iommu, desc, 3, QI_OPT_WAIT_DRAIN); 797 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) { 798 wait_for_completion(&iommu->prq_complete); 799 goto qi_retry; 800 } 801 } 802 803 static irqreturn_t prq_event_thread(int irq, void *d) 804 { 805 struct intel_iommu *iommu = d; 806 struct intel_svm *svm = NULL; 807 int head, tail, handled = 0; 808 809 /* Clear PPR bit before reading head/tail registers, to 810 * ensure that we get a new interrupt if needed. */ 811 writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); 812 813 tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; 814 head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; 815 while (head != tail) { 816 struct intel_svm_dev *sdev; 817 struct vm_area_struct *vma; 818 struct page_req_dsc *req; 819 struct qi_desc resp; 820 int result; 821 vm_fault_t ret; 822 u64 address; 823 824 handled = 1; 825 826 req = &iommu->prq[head / sizeof(*req)]; 827 828 result = QI_RESP_FAILURE; 829 address = (u64)req->addr << VTD_PAGE_SHIFT; 830 if (!req->pasid_present) { 831 pr_err("%s: Page request without PASID: %08llx %08llx\n", 832 iommu->name, ((unsigned long long *)req)[0], 833 ((unsigned long long *)req)[1]); 834 goto no_pasid; 835 } 836 837 if (!svm || svm->pasid != req->pasid) { 838 rcu_read_lock(); 839 svm = ioasid_find(NULL, req->pasid, NULL); 840 /* It *can't* go away, because the driver is not permitted 841 * to unbind the mm while any page faults are outstanding. 842 * So we only need RCU to protect the internal idr code. */ 843 rcu_read_unlock(); 844 if (IS_ERR_OR_NULL(svm)) { 845 pr_err("%s: Page request for invalid PASID %d: %08llx %08llx\n", 846 iommu->name, req->pasid, ((unsigned long long *)req)[0], 847 ((unsigned long long *)req)[1]); 848 goto no_pasid; 849 } 850 } 851 852 result = QI_RESP_INVALID; 853 /* Since we're using init_mm.pgd directly, we should never take 854 * any faults on kernel addresses. */ 855 if (!svm->mm) 856 goto bad_req; 857 858 /* If address is not canonical, return invalid response */ 859 if (!is_canonical_address(address)) 860 goto bad_req; 861 862 /* If the mm is already defunct, don't handle faults. */ 863 if (!mmget_not_zero(svm->mm)) 864 goto bad_req; 865 866 mmap_read_lock(svm->mm); 867 vma = find_extend_vma(svm->mm, address); 868 if (!vma || address < vma->vm_start) 869 goto invalid; 870 871 if (access_error(vma, req)) 872 goto invalid; 873 874 ret = handle_mm_fault(vma, address, 875 req->wr_req ? FAULT_FLAG_WRITE : 0); 876 if (ret & VM_FAULT_ERROR) 877 goto invalid; 878 879 result = QI_RESP_SUCCESS; 880 invalid: 881 mmap_read_unlock(svm->mm); 882 mmput(svm->mm); 883 bad_req: 884 /* Accounting for major/minor faults? */ 885 rcu_read_lock(); 886 list_for_each_entry_rcu(sdev, &svm->devs, list) { 887 if (sdev->sid == req->rid) 888 break; 889 } 890 /* Other devices can go away, but the drivers are not permitted 891 * to unbind while any page faults might be in flight. So it's 892 * OK to drop the 'lock' here now we have it. */ 893 rcu_read_unlock(); 894 895 if (WARN_ON(&sdev->list == &svm->devs)) 896 sdev = NULL; 897 898 if (sdev && sdev->ops && sdev->ops->fault_cb) { 899 int rwxp = (req->rd_req << 3) | (req->wr_req << 2) | 900 (req->exe_req << 1) | (req->pm_req); 901 sdev->ops->fault_cb(sdev->dev, req->pasid, req->addr, 902 req->priv_data, rwxp, result); 903 } 904 /* We get here in the error case where the PASID lookup failed, 905 and these can be NULL. Do not use them below this point! */ 906 sdev = NULL; 907 svm = NULL; 908 no_pasid: 909 if (req->lpig || req->priv_data_present) { 910 /* 911 * Per VT-d spec. v3.0 ch7.7, system software must 912 * respond with page group response if private data 913 * is present (PDP) or last page in group (LPIG) bit 914 * is set. This is an additional VT-d feature beyond 915 * PCI ATS spec. 916 */ 917 resp.qw0 = QI_PGRP_PASID(req->pasid) | 918 QI_PGRP_DID(req->rid) | 919 QI_PGRP_PASID_P(req->pasid_present) | 920 QI_PGRP_PDP(req->pasid_present) | 921 QI_PGRP_RESP_CODE(result) | 922 QI_PGRP_RESP_TYPE; 923 resp.qw1 = QI_PGRP_IDX(req->prg_index) | 924 QI_PGRP_LPIG(req->lpig); 925 926 if (req->priv_data_present) 927 memcpy(&resp.qw2, req->priv_data, 928 sizeof(req->priv_data)); 929 resp.qw2 = 0; 930 resp.qw3 = 0; 931 qi_submit_sync(iommu, &resp, 1, 0); 932 } 933 head = (head + sizeof(*req)) & PRQ_RING_MASK; 934 } 935 936 dmar_writeq(iommu->reg + DMAR_PQH_REG, tail); 937 938 /* 939 * Clear the page request overflow bit and wake up all threads that 940 * are waiting for the completion of this handling. 941 */ 942 if (readl(iommu->reg + DMAR_PRS_REG) & DMA_PRS_PRO) 943 writel(DMA_PRS_PRO, iommu->reg + DMAR_PRS_REG); 944 945 if (!completion_done(&iommu->prq_complete)) 946 complete(&iommu->prq_complete); 947 948 return IRQ_RETVAL(handled); 949 } 950 951 #define to_intel_svm_dev(handle) container_of(handle, struct intel_svm_dev, sva) 952 struct iommu_sva * 953 intel_svm_bind(struct device *dev, struct mm_struct *mm, void *drvdata) 954 { 955 struct iommu_sva *sva = ERR_PTR(-EINVAL); 956 struct intel_svm_dev *sdev = NULL; 957 int flags = 0; 958 int ret; 959 960 /* 961 * TODO: Consolidate with generic iommu-sva bind after it is merged. 962 * It will require shared SVM data structures, i.e. combine io_mm 963 * and intel_svm etc. 964 */ 965 if (drvdata) 966 flags = *(int *)drvdata; 967 mutex_lock(&pasid_mutex); 968 ret = intel_svm_bind_mm(dev, flags, NULL, mm, &sdev); 969 if (ret) 970 sva = ERR_PTR(ret); 971 else if (sdev) 972 sva = &sdev->sva; 973 else 974 WARN(!sdev, "SVM bind succeeded with no sdev!\n"); 975 976 mutex_unlock(&pasid_mutex); 977 978 return sva; 979 } 980 981 void intel_svm_unbind(struct iommu_sva *sva) 982 { 983 struct intel_svm_dev *sdev; 984 985 mutex_lock(&pasid_mutex); 986 sdev = to_intel_svm_dev(sva); 987 intel_svm_unbind_mm(sdev->dev, sdev->pasid); 988 mutex_unlock(&pasid_mutex); 989 } 990 991 int intel_svm_get_pasid(struct iommu_sva *sva) 992 { 993 struct intel_svm_dev *sdev; 994 int pasid; 995 996 mutex_lock(&pasid_mutex); 997 sdev = to_intel_svm_dev(sva); 998 pasid = sdev->pasid; 999 mutex_unlock(&pasid_mutex); 1000 1001 return pasid; 1002 } 1003